diff --git a/ChangeLog.md b/ChangeLog.md index fb7d45237..618900f1a 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -13,14 +13,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed ### Added - +- #194 Reliable delivery of readings to MMW. Also called batch or queued messaege delivery. Readings can be queued for later send. This can accumulate a number of readings, and on sending to MMW if they don't receive a SUCCESS http 201, automatically go into a queue for next time a connection is made. ### Removed ### Fixed -- Fixed GitHub actions for pull requests from forks. *** + +## [0.35.0] + +### Added +- Support [GroPoint Profile GPLP-8 Eight-Segment Soil Moisture and Temperature Profiling Probe](https://www.gropoint.com/products/soil-sensors/gropoint-profile) + ## [0.34.1] ### Changed @@ -32,6 +37,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 By default, all polling is off, but polling is enabled for a modem sensor when a sensor is created and attached to a modem. This functionailty is inspired from [neilh10](https://github.com/EnviroDIY/ModularSensors/commits?author=neilh10). +### Fixed +- Fixed GitHub actions for pull requests from forks. + ## [0.34.0] ### Changed diff --git a/a/mayfly1_wifi_wroom/.gitignore b/a/mayfly1_wifi_wroom/.gitignore new file mode 100644 index 000000000..89cc49cbd --- /dev/null +++ b/a/mayfly1_wifi_wroom/.gitignore @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/a/mayfly1_wifi_wroom/ReadMe.md b/a/mayfly1_wifi_wroom/ReadMe.md new file mode 100644 index 000000000..e62cf7e26 --- /dev/null +++ b/a/mayfly1_wifi_wroom/ReadMe.md @@ -0,0 +1,18 @@ +# DRWI Sites with a Mayfly 1.x and EnviroDIY ESP32 WiFi Bees +Alpha test sketch for transmitting over WiFi ESP32 WROOM + +The hardware configuration used in this example: + * Mayfly v1.1 board + * EnviroDIY ESP32 WiFi module + +This is used to intergrate in Reliable Delivery +https://github.com/EnviroDIY/ModularSensors/issues/194 +Setup ms_cfg.h for your network parameters + +230710 Overnight testing seemed to go well. Going to submit the PR +230709 - 24hrs expanded "AT" debug tty230708-1539_dvlp_esp32.txt +230708: Switched to 57600 to wroom and testing +230708: Initial Testing sames as S6B at 9600baud + + + diff --git a/a/mayfly1_wifi_wroom/git_rev_macro.py b/a/mayfly1_wifi_wroom/git_rev_macro.py new file mode 100644 index 000000000..aa4fb114b --- /dev/null +++ b/a/mayfly1_wifi_wroom/git_rev_macro.py @@ -0,0 +1,7 @@ +import subprocess + +git_rev = str(subprocess.check_output(["git", "rev-parse","--abbrev-ref", "HEAD"]).strip()) +git_usr = str(subprocess.check_output(["git", "config","user.name"]).strip()) + +print('-DPIO_SRC_REV={'+(''.join('0x%02x,' % ord(c) for c in git_rev) ) +'0x0}') +print('-DPIO_SRC_USR={'+(''.join('0x%02x,' % ord(c) for c in git_usr) ) +'0x0}') \ No newline at end of file diff --git a/a/mayfly1_wifi_wroom/platformio.ini b/a/mayfly1_wifi_wroom/platformio.ini new file mode 100644 index 000000000..c2dd47e51 --- /dev/null +++ b/a/mayfly1_wifi_wroom/platformio.ini @@ -0,0 +1,204 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; http://docs.platformio.org/page/projectconf.html + +[platformio] +description = enviroDIY/ModularSensors a\mayfly1_wifi_wroom +;src_dir =. +default_envs = mayfly + +[common] +cmn_lib_deps= + ; There are 3 levels of code production + ; The stream ( rel1_dvlp1m) is used in alpha coding and development + ; alpha - this platformio.ini - which enables (rel1_dvlp1m) editing in ModularSensors\src & a\ex + ; development see (rel1_dvlp1m) examples\tu_ec + ; production see (release1) examples\tu_ex + ; + ; For alpha coding, + ; (2023-Jan-02 : this worked, however PIO can change, and now seems to be doing realtime analysis of dependencies) + ; and periodically when need to update all libs including ModularSensors #rel1_dvlp1m + ; for clean lib - delete .pio, + ; pio pkg install - execute + ; leave .pio/libdeps/xxx/EnviroDIY_modularSensors as doing realtime dependencie checks + ;https://github.com/neilh10/ModularSensors#rel1_dvlp1m + https://github.com/EnviroDIY/ModularSensors.git#develop +; ^^ Use this when you want to pull from a specific branch + ; These are latest beyond EnviroDIY releases + https://github.com/vshymanskyy/StreamDebugger ;Debug when needed + ; StreamDebugger ; Same as above + ;https://github.com/neilh10/SensorModbusMaster ; default release1 + ;https://github.com/enviroDIY/SensorModbusMaster + ;https://github.com/neilh10/Adafruit_INA219.git + ; del https://github.com/adafruit/SdFat ;- need adafruit/SdFat for multiple SDx + ;https://github.com/greiman/SdFat.git ;- + ;https://github.com/neilh10/EnableInterrupt - old v1.0.0 need v1.1 + ;https://github.com/GreyGnome/EnableInterrupt + ; del https://github.com/arduino-libraries/NTPClient - repalced + ;; see lib_samd_deps for Adafruit_SPIFlash & Adafruit_TinyUSB_Arduino & Adafruit-GFX-Library Adafruit_NeoPixel + ; Need extra libs, so enable STD_LIBS on 1st pass enable ModularSensors then build pulling in ref libs, + ; +; STD_LIBS pulled in by ModularSensors, needed if not enabling ModularSensors to Historical Ref + ;https://github.com/soligen2010/Adafruit_ADS1X15 + ;https://github.com/adafruit/Adafruit_AM2315 + ;https://github.com/adafruit/Adafruit_AM2320 + ;https://github.com/adafruit/Adafruit_BME280_Library + ;https://github.com/adafruit/Adafruit_MPL115A2 + ;https://github.com/adafruit/DHT-sensor-library + ;https://github.com/adafruit/Adafruit_Sensor + ;https://github.com/milesburton/Arduino-Temperature-Control-Library ;Pulls in DallasTemperature + ;https://github.com/neilh10/KellerModbus + ;https://github.com/EnviroDIY/KellerModbus + ;https://github.com/NorthernWidget/MS5803 + ;https://github.com/PaulStoffregen/OneWire + ;https://github.com/knolleary/pubsubclient.git + ;bug https://github.com/EnviroDIY/TinyGSM.git ; bug EnviroDIY/ModularSensors/issues 311 + ;https://github.com/neilh10/TinyGSM.git#rel1 ;Bug fixes over Envirodiy + https://github.com/vshymanskyy/TinyGSM + ;https://github.com/EnviroDIY/YosemitechModbus#develop ;https://github.com/EnviroDIY/YosemitechModbus/issues/32 + https://github.com/neilh10/Arduino-SDI-12#release1 + ;https://github.com/EnviroDIY/Arduino-SDI-12.git bug with extraWakeTime=10 + ;https://github.com/EnviroDIY/Tally_Library.git#Dev_I2C + ; + ;Historical reference + ;https://github.com/greiman/SdFat.git ;- need adafruit/SdFat for multiple SDx + ;https://github.com/neilh10/STC3100arduino.git # For Mayfly board Knh002rev7 + ;https://github.com/neilh10/Arduino-SerialCommand + ;https://github.com/neilh10/InsituModbus + ;https://github.com/neilh10/KellerModbus + +; Ignore these folders or PlatformIO will double count all the dependencies +; Development: ModularSensors Release1: src ? +; ?? .pioenvs, .piolibdeps, .vscode, include, doc, examples, sensor_tests, compile_tests, pioScripts +cmn_lib_ignore = .git, doc, examples, arduino_update, sensor_tests, + ;EnviroDIY_DS3231 + ;SensorModbusMaster + Arduino-SDI-12 + ;TinyGSM + Adafruit_INA21 + EnviroDIY_ModularSensors + ModularSensors + +lib_avr_deps = ;Specific Mayfly board goes in [Mayfly] + +; for development, can edit directly in these directories ... relative to src_dir +; Use the src filter to ensure subfolders are built +; If enabled, disable in cmn_lib_deps=[]ModularSensors, and may need to delete duplicate directory .libdeps/ModularSensors +cmn_src_filter = +<*> +<../../../src> +<../../../src/sensors> +<../../../src/publishers> +<../../../src/modems> + +#build_flags -v for detailed cc verbose, single threaded takes a long time +cmn_build_flags = + -Isrc + -I../../src ; .h For easy debug + -I../../src/sensors ; .h For easy debug + !python git_rev_macro.py ;returns -DPIO_SRC_REV= + ;-DMQTT_MAX_PACKET_SIZE=240 + -DTINY_GSM_YIELD_MS=2 + ;-DTINY_GSM_RX_BUFFER=64 ;is this needed? + ;-DNO_FIRST_SYNC_WITH_NIST + -DMS_NETWORK_LAYER + -DNIST_SYNC_HOURLY + -DMS_DISCARD_HTTP_500 + ;-DUSE_RTCLIB=RTC_DS3231 ;not compiling + ;-DMS_TU_XX_DEBUG + ;-DMS_TU_XX_DEBUG_DEEP + ;-DSTREAMDEBUGGER_DBG + ;-DMS_DUMP_FREE_RAM + ;-DMS_TTY_USER_INPUT + ;-DSerialCommand_inFlash ;requires MS_TTY_USER_INPUT + ;-DMS_TTY_INPUT_COUNT ;only if no MS_TTY_USER_INPUT + ;-DSERIALCOMMAND_DEBUG + -DMS_LOGGERBASE_POSTS + ;-DMS_LOGGERBASE_SLEEP_DEBUG ;Need or below + ;-DMS_LOGGERBASE_DEBUG + ;-DMS_LOGGERBASE_DEEP_DEBUG + ;-DMS_LOGGERMODEM_DEBUG + ;-DMS_LOGGERMODEM_DEBUG_DEEP + ;-DMS_SENSORBASE_DEBUG + ;-DMS_VARIABLEARRAY_DEBUG + ;-DMS_VARIABLEARRAY_DEBUG_DEEP + ;-DMS_VARIABLEBASE_DEBUG + ;-DMS_VARIABLEBASE_DEBUG_DEEP + ;-DMS_DATAPUBLISHERBASE_DEBUG + ;-DMS_ENVIRODIYPUBLISHER_DEBUG + ;-DMS_THINGSPEAKPUBLISHER_DEBUG + ;-DMS_UBIDOTSPUBLISHER_DEBUG + ;-DMS_DIGIXBEEWIFI_DEBUG + ;-DMS_DIGIXBEEWIFI_DEBUG_DEEP + ;-DMS_DIGIXBEECELLULARTRANSPARENT_DEBUG + ;-DMS_DIGIXBEECELLULARTRANSPARENT_DEBUG_DEEP + ;-DMS_DIGIXBEE_DEBUG + ;-DTINY_GSM_DEBUG=Serial + ;-DTinyGsmClientXbee_DBG=Serial + ;-DMS_ESPRESSIFESP8266_DEBUG + ;-DMS_ESPRESSIFESP8266_DEBUG_DEEP + ;-DMS_STSTC3100SENSOR_DEBUG + ;-DMS_BATTERYMANAGEMENT_DEBUG + ;-DMS_PROCESSORSTATS_DEBUG + ;-DMS_PROCESSORADC_DEBUG + ;-DMS_TIINA219M_DEBUG + ;-DMS_AOSONGAM2315_DEBUG + ;-DMS_EXTERNALVOLTAGE_DEBUG + ;-DMS_EXTERNALVOLTAGE_DEBUG_DEEP + ;-DMS_KELLERPARENT_DEBUG + ;-DMS_KELLERPARENT_DEBUG_DEEP has problem + ;-DMS_MODBUS_DEBUG + -DSENSORMODBUSMASTER_NO_DBG + ;-DMS_ANALOGELECCONDUCTIVITY_DEBUG + ;-DMS_ANALOGELECCONDUCTIVITY_DEBUG_DEEP + ;-DMS_SDI12SENSORS_DEBUG ;LT500 sometimes responding + ;-DMS_SDI12SENSORS_DEBUG_DEEP + ;For -DMS_SDI12 options: NONE or MS_SDI12_SINGLE_LINE_RESPONSE OR MS_SDI12_NON_CONCURRENT not BOTH + ;-DMS_SDI12_SINGLE_LINE_RESPONSE + ;-DMS_SDI12_NON_CONCURRENT + ;-DENVIRODIY_SDI12_USE_CRC + ;-DMS_ENVIRODIYPUBLISHER_DEBUG + ;-DMS_ENVIRODIYPUBLISHER_DEBUG_DEEP + ;-DSodaq_DS3231_DEBUG + -DMS_WATCHDOGAVR_DEBUG + -DMS_WATCHDOGAVR_DEBUG_DEEP + +[env:mayfly] +;upload_port = COM11 +monitor_speed = 115200 +board = mayfly +platform = atmelavr +framework = arduino +;debug_tool = blackmagic +;debug_port = test01u:4242 + +lib_compat_mode = strict +lib_ldf_mode = deep+ +lib_ignore = ${common.cmn_lib_ignore}, RTCZero +build_src_filter = ${common.cmn_src_filter} +;build_unflags = -Os +build_flags = + ${common.cmn_build_flags} + -DSDI12_EXTERNAL_PCINT + -DNEOSWSERIAL_EXTERNAL_PCINT + -fmax-errors=5 + ;-Wl,-Map,.pio/build/mayfly/firmware.map + +lib_deps =${common.cmn_lib_deps} ${common.lib_avr_deps} + ;https://github.com/neilh10/AltSoftSerial ; Need this for managing pwr - data pin off as well. + ;https://github.com/PaulStoffregen/AltSoftSerial.git + ; + ;https://github.com/EnviroDIY/SoftwaterSerial_ExternalInts.git + ;https://github.com/SRGDamia1/NeoSWSerial.git + ;https://github.com/neilh10/Sodaq_DS3231#release1 ; Bug fix #34 - change to RTClibnh with DS3231 + +monitor_filters = log2file, default, time +; +; The following monitor_flags setting is needed becasue in transparent mode the XBee3 +; uses CR line endings instead of the more typical CR+LF. This setting enables you +; to see all back and forth communication. +; (https://www.envirodiy.org/topic/connecting-xbee3-lte-to-the-internet/#post-13312) +monitor_flags = + --eol + CR diff --git a/a/mayfly1_wifi_wroom/src/mayfly1_wifi_wroom.cpp b/a/mayfly1_wifi_wroom/src/mayfly1_wifi_wroom.cpp new file mode 100644 index 000000000..972121721 --- /dev/null +++ b/a/mayfly1_wifi_wroom/src/mayfly1_wifi_wroom.cpp @@ -0,0 +1,456 @@ +/** ========================================================================= + * @file DRWI_Mayfly1_WiFi_wroom.ino + * @brief Adapted for for testing ReliableDelivery/ WiFi . + * + * This example shows proper settings for the following configuration: + * + * Mayfly v1.x board + * EnviroDIY ESP32 Wifi Bee module + * Internal sensors + * + * @author Neil Hancock & Sara Geleskie Damiano + * @copyright (c) 2017-2022 Stroud Water Research Center (SWRC) + * and the EnviroDIY Development Team + * This example is published under the BSD-3 license. + * + + * Hardware Platform: EnviroDIY Mayfly Arduino Datalogger + * + * DISCLAIMER: + * THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. + * ======================================================================= */ + +// ========================================================================== +// Defines for the Arduino IDE +// NOTE: These are ONLY needed to compile with the Arduino IDE. +// If you use PlatformIO, you should set these build flags in your +// platformio.ini +// ========================================================================== +/** Start [defines] */ +#ifndef TINY_GSM_RX_BUFFER +#define TINY_GSM_RX_BUFFER 256 +#endif +#ifndef TINY_GSM_YIELD_MS +#define TINY_GSM_YIELD_MS 2 +#endif +/** End [defines] */ + +// ========================================================================== +// Include the libraries required for any data logger +// ========================================================================== +/** Start [includes] */ +// The Arduino library is needed for every Arduino program. +#include +//Site https://monitormywatershed.org/sites/bq_test01/ +#include "ms_cfg.h" + +// EnableInterrupt is used by ModularSensors for external and pin change +// interrupts and must be explicitly included in the main program. +#include + +// Include the main header for ModularSensors +#include +/** End [includes] */ + + +// ========================================================================== +// Data Logging Options +// ========================================================================== +// Details for this build +extern const String build_ref = "a\\" __FILE__ " " __DATE__ " " __TIME__ " "; +#ifdef PIO_SRC_REV +const char git_branch[] = PIO_SRC_REV; +#else +const char git_branch[] = "brnch"; +#endif +#ifdef PIO_SRC_USR +const char git_usr[] = PIO_SRC_USR; +#else +const char git_usr[] = "usr"; +#endif +/** Start [logging_options] */ +// The name of this program file +const char* sketchName = "mayfly1_wifi_wroom.cpp"; +// Logger ID, also becomes the prefix for the name of the data file on SD card +const char* LoggerID = "reldlv1"; +// How frequently (in minutes) to log data +const uint8_t loggingInterval = 2; +// Your logger's timezone. +const int8_t timeZone = -8; // PST +// NOTE: Daylight savings time will not be applied! Please use standard time! +#define SerialStd STANDARD_SERIAL_OUTPUT + +// Set the input and output pins for the logger +// NOTE: Use -1 for pins that do not apply +const int32_t serialBaud = 115200; // Baud rate for debugging +const int8_t greenLED = 8; // Pin for the green LED +const int8_t redLED = 9; // Pin for the red LED +const int8_t buttonPin = 21; // Pin for debugging mode (ie, button pin) +const int8_t wakePin = 31; // MCU interrupt/alarm pin to wake from sleep +// Mayfly 0.x, 1.x D31 = A7 +const int8_t sdCardPwrPin = -1; // MCU SD card power pin +const int8_t sdCardSSPin = 12; // SD card chip select/slave select pin +const int8_t sensorPowerPin = 22; // MCU pin controlling main sensor power +/** End [logging_options] */ + + +// ========================================================================== +// Wifi/Cellular Modem Options +// ========================================================================== +/** Start [espressif_esp32] */ +#include + +// Create a reference to the serial port for the modem + +HardwareSerial& modemSerial = Serial1; // Use hardware serial if possible +#define ESP32_MODEM_115K_BAUD 115200 +#define ESP32_MODEM_57K_BAUD 57600 +#define ESP32_MODEM_9K6_BAUD 9600 +#define ESP32_MODEM_DEF_BAUD ESP32_MODEM_57K_BAUD + const int32_t modemBaud = ESP32_MODEM_DEF_BAUD; // Communication speed of the modem +// NOTE: This baud rate too fast for the Mayfly. We'll slow it down in the +// setup. + +// Modem Pins - Describe the physical pin connection of your modem to your board +// NOTE: Use -1 for pins that do not apply +// Example pins here are for a EnviroDIY ESP32 Bluetooth/Wifi Bee with +// Mayfly 1.1 +const int8_t modemVccPin = 18; // MCU pin controlling modem power +const int8_t modemResetPin = -1; // MCU pin connected to modem reset pin +const int8_t modemLEDPin = redLED; // MCU pin connected an LED to show modem + // status + +// Network connection information +const char* wifiId = WIFIID_SSID_DEF; // WiFi access point name +const char* wifiPwd = WIFIPWD_DEF; // WiFi password (WPA2) + +// Create the modem object +EspressifESP32 modemESP(&modemSerial, modemVccPin, modemResetPin, wifiId, + wifiPwd); +// Create an extra reference to the modem by a generic name +EspressifESP32 modemPhy = modemESP; +/** End [espressif_esp32] */ + + +// ========================================================================== +// Using the Processor as a Sensor +// ========================================================================== +/** Start [processor_sensor] */ +#include + +// Create the main processor chip "sensor" - for general metadata +const char* mcuBoardVersion = "v1.1"; +ProcessorStats mcuBoard(mcuBoardVersion); +/** End [processor_sensor] */ + + +// ========================================================================== +// Maxim DS3231 RTC (Real Time Clock) +// Built in on all versions of the Mayfly +// ========================================================================== +/** Start [ds3231] */ +#if 0 +#include + +// Create a DS3231 sensor object +MaximDS3231 ds3231(1); +#endif +/** End [ds3231] */ + +#if defined SENSIRION_SHT4X_UUID +// ========================================================================== +// Sensirion SHT4X Digital Humidity and Temperature Sensor +// Built in on Mayfly 1.x +// ========================================================================== +/** Start [sensirion_sht4x] */ +#include + +// NOTE: Use -1 for any pins that don't apply or aren't being used. +const int8_t SHT4xPower = sensorPowerPin; // Power pin +const bool SHT4xUseHeater = true; + +// Create an Sensirion SHT4X sensor object +SensirionSHT4x sht4x(SHT4xPower, SHT4xUseHeater); +/** End [sensirion_sht4x] */ +#endif //SENSIRION_SHT4X_UUID + +// ========================================================================== +// Creating the Variable Array[s] and Filling with Variable Objects +// ========================================================================== +/** Start [variable_arrays] */ +//#include "ms_cfg.h" +Variable* variableList[] = { + // Should follow UUIDs + new ProcessorStats_SampleNumber(&mcuBoard), + new ProcessorStats_Battery(&mcuBoard), // Battery voltage (EnviroDIY_Mayfly_Batt) + new SensirionSHT4x_Temp(&sht4x), // Temperature (Sensirion_SHT40_Temperature) + new SensirionSHT4x_Humidity(&sht4x), // Relative humidity (Sensirion_SHT40_Humidity) +}; + +// All UUID's, device registration, and sampling feature information can be +// pasted directly from Monitor My Watershed. +// To get the list, click the "View token UUID list" button on the upper right +// of the site page. + +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** +// Check the order of your variables in the variable list!!! +// Be VERY certain that they match the order of your UUID's! +// Rearrange the variables in the variable list ABOVE if necessary to match! +// Do not change the order of the variables in the section below. +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** + +// Replace all of the text in the following section with the UUID array from +// MonitorMyWatershed + +/* clang-format off */ +// --------------------- Beginning of Token UUID List --------------------- +// see ms_cfg.h initially setup for https://monitormywatershed.org/sites/bq_test01/ + + +// ----------------------- End of Token UUID List ----------------------- +/* clang-format on */ + +// Count up the number of pointers in the array +int variableCount = sizeof(variableList) / sizeof(variableList[0]); + +// Create the VariableArray object +VariableArray varArray(variableCount, variableList,UUIDs); +/** End [variable_arrays] */ + + +// ========================================================================== +// The Logger Object[s] +// ========================================================================== +/** Start [loggers] */ +// Create a new logger instance +Logger dataLogger(LoggerID, loggingInterval, &varArray); +/** End [loggers] */ + + +// ========================================================================== +// Creating Data Publisher[s] +// ========================================================================== +/** Start [publishers] */ +// Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint +#include +EnviroDIYPublisher EnviroDIYPOST(dataLogger, &modemPhy.gsmClient, + registrationToken, samplingFeature); +/** End [publishers] */ + + +// ========================================================================== +// Working Functions +// ========================================================================== +/** Start [working_functions] */ +// Flashes the LED's on the primary board +void greenredflash(uint8_t numFlash = 4, uint8_t rate = 75) { + for (uint8_t i = 0; i < numFlash; i++) { + digitalWrite(greenLED, HIGH); + digitalWrite(redLED, LOW); + delay(rate); + digitalWrite(greenLED, LOW); + digitalWrite(redLED, HIGH); + delay(rate); + } + digitalWrite(redLED, LOW); +} + +// Reads the battery voltage +// NOTE: This will actually return the battery level from the previous update! +float getBatteryVoltage() { + if (mcuBoard.sensorValues[0] == -9999) mcuBoard.update(); + return mcuBoard.sensorValues[0]; +} + + +// ========================================================================== +// Arduino Setup Function +// ========================================================================== +/** Start [setup] */ +void setup() { + // Start the Debug serial connection + SerialStd.begin(serialBaud); + + // Print a start-up note to the first serial port + SerialStd.print(F("\n---Boot. Sw Build: ")); + SerialStd.print(build_ref); + SerialStd.print(" "); + SerialStd.println(git_usr); + SerialStd.print(" "); + SerialStd.println(git_branch); + + SerialStd.print(F("Sw Name: ")); + SerialStd.print(sketchName); + SerialStd.print(F(" on Logger ")); + SerialStd.println(LoggerID); + SerialStd.println(); + + SerialStd.print(F("Using ModularSensors Library version ")); + SerialStd.println(MODULAR_SENSORS_VERSION); + SerialStd.print(F("TinyGSM Library version ")); + SerialStd.println(TINYGSM_VERSION); + SerialStd.println(); + + // Start the serial connection with the modem + modemSerial.begin(modemBaud); + + // Set up pins for the LED's + pinMode(greenLED, OUTPUT); + digitalWrite(greenLED, LOW); + pinMode(redLED, OUTPUT); + digitalWrite(redLED, LOW); + // Blink the LEDs to show the board is on and starting up + greenredflash(); + + pinMode(20, OUTPUT); // for proper operation of the onboard flash memory + // chip's ChipSelect (Mayfly v1.0 and later) + + // Set the timezones for the logger/data and the RTC + // Logging in the given time zone + Logger::setLoggerTimeZone(timeZone); + // It is STRONGLY RECOMMENDED that you set the RTC to be in UTC (UTC+0) + Logger::setRTCTimeZone(0); + + // Attach the modem and information pins to the logger + dataLogger.attachModem(modemPhy); + modemPhy.setModemLED(modemLEDPin); + dataLogger.setLoggerPins(wakePin, sdCardSSPin, sdCardPwrPin, buttonPin, + greenLED); + + // Begin the logger + dataLogger.begin(); + + // Note: Please change these battery voltages to match your battery + // Set up the sensors, except at lowest battery level + //if (getBatteryVoltage() > 3.4) + { + SerialStd.println(F("Setting up sensors...")); + varArray.setupSensors(); + } + + /** Start [setup_esp] */ + // Modem wroom default baud is 115200 + // Mayfly TinyGSM read() processing doesn't work at 115200. + // It needs to be slowed down. + // On a newly installed modem, it will be at 115200, + // however previously programmed modems could be 57600 or 9600 + uint32_t cfgMdmBaud = modemBaud; + SerialStd.print("ModemESP32 init default "); + SerialStd.println(cfgMdmBaud ); + //modemSerial.end(); + modemSerial.begin(cfgMdmBaud ); + + for (uint8_t ntries = 0; ntries<5; ntries++) { + // This will also verify communication and set up the modem + if (modemPhy.modemWake()) break; + + // if that didn't work, try changing baud rate + cfgMdmBaud= ESP32_MODEM_115K_BAUD; + SerialStd.print(ntries); + SerialStd.print("] ModemESP32 init "); + SerialStd.println(cfgMdmBaud); + modemPhy.gsmModem.sendAT(GF("+UART_DEF=115200,8,1,0,0")); + modemPhy.gsmModem.waitResponse(); + modemSerial.end(); + modemSerial.begin(cfgMdmBaud); + if (modemPhy.modemWake()) break; + + // if that didn't work, try changing baud rate + cfgMdmBaud= ESP32_MODEM_57K_BAUD; + SerialStd.print(ntries); + SerialStd.print("] ModemESP32 init "); + SerialStd.println(cfgMdmBaud); + modemPhy.gsmModem.sendAT(GF("+UART_DEF=57600,8,1,0,0")); + modemPhy.gsmModem.waitResponse(); + modemSerial.end(); + modemSerial.begin(cfgMdmBaud); + if (modemPhy.modemWake()) break; + + + cfgMdmBaud=ESP32_MODEM_9K6_BAUD; + SerialStd.print(ntries); + SerialStd.print("] ModemESP32 init "); + SerialStd.println(cfgMdmBaud ); + modemPhy.gsmModem.sendAT(GF("+UART_DEF=9600,8,1,0,0")); + modemPhy.gsmModem.waitResponse(); + modemSerial.end(); + modemSerial.begin(cfgMdmBaud); + } + // set BAUD if not expected value + if (ESP32_MODEM_DEF_BAUD== cfgMdmBaud ) { + cfgMdmBaud= ESP32_MODEM_57K_BAUD; + modemPhy.gsmModem.sendAT(GF("+UART_DEF=57600,8,1,0,0")); + modemPhy.gsmModem.waitResponse(); + modemSerial.end(); + modemSerial.begin(cfgMdmBaud); + } + SerialStd.print("ModemESP32 connected at baud "); + SerialStd.println(cfgMdmBaud); + + modemPhy.gsmModem.sendAT(GF("+GMR")); + //String MdmRsp; + modemPhy.gsmModem.waitResponse(); + modemPhy.gsmModem.sendAT(GF("+UART_DEF?")); + modemPhy.gsmModem.waitResponse(); + //modemPhy.gsmModem.sendAT(GF("+UART_DEF=115200,8,1,0,0")); + //modemPhy.gsmModem.waitResponse(); + modemPhy.gsmModem.sendAT(GF("+UART_CUR?")); + modemPhy.gsmModem.waitResponse(); + /** End [setup_esp] */ + + + // Sync the clock if it isn't valid or we have battery to spare + //if (getBatteryVoltage() > 3.55 || !dataLogger.isRTCSane()) + { + // Synchronize the RTC with NIST + // This will also set up the modem + dataLogger.syncRTC(); + } + + // Create the log file, adding the default header to it + // Do this last so we have the best chance of getting the time correct + // and all sensor names correct Writing to the SD card can be power + // intensive, so if we're skipping the sensor setup we'll skip this too. + //if (getBatteryVoltage() > 3.4) + { + SerialStd.println(F("Setting up file on SD card")); + dataLogger.turnOnSDcard( + true); // true = wait for card to settle after power up + dataLogger.createLogFile(true); // true = write a new header + dataLogger.turnOffSDcard( + true); // true = wait for internal housekeeping after write + } + #if defined MS_NETWORK_LAYER + EnviroDIYPOST.setQueuedState(true); + EnviroDIYPOST.setTimerPostTimeout_mS(9876); //9.876Sec + EnviroDIYPOST.setTimerPostPacing_mS(500); + + dataLogger.setLoggingInterval(2); //Set every minute, default 5min + dataLogger.setSendOffset(0); + dataLogger._sendEveryX_cnt=1; + dataLogger.setPostMax_num(5); + dataLogger.logDataAndPubReliably(0x08 |0x03); + #endif // #if defined MS_NETWORK_LAYER + // Call the processor sleep + SerialStd.println(F("Putting processor to sleep\n")); + dataLogger.systemSleep(); +} +/** End [setup] */ + + +// ========================================================================== +// Arduino Loop Function +// ========================================================================== +/** Start [loop] */ +// Use this short loop for simple data logging and sending +void loop() { + + { + #if !defined MS_NETWORK_LAYER + dataLogger.logDataAndPublish(); + #else + dataLogger.logDataAndPubReliably(); //TCP / RTL !there + #endif + } +} +/** End [loop] */ diff --git a/a/mayfly1_wifi_wroom/src/ms_cfg.h b/a/mayfly1_wifi_wroom/src/ms_cfg.h new file mode 100644 index 000000000..0d5dd157d --- /dev/null +++ b/a/mayfly1_wifi_wroom/src/ms_cfg.h @@ -0,0 +1,40 @@ +/***************************************************************************** +ms_cfg.h_wio_wifi - ModularSensors Config - MMW _Wio/Mayfly WiFi +Status 220617: 0.33.1.abaa +Written By: Neil Hancock www.envirodiy.org/members/neilh20/ +Development Environment: PlatformIO +Hardware Platform(s): EnviroDIY Mayfly Arduino Datalogger+RS485 Wingboard + +Software License: BSD-3. + Copyright (c) 2022, Neil Hancock - all rights assigned to Stroud Water +Research Center (SWRC) and they may change this title to Stroud Water Research +Center as required and the EnviroDIY Development Team + + +DISCLAIMER: +THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. +*****************************************************************************/ +#ifndef mayfly1_wifi_wroom_ms_cfg_h +#define mayfly1_wifi_wroom_ms_cfg_h + +#define WIFIID_SSID_DEF "WiFiSsid" +#define WIFIPWD_DEF "pwd" +// Defaults for data.envirodiy.org +#define SENSIRION_SHT4X_UUID 1 + +// Defaults for data.envirodiy.org +#define LOGGERID_DEF_STR "itst01" +//https://monitormywatershed.org/sites/intg_test01/ + +const char *UUIDs[] = // UUID array for device sensors +{ + "a61813cd-ac96-49a3-b374-fec5dffb42xx", // Sequence number (EnviroDIY_Mayfly_SampleNum) + "597108b0-c147-49a2-a71d-704539d774xx", // Battery voltage (EnviroDIY_Mayfly_Batt) + "09fbbfde-5b9a-48c9-88a3-39d68463e0xx", // Temperature (Sensirion_SHT40_Temperature) + "e1f4579e-2d6d-4382-a6e8-a8b054fb02xx" // Relative humidity (Sensirion_SHT40_Humidity) +}; +const char *registrationToken = "945af9f2-709c-4d09-ab49-fc7070c337xx"; // Device registration token +const char *samplingFeature = "63bec1f9-b704-435c-bf67-4b8a04a0b6xx"; // Sampling feature UUID + + +#endif // mayfly1_wifi_wroom_ms_cfg_h \ No newline at end of file diff --git a/a/mayfly2_wifi_s6b/.clang-format b/a/mayfly2_wifi_s6b/.clang-format new file mode 100644 index 000000000..6e42a7597 --- /dev/null +++ b/a/mayfly2_wifi_s6b/.clang-format @@ -0,0 +1,127 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -3 +AlignAfterOpenBracket: Align +AlignConsecutiveMacros: false +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: true +AlignEscapedNewlines: Left +AlignOperands: false +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Always +AllowShortCaseLabelsOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BreakAfterJavaFieldAnnotations: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Leave +EmptyLineBeforeAccessModifier: Leave +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: 'TinyGsmClient.h' + Priority: -1 + - Regex: 'VariableBase.h' + Priority: -1 + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentAccessModifiers: false +IndentCaseBlocks: false +IndentCaseLabels: true +IndentExternBlock: Indent +IndentGotoLabels: true +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 2 +NamespaceIndentation: None +# ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PackConstructorInitializers: CurrentLine +PenaltyBreakAssignment: 25 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 600 +PenaltyReturnTypeOnItsOwnLine: 50 +PointerAlignment: Left +PointerBindsToType: true +QualifierAlignment: Left +ReferenceAlignment: Left +ReflowComments: true +SeparateDefinitionBlocks: Leave +SortIncludes: false +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInConditionalStatement: false +SpacesInLineCommentPrefix: + Minimum: 1 +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +TabWidth: 4 +UseTab: Never +--- diff --git a/a/mayfly2_wifi_s6b/.gitignore b/a/mayfly2_wifi_s6b/.gitignore new file mode 100644 index 000000000..ca3297356 --- /dev/null +++ b/a/mayfly2_wifi_s6b/.gitignore @@ -0,0 +1,109 @@ +# Windows image file caches +Thumbs.db +ehthumbs.db +~* + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# ========================= +# Operating System Files +# ========================= + +# OSX +# ========================= + +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# PyCharm +.idea/ + +# Atom / PlatformIO +.pio +.pio_del +.pioenvs +.piolibdeps +.pio +.pio/libdeps +.pio/build +.pio/* +.clang_complete +.gcc-flags.json +.atomrc.cson +lib/* +include/* +.sconsign.dblite + +# VSCode +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode +.history + +# Other +compile_tests/ +logger_test*/ +YosemitechDO/ +LongTermLogger/ +barometric_correction/ +3G_Test/ + +__pycache__/ +runDoxygen.bat +docs/examples/* +output_doxygen.log +output_doxygen_run.log +output_mcss_run.log +output_mcss.log +docs/examples.dox_x +platformio_extra_envs.ini +src/sensors/table.md +cache +docs/output_copyFunctions.log +docs/output_documentExamples.log +docs/output_fixSectionsInXml.log +examples/DRWI_Mayfly1_Wifi_5tm_ds18b20_1/DRWI_Mayfly1_Wifi_5tm_ds18b20_1.ino +clang_format_all.bat +arduino_lint.md +arduino_lint.json +**/home/arduino/* +examples/test_code/* +compile_results.md +arduino_cli_log.log +continuous_integration/arduino_cli_local.yaml +compile_results.log +continuous_integration_artifacts/* +arduino_cli.log diff --git a/a/mayfly2_wifi_s6b/ReadMe.md b/a/mayfly2_wifi_s6b/ReadMe.md new file mode 100644 index 000000000..dde06488e --- /dev/null +++ b/a/mayfly2_wifi_s6b/ReadMe.md @@ -0,0 +1,31 @@ +# DRWI Sites with a Mayfly 1.x and EnviroDIY ESP32 WiFi Bees +Alpha test sketch for transmitting over WiFi Digi S6B + +The hardware configuration used in this example: + * Mayfly v1.1 board + * EnviroDIY Digi WiFi S6B module + +This is used to intergrate in Reliable Delivery +https://github.com/EnviroDIY/ModularSensors/issues/194 +Setup ms_cfg.h for your network parameters + +Merge PR Notes + setFileTimestamp() ~ assume all local view is at local time +on uSD internet connection synospsis DBGyymm.log + SdFat sd1_card_fatfs; - renamed to be readable and findable + +rtc - renamed to rtcExtPhy + going with compatible versions for SAMDxx + +Note if "old date" that forces MMW to decompress +gets 400 "bad request" +-- Response Code -- 400 waited 108 mS Timeout 15432 +data needs to be deleted +remove /* atl_extension */ + +230710 Overnight testing seemed to go well. Going to submit the PR +230707-1600 First pass testing +tested bad WIFi signal and then resend on good +tested timeout - and generates 504 and puts in QUE0.txt +tested sample and delay and stores in RDELAY.txt +tested for send limit of 5, disabled WiFi allowed, number to buld up, and then send in groups of 5 diff --git a/a/mayfly2_wifi_s6b/git_rev_macro.py b/a/mayfly2_wifi_s6b/git_rev_macro.py new file mode 100644 index 000000000..aa4fb114b --- /dev/null +++ b/a/mayfly2_wifi_s6b/git_rev_macro.py @@ -0,0 +1,7 @@ +import subprocess + +git_rev = str(subprocess.check_output(["git", "rev-parse","--abbrev-ref", "HEAD"]).strip()) +git_usr = str(subprocess.check_output(["git", "config","user.name"]).strip()) + +print('-DPIO_SRC_REV={'+(''.join('0x%02x,' % ord(c) for c in git_rev) ) +'0x0}') +print('-DPIO_SRC_USR={'+(''.join('0x%02x,' % ord(c) for c in git_usr) ) +'0x0}') \ No newline at end of file diff --git a/a/mayfly2_wifi_s6b/platformio.ini b/a/mayfly2_wifi_s6b/platformio.ini new file mode 100644 index 000000000..b8a794c46 --- /dev/null +++ b/a/mayfly2_wifi_s6b/platformio.ini @@ -0,0 +1,204 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; http://docs.platformio.org/page/projectconf.html + +[platformio] +description = enviroDIY/ModularSensors a\mayfly2_wifi_s6b +;src_dir =. +default_envs = mayfly + +[common] +cmn_lib_deps= + ; There are 3 levels of code production + ; The stream ( rel1_dvlp1m) is used in alpha coding and development + ; alpha - this platformio.ini - which enables (rel1_dvlp1m) editing in ModularSensors\src & a\ex + ; development see (rel1_dvlp1m) examples\tu_ec + ; production see (release1) examples\tu_ex + ; + ; For alpha coding, + ; (2023-Jan-02 : this worked, however PIO can change, and now seems to be doing realtime analysis of dependencies) + ; and periodically when need to update all libs including ModularSensors #rel1_dvlp1m + ; for clean lib - delete .pio, + ; pio pkg install - execute + ; leave .pio/libdeps/xxx/EnviroDIY_modularSensors as doing realtime dependencie checks + ;https://github.com/neilh10/ModularSensors#rel1_dvlp1m + https://github.com/EnviroDIY/ModularSensors.git#develop +; ^^ Use this when you want to pull from a specific branch + ; These are latest beyond EnviroDIY releases + https://github.com/vshymanskyy/StreamDebugger ;Debug when needed + ; StreamDebugger ; Same as above + ;https://github.com/neilh10/SensorModbusMaster ; default release1 + ;https://github.com/enviroDIY/SensorModbusMaster + ;https://github.com/neilh10/Adafruit_INA219.git + ; del https://github.com/adafruit/SdFat ;- need adafruit/SdFat for multiple SDx + ;https://github.com/greiman/SdFat.git ;- + ;https://github.com/neilh10/EnableInterrupt - old v1.0.0 need v1.1 + ;https://github.com/GreyGnome/EnableInterrupt + ; del https://github.com/arduino-libraries/NTPClient - repalced + ;; see lib_samd_deps for Adafruit_SPIFlash & Adafruit_TinyUSB_Arduino & Adafruit-GFX-Library Adafruit_NeoPixel + ; Need extra libs, so enable STD_LIBS on 1st pass enable ModularSensors then build pulling in ref libs, + ; +; STD_LIBS pulled in by ModularSensors, needed if not enabling ModularSensors to Historical Ref + ;https://github.com/soligen2010/Adafruit_ADS1X15 + ;https://github.com/adafruit/Adafruit_AM2315 + ;https://github.com/adafruit/Adafruit_AM2320 + ;https://github.com/adafruit/Adafruit_BME280_Library + ;https://github.com/adafruit/Adafruit_MPL115A2 + ;https://github.com/adafruit/DHT-sensor-library + ;https://github.com/adafruit/Adafruit_Sensor + ;https://github.com/milesburton/Arduino-Temperature-Control-Library ;Pulls in DallasTemperature + ;https://github.com/neilh10/KellerModbus + ;https://github.com/EnviroDIY/KellerModbus + ;https://github.com/NorthernWidget/MS5803 + ;https://github.com/PaulStoffregen/OneWire + ;https://github.com/knolleary/pubsubclient.git + ;bug https://github.com/EnviroDIY/TinyGSM.git ; bug EnviroDIY/ModularSensors/issues 311 + ;https://github.com/neilh10/TinyGSM.git#rel1 ;Bug fixes over Envirodiy + https://github.com/vshymanskyy/TinyGSM + ;https://github.com/EnviroDIY/YosemitechModbus#develop ;https://github.com/EnviroDIY/YosemitechModbus/issues/32 + https://github.com/neilh10/Arduino-SDI-12#release1 + ;https://github.com/EnviroDIY/Arduino-SDI-12.git bug with extraWakeTime=10 + ;https://github.com/EnviroDIY/Tally_Library.git#Dev_I2C + ; + ;Historical reference + ;https://github.com/greiman/SdFat.git ;- need adafruit/SdFat for multiple SDx + ;https://github.com/neilh10/STC3100arduino.git # For Mayfly board Knh002rev7 + ;https://github.com/neilh10/Arduino-SerialCommand + ;https://github.com/neilh10/InsituModbus + ;https://github.com/neilh10/KellerModbus + +; Ignore these folders or PlatformIO will double count all the dependencies +; Development: ModularSensors Release1: src ? +; ?? .pioenvs, .piolibdeps, .vscode, include, doc, examples, sensor_tests, compile_tests, pioScripts +cmn_lib_ignore = .git, doc, examples, arduino_update, sensor_tests, + ;EnviroDIY_DS3231 + ;SensorModbusMaster + Arduino-SDI-12 + ;TinyGSM + Adafruit_INA21 + EnviroDIY_ModularSensors + ModularSensors + +lib_avr_deps = ;Specific Mayfly board goes in [Mayfly] + +; for development, can edit directly in these directories ... relative to src_dir +; Use the src filter to ensure subfolders are built +; If enabled, disable in cmn_lib_deps=[]ModularSensors, and may need to delete duplicate directory .libdeps/ModularSensors +cmn_src_filter = +<*> +<../../../src> +<../../../src/sensors> +<../../../src/publishers> +<../../../src/modems> + +#build_flags -v for detailed cc verbose, single threaded takes a long time +cmn_build_flags = + -Isrc + -I../../src ; .h For easy debug + -I../../src/sensors ; .h For easy debug + !python git_rev_macro.py ;returns -DPIO_SRC_REV= + ;-DMQTT_MAX_PACKET_SIZE=240 + -DTINY_GSM_YIELD_MS=2 + ;-DTINY_GSM_RX_BUFFER=64 ;is this needed? + ;-DNO_FIRST_SYNC_WITH_NIST + -DNIST_SYNC_HOURLY + -DMS_DISCARD_HTTP_500 + -DMS_NETWORK_LAYER + ;-DUSE_RTCLIB=RTC_DS3231 ;not compiling + ;-DMS_TU_XX_DEBUG + ;-DMS_TU_XX_DEBUG_DEEP + ;-DSTREAMDEBUGGER_DBG + ;-DMS_DUMP_FREE_RAM + ;-DMS_TTY_USER_INPUT + ;-DSerialCommand_inFlash ;requires MS_TTY_USER_INPUT + ;-DMS_TTY_INPUT_COUNT ;only if no MS_TTY_USER_INPUT + ;-DSERIALCOMMAND_DEBUG + -DMS_LOGGERBASE_POSTS + ;-DMS_LOGGERBASE_SLEEP_DEBUG ;Need or below + ;-DMS_LOGGERBASE_DEBUG + ;-DMS_LOGGERBASE_DEEP_DEBUG + ;-DMS_LOGGERMODEM_DEBUG + ;-DMS_LOGGERMODEM_DEBUG_DEEP + ;-DMS_SENSORBASE_DEBUG + ;-DMS_VARIABLEARRAY_DEBUG + ;-DMS_VARIABLEARRAY_DEBUG_DEEP + ;-DMS_VARIABLEBASE_DEBUG + ;-DMS_VARIABLEBASE_DEBUG_DEEP + ;-DMS_DATAPUBLISHERBASE_DEBUG + ;-DMS_ENVIRODIYPUBLISHER_DEBUG + ;-DMS_THINGSPEAKPUBLISHER_DEBUG + ;-DMS_UBIDOTSPUBLISHER_DEBUG + ;-DMS_DIGIXBEEWIFI_DEBUG + ;-DMS_DIGIXBEEWIFI_DEBUG_DEEP + ;-DMS_DIGIXBEECELLULARTRANSPARENT_DEBUG + ;-DMS_DIGIXBEECELLULARTRANSPARENT_DEBUG_DEEP + ;-DMS_DIGIXBEE_DEBUG + ;-DTINY_GSM_DEBUG=Serial + ;-DTinyGsmClientXbee_DBG=Serial + ;-DMS_STSTC3100SENSOR_DEBUG + ;-DSTC3100DM_DEBUG ; device manager + ;-DSTC3100DD_DEBUG ;device driver + ;-DMS_BATTERYMANAGEMENT_DEBUG + ;-DMS_PROCESSORSTATS_DEBUG + ;-DMS_PROCESSORADC_DEBUG + ;-DMS_TIINA219M_DEBUG + ;-DMS_AOSONGAM2315_DEBUG + ;-DMS_EXTERNALVOLTAGE_DEBUG + ;-DMS_EXTERNALVOLTAGE_DEBUG_DEEP + ;-DMS_KELLERPARENT_DEBUG + ;-DMS_KELLERPARENT_DEBUG_DEEP has problem + ;-DMS_MODBUS_DEBUG + -DSENSORMODBUSMASTER_NO_DBG + ;-DMS_ANALOGELECCONDUCTIVITY_DEBUG + ;-DMS_ANALOGELECCONDUCTIVITY_DEBUG_DEEP + ;-DMS_SDI12SENSORS_DEBUG ;LT500 sometimes responding + ;-DMS_SDI12SENSORS_DEBUG_DEEP + ;For -DMS_SDI12 options: NONE or MS_SDI12_SINGLE_LINE_RESPONSE OR MS_SDI12_NON_CONCURRENT not BOTH + ;-DMS_SDI12_SINGLE_LINE_RESPONSE + ;-DMS_SDI12_NON_CONCURRENT + ;-DENVIRODIY_SDI12_USE_CRC + ;-DMS_ENVIRODIYPUBLISHER_DEBUG + ;-DMS_ENVIRODIYPUBLISHER_DEBUG_DEEP + ;-DSodaq_DS3231_DEBUG + -DMS_WATCHDOGAVR_DEBUG + -DMS_WATCHDOGAVR_DEBUG_DEEP + +[env:mayfly] +;upload_port = COM11 +monitor_speed = 115200 +board = mayfly +platform = atmelavr +framework = arduino +;debug_tool = blackmagic +;debug_port = test01u:4242 + +lib_compat_mode = strict +lib_ldf_mode = deep+ +lib_ignore = ${common.cmn_lib_ignore}, RTCZero +build_src_filter = ${common.cmn_src_filter} +;build_unflags = -Os +build_flags = + ${common.cmn_build_flags} + -DSDI12_EXTERNAL_PCINT + -DNEOSWSERIAL_EXTERNAL_PCINT + -fmax-errors=5 + ;-Wl,-Map,.pio/build/mayfly/firmware.map + +lib_deps =${common.cmn_lib_deps} ${common.lib_avr_deps} + ;https://github.com/neilh10/AltSoftSerial ; Need this for managing pwr - data pin off as well. + ;https://github.com/PaulStoffregen/AltSoftSerial.git + ; + ;https://github.com/EnviroDIY/SoftwaterSerial_ExternalInts.git + ;https://github.com/SRGDamia1/NeoSWSerial.git + ;https://github.com/neilh10/Sodaq_DS3231#release1 ; Bug fix #34 - change to RTClibnh with DS3231 + +monitor_filters = log2file, default, time +; +; The following monitor_flags setting is needed becasue in transparent mode the XBee3 +; uses CR line endings instead of the more typical CR+LF. This setting enables you +; to see all back and forth communication. +; (https://www.envirodiy.org/topic/connecting-xbee3-lte-to-the-internet/#post-13312) +monitor_flags = + --eol + CR diff --git a/a/mayfly2_wifi_s6b/src/mayfly2_wifi_s6b.cpp b/a/mayfly2_wifi_s6b/src/mayfly2_wifi_s6b.cpp new file mode 100644 index 000000000..d01ddf196 --- /dev/null +++ b/a/mayfly2_wifi_s6b/src/mayfly2_wifi_s6b.cpp @@ -0,0 +1,449 @@ +/** ========================================================================= + * @file Mayfly2_wifi_s6b_.cpp + * @brief Adapted for for testing ReliableDelivery/ WiFi . + * + * This example shows proper settings for the following configuration: + * + * Mayfly v1.x board + * EnviroDIY Digi WiFi S6B + * Internal sensors + * + * @author Neil Hancock & Sara Geleskie Damiano + * @copyright (c) 2017-2022 Stroud Water Research Center (SWRC) + * and the EnviroDIY Development Team + * This example is published under the BSD-3 license. + * + + * Hardware Platform: EnviroDIY Mayfly Arduino Datalogger + * + * DISCLAIMER: + * THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. + * ======================================================================= */ + +// ========================================================================== +// Defines for the Arduino IDE +// NOTE: These are ONLY needed to compile with the Arduino IDE. +// If you use PlatformIO, you should set these build flags in your +// platformio.ini +// ========================================================================== +/** Start [defines] */ +#ifndef TINY_GSM_RX_BUFFER +#define TINY_GSM_RX_BUFFER 64 +#endif +#ifndef TINY_GSM_YIELD_MS +#define TINY_GSM_YIELD_MS 2 +#endif +/** End [defines] */ + +// ========================================================================== +// Include the libraries required for any data logger +// ========================================================================== +/** Start [includes] */ +// The Arduino library is needed for every Arduino program. +#include +//Site https://monitormywatershed.org/sites/bq_test02/ +#include "ms_cfg.h" + +// EnableInterrupt is used by ModularSensors for external and pin change +// interrupts and must be explicitly included in the main program. +#include + +// Include the main header for ModularSensors +#include +/** End [includes] */ + + +// ========================================================================== +// Data Logging Options +// ========================================================================== +// Details for this build +extern const String build_ref = "a\\" __FILE__ " " __DATE__ " " __TIME__ " "; +#ifdef PIO_SRC_REV +const char git_branch[] = PIO_SRC_REV; +#else +const char git_branch[] = "brnch"; +#endif +#ifdef PIO_SRC_USR +const char git_usr[] = PIO_SRC_USR; +#else +const char git_usr[] = "usr"; +#endif +/** Start [logging_options] */ +// The name of this program file +const char* sketchName = "a/mayfly2_wifi_s6b.cpp"; +// Logger ID, also becomes the prefix for the name of the data file on SD card +const char* LoggerID = "reldlv2"; +// How frequently (in minutes) to log data +const uint8_t loggingInterval = 2; +// Your logger's timezone. +const int8_t timeZone = -8; // PST +// NOTE: Daylight savings time will not be applied! Please use standard time! +#define SerialStd STANDARD_SERIAL_OUTPUT + +// Set the input and output pins for the logger +// NOTE: Use -1 for pins that do not apply +const int32_t serialBaud = 115200; // Baud rate for debugging +const int8_t greenLED = 8; // Pin for the green LED +const int8_t redLED = 9; // Pin for the red LED +const int8_t buttonPin = 21; // Pin for debugging mode (ie, button pin) +const int8_t wakePin = 31; // MCU interrupt/alarm pin to wake from sleep +// Mayfly 0.x, 1.x D31 = A7 +const int8_t sdCardPwrPin = -1; // MCU SD card power pin +const int8_t sdCardSSPin = 12; // SD card chip select/slave select pin +const int8_t sensorPowerPin = 22; // MCU pin controlling main sensor power +/** End [logging_options] */ + + +// ========================================================================== +// Wifi/Cellular Modem Options +// ========================================================================== +HardwareSerial& modemSerial = Serial1; // Use hardware serial if possible +/*#if defined STREAMDEBUGGER_DBG +#include +StreamDebugger modemDebugger(modemSerial, STANDARD_SERIAL_OUTPUT); +#define modemSerHw modemDebugger +#else */ +#define modemSerHw modemSerial + +//#endif // STREAMDEBUGGER_DBG + +#define sim_com_xbee_wifi +#if defined sim_com_sim7080 +/** Start [sim_com_sim7080] */ + +// For almost anything based on the SIMCom SIM7080G +#include + +// Create a reference to the serial port for the modem +const int32_t modemBaud = 9600; // SIM7080 does auto-bauding by default, but + // for simplicity we set to 9600 + +// Modem Pins - Describe the physical pin connection of your modem to your board +// NOTE: Use -1 for pins that do not apply + +const int8_t modemVccPin = + 18; // MCU pin controlling modem power --- + // Pin 18 is the power enable pin + // for the bee socket on Mayfly v1.0, + // use -1 if using Mayfly 0.5b or if the bee socket is constantly + // powered (ie you changed SJ18 on Mayfly 1.x to 3.3v) +const int8_t modemStatusPin = 19; // MCU pin used to read modem status +const int8_t modemSleepRqPin = 23; // MCU pin for modem sleep/wake request +const int8_t modemLEDPin = redLED; // MCU pin connected an LED to show modem + // status + +// Network connection information +const char* apn = + "hologram"; // APN connection name, typically Hologram unless you have a + // different provider's SIM card. Change as needed + +// Create the modem object +SIMComSIM7080 modem7080(&modemSerHw, modemVccPin, modemStatusPin, + modemSleepRqPin, apn); +// Create an extra reference to the modem by a generic name +SIMComSIM7080 modem = modem7080; +/** End [sim_com_sim7080] */ +#elif defined sim_com_xbee_wifi +/** Start [sim_com_xbee_wifi] */ +// For the Digi Wifi XBee (S6B) +#include +// Create a reference to the serial port for the modem + +const int32_t modemBaud = 9600; // All XBee's use 9600 by default + +// Modem Pins - Describe the physical pin connection of your modem to your board +// NOTE: Use -1 for pins that do not apply +const int8_t modemVccPin = 18; // Mayfly1.1 pin controlling modem power +const int8_t modemStatusPin = 19; // MCU pin used to read modem status +const bool useCTSforStatus = true; // Flag to use the modem CTS pin for status +const int8_t modemResetPin = 20; // MCU pin connected to modem reset pin +const int8_t modemSleepRqPin = 23; // MCU pin for modem sleep/wake request +const int8_t modemLEDPin = redLED; // MCU pin connected an LED to show modem + // status (-1 if unconnected) + +// Network connection information +const char* wifiId = WIFIID_SSID_DEF ; // WiFi access point, unnecessary for GPRS +const char* wifiPwd = WIFIPWD_DEF ; // WiFi password, unnecessary for GPRS + +DigiXBeeWifi modemXBWF(&modemSerHw, modemVccPin, modemStatusPin, + useCTSforStatus, modemResetPin, modemSleepRqPin, wifiId, + wifiPwd); +// Create an extra reference to the modem by a generic name +DigiXBeeWifi modemPhy = modemXBWF; +/** End [sim_com_xbee_wifi] */ +#endif //Modem options + +// ========================================================================== +// Using the Processor as a Sensor +// ========================================================================== +/** Start [processor_sensor] */ +#include + +// Create the main processor chip "sensor" - for general metadata +const char* mcuBoardVersion = "v1.1"; +ProcessorStats mcuBoard(mcuBoardVersion); +/** End [processor_sensor] */ + + +// ========================================================================== +// Maxim DS3231 RTC (Real Time Clock) +// Built in on all versions of the Mayfly +// ========================================================================== +/** Start [ds3231] */ +#if 0 +#include + +// Create a DS3231 sensor object +MaximDS3231 ds3231(1); +#endif +/** End [ds3231] */ + +#if defined SENSIRION_SHT4X_UUID +// ========================================================================== +// Sensirion SHT4X Digital Humidity and Temperature Sensor +// Built in on Mayfly 1.x +// ========================================================================== +/** Start [sensirion_sht4x] */ +#include + +// NOTE: Use -1 for any pins that don't apply or aren't being used. +const int8_t SHT4xPower = sensorPowerPin; // Power pin +const bool SHT4xUseHeater = true; + +// Create an Sensirion SHT4X sensor object +SensirionSHT4x sht4x(SHT4xPower, SHT4xUseHeater); +/** End [sensirion_sht4x] */ +#endif //SENSIRION_SHT4X_UUID + +// ========================================================================== +// Creating the Variable Array[s] and Filling with Variable Objects +// ========================================================================== +/** Start [variable_arrays] */ + +Variable* variableList[] = { + // Should follow UUIDs + new ProcessorStats_SampleNumber(&mcuBoard), + new ProcessorStats_Battery(&mcuBoard ), + new SensirionSHT4x_Temp(&sht4x), + new SensirionSHT4x_Humidity(&sht4x) + + // Fut Sensiron Temperature + // Fut Sensirom Humidity + //new Modem_SignalPercent(&modem), +}; + +// All UUID's, device registration, and sampling feature information can be +// pasted directly from Monitor My Watershed. +// To get the list, click the "View token UUID list" button on the upper right +// of the site page. + + +// Replace all of the text in the following section with the UUID array from +// MonitorMyWatershed + +/* clang-format off */ +// --------------------- Beginning of Token UUID List --------------------- +// see ms_cfg.h initially setup for https://monitormywatershed.org/sites/bq_test02/ + + +// ----------------------- End of Token UUID List ----------------------- +/* clang-format on */ + +// Count up the number of pointers in the array +int variableCount = sizeof(variableList) / sizeof(variableList[0]); + +// Create the VariableArray object +VariableArray varArray(variableCount, variableList,UUIDs); +/** End [variable_arrays] */ + + +// ========================================================================== +// The Logger Object[s] +// ========================================================================== +/** Start [loggers] */ +// Create a new logger instance +Logger dataLogger(LoggerID, loggingInterval, &varArray); +/** End [loggers] */ + + +// ========================================================================== +// Creating Data Publisher[s] +// ========================================================================== +/** Start [publishers] */ +// Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint +#include +EnviroDIYPublisher EnviroDIYPOST(dataLogger, &modemPhy.gsmClient, + registrationToken, samplingFeature); +/** End [publishers] */ + + +// ========================================================================== +// Working Functions +// ========================================================================== +/** Start [working_functions] */ +// Flashes the LED's on the primary board +void greenredflash(uint8_t numFlash = 4, uint8_t rate = 75) { + for (uint8_t i = 0; i < numFlash; i++) { + digitalWrite(greenLED, HIGH); + digitalWrite(redLED, LOW); + delay(rate); + digitalWrite(greenLED, LOW); + digitalWrite(redLED, HIGH); + delay(rate); + } + digitalWrite(redLED, LOW); +} + +// Reads the battery voltage +// NOTE: This will actually return the battery level from the previous update! +float getBatteryVoltage() { + if (mcuBoard.sensorValues[0] == -9999) mcuBoard.update(); + return mcuBoard.sensorValues[0]; +} + + +// ========================================================================== +// Arduino Setup Function +// ========================================================================== +/** Start [setup] */ +void setup() { + // Start the Debug serial connection + SerialStd.begin(serialBaud); + + // Print a start-up note to the first serial port + SerialStd.print(F("\n---Boot. Sw Build: ")); + SerialStd.print(build_ref); + SerialStd.print(" "); + SerialStd.println(git_usr); + SerialStd.print(" "); + SerialStd.println(git_branch); + + SerialStd.print(F("Sw Name: ")); + SerialStd.print(sketchName); + SerialStd.print(F(" on Logger ")); + SerialStd.println(LoggerID); + SerialStd.println(); + + SerialStd.print(F("Using ModularSensors Library version ")); + SerialStd.println(MODULAR_SENSORS_VERSION); + SerialStd.print(F("TinyGSM Library version ")); + SerialStd.println(TINYGSM_VERSION); + SerialStd.println(); + + // Start the serial connection with the modem + modemSerial.begin(modemBaud); + + // Set up pins for the LED's + pinMode(greenLED, OUTPUT); + digitalWrite(greenLED, LOW); + pinMode(redLED, OUTPUT); + digitalWrite(redLED, LOW); + // Blink the LEDs to show the board is on and starting up + greenredflash(); + + pinMode(20, OUTPUT); // for proper operation of the onboard flash memory + // chip's ChipSelect (Mayfly v1.0 and later) + + // Set the timezones for the logger/data and the RTC + // Logging in the given time zone + Logger::setLoggerTimeZone(timeZone); + // It is STRONGLY RECOMMENDED that you set the RTC to be in UTC (UTC+0) + Logger::setRTCTimeZone(0); + + // Attach the modem and information pins to the logger + dataLogger.attachModem(modemPhy); + modemPhy.setModemLED(modemLEDPin); + dataLogger.setLoggerPins(wakePin, sdCardSSPin, sdCardPwrPin, buttonPin, + greenLED); + + // Begin the logger + dataLogger.begin(); + + // Note: Please change these battery voltages to match your battery + // Set up the sensors, except at lowest battery level + //if (getBatteryVoltage() > 3.4) + { + SerialStd.println(F("Setting up sensors...")); + varArray.setupSensors(); + } + + #if defined sim_com_sim700 + /** Start [setup_sim7080] */ + modem.setModemWakeLevel(HIGH); // ModuleFun Bee inverts the signal + modem.setModemResetLevel(HIGH); // ModuleFun Bee inverts the signal + SerialStd.println(F("Waking modem and setting Cellular Carrier Options...")); + modem.modemWake(); // NOTE: This will also set up the modem + modem.gsmModem.setBaud(modemBaud); // Make sure we're *NOT* auto-bauding! + modem.gsmModem.setNetworkMode(38); // set to LTE only + // 2 Automatic + // 13 GSM only + // 38 LTE only + // 51 GSM and LTE only + modem.gsmModem.setPreferredMode(1); // set to CAT-M + // 1 CAT-M + // 2 NB-IoT + // 3 CAT-M and NB-IoT + /** End [setup_sim7080] */ + #elif defined sim_com_xbee_wifi + /** Start [setup_sim7080] */ + + SerialStd.println(F("Waking modem WiFi ...")); + modemPhy.modemWake(); // NOTE: This will also set up the modem + modemPhy.gsmModem.setBaud(modemBaud); // Make sure we're *NOT* auto-bauding! + #endif //Modem setup + + // Sync the clock if it isn't valid or we have battery to spare + //if (getBatteryVoltage() > 3.55 || !dataLogger.isRTCSane()) + { + // Synchronize the RTC with NIST + // This will also set up the modem + dataLogger.syncRTC(); + } + + // Create the log file, adding the default header to it + // Do this last so we have the best chance of getting the time correct + // and all sensor names correct Writing to the SD card can be power + // intensive, so if we're skipping the sensor setup we'll skip this too. + //if (getBatteryVoltage() > 3.4) + { + SerialStd.println(F("Setting up file on SD card")); + dataLogger.turnOnSDcard( + true); // true = wait for card to settle after power up + dataLogger.createLogFile(true); // true = write a new header + dataLogger.turnOffSDcard( + true); // true = wait for internal housekeeping after write + } + #if defined MS_NETWORK_LAYER + EnviroDIYPOST.setQueuedState(true); + EnviroDIYPOST.setTimerPostTimeout_mS(9876); //9.876Sec + EnviroDIYPOST.setTimerPostPacing_mS(500); + + dataLogger.setLoggingInterval(2); //Set every minute, default 5min + dataLogger.setSendOffset(0); + dataLogger._sendEveryX_cnt=1; + dataLogger.setPostMax_num(5); + dataLogger.logDataAndPubReliably(0x08 |0x03); + #endif // #if defined MS_NETWORK_LAYER + // Call the processor sleep + SerialStd.println(F("Putting processor to sleep\n")); + dataLogger.systemSleep(); +} +/** End [setup] */ + + +// ========================================================================== +// Arduino Loop Function +// ========================================================================== +/** Start [loop] */ +// Use this short loop for simple data logging and sending +void loop() { + + { + #if !defined MS_NETWORK_LAYER + dataLogger.logDataAndPublish(); + #else + dataLogger.logDataAndPubReliably(); //TCP / RTL !there + #endif + } +} +/** End [loop] */ diff --git a/a/mayfly2_wifi_s6b/src/ms_cfg.h b/a/mayfly2_wifi_s6b/src/ms_cfg.h new file mode 100644 index 000000000..ebd63e56a --- /dev/null +++ b/a/mayfly2_wifi_s6b/src/ms_cfg.h @@ -0,0 +1,40 @@ +/***************************************************************************** +ms_cfg.h - ModularSensors Config - remove this file to delete personalization info + +dvlpRelDlvry1\ModularSensors\a\mayfly2_wifi_s6b\src +Status 230629: 0.34.2.aaa Unique tracker +Written By: Neil Hancock www.envirodiy.org/members/neilh20/ +Development Environment: PlatformIO +Hardware Platform(s): EnviroDIY Mayfly Arduino Datalogger + +Software License: BSD-3. + Copyright (c) 2023, Neil Hancock - all rights assigned to Stroud Water +Research Center (SWRC) and they may change this title to Stroud Water Research +Center as required and the EnviroDIY Development Team + + +DISCLAIMER: +THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. +*****************************************************************************/ +#ifndef mayfly2_wifi_s6b_ms_cfg_h +#define mayfly2_wifi_s6b_ms_cfg_h + +#define WIFIID_SSID_DEF "WiFiSsid" +#define WIFIPWD_DEF "WiFi pwd" +// Defaults for data.envirodiy.org +#define SENSIRION_SHT4X_UUID 1 + +#define LOGGERID_DEF_STR "itst02" +//https://monitormywatershed.org/sites/intg_test02/ + +const char *UUIDs[] = // UUID array for device sensors +{ + "c317f9ee-d0ca-410b-b051-11cb683c89xx", // Sequence number (EnviroDIY_Mayfly_SampleNum) + "135714b5-30fa-4fd9-83f4-3d70edce3xxx", // Battery voltage (EnviroDIY_Mayfly_Batt) + "f24f4af6-c469-4597-893b-9e9eee9e2bxx", // Temperature (Sensirion_SHT40_Temperature) + "2b8d3edb-e0ac-455e-99a4-09b2a859d6xx" // Relative humidity (Sensirion_SHT40_Humidity) +}; +const char *registrationToken = "8467bd56-3659-41a7-9fad-bd0c93b6bfxx"; // Device registration token +const char *samplingFeature = "8eed720a-345d-4587-9a68-24cd77771cxx"; // Sampling feature UUID + +#endif // mayfly2_wifi_s6b_ms_cfg_h diff --git a/sensors_test/DRWI_SIM7080LTE/.gitignore b/sensors_test/DRWI_SIM7080LTE/.gitignore new file mode 100644 index 000000000..89cc49cbd --- /dev/null +++ b/sensors_test/DRWI_SIM7080LTE/.gitignore @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/sensors_test/DRWI_SIM7080LTE/platformio.ini b/sensors_test/DRWI_SIM7080LTE/platformio.ini new file mode 100644 index 000000000..dc885e670 --- /dev/null +++ b/sensors_test/DRWI_SIM7080LTE/platformio.ini @@ -0,0 +1,43 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; http://docs.platformio.org/page/projectconf.html + +[platformio] +description = ModularSensors example intended for DRWI users with CTD, turbidity, and a EnviroDIY SIM7080G LTE modem + +[env:mayfly] +monitor_speed = 115200 +board = mayfly +platform = atmelavr +framework = arduino +lib_ldf_mode = deep+ +lib_ignore = + RTCZero + Adafruit NeoPixel + Adafruit GFX Library + Adafruit SSD1306 + Adafruit ADXL343 + Adafruit STMPE610 + Adafruit TouchScreen + Adafruit ILI9341 +build_flags = + -DSDI12_EXTERNAL_PCINT + -D STREAMDEBUGGER_DBG + -D TinyGsmClientXbee_DBG=Serial + -D MS_DIGIXBEEWIFI_DEBUG + -D MS_DIGIXBEE_DEBUG + ;-D MS_LOGGERMODEM_DEBUG +lib_deps = + vshymanskyy/StreamDebugger + ;https://github.com/vshymanskyy/TinyGSM +; envirodiy/EnviroDIY_ModularSensors +; ^^ Use this when working from an official release of the library + https://github.com/EnviroDIY/ModularSensors.git#develop +; https://github.com/neilh10/ModularSensors.git#digi_wifi_s6b_stability +; ^^ Use this when if you want to pull from the develop branch diff --git a/sensors_test/DRWI_SIM7080LTE/src/DRWI_SIM7080LTE.cpp b/sensors_test/DRWI_SIM7080LTE/src/DRWI_SIM7080LTE.cpp new file mode 100644 index 000000000..95c8f15eb --- /dev/null +++ b/sensors_test/DRWI_SIM7080LTE/src/DRWI_SIM7080LTE.cpp @@ -0,0 +1,481 @@ +/** ========================================================================= + * @file DRWI_SIM7080LTE.cpp + * @brief Adapted for for testing WiFi sites. + * + * This example shows proper settings for the following configuration: + * + * Mayfly v1.0 board + * EnviroDIY or WiFi SIM7080 LTE module (with Hologram SIM card) + * + * @author Neil Hancock & Sara Geleskie Damiano + * @copyright (c) 2017-2022 Stroud Water Research Center (SWRC) + * and the EnviroDIY Development Team + * This example is published under the BSD-3 license. + * + + * Hardware Platform: EnviroDIY Mayfly Arduino Datalogger + * + * DISCLAIMER: + * THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. + * ======================================================================= */ + +// ========================================================================== +// Defines for the Arduino IDE +// NOTE: These are ONLY needed to compile with the Arduino IDE. +// If you use PlatformIO, you should set these build flags in your +// platformio.ini +// ========================================================================== +/** Start [defines] */ +#ifndef TINY_GSM_RX_BUFFER +#define TINY_GSM_RX_BUFFER 64 +#endif +#ifndef TINY_GSM_YIELD_MS +#define TINY_GSM_YIELD_MS 2 +#endif +/** End [defines] */ + +// ========================================================================== +// Include the libraries required for any data logger +// ========================================================================== +/** Start [includes] */ +// The Arduino library is needed for every Arduino program. +#include +#include "ms_cfg.h" + +// EnableInterrupt is used by ModularSensors for external and pin change +// interrupts and must be explicitly included in the main program. +#include + +// Include the main header for ModularSensors +#include +/** End [includes] */ + + +// ========================================================================== +// Data Logging Options +// ========================================================================== +// Details for this build +extern const String build_ref = "a\\" __FILE__ " " __DATE__ " " __TIME__ " "; +#ifdef PIO_SRC_REV +const char git_branch[] = PIO_SRC_REV; +#else +const char git_branch[] = "brnch"; +#endif +#ifdef PIO_SRC_USR +const char git_usr[] = PIO_SRC_USR; +#else +const char git_usr[] = "usr"; +#endif +/** Start [logging_options] */ +// The name of this program file +const char* sketchName = "sensorTest/DRWI_SIM7080LTE.cpp"; +// Logger ID, also becomes the prefix for the name of the data file on SD card +const char* LoggerID = "snsrTst"; +// How frequently (in minutes) to log data +const uint8_t loggingInterval = 2; +// Your logger's timezone. +const int8_t timeZone = -5; // Eastern Standard Time +// NOTE: Daylight savings time will not be applied! Please use standard time! + +// Set the input and output pins for the logger +// NOTE: Use -1 for pins that do not apply +const int32_t serialBaud = 115200; // 57600 Baud rate for debugging +const int8_t greenLED = 8; // Pin for the green LED +const int8_t redLED = 9; // Pin for the red LED +const int8_t buttonPin = 21; // Pin for debugging mode (ie, button pin) +const int8_t wakePin = 31; // MCU interrupt/alarm pin to wake from sleep +// Mayfly 0.x D31 = A7 +const int8_t sdCardPwrPin = -1; // MCU SD card power pin +const int8_t sdCardSSPin = 12; // SD card chip select/slave select pin +const int8_t sensorPowerPin = 22; // MCU pin controlling main sensor power +/** End [logging_options] */ + + +// ========================================================================== +// Wifi/Cellular Modem Options +// ========================================================================== +HardwareSerial& modemSerial = Serial1; // Use hardware serial if possible +#if defined STREAMDEBUGGER_DBG +#include +StreamDebugger modemDebugger(modemSerial, STANDARD_SERIAL_OUTPUT); +#define modemSerHw modemDebugger +#else +#define modemSerHw modemSerial +#endif // STREAMDEBUGGER_DBG + +#define sim_com_xbee_wifi +#if defined sim_com_sim7080 +/** Start [sim_com_sim7080] */ + +// For almost anything based on the SIMCom SIM7080G +#include + +// Create a reference to the serial port for the modem +const int32_t modemBaud = 9600; // SIM7080 does auto-bauding by default, but + // for simplicity we set to 9600 + +// Modem Pins - Describe the physical pin connection of your modem to your board +// NOTE: Use -1 for pins that do not apply + +const int8_t modemVccPin = + 18; // MCU pin controlling modem power --- + // Pin 18 is the power enable pin + // for the bee socket on Mayfly v1.0, + // use -1 if using Mayfly 0.5b or if the bee socket is constantly + // powered (ie you changed SJ18 on Mayfly 1.x to 3.3v) +const int8_t modemStatusPin = 19; // MCU pin used to read modem status +const int8_t modemSleepRqPin = 23; // MCU pin for modem sleep/wake request +const int8_t modemLEDPin = redLED; // MCU pin connected an LED to show modem + // status + +// Network connection information +const char* apn = + "hologram"; // APN connection name, typically Hologram unless you have a + // different provider's SIM card. Change as needed + +// Create the modem object +SIMComSIM7080 modem7080(&modemSerHw, modemVccPin, modemStatusPin, + modemSleepRqPin, apn); +// Create an extra reference to the modem by a generic name +SIMComSIM7080 modem = modem7080; +/** End [sim_com_sim7080] */ +#elif defined sim_com_xbee_wifi +/** Start [sim_com_xbee_wifi] */ +// For the Digi Wifi XBee (S6B) +#include +// Create a reference to the serial port for the modem + +const int32_t modemBaud = 9600; // All XBee's use 9600 by default + +// Modem Pins - Describe the physical pin connection of your modem to your board +// NOTE: Use -1 for pins that do not apply +const int8_t modemVccPin = 18; // Mayfly1.1 pin controlling modem power +const int8_t modemStatusPin = 19; // MCU pin used to read modem status +const bool useCTSforStatus = true; // Flag to use the modem CTS pin for status +const int8_t modemResetPin = 20; // MCU pin connected to modem reset pin +const int8_t modemSleepRqPin = 23; // MCU pin for modem sleep/wake request +const int8_t modemLEDPin = redLED; // MCU pin connected an LED to show modem + // status (-1 if unconnected) + +// Network connection information +const char* wifiId = "ArthurGuestSsid"; // WiFi access point, unnecessary for GPRS +const char* wifiPwd = "Arthur8166"; // WiFi password, unnecessary for GPRS + +DigiXBeeWifi modemXBWF(&modemSerHw, modemVccPin, modemStatusPin, + useCTSforStatus, modemResetPin, modemSleepRqPin, wifiId, + wifiPwd); +// Create an extra reference to the modem by a generic name +DigiXBeeWifi modemPhy = modemXBWF; +/** End [sim_com_xbee_wifi] */ +#endif //Modem options + +// ========================================================================== +// Using the Processor as a Sensor +// ========================================================================== +/** Start [processor_sensor] */ +#include + +// Create the main processor chip "sensor" - for general metadata +const char* mcuBoardVersion = "v1.1"; +ProcessorStats mcuBoard(mcuBoardVersion); +/** End [processor_sensor] */ + + +// ========================================================================== +// Maxim DS3231 RTC (Real Time Clock) +// ========================================================================== +/** Start [ds3231] */ +#include + +// Create a DS3231 sensor object +MaximDS3231 ds3231(1); +/** End [ds3231] */ + +//#define SENSORS_EXTERNAL +#if defined SENSORS_EXTERNAL +// ========================================================================== +// Meter Hydros 21 Conductivity, Temperature, and Depth Sensor +// ========================================================================== +/** Start [hydros21] */ +#include + +const char* hydrosSDI12address = "1"; // The SDI-12 Address of the Hydros 21 +const uint8_t hydrosNumberReadings = 6; // The number of readings to average +const int8_t SDI12Power = sensorPowerPin; // Power pin (-1 if unconnected) +const int8_t SDI12Data = 7; // The SDI12 data pin + +// Create a Meter Hydros 21 sensor object +MeterHydros21 hydros(*hydrosSDI12address, SDI12Power, SDI12Data, + hydrosNumberReadings); +/** End [hydros21] */ + + +// ========================================================================== +// Campbell OBS 3 / OBS 3+ Analog Turbidity Sensor +// ========================================================================== +/** Start [obs3] */ +#include + +const int8_t OBS3Power = sensorPowerPin; // Power pin (-1 if unconnected) +const uint8_t OBS3NumberReadings = 10; +const uint8_t ADSi2c_addr = 0x48; // The I2C address of the ADS1115 ADC +// Campbell OBS 3+ *Low* Range Calibration in Volts +const int8_t OBSLowADSChannel = 0; // ADS channel for *low* range output +const float OBSLow_A = 0.000E+00; // "A" value (X^2) [*low* range] +const float OBSLow_B = 1.000E+00; // "B" value (X) [*low* range] +const float OBSLow_C = 0.000E+00; // "C" value [*low* range] + +// Create a Campbell OBS3+ *low* range sensor object +CampbellOBS3 osb3low(OBS3Power, OBSLowADSChannel, OBSLow_A, OBSLow_B, OBSLow_C, + ADSi2c_addr, OBS3NumberReadings); + + +// Campbell OBS 3+ *High* Range Calibration in Volts +const int8_t OBSHighADSChannel = 1; // ADS channel for *high* range output +const float OBSHigh_A = 0.000E+00; // "A" value (X^2) [*high* range] +const float OBSHigh_B = 1.000E+00; // "B" value (X) [*high* range] +const float OBSHigh_C = 0.000E+00; // "C" value [*high* range] + +// Create a Campbell OBS3+ *high* range sensor object +CampbellOBS3 osb3high(OBS3Power, OBSHighADSChannel, OBSHigh_A, OBSHigh_B, + OBSHigh_C, ADSi2c_addr, OBS3NumberReadings); +/** End [obs3] */ + +#endif // SENSORS_EXTERNAL +// ========================================================================== +// Creating the Variable Array[s] and Filling with Variable Objects +// ========================================================================== +/** Start [variable_arrays] */ +Variable* variableList[] = { + #if defined SENSORS_EXTERNAL + new MeterHydros21_Cond(&hydros), + new MeterHydros21_Depth(&hydros), + new MeterHydros21_Temp(&hydros), + new CampbellOBS3_Turbidity(&osb3low, "", "TurbLow"), + new CampbellOBS3_Turbidity(&osb3high, "", "TurbHigh"), + new ProcessorStats_Battery(&mcuBoard), + #endif //SENSORS_EXTERNAL + new ProcessorStats_SampleNumber(&mcuBoard,SEQUENCE_NUMBER_UUID), + new ProcessorStats_Battery(&mcuBoard, BAT_VOLTAGE_UUID ) + // Fut Sensiron Temperature + // Fut Sensirom Humidity + //new Modem_SignalPercent(&modem), +}; + +// All UUID's, device registration, and sampling feature information can be +// pasted directly from Monitor My Watershed. +// To get the list, click the "View token UUID list" button on the upper right +// of the site page. + +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** +// Check the order of your variables in the variable list!!! +// Be VERY certain that they match the order of your UUID's! +// Rearrange the variables in the variable list ABOVE if necessary to match! +// Do not change the order of the variables in the section below. +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** + +// Replace all of the text in the following section with the UUID array from +// MonitorMyWatershed + +/* clang-format off */ +// --------------------- Beginning of Token UUID List --------------------- +// see ms_cfg.h initially setup for https://monitormywatershed.org/sites/bq_test01/ + +const char* registrationToken = registrationToken_UUID; // Device registration token +const char* samplingFeature = samplingFeature_UUID ; // Sampling feature UUID + + +// ----------------------- End of Token UUID List ----------------------- +/* clang-format on */ + +// Count up the number of pointers in the array +int variableCount = sizeof(variableList) / sizeof(variableList[0]); + +// Create the VariableArray object +VariableArray varArray(variableCount, variableList); +/** End [variable_arrays] */ + + +// ========================================================================== +// The Logger Object[s] +// ========================================================================== +/** Start [loggers] */ +// Create a new logger instance +Logger dataLogger(LoggerID, loggingInterval, &varArray); +/** End [loggers] */ + + +// ========================================================================== +// Creating Data Publisher[s] +// ========================================================================== +/** Start [publishers] */ +// Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint +#include +EnviroDIYPublisher EnviroDIYPOST(dataLogger, &modemPhy.gsmClient, + registrationToken, samplingFeature); +/** End [publishers] */ + + +// ========================================================================== +// Working Functions +// ========================================================================== +/** Start [working_functions] */ +// Flashes the LED's on the primary board +void greenredflash(uint8_t numFlash = 4, uint8_t rate = 75) { + for (uint8_t i = 0; i < numFlash; i++) { + digitalWrite(greenLED, HIGH); + digitalWrite(redLED, LOW); + delay(rate); + digitalWrite(greenLED, LOW); + digitalWrite(redLED, HIGH); + delay(rate); + } + digitalWrite(redLED, LOW); +} + +// Reads the battery voltage +// NOTE: This will actually return the battery level from the previous update! +float getBatteryVoltage() { + if (mcuBoard.sensorValues[0] == -9999) mcuBoard.update(); + return mcuBoard.sensorValues[0]; +} + + +// ========================================================================== +// Arduino Setup Function +// ========================================================================== +/** Start [setup] */ +void setup() { + // Start the primary serial connection + Serial.begin(serialBaud); + Serial.print(F("\n---Boot. Sw Build: ")); + Serial.print(build_ref); + Serial.print(" "); + Serial.println(git_usr); + Serial.print(" "); + Serial.println(git_branch); + + // Print a start-up note to the first serial port + Serial.print(F("\nNow running ")); + Serial.print(sketchName); + Serial.print(F(" on Logger ")); + Serial.println(LoggerID); + Serial.println(); + + Serial.print(F("Using ModularSensors Library version ")); + Serial.println(MODULAR_SENSORS_VERSION); + Serial.print(F("TinyGSM Library version ")); + Serial.println(TINYGSM_VERSION); + Serial.println(); + + // Start the serial connection with the modem + modemSerial.begin(modemBaud); + + // Set up pins for the LED's + pinMode(greenLED, OUTPUT); + digitalWrite(greenLED, LOW); + pinMode(redLED, OUTPUT); + digitalWrite(redLED, LOW); + // Blink the LEDs to show the board is on and starting up + greenredflash(); + + pinMode(20, OUTPUT); // for proper operation of the onboard flash memory + // chip's ChipSelect (Mayfly v1.0 and later) + + // Set the timezones for the logger/data and the RTC + // Logging in the given time zone + Logger::setLoggerTimeZone(timeZone); + // It is STRONGLY RECOMMENDED that you set the RTC to be in UTC (UTC+0) + Logger::setRTCTimeZone(0); + + // Attach the modem and information pins to the logger + dataLogger.attachModem(modemPhy); + modemPhy.setModemLED(modemLEDPin); + dataLogger.setLoggerPins(wakePin, sdCardSSPin, sdCardPwrPin, buttonPin, + greenLED); + + // How to turn of meta data dta polling modemPhy.updateModemMetadata(0); + // Begin the logger + dataLogger.begin(); + + // Note: Please change these battery voltages to match your battery + // Set up the sensors, except at lowest battery level + if (getBatteryVoltage() > 3.4) { + Serial.println(F("Setting up sensors...")); + varArray.setupSensors(); + } + + #if defined sim_com_sim700 + /** Start [setup_sim7080] */ + modem.setModemWakeLevel(HIGH); // ModuleFun Bee inverts the signal + modem.setModemResetLevel(HIGH); // ModuleFun Bee inverts the signal + Serial.println(F("Waking modem and setting Cellular Carrier Options...")); + modem.modemWake(); // NOTE: This will also set up the modem + modem.gsmModem.setBaud(modemBaud); // Make sure we're *NOT* auto-bauding! + modem.gsmModem.setNetworkMode(38); // set to LTE only + // 2 Automatic + // 13 GSM only + // 38 LTE only + // 51 GSM and LTE only + modem.gsmModem.setPreferredMode(1); // set to CAT-M + // 1 CAT-M + // 2 NB-IoT + // 3 CAT-M and NB-IoT + /** End [setup_sim7080] */ + #elif defined sim_com_xbee_wifi + /** Start [setup_sim7080] */ + + Serial.println(F("Waking modem WiFi ...")); + modemPhy.modemWake(); // NOTE: This will also set up the modem + modemPhy.gsmModem.setBaud(modemBaud); // Make sure we're *NOT* auto-bauding! + #endif //Modem setup + + // Sync the clock if it isn't valid or we have battery to spare + if (getBatteryVoltage() > 3.55 || !dataLogger.isRTCSane()) { + // Synchronize the RTC with NIST + // This will also set up the modem + dataLogger.syncRTC(); + } + + // Create the log file, adding the default header to it + // Do this last so we have the best chance of getting the time correct and + // all sensor names correct + // Writing to the SD card can be power intensive, so if we're skipping + // the sensor setup we'll skip this too. + if (getBatteryVoltage() > 3.4) { + Serial.println(F("Setting up file on SD card")); + dataLogger.turnOnSDcard( + true); // true = wait for card to settle after power up + dataLogger.createLogFile(true); // true = write a new header + dataLogger.turnOffSDcard( + true); // true = wait for internal housekeeping after write + } + + // Call the processor sleep + Serial.println(F("Putting processor to sleep\n")); + dataLogger.systemSleep(); +} +/** End [setup] */ + + +// ========================================================================== +// Arduino Loop Function +// ========================================================================== +/** Start [loop] */ +// Use this short loop for simple data logging and sending +void loop() { + // Note: Please change these battery voltages to match your battery + // At very low battery, just go back to sleep + if (getBatteryVoltage() < 3.4) { + dataLogger.systemSleep(); + } + // At moderate voltage, log data but don't send it over the modem + else if (getBatteryVoltage() < 3.55) { + dataLogger.logData(); + } + // If the battery is good, send the data to the world + else { + dataLogger.logDataAndPublish(); + } +} +/** End [loop] */ diff --git a/sensors_test/DRWI_SIM7080LTE/src/ms_cfg.h b/sensors_test/DRWI_SIM7080LTE/src/ms_cfg.h new file mode 100644 index 000000000..2abffd99a --- /dev/null +++ b/sensors_test/DRWI_SIM7080LTE/src/ms_cfg.h @@ -0,0 +1,37 @@ +/***************************************************************************** +ms_cfg.h - ModularSensors Config - +Testing envPr2a\ModularSensors\sensors_test\DRWI_SIM7080LTE\src +Status 230630: part of (develop) for 0.34.1 +Written By: Neil Hancock www.envirodiy.org/members/neilh20/ +Development Environment: PlatformIO +Hardware Platform(s): EnviroDIY Mayfly Arduino Datalogger + +Software License: BSD-3. + Copyright (c) 2023, Neil Hancock - all rights assigned to Stroud Water +Research Center (SWRC) and they may change this title to Stroud Water Research +Center as required and the EnviroDIY Development Team + + +DISCLAIMER: +THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. +*****************************************************************************/ +#ifndef DRWI_SIM7080LTE_ms_cfg_h +#define ms_cfg_h +#define WIFIID_SSID_DEF "ArthurGuestSsid" +#define WIFIPWD_DEF "Arthur8166" +// Defaults for data.envirodiy.org +//Site https://monitormywatershed.org/sites/bq_test01/ +#define LOGGERID_DEF_STR "btest01" +#define NEW_LOGGERID_MAX_SIZE 40 +#define registrationToken_UUID "22752220-5925-4a2c-aeb1-a57b58e1c246" +#define samplingFeature_UUID "747478ef-4e80-4cc8-921e-89172d05ea42" +#define SEQUENCE_NUMBER_UUID "e0d7b81b-0241-4017-b5dc-e90ecdb7c279" + +//A tempororay repurpose of above UUIDs +//#define SENSIRON_HUMIDITY_UUID "" +//#define SENSIRON_TEMPERATURE_UUID "" +//#define TEMPERATURE_D_UUID "" + +#define BAT_VOLTAGE_UUID "9fdcefc1-b43f-4c3c-8d46-ca0e90845153" + +#endif // DRWI_SIM7080LTE_ms_cfg_h \ No newline at end of file diff --git a/src/LoggerBase.cpp b/src/LoggerBase.cpp index 2e1a54c5a..19a232817 100644 --- a/src/LoggerBase.cpp +++ b/src/LoggerBase.cpp @@ -39,6 +39,18 @@ volatile bool Logger::startTesting = false; RTCZero Logger::zero_sleep_rtc; #endif +#if defined USE_RTCLIB +// or change to +// USE_RTC_EXTPHY rtcExtPhy; +USE_RTCLIB rtcExtPhy; +// For RTClib.h:DateTime(uint32_t) use secs since 1970 +#define DateTimeClass(varNam, epochTime) DateTime varNam(epochTime); +#else +// For Sodaq_DS3231.h:DateTime(long) uses secs since 2000 +#define DateTimeClass(varNam, epochTime) \ + DateTime varNam((long)((uint64_t)(epochTime - EPOCH_TIME_DTCLASS))); +#endif // USE_RTCLIB +#define TEMP_BUFFER_SZ 37 // Constructors Logger::Logger(const char* loggerID, uint16_t loggingIntervalMinutes, @@ -347,10 +359,12 @@ void Logger::registerDataPublisher(dataPublisher* publisher) { void Logger::publishDataToRemotes(void) { + // Assumes that there is an internet connection MS_DBG(F("Sending out remote data.")); for (uint8_t i = 0; i < MAX_NUMBER_SENDERS; i++) { if (dataPublishers[i] != nullptr) { + _dataPubInstance = i; PRINTOUT(F("\nSending data to ["), i, F("]"), dataPublishers[i]->getEndpoint()); dataPublishers[i]->publishData(); @@ -592,24 +606,61 @@ void Logger::markTime(void) { // This checks to see if the CURRENT time is an even interval of the logging // rate -bool Logger::checkInterval(void) { - bool retval; +uint8_t Logger::checkInterval(void) { + uint8_t retval = CIA_NOACTION; uint32_t checkTime = getNowLocalEpoch(); - MS_DBG(F("Current Unix Timestamp:"), checkTime, F("->"), + int modulus_time_sec =checkTime % (_loggingIntervalMinutes * 60); + MS_DBG(F("Current Epoch local Timestamp:"), checkTime, F("->"), formatDateTime_ISO8601(checkTime)); MS_DBG(F("Logging interval in seconds:"), (_loggingIntervalMinutes * 60)); MS_DBG(F("Mod of Logging Interval:"), - checkTime % (_loggingIntervalMinutes * 60)); + modulus_time_sec); + + if (_sendOffset_act) { + // A Timer is counting down to perform delayed Post Readings + if (0 >= --_sendOffset_cnt) { + // Timer has expired + _sendOffset_act = false; + retval |= CIA_POST_READINGS; + MS_DBG(F("sendOffset Post Readings")); + } else { + MS_DBG(F("sendOffset Timer "), _sendOffset_cnt); + } + } - if (checkTime % (_loggingIntervalMinutes * 60) == 0) { + if (modulus_time_sec < 59 ) { // Update the time variables with the current time markTime(); - MS_DBG(F("Time marked at (unix):"), Logger::markedLocalEpochTime); - MS_DBG(F("Time to log!")); - retval = true; + MS_DBG(F("Take Sensor readings. Epoch:"), Logger::markedLocalEpochTime); + + // Check what actions for this time period + retval |= CIA_NEW_READING; + if (1 < _sendEveryX_num) { + _sendEveryX_cnt++; + if (_sendEveryX_cnt >= _sendEveryX_num) { + _sendEveryX_cnt = 0; + // Check if delay ~ offset to Send Readings + if (0 == _sendOffset_min) { + // No dealy ~ send readings now + retval |= CIA_POST_READINGS; + MS_DBG(F("sendEveryX Post Readings")); + } else { + // delayed retval |= CIA_POST_READINGS; + _sendOffset_act = true; + _sendOffset_cnt = _sendOffset_min; + MS_DBG(F("sendEveryX Timer sendOffset started "), + _sendOffset_min); + } + } else { + MS_DBG(F("sendEveryX "), _sendEveryX_cnt, F("counting to "), + _sendEveryX_num); + } + } else { + retval |= CIA_POST_READINGS; + MS_DBG(F("Post readings.")); + } } else { MS_DBG(F("Not time yet.")); - retval = false; } if (!isRTCSane(checkTime)) { PRINTOUT(F("----- WARNING ----- !!!!!!!!!!!!!!!!!!!!")); @@ -632,7 +683,8 @@ bool Logger::checkInterval(void) { delay(25); alertOff(); delay(25); - PRINTOUT(F("The current clock timestamp is not valid!")); + PRINTOUT(F("The current clock timestamp is not valid!"), + formatDateTime_ISO8601(getNowUTCEpoch()).substring(0, 10)); alertOn(); delay(25); alertOff(); @@ -1037,7 +1089,7 @@ bool Logger::initializeSDCard(void) { return false; } // Initialise the SD card - if (!sd.begin(_SDCardSSPin, SPI_FULL_SPEED)) { + if (!sd1_card_fatfs.begin(_SDCardSSPin, SPI_FULL_SPEED)) { PRINTOUT(F("Error: SD card failed to initialize or is missing.")); PRINTOUT(F("Data will not be saved!")); return false; @@ -1052,13 +1104,29 @@ bool Logger::initializeSDCard(void) { // Protected helper function - This sets a timestamp on a file -void Logger::setFileTimestamp(File fileToStamp, uint8_t stampFlag) { - fileToStamp.timestamp(stampFlag, dtFromEpoch(getNowLocalEpoch()).year(), - dtFromEpoch(getNowLocalEpoch()).month(), - dtFromEpoch(getNowLocalEpoch()).date(), - dtFromEpoch(getNowLocalEpoch()).hour(), - dtFromEpoch(getNowLocalEpoch()).minute(), - dtFromEpoch(getNowLocalEpoch()).second()); +void Logger::setFileTimestamp(File fileToStamp, uint8_t stampFlag, bool localTime) { + if (false == localTime) { + fileToStamp.timestamp(stampFlag, dtFromEpoch(getNowLocalEpoch()).year(), + dtFromEpoch(getNowLocalEpoch()).month(), + dtFromEpoch(getNowLocalEpoch()).date(), + dtFromEpoch(getNowLocalEpoch()).hour(), + dtFromEpoch(getNowLocalEpoch()).minute(), + dtFromEpoch(getNowLocalEpoch()).second()); + }else { + + DateTime markedDtTz(getNowLocalEpoch()- EPOCH_TIME_DTCLASS ); + + MS_DEEP_DBG(F("setFTTz"),markedDtTz.year(),markedDtTz.month(), markedDtTz.date(), + markedDtTz.hour(), markedDtTz.minute(), markedDtTz.second()); + bool crStat = fileToStamp.timestamp( + stampFlag, markedDtTz.year(), markedDtTz.month(), markedDtTz.date(), + markedDtTz.hour(), markedDtTz.minute(), markedDtTz.second()); + if (!crStat) { + PRINTOUT(F("setFTTz err for "), markedDtTz.year(), markedDtTz.month(), + markedDtTz.date(), markedDtTz.hour(), markedDtTz.minute(), + markedDtTz.second()); + } + } } @@ -1558,3 +1626,1110 @@ void Logger::logDataAndPublish(void) { // Call the processor sleep systemSleep(); } + +void Logger::forceSysReset(uint8_t source, uint16_t simpleMagicNumber) { + + if (4567 !=simpleMagicNumber) return; + + PRINTOUT(F("Forcing reset"), source); + delay(20); + watchDogTimer.setupWatchDog(1); + watchDogTimer.enableWatchDog(); + delay(100000); //Expect watchdog to kick in within 8secs +} //forceReset + +// ===================================================================== // +// Reliable Delivery functions +// see class headers +// ===================================================================== // + +void Logger::logDataAndPubReliably(uint8_t cia_val_override) { + + if (cia_val_override & CIA_NO_SLEEP) { + cia_val_override &= ~CIA_NO_SLEEP; + } else { + // Sleep at start of cycle, so data is available for caller at the end + systemSleep(); + } + + // Reset the watchdog + watchDogTimer.resetWatchDog(); + + + // Assuming we were woken up by the clock, check if the current time is + // an even interval of the logging interval + uint8_t cia_val = checkInterval(); + if (cia_val_override) { + cia_val = cia_val_override; + wakeUpTime_secs = getNowLocalEpoch();//Set reference time + markTime(); + PRINTOUT(F("logDataAndPubReliably - overide with "),cia_val); + } + + if (NULL != _bat_handler_atl) { + _bat_handler_atl(LB_PWR_USEABLE_REQ); // Set battery status + if (!_bat_handler_atl(LB_PWR_SENSOR_USE_REQ)) { + // Squash any activity + //PRINTOUT(F("logDataAndPubReliably - all cancelled")); + const static char ALL_CANCELLED_pm[] EDIY_PROGMEM = + "logDataAndPubReliably - all cancelled"; + // njh PRINT_LOGLINE_P(ALL_CANCELLED_pm); + cia_val = 0; + } + if (!_bat_handler_atl(LB_PWR_MODEM_USE_REQ)) { + if (CIA_POST_READINGS & cia_val) { + // Change publish attempt to saving for next publish attempt + cia_val &= ~CIA_POST_READINGS; + cia_val |= CIA_RLB_READINGS; // + //PRINTOUT(F("logDataAndPubReliably - tx cancelled")); + const static char TX_CANCELLED_pm[] EDIY_PROGMEM = + "logDataAndPubReliably - tx cancelled"; + // njh PRINT_LOGLINE_P(TX_CANCELLED_pm); + } + } + } + +#if 1 + if (cia_val) { + // Flag to notify that we're in already awake and logging a point + Logger::isLoggingNow = true; + // Reset the watchdog + watchDogTimer.resetWatchDog(); + + // Print a line to show new reading + //PRINTOUT(F("---logDataAndPubReliably ("),cia_val,F(")----")); + STANDARD_SERIAL_OUTPUT.print(F("---logDataAndPubReliably (0x")); + STANDARD_SERIAL_OUTPUT.print(cia_val,HEX); + STANDARD_SERIAL_OUTPUT.println(F(")----")); + // Turn on the LED to show we're taking a reading + alertOn(); + // Power up the SD Card + // TODO(SRGDamia1): Decide how much delay is needed between turning on + // the card and writing to it. Could we turn it on just before writing? + turnOnSDcard(false); + if (cia_val & CIA_NEW_READING) { + // Do a complete update on the variable array. + // This this includes powering all of the sensors, getting + // updated values, and turing them back off. NOTE: The wake + // function for each sensor should force sensor setup to run if + // the sensor was not previously set up. + PRINTOUT(F("Read sensors...")); + watchDogTimer.resetWatchDog(); + _internalArray->completeUpdate(); + watchDogTimer.resetWatchDog(); + + // Create a csv data record and save it to the log file + logToSD(); + + serzRdel_Line(); // Start Queue + } + if (cia_val & CIA_POST_READINGS) { + if (_logModem != NULL) { + MS_DBG(F("Waking up"), _logModem->getModemName(), F("...")); + if (_logModem->modemWake()) { + // Connect to the network + watchDogTimer.resetWatchDog(); + PRINTOUT(F("Connecting to the Internet with"),_logModem->getModemName()); + if (_logModem->connectInternet()) { + const static char CONNECT_INTERNET_pm[] EDIY_PROGMEM = + "Connected Internet"; + PRINT_LOGLINE_P(CONNECT_INTERNET_pm); + // be nice to add _logModem->getModemName() + //This doesn't work PRINT_LOGLINE_P2(CONNECT_INTERNET_pm,_logModem->getModemName().c_str()); + // Publish data to remotes + watchDogTimer.resetWatchDog(); + publishDataQueuedToRemotes(true); + watchDogTimer.resetWatchDog(); + +// Sync the clock at midnight or on the hour +#define NIST_SYNC_DAY 86400 +#define NIST_SYNC_HR 3600 +#if defined NIST_SYNC_HOURLY +#define NIST_SYNC_RATE NIST_SYNC_HR +#else +#define NIST_SYNC_RATE NIST_SYNC_DAY +#endif //NIST_SYNC_HOURLY + uint32_t logIntvl_sec = _loggingIntervalMinutes * 60; + uint32_t timeToday_sec = markedLocalEpochTime % NIST_SYNC_RATE; + bool doSyncTimeCheck = (timeToday_sec< logIntvl_sec); + /*MS_DBG*/PRINTOUT(F("SyncTimeCheck "),doSyncTimeCheck," modulo_sec",timeToday_sec," Time",Logger::markedLocalEpochTime); + if (doSyncTimeCheck) { + MS_DBG(F("Running an NIST clock sync...")); + if(setRTClock(_logModem->getNISTTime())) { + const static char CLOCK_NIST_OK_pm[] EDIY_PROGMEM ="Clock Nist Synced"; + PRINT_LOGLINE_P(CLOCK_NIST_OK_pm); + } else { + const static char CLOCK_NIST_FAIL_pm[] EDIY_PROGMEM ="Clock Nist Failed"; + PRINT_LOGLINE_P(CLOCK_NIST_FAIL_pm); + } + } + watchDogTimer.resetWatchDog(); + + // Update the modem metadata + MS_DBG(F("Updating modem metadata...")); + _logModem->updateModemMetadata(); + + // Disconnect from the network + MS_DBG(F("Disconnecting from the Internet...")); + _logModem->disconnectInternet(); + } else { + //RINTOUT(F("Connect to the internet failed with"),_logModem->getModemName()); + const static char CONNECT_FAILED_pm[] EDIY_PROGMEM = + "Connected Internet Failed"; + PRINT_LOGLINE_P(CONNECT_FAILED_pm); + watchDogTimer.resetWatchDog(); + } + } else { + PRINTOUT(F("Failed to wake "), _logModem->getModemName()); + } + // Turn the modem off + _logModem->modemSleepPowerDown(); + } else + PRINTOUT(F("Internet failed, no _logModem ")); + } else if (cia_val & CIA_RLB_READINGS) { + // Values not transmitted, save readings for later transmission + PRINTOUT(F("logDataAndPubReliably - store readings, no pub")); + publishDataQueuedToRemotes(false); + } + + + // TODO(SRGDamia1): Do some sort of verification that minimum 1 sec has + // passed for internal SD card housekeeping before cutting power It + // seems very unlikely based on my testing that less than one second + // would be taken up in publishing data to remotes + // Cut power from the SD card - without additional housekeeping wait + turnOffSDcard(false); + + // Turn off the LED + alertOff(); + // Print a line to show reading ended + PRINTOUT(F("---logDataAndPubReliably Complete----------")); + + // Unset flag + Logger::isLoggingNow = false; + Logger::startTesting = false; //Interrupt going off + // njh dumpFreeRam(8256); //large Number + } + + // Check if it was instead the testing interrupt that woke us up + if (Logger::startTesting) testingMode(); + + // Call the processor sleep + //systemSleep(); + #endif //0 +} // logDataAndPubReliably + +bool Logger::publishRspCodeAccepted(int16_t rspCode) { + if (HTTPSTATUS_CREATED_201 == rspCode) return true; + //return (HTTPSTATUS_CREATED_201 == rspCode); + #if defined MS_DISCARD_HTTP_500 + if (HTTPSTATUS_GT_500 == rspCode) { + //As of 2022Sept15 this error is repetitive and prevents more messages being sent + // https://github.com/ODM2/ODM2DataSharingPortal/issues/628 + //https://github.com/neilh10/ModularSensors/issues/119 + // Unfortunately throw away this reading + PRINTOUT(F("pubRspCode SERVER ERROR discard reading")); + return true; + } + #endif //MS_DISCARD_HTTP_500 + return false; +} //publishRspCodeAccepted + +void Logger::publishDataQueuedToRemotes(bool internetPresent) { + // Assumes that there is an internet connection + // bool useQueue = false; + int16_t rspCode = 0; + uint32_t tmrGateway_ms; + bool dslStatus = false; + bool retVal = false; + // MS_DBG(F("Pub Data Queued")); + MS_DBG(F("pubDQTR from"), serzRdelFn_str, internetPresent); + + // Open debug file +#if defined MS_LOGGERBASE_POSTS + retVal = postLogOpen(postsLogFn_str); +#endif // MS_LOGGERBASE_POSTS + + for (uint8_t i = 0; i < MAX_NUMBER_SENDERS; i++) { + if (dataPublishers[i] != NULL) { + _dataPubInstance = i; + PRINTOUT(F("\npubDQTR Sending data to ["), i, F("]"), + dataPublishers[i]->getEndpoint()); + // open the qued file for serialized readings + // (char*)serzQueuedFn_str + + + // dataPublishers[i]->publishData(_logModem->getClient()); + // Need to manage filenames[i] + + /* TODO njh check power availability + ps_Lbatt_status_t Lbatt_status; + Lbatt_status = + mcuBoard.isBatteryStatusAbove(true,PS_PWR_USEABLE_REQ); + if (no power) break out for loop; + */ + + if (dataPublishers[i]->getQueuedStatus()) { + uint16_t delay_posted_pacing_ms = dataPublishers[i]->getTimerPostPacing_mS(); + uint16_t published_this_pass =0; + serzQueuedStart((char)('0' + i)); + deszRdelStart(); + // MS_START_DEBUG_TIMER; + tmrGateway_ms = millis(); + uint32_t tmrThisPublish_ms; + bool attemptPostStatus=true; + uint16_t attemptPostCnt=0; + uint16_t attemptPostFailedCnt=0; + while ((dslStatus = deszRdelLine())) { + attemptPostCnt++; + tmrThisPublish_ms = millis(); + if (internetPresent && attemptPostStatus) { + rspCode = dataPublishers[i]->publishData(); + } else { + if (internetPresent) { + rspCode = HTTPSTATUS_NC_901; + } else { + //then must be attemptPostStatus==false + rspCode = HTTPSTATUS_NC_904; + } + } + + watchDogTimer.resetWatchDog(); + // MS_DBG(F("Rsp"), rspCode, F(", in"), + // MS_PRINT_DEBUG_TIMER, F("ms\n")); + postLogLine( (millis() -tmrThisPublish_ms), rspCode); + + if (false == publishRspCodeAccepted(rspCode)) { +#define DESLZ_STATUS_UNACK '1' +#define DESLZ_STATUS_MAX '8' +#define DESLZ_STATUS_POS 0 +#if 0 + if (++deszq_line[0] > '8') { + deszq_line[DESLZ_STATUS_POS] = DESLZ_STATUS_UNACK; + } +#endif // if x +#if defined(USE_PS_modularSensorsNetwork) + if ((desz_pending_records >= _sendQueueSz_num)&&(MMWGI_SEND_QUE_SZ_NUM_NOP != _sendQueueSz_num )) { + PRINTOUT(F("pubDQTR QueuedFull, skip reading. sendQueue "), _sendQueueSz_num); + postLogLine(0,rspCode); //Log skipped readings + } else + #endif // USE_PS_modularSensorsNetwork + { + retVal = serzQueuedFile.print(deszq_line); + if (0 >= retVal) { + PRINTOUT(F("pubDQTR serzQueuedFil err"), retVal); + } + desz_pending_records++; + } + } else { + /*A publish has been sucessfull. + * Slow Down sending based on publishers acceptance rate + * Each publish creates and tears down a TCP connection */ + /*TODO njh create intergrate all POSTS to one tcp/ip connection */ + published_this_pass++; + attemptPostFailedCnt=0; + MS_DBG(F("pubDQTR1 delay"),delay_posted_pacing_ms ,F("mS : posted"), published_this_pass); + delay(delay_posted_pacing_ms); + } + + //Check for any limits that might have been exceeded + if ((attemptPostCnt >= _postMax_num) && (0 != _postMax_num)) { + //Exceeded number to attempt, force write to serzQueue + attemptPostStatus = false; + } + + if (++attemptPostFailedCnt > RDELAY_FAILED_POSTS_THRESHOLD ) { + //Exceeded number of consecutive failures, force write to serzQueue + attemptPostStatus = false; + } + } // while reading line + deszRdelClose(true); + serzQueuedCloseFile(false); + // retVal = serzQueuedFile.close(); + // if (!retVal) + // PRINTOUT( + // F("publishDataQueuedToRemote serzQueuedFile.close err")); + + PRINTOUT(F("Sent"), deszLinesRead, F("readings in"), + ((float)(millis() - tmrGateway_ms)) / 1000, + F("sec. Queued readings="), desz_pending_records); + + if (true == publishRspCodeAccepted(rspCode)) { + // Do retrys through publisher - if file exists + if (sd1_card_fatfs.exists(serzQueuedFn)) { + uint16_t tot_posted = 0; + uint16_t cnt_for_pwr_analysis = 1; + MS_DBG(F("pubDQTR retry from"), serzQueuedFn); + deszQueuedStart(); + while ((dslStatus = deszQueuedLine()) ) { + + /*At least one publish has been sucessfull. + * Slow Down sending based on publishers acceptance rate + * Each publish creates and tears down a TCP connection */ + MS_DBG(F("pubDQTR2 delay"),delay_posted_pacing_ms ,F("mS : total posted"), published_this_pass); + delay(delay_posted_pacing_ms); + + // setup for publisher to call deszqNextCh() + rspCode = dataPublishers[i]->publishData(); + watchDogTimer.resetWatchDog(); + postLogLine(i, rspCode); + if (false == publishRspCodeAccepted(rspCode)) break; + + tot_posted++; + published_this_pass++; + + deszq_line[0] = 0; // Show completed + + // Check for enough battery power + if (cnt_for_pwr_analysis++ >= + _sendAtOneTimeMaxX_num) { + if (NULL != _bat_handler_atl) { + // Measure battery + _bat_handler_atl(LB_PWR_USEABLE_REQ); + if (!_bat_handler_atl( + LB_PWR_MODEM_USE_REQ)) { + // stop transmission + cnt_for_pwr_analysis = 0; + PRINTOUT(F("pubDQTR not enough power available")); + break; + } + } + cnt_for_pwr_analysis = 1; + } + if ((tot_posted >= _postMax_num) && (0 != _postMax_num)) { + PRINTOUT(F("pubDQTR POST_MAX_NUM reached"), tot_posted); + break; /// unsent lines are copied through + } + } //while +// increment status of number attempts +#if 0 + if (deszq_line[DESLZ_STATUS_POS]++ >= + DESLZ_STATUS_MAX) { + deszq_line[DESLZ_STATUS_POS] = DESLZ_STATUS_MAX; + } +#endif // if z + // deszQueuedCloseFile() is serzQueuedCloseFile(true) + if (tot_posted) { + // At least one POST was accepted, if 2 or more, the last may have failed + // and still be in deszq_line + serzQueuedCloseFile(true); + } else { + serzQueuedCloseFile(false); + } + } else { MS_DBG(F("pubDQTR no queued file"), serzQueuedFn);} + } else { + MS_DBG(F("pubDQTR drop retrys. rspCode"), rspCode); + } + } + } + } + postLogClose(); +} // publishDataQueuedToRemotes + +// ===================================================================== // +// Serialize/deserialize functions +// see class headers +// ===================================================================== // + +#define DELIM_CHAR2 ',' +#define SERZQUED_OFLAGS +bool Logger::serzQueuedStart(char uniqueId) { + strcpy(serzQueuedFn, serzQueuedFn_str); + strncat(serzQueuedFn, &uniqueId, 1); + strcat(serzQueuedFn, ".TXT"); + + if (!serzQueuedFile.open(serzQueuedFn, (O_WRITE | O_CREAT | O_AT_END))) { + PRINTOUT(F("serzQueuedStart open err")); + return false; + } else { + MS_DEEP_DBG(F("serzQueuedStart open"), serzQueuedFn); + } + return true; +} + +bool Logger::serzQueuedCloseFile(bool flush) { + /* This closes the file, removing the sent messages + Assumes serzQueuedFile points incoming file if flush==true + */ + bool retBool=true; + + if (flush) { + // There may be 0, or more of unsent records left in serzQueued + uint16_t num_lines = serzQueuedFlushFile(); + + PRINTOUT(F("seQCF Queue for next pass unsent records"), num_lines); + desz_pending_records = num_lines; + + } else { // !flush simple clean + retBool = serzQueuedFile.close(); + if (!retBool) { + sd1_Err("seQCF serzQueuedFile.close2 err"); + return false; + } + } + return retBool; +} + +#define TEMP_BASE_FN_STR "TMP01.TXT" +#define QUEOLD_BASE_FN_STR "QUEDEL01.TXT" +inline uint16_t Logger::serzQueuedFlushFile() { + /* The flush algorithim is, + copy unsent lines to a temporary_file up to _sendQueueSz_num, and then discard rest + Assumes serzQueuedFile points incoming file + when complete rename serzQueuedFile to delete_file + rename temporary_file to serzQueuedFile to complete flush + */ + const char* tempFn = TEMP_BASE_FN_STR; + const char* queDelFn = QUEOLD_BASE_FN_STR; + File tgtoutFile; + int16_t retNum; + int16_t num_char ; + uint16_t num_lines = 0; + uint16_t num_skipped=0; + bool retBool; + + // Check if exists and delete + if (sd1_card_fatfs.exists(tempFn)) { + if (!sd1_card_fatfs.remove(tempFn)) { + PRINTOUT(F("seQFF remove1 err"), tempFn); + sd1_Err("seQFF err6 remove"); + } else { + MS_DEEP_DBG(F("seQFF remove "), tempFn); + } + } + retBool = tgtoutFile.open(tempFn, (O_WRITE | O_CREAT)); + if (!retBool) { + PRINTOUT(F("seQFF open2 err"), tempFn); + // sd1_Err("seQCF open4"); + //todo close all other files + return 0; + } else { + MS_DEEP_DBG(F("seQFF opened "), tempFn); + } + + num_char = strlen(deszq_line); + if (num_char) { // Test could be min size, but this unknown + MS_DBG(F("seQFF Last POST Failed "), deszq_line); + retNum = tgtoutFile.write(deszq_line, num_char); + if (retNum != num_char) { + PRINTOUT(F("seQFF tgtoutFile write1 err"), num_char); + // sd1_Err("seQCF write2"); + } + } + + MS_DBG(F("seQFF cpy lines across")); + while (0 < (num_char = serzQueuedFile.fgets(deszq_line, + QUEFILE_MAX_LINE))) { + +#if defined(USE_PS_modularSensorsNetwork) + if ((num_lines>=_sendQueueSz_num)&&(MMWGI_SEND_QUE_SZ_NUM_NOP != _sendQueueSz_num )) { + /*Limit sendQueueSz on Copy, implicitly this not on creation + This is the first pass at limiting the size of the que by dumping the newest. + FIFO. + Future may want to keep the latest readings + */ + postLogLine((MAX_NUMBER_SENDERS+1),HTTPSTATUS_NC_903); + num_skipped++; + } else +#endif // USE_PS_modularSensorsNetwork + { + + retNum = tgtoutFile.write(deszq_line, num_char); + // Squelch last char LF + deszq_line[sizeof(deszq_line) - 1] = 0; + MS_DBG(deszq_line); + if (retNum != num_char) { + PRINTOUT(F("seQFF tgtoutFile write3 err"), num_char, + retNum); + // sd1_Err("seQFF write4"); + break; + } + num_lines++; + } + } + if (num_skipped){ + PRINTOUT(F("seQFF sendQueue Size "), _sendQueueSz_num, F(",queued"),num_lines, F(",latest readings discarded"),num_skipped); + }; + //Cleanup flushed serzQueuedFile to del_file as debugging aid + if (sd1_card_fatfs.exists(queDelFn)) { + if (!sd1_card_fatfs.remove(queDelFn)) { + PRINTOUT(F("seQFF remove2 err"), queDelFn); + sd1_Err("seQFF err7 remove"); + } + if (sd1_card_fatfs.exists(queDelFn)) { + PRINTOUT(F("seQFF err failed remove"), queDelFn); + } + } + + retBool = serzQueuedFile.rename(queDelFn); + if (!retBool) { + PRINTOUT(F("seQFF REBOOT rename1 err"), queDelFn); + //Problem - unrecoverable, so reboot + retBool = serzQueuedFile.close(); + if (!retBool) { + PRINTOUT(F("seQFF close1 failed err"), serzQueuedFn); + } + forceSysReset(1,4567); + //sd1_card_fatfs.remove(serzQueuedFn); + // sd1_Err("seQFF rename2"); + //return num_lines; + } else { + MS_DBG(F("seQFF cleanup rename "), serzQueuedFn, F("to"), queDelFn); + + retBool = serzQueuedFile.close(); + if (!retBool) { + sd1_Err("seQFF serzQueuedFile.close2 err"); + return num_lines; + } else {MS_DEEP_DBG(F("seQFF close serzQueuedFile")); } + + retBool = tgtoutFile.rename(serzQueuedFn); + if (!retBool) { + sd1_Err("seQFF tgtoutFile.rename err"); + return num_lines; + } else {MS_DEEP_DBG(F("seQFF rename "), tempFn, F("to"), serzQueuedFn); } + + retBool = tgtoutFile.close(); + if (!retBool) { + sd1_Err("seQFF tgtoutFile.close1 err"); + return num_lines; + } else {MS_DEEP_DBG(F("seQFF closed tgtoutFile")); } + } + + return num_lines; +} //serzQueuedFlushFile + +/* +For serialize, create ASCII CSV records of the form +status, n*[<,values>] +*/ +#define RDEL_OFLAG (O_WRITE | O_CREAT | O_AT_END) +bool Logger::serzRdel_Line() { + if (serzRdelFile.open(serzRdelFn_str, RDEL_OFLAG)) { + uint16_t outputSz; + // String csvString(Logger::markedLocalEpochTime); + outputSz = serzRdelFile.print("0,"); // Start READINGS_STATUS + outputSz += serzRdelFile.print(Logger::markedLocalEpochTime); + for (uint8_t i = 0; i < getArrayVarCount(); i++) { + // csvString += ','; + outputSz += serzRdelFile.print(',' + getValueStringAtI(i)); + } + outputSz += serzRdelFile.println(); + // setFileAccessTime(serzRdelFile); + serzRdelFile.close(); + MS_DEEP_DBG(F("serzRdel_Line on"), serzRdelFn_str, F(" at "), + markedLocalEpochTime, F(" size="), outputSz); + } else { + PRINTOUT(F("serzRdel_Line; No file"), serzRdelFn_str); + return false; + } + return true; +} // Logger::serzLine + + +/* Deserializer functions + +For deserialize, read ASCII CSV records of the form + n*[<,values>] + +deszRdelStart() ~ to open file +deszLine() to populate + deszq_epochTime & + +*/ + +/* Find fixed delimeter + * behave as strchrnul() if goes past end of string + */ +char* Logger::deszFind(const char* in_line, char caller_id) { + char* retResult = strchr(in_line, DELIM_CHAR2); + if (NULL != retResult) return retResult; + MS_DEEP_DBG(F("deszFind NULL found on "), caller_id); + // For NULL return pointer as per strchrnul + // should only occur on last search + return (char*)(in_line + strlen(in_line)); + + /* The strchrnul() function is like strchr() except that if \p c is not + found in \p s, then it returns a pointer to the null byte at the end + of \p s, rather than \c NULL. (Glibc, GNU extension.) + + \return The strchrnul() function returns a pointer to the matched + character, or a pointer to the null byte at the end of \p s (i.e., + \c s+strlen(s)) if the character is not found. + //char *strchrnul(const char *in, int delim_char) */ +} + + +bool Logger::deszRdelStart() { + deszLinesRead = deszLinesUnsent = 0; + + deszq_nextChar = deszq_line; + // Open - RD & WR. WR needed to be able to delete when complete. + if (!serzRdelFile.open(serzRdelFn_str, (O_RDWR | O_CREAT))) { + PRINTOUT(F("deRS; No file "), serzRdelFn_str); + return false; + } else { + MS_DEEP_DBG(F("deRS open RDWR"), serzRdelFn_str); + } + return true; +} + +bool Logger::deszQueuedStart() { + deszLinesRead = deszLinesUnsent = 0; + + deszq_nextChar = deszq_line; + // Open - RD & WR. WR needed to be able to delete when complete. + // Expect serzQueuedFn to be setup in serzQueuedStart + if (!serzQueuedFile.open(serzQueuedFn, O_RDWR)) { + // This could be that there aren;t any Queued readings + MS_DEEP_DBG(F("deQS; No file "), serzQueuedFn); + // sd1_card_fatfs.ls(); + return false; + } else { + MS_DEEP_DBG(F("deQS open READ"), serzQueuedFn); + } + + return true; +} +bool Logger::deszLine(File* filep) { + char* errCheck; + /* Scan through one line. Expect format + , representing integer STATUS + , represnting inteeger marked Epoch Time + .... representing reading values + + Not renetrant, assumption: there is only deserialize going on at a time. + Uses + char deszq_line[], + uint8_t deszq_status + long deszq_epochTime + char *deszq_nextChar + deszq_nextCharSz + */ + + uint16_t num_char = filep->fgets(deszq_line, QUEFILE_MAX_LINE); + char* orig_nextChar; + + if (0 == num_char) return false; + deszLinesRead++; + // First is the Status of record + deszq_status = strtol(deszq_line, &errCheck, 10); + if (errCheck == deszq_line) { + PRINTOUT(F("deszLine Status err'"), deszq_line, F("'")); + return false; // EIO; + } + // Find next DELIM and go past it + deszq_nextChar = 1 + deszFind(deszq_line, '1'); + if (deszq_nextChar == deszq_line) { + PRINTOUT(F("deszLine epoch start not found"), deszq_line, F("'")); + deszq_nextCharSz = 0; + return false; + } + // Second is the epochTime, + deszq_epochTime = strtol(deszq_nextChar, &errCheck, 10); + if (errCheck == deszq_line) { + PRINTOUT(F("deszLine Epoch err'"), deszq_line, F("'")); + return false; // EIO; + } + // Find next DELIM and go past it + orig_nextChar = deszq_nextChar; + deszq_nextChar = 1 + deszFind(deszq_nextChar, '2'); + if (orig_nextChar == deszq_nextChar) { + PRINTOUT(F("deszLine readung start not found"), deszq_line, F("'")); + deszq_nextCharSz = 0; + return false; + } + // Find sz of this field + char* nextCharEnd = deszFind(deszq_nextChar, '3'); + deszq_nextCharSz = nextCharEnd - deszq_nextChar; + + deszq_timeVariant_sz = strlen(deszq_nextChar) - 1; + MS_DBG(F("deszLine Reading sz"), deszq_timeVariant_sz, F(":"), deszq_nextChar, + F(":")); + return true; +} + +bool Logger::deszqNextCh(void) { + char* deszq_old = deszq_nextChar; + // Find next DELIM and go past it + deszq_nextChar = 1 + deszFind(deszq_nextChar, 'L'); + if ((deszq_old == deszq_nextChar)) { + deszq_nextCharSz = 0; + PRINTOUT(F("deszqNextCh 1error:"), deszq_nextChar, F("'")); + return false; + } + /* Find sz of this field + either + ,[..] + EOL + EOF + */ + char* nextCharEnd = strchr(deszq_nextChar, DELIM_CHAR2); + deszq_nextCharSz = strlen(deszq_nextChar); + if ((0 == deszq_nextCharSz)) { + // Found end of line + MS_DBG(F("dSRN unexpected EOL ")); + return false; + } else if (NULL == nextCharEnd) { + // Found EOF ~ nextSr_sz is valid + deszq_nextCharSz -= 1; // take off turds + MS_DEEP_DBG(F("dSRN info "), deszq_nextCharSz, " '", deszq_nextChar, + "'"); + // return true + } else { + // expect to have found ,[..] + // if found ,, then invalid and finish + deszq_nextCharSz = nextCharEnd - deszq_nextChar; + if (0 == deszq_nextCharSz) { + MS_DEEP_DBG(F("dSRN unexpected 0 bytes ")); + return false; + } + } + return true; +} + +bool Logger::deszRdelClose(bool deleteFile) { + bool retVal = false; + + if (!(retVal = serzRdelFile.close())) { + PRINTOUT(F("deSRC close err"), serzRdelFn_str); + sd1_Err("serzBegin err close"); + } else { + MS_DEEP_DBG(F("deSRC closed"), serzRdelFn_str); + } + if (deleteFile) { + // if (!(retVal = serzRdelFile.remove())) { + if (!(retVal = sd1_card_fatfs.remove(serzRdelFn_str))) { + PRINTOUT(F("deSRC remove err"), serzRdelFn_str); + sd1_Err("serzBegin err remove"); + } + MS_DEEP_DBG(F("deSRC removed"), serzRdelFn_str); + } + + return retVal; +} + +/* Prototyping des example + */ + +uint16_t serialCnt = 0; +// SdFat sdfat_phy; +// SdFile rootDir; +bool Logger::deszDbg(void) { + // char* next_token; +#define TEMPBUF_SZ 37 + char tempBuffer[TEMPBUF_SZ] = ""; + if (++serialCnt >= SERIALIZE_sendEveryX_NUM) { + String d_str(80); + serialCnt = 0; + deszRdelStart(); + while (deszRdelLine()) { + d_str = formatDateTime_ISO8601(deszq_epochTime) + ';'; + // next_token = find_chars_or_comment(deszq_nextChar, + // DELIM_CHAR2); + tempBuffer[0] = 0; + strncat(tempBuffer, deszq_nextChar, deszq_nextCharSz); + strcat(tempBuffer, ";"); + // PRINTOUT("Sn=", tempBuffer); + d_str.concat(tempBuffer); + // getline + while (deszqNextCh()) { + tempBuffer[0] = 0; + strncat(tempBuffer, deszq_nextChar, deszq_nextCharSz); + strcat(tempBuffer, ";"); + d_str.concat(tempBuffer); + // PRINTOUT("t='", tempBuffer, F("'")); + } + PRINTOUT("L=", d_str, "Stat=", deszq_status); + } + deszRdelClose(true); // Delete serial file + } + return true; +} + +bool Logger::postLogOpen(const char* postLogNam_str) { + bool retVal = false; +#if defined MS_LOGGERBASE_POSTS + // Generate the file name from logger ID and date + // Create rotating log of 4 chars YYMM - formatDateTime is YYYY MM DD + String nameTemp = formatDateTime_str(getNowLocalEpoch()); + + // Drop middle _ and get YYMM + String fileName = String(postLogNam_str + nameTemp.substring(2, 4) + nameTemp.substring(5, 7) + ".log"); + + // Convert the string filename to a character file name for SdFat + uint16_t fileNameLength = fileName.length()+2; + MS_DBG(F("PLO postLog file"), fileName, F("res len"),fileNameLength); + char charFileName[fileNameLength]; + fileName.toCharArray(charFileName, fileNameLength); + + // Attempt to open an existing file + retVal = postsLogHndl.open(charFileName, (O_WRITE | O_AT_END)); + if (!retVal) { + retVal = postsLogHndl.open(charFileName, + (O_CREAT | O_WRITE | O_AT_END)); + if (!retVal) { + PRINTOUT(F("logPLO err opening"), charFileName); + + } else { + setFileTimestamp(postsLogHndl, T_CREATE,true); + MS_DBG(F("logPLO new file"), charFileName); + } + } +#endif // MS_LOGGERBASE_POSTS + return retVal; +} +bool Logger::postLogOpen() { + bool retVal = false; +#if defined MS_LOGGERBASE_POSTS + retVal =postLogOpen(postsLogFn_str); +#endif // + return retVal; +} +void Logger::postLogClose() { +#if defined MS_LOGGERBASE_POSTS + + setFileTimestamp(postsLogHndl, (T_WRITE),true); //| T_ACCESS + postsLogHndl.close(); + + +#endif // MS_LOGGERBASE_POSTS +} + +void Logger::postLogLine(uint32_t tmr_ms, int16_t rspParam) { +// If debug ...keep record +#if defined MS_LOGGERBASE_POSTS +#if 0 + if (0 == postsLogHndl.print(getNowEpochUTC())) { + PRINTOUT(F("publishDataQueuedToRemote postsLog err")); + } +#else + + char tempBuffer[TEMP_BUFFER_SZ]; + //Print internal time + formatDateTime_str(getNowLocalEpoch()) + .toCharArray(tempBuffer, TEMP_BUFFER_SZ); + postsLogHndl.print(tempBuffer); +#endif + postsLogHndl.print(F(",POST,")); + itoa(rspParam, tempBuffer, 10); + postsLogHndl.print(tempBuffer); + postsLogHndl.print(F(",")); + itoa(tmr_ms, tempBuffer, 10); + postsLogHndl.print(tempBuffer); + postsLogHndl.print(F(",")); + postsLogHndl.print(deszq_line); +#endif //#if defined MS_LOGGERBASE_POSTS +} + +void Logger::postLogLine(const char *logMsg,bool addCRNL) { +#if defined MS_LOGGERBASE_POSTS + bool wasOpen =true; + if (!postsLogHndl.isOpen()) { + wasOpen=false; + if (!postLogOpen()) { + PRINTOUT(F("postLogLine can't open file")); + //TODO possible reboot + return; + } + } + char tempBuffer[TEMP_BUFFER_SZ]; + //Print internal time + formatDateTime_str(getNowLocalEpoch()) + .toCharArray(tempBuffer, TEMP_BUFFER_SZ); + postsLogHndl.print(tempBuffer); + postsLogHndl.print(F(",MSG,")); + postsLogHndl.print(logMsg); + if (addCRNL)postsLogHndl.println(); + + if (!wasOpen) postLogClose(); +#endif //MS_LOGGERBASE_POSTS +} +/* +Cleanup if necessary +*/ + +bool Logger::listFile(File* filep, char* fn_str, char* uid) { + char loc_line[QUEFILE_MAX_LINE]; + int16_t num_char; + int16_t num_cnt = 0; + + if (!filep->open(fn_str, O_READ)) { + PRINTOUT(F("listFile; No file "), fn_str); + sd1_Err("listFile: no file2"); + return false; + } else { + MS_DBG(F("listFile"), fn_str, uid, F("")); + } + + while (0 < (num_char = filep->fgets(loc_line, QUEFILE_MAX_LINE))) { + PRINTOUT(++num_cnt, loc_line); + } + if (0 > num_char) { + PRINTOUT(F("listFile err"), num_char); + sd1_Err("listFile err2"); + } + if (!filep->close()) { + PRINTOUT(F("listFile; close err "), fn_str); + sd1_Err("listFile close err2"); + return false; + } + MS_DBG(F("listFile"), uid, F("")); + return true; +} + +/* This tests all the primitives used to access the SD card. + */ +bool Logger::serzBegin(void) { + bool dslStat_bool; + + MS_DBG(F("serzBegin list1---")); + if (!sd1_card_fatfs.ls()) { + // MS_DBG(F("serzBegin ls err")); + sd1_Err("serzBegin err ls"); + } else { + MS_DBG(F("---1Complete")); + } + + // Test RDELAY.TXT + serzRdelFile.open(serzRdelFn_str, RDEL_OFLAG); + serzRdelFile.println(F("1,1595653100,1,4.730,-38")); + serzRdelFile.println(F("1,1595653200,2,4.730,-38")); + serzRdelFile.close(); + + serzRdelFile.open(serzRdelFn_str, RDEL_OFLAG); + serzRdelFile.println(F("1,1595653300,3,4.730,-38")); + serzRdelFile.println(F("1,1595653400,4,4.730,-38")); + serzRdelFile.close(); + + PRINTOUT(F("serzBegin list2---")); + if (!sd1_card_fatfs.ls()) { + // MS_DBG(F("serzBegin ls err")); + sd1_Err("serzBegin err ls"); + } else { + PRINTOUT(F("---2Complete")); + } + listFile(&serzRdelFile, (char*)serzRdelFn_str, (char*)"1"); + + + deszRdelStart(); + int16_t cnt_num = 0; + while (0 < (dslStat_bool = deszRdelLine())) { + PRINTOUT(++cnt_num, F("] "), dslStat_bool, deszq_line); + } + + deszRdelClose(true); // Test for delete + PRINTOUT(F("serzBegin list3---")); + if (!sd1_card_fatfs.ls()) { + // MS_DBG(F("serzBegin ls err")); + sd1_Err("serzBegin err ls"); + } else { + PRINTOUT(F("---3Complete")); + } + +// Test QUED algorithims ~ use QUE7.txt +#define QUE_TST '7' +#define TESTQ_FN_STR "QUE7.TXT" + MS_DBG(F("TESTQ START")); + if (sd1_card_fatfs.exists(TESTQ_FN_STR)) { + // PRINTOUT(F("serzBegin removing "), TESTQ_FN_STR); + if (!sd1_card_fatfs.remove(TESTQ_FN_STR)) { + PRINTOUT(F("serzBegin err remove"), TESTQ_FN_STR); + sd1_Err("serzBegin remove"); + } + } else { + MS_DBG(F("serzBegin no "), TESTQ_FN_STR); + } + // Test1 ** QUED new file name & update + MS_DBG(F("TESTQ1")); + serzQueuedStart((char)(QUE_TST)); + serzQueuedFile.println(F("1,1595654100,1,4.7,-38")); + serzQueuedFile.println(F("1,1595654200,2,4.7,-38")); + serzQueuedCloseFile(false); + + + // Test2 ** QUED file update + MS_DBG(F("TESTQ2")); + if (!serzQueuedFile.open(serzQueuedFn, (O_WRITE | O_AT_END))) { + // Could be that there are no retrys. + PRINTOUT(F("serzQueuedFile.open err"), serzQueuedFn); + sd1_Err("serzQueuedFile.open err2"); + return false; + } else { + PRINTOUT(F("Testq2 Opened"), serzQueuedFn); + } + serzQueuedFile.println(F("1,1595654300,3,4.7,-38")); + serzQueuedFile.println(F("1,1595654400,4,4.7,-38")); + if (!serzQueuedCloseFile(false)) return false; + + PRINTOUT(F("serzBegin list4---")); + if (!sd1_card_fatfs.ls()) { + // MS_DBG(F("serzBegin ls err")); + sd1_Err("serzBegin err4 ls"); + return false; + } else { + PRINTOUT(F("---4Complete")); + } + listFile(&serzQueuedFile, serzQueuedFn, (char*)"2"); + + // Test3 ** QUED file rollover + MS_DBG(F("TESTQ3")); + if (!deszQueuedStart()) return false; + + dslStat_bool = deszQueuedLine(); + MS_DBG(F("1: deszq_line"), dslStat_bool, deszq_line); + if (!dslStat_bool) return false; + dslStat_bool = deszQueuedLine(); + MS_DBG(F("2: deszq_line"), dslStat_bool, deszq_line); + if (!dslStat_bool) return false; + // only the 1: should be dropped + dslStat_bool = serzQueuedCloseFile(true); + PRINTOUT(F("serzBegin list5---")); + if (!sd1_card_fatfs.ls()) { + // MS_DBG(F("serzBegin ls err")); + sd1_Err("serzBegin err5 ls"); + return false; + } else { + PRINTOUT(F("---5Complete")); + } + listFile(&serzQueuedFile, serzQueuedFn, (char*)"3"); + if (!dslStat_bool) return false; + + if (sd1_card_fatfs.exists(serzQueuedFn)) { + PRINTOUT(F("serzBegin removing "), serzQueuedFn); + if (!sd1_card_fatfs.remove(serzQueuedFn)) { + PRINTOUT(F("serzBegin err remove"), serzQueuedFn); + sd1_Err("serzBegin err6 remove"); + } + } else { + PRINTOUT(F("serzBegin no "), TEMP_BASE_FN_STR); + } + + // Cleanup + + MS_DBG(F("TESTQ CLEANUP")); + if (sd1_card_fatfs.exists(TEMP_BASE_FN_STR)) { + PRINTOUT(F("serzBegin removing "), TEMP_BASE_FN_STR); + if (!sd1_card_fatfs.remove(TEMP_BASE_FN_STR)) { + PRINTOUT(F("serzBegin err remove"), TEMP_BASE_FN_STR); + sd1_Err("serzBegin err6 remove"); + } + } else { + MS_DBG(F("serzBegin no "), TEMP_BASE_FN_STR); + } /* */ + MS_DBG(F("TESTQ END END END \n\n")); + return true; +} + +// Convert a date-time object into a formatted string +String Logger::formatDateTime_str(DateTime& dt) { + String dateTimeStr; + dt.addToString(dateTimeStr); + return dateTimeStr; +} + +// Convert an epoch time into a formatted string +String Logger::formatDateTime_str(uint32_t epochTime) { + DateTimeClass(dt1, epochTime); + return formatDateTime_str(dt1); +} +// End LoggerBase.cpp diff --git a/src/LoggerBase.h b/src/LoggerBase.h index 36040f426..a24c470da 100644 --- a/src/LoggerBase.h +++ b/src/LoggerBase.h @@ -31,6 +31,8 @@ #undef MS_DEBUGGING_DEEP #include "VariableArray.h" #include "LoggerModem.h" +#include "ms_common.h" +#include // The base Arduino library // Bring in the libraries to handle the processor sleep/standby modes // The SAMD library can also the built-in clock on those modules @@ -46,6 +48,9 @@ // Bring in the library to communicate with an external high-precision real time // clock This also implements a needed date/time class #include +//FUT using namespace sodaq_DS3231_nm; +//This is a fudge for handling time between DS3231 and SAMD +#define EPOCH_TIME_DTCLASS EPOCH_TIME_OFF /** * @brief January 1, 2000 00:00:00 in "epoch" time @@ -58,6 +63,15 @@ #include // To communicate with the SD card +typedef enum { + LB_PWR_USEABLE_REQ = 0, + LB_PWR_SENSOR_USE_REQ, + LB_PWR_MODEM_USE_REQ, + LB_PWR_END +} lb_pwr_req_t; +typedef bool (*bat_handler_atl)(lb_pwr_req_t reqBatState); + + /** * @brief The largest number of variables from a single sensor */ @@ -808,7 +822,12 @@ class Logger { * @return **bool** True if the current time on the RTC is an even interval * of the logging rate. */ - bool checkInterval(void); + uint8_t checkInterval(void); + static const uint8_t CIA_NOACTION = 0x0; + static const uint8_t CIA_NEW_READING = 0x01; + static const uint8_t CIA_POST_READINGS = 0x02; + static const uint8_t CIA_RLB_READINGS = 0x04; // store readings, no pub" + static const uint8_t CIA_NO_SLEEP = 0x08; // /** * @brief Check if the MARKED time is an even interval of the logging rate - @@ -1022,7 +1041,7 @@ class Logger { /** * @brief An internal reference to SdFat for SD card control */ - SdFat sd; + SdFat sd1_card_fatfs; /** * @brief An internal reference to an SdFat file instance */ @@ -1057,7 +1076,7 @@ class Logger { * @param stampFlag The "flag" of the timestamp to change - should be * T_CREATE, T_WRITE, or T_ACCESS */ - void setFileTimestamp(File fileToStamp, uint8_t stampFlag); + void setFileTimestamp(File fileToStamp, uint8_t stampFlag, bool localTime=false); /** * @brief Open or creates a file, converting a string file name to a @@ -1198,6 +1217,250 @@ class Logger { */ static volatile bool startTesting; /**@}*/ + + protected: + /** + * @brief Active instance of the attached data publishers + */ + uint8_t _dataPubInstance; +public: +/* Provides a method of getting battery voltage + to determine if enough power to continue +*/ +bat_handler_atl _bat_handler_atl = NULL; + +void setBatHandler(bool (*bat_handler_atl)(lb_pwr_req_t reqBatState)); + + +#if !defined SERIALIZE_POST_MAX_READINGS +#define SERIALIZE_POST_MAX_READINGS 20 +#endif // SERIALIZE_POST_MAX_READINGS +uint16_t _sendAtOneTimeMaxX_num = SERIALIZE_POST_MAX_READINGS; +void setSendEveryX( + uint8_t sendEveryX_num, + uint16_t sendAtOneTimeMaxX_num = SERIALIZE_POST_MAX_READINGS) { + _sendEveryX_num = sendEveryX_num; + // Check range, if too small set to max + if (sendAtOneTimeMaxX_num > 3) { + _sendAtOneTimeMaxX_num = sendAtOneTimeMaxX_num; + } else { + _sendAtOneTimeMaxX_num = -1; // Max + } +} +uint8_t getSendEveryX(void) { + return _sendEveryX_num; +} +#if !defined SERIALIZE_sendEveryX_NUM +#define SERIALIZE_sendEveryX_NUM 2 +#endif // SERIALIZE_sendEveryX_NUM +// Range typically 0-20 +uint8_t _sendEveryX_num = SERIALIZE_sendEveryX_NUM; +int8_t _sendEveryX_cnt = 0; // Counter to track status + +void setSendOffset(uint8_t param1) { + // Might also have check for SampleTime * _sendEveryX_num < param1 + if (_sendOffset_MAX >= param1) { + _sendOffset_min = param1; + } else { + _sendOffset_min = _sendOffset_MAX; + } +} +uint8_t getSendOffset(void) { + return _sendOffset_min; +} +#if !defined SERIALIZE_sendOffset_min +#define SERIALIZE_sendOffset_min 0 +#endif // SERIALIZE_sendOffset_min +uint8_t _sendOffset_min = SERIALIZE_sendOffset_min; +const uint8_t _sendOffset_MAX = 14; // Max allowed +bool _sendOffset_act = false; +uint8_t _sendOffset_cnt = 0; + +void setSendPacingDelay(uint8_t param) { + _sendPacingDelay_mSec = param; +} +uint8_t getSendPacingDelay(void) { + return _sendPacingDelay_mSec; +} +#define SERIALIZE_sendPacingDelay_mSec 2 +uint16_t _sendPacingDelay_mSec = SERIALIZE_sendPacingDelay_mSec; + + uint16_t setSendQueueSz_num(uint16_t sqz_num) { + MS_DBG(F("setSendQueueSz_num"), sqz_num); + return _sendQueueSz_num = sqz_num; + } + + uint16_t getSendQueueSz_num() { + MS_DBG(F("getSendQueueSz_num"), _sendQueueSz_num); + return _sendQueueSz_num; + } +#if !defined LB_SENDQUESZ_NUM_DEF +#define LB_SENDQUESZ_NUM_DEF 2800L //MMWGI_SEND_QUE_SZ_NUM_DEF +#endif // LB_SENDQUESZ_NUM_DEF + uint16_t _sendQueueSz_num = LB_SENDQUESZ_NUM_DEF ; //See MMMWGI_SEND_QUE_SZ_NUM_DEF + + uint16_t setPostMax_num(uint16_t mp_num) { + MS_DBG(F("setMaxPost_num"), mp_num); + return _postMax_num = mp_num; + } + + uint16_t getPostMax_num() { + MS_DBG(F("getPostMax_num"), _postMax_num); + return _postMax_num; + } +#if !defined LB_POSTMAX_NUM_DEF +#define LB_POSTMAX_NUM_DEF 96L //MMWGI_POST_MAX_RECS_MUM_DEF +#endif // LB_POSTMAX_NUM_DEF + uint16_t _postMax_num = LB_POSTMAX_NUM_DEF; //See MMWGI_POST_MAX_RECS_MUM_DEF + +// Time woken up +uint32_t wakeUpTime_secs; + + +public: + +void forceSysReset(uint8_t source, uint16_t simpleCheck); +/** + * @brief Process queued readings to send to remote if internet available. + * + * If previously registered, it will determine if battery power is available + * It uses an algorithim to reliably deliver the readings. + * + */ +#define LOGGER_RELIABLE_POST (Logger::CIA_NEW_READING|Logger::CIA_POST_READINGS) +#define LOGGER_NEW_READING (Logger::CIA_NEW_READING) +void logDataAndPubReliably(uint8_t cia_val_override =0); + +/** + * @brief Process queued readings to send to remote if internet available. + * + * @param internetPresent true if an internet connection is present. + * For false store the readings for later transmission + * This reads from stored files + * 1) RDELAY.txt + * 2) QUE0.txt + * For data in RDELAY.txt, an attempt is made to transmit each line, + * if not sucessful then it is stored at the end of QUE0.txt + * Once RDELAY.txt is complete, + * any readings in QUE0.txt are attempted, + * + * The format of he readings in the file are dependent on sensor configuration, + * so they maynot be compatible if the builds sensor configuration changes. + * + * The forwarding of RDELAY.txt to QUE0.txt is only up to specific thresholds + * POST_MAX_NUM + * RDELAY_FAILED_POSTS_THRESHOLD + */ +#define RDELAY_FAILED_POSTS_THRESHOLD 7 +void publishDataQueuedToRemotes(bool internetPresent); + +/** + * @brief Set up for sleep. + * + * For external clock chip need to ensure the data is set correctly + * + */ +void setExtRtcSleep(); + +private: +/** + * @brief Check HTTP status response + * + * @param rspCode true if server is deemed to have received message + * or nothing further can be done with message. + */ +bool publishRspCodeAccepted(int16_t rspCode); + +// ===================================================================== // +/* Serializing/Deserialing + A common set of functions that operate on files + serzRdelFn_str + serzQueuedFn +*/ +// ===================================================================== // +public: +bool deszqNextCh(void); +long deszq_epochTime = 0; // Marked Epoch Time +char* deszq_nextChar; +uint16_t deszq_nextCharSz; + +// Calculated length of timeVariant data fields as ASCII+ delimiter, except +// for last data field +uint16_t deszq_timeVariant_sz; + +private: +#define sd1_Err(s) sd1_card_fatfs.errorPrint(F(s)) +uint16_t deszq_status = 0; // Bit wise status of reading +uint16_t deszLinesRead = 0; +uint16_t deszLinesUnsent = 0; +#define QUEFILE_MAX_LINE 100 +char deszq_line[QUEFILE_MAX_LINE] = ""; +uint16_t desz_pending_records = 0; + +// Qu SdFat/sd1_card_fatfs connects to Physical pins or File/logFile or +// keep to LFN - capitals https://en.wikipedia.org/wiki/8.3_filename + +#if defined MS_LOGGERBASE_POSTS +File postsLogHndl; // Record all POSTS when enabled +const char* postsLogFn_str = "DBG"; // Not more than 8.3 total + +#endif // MS_LOGGERBASE_POSTS + +// que Readings DELAYed (RDEL) ~ serialize/deserialize +File serzRdelFile; +const char* serzRdelFn_str = "RDELAY.TXT"; + +// QUEueD for reliable delivery +// first POST didn't suceed to serialize/deserialize +// Potentially multiple versions of files based on dataPublisher[] +File serzQueuedFile; +#define FN_BUFFER_SZ 13 +char serzQueuedFn[FN_BUFFER_SZ] = ""; +const char* serzQueuedFn_str = "QUE"; // begin of name, keep 8.3 + + +// perform a serialize to RdelFile +bool serzRdel_Line(void); +// Uses serzRdelFn_str, File serzRdelFile +bool deszRdelStart(); +char* deszFind(const char* in_line, char caller_id); +#define deszRdelLine() deszLine(&serzRdelFile) +bool deszRdelClose(bool deleteFile = false); + +// Uses serzQueuedFn_str, File serzQueuedFile +bool serzQueuedStart(char uniqueId); // Use 1st, & sets filename +bool deszQueuedStart(void); +#define deszQueuedLine() deszLine(&serzQueuedFile) +uint16_t serzQueuedFlushFile(); +bool serzQueuedCloseFile(bool action); +/* +bool deszQueuedCleanup(bool debug = false); +*/ +// This does the work +bool deszLine(File* filep); + +// Utility resources +//void setFileTimeStampMet(File fileToStamp, uint8_t stampFlag); +bool deszDbg(void); +bool postLogOpen(); +bool postLogOpen(const char* postsLogNm_str); +void postLogLine(uint32_t tmr_ms, int16_t rspParam); +void postLogLine(const char *logMsg,bool addCR=true); +// Macro to print to TTY and log on uSD +#define PRINT_LOGLINE_P(msg_parm) \ + char tttbuf[sizeof(msg_parm)+1]; \ + strcpy_P(tttbuf,msg_parm);\ + PRINTOUT(tttbuf);\ + postLogLine(tttbuf); +void postLogClose(); +bool listFile(File* filep, char* fn_str, char* uid); + +public: +// A simple data-time formatting, compatible with reading into excel spreadsheet - see also formatDateTime_ISO8601() +String formatDateTime_str(DateTime& dt); +String formatDateTime_str(uint32_t epochTime); +bool serzBegin(void); + }; #endif // SRC_LOGGERBASE_H_ diff --git a/src/ModularSensors.h b/src/ModularSensors.h index 04563fdf7..9ab625b6b 100644 --- a/src/ModularSensors.h +++ b/src/ModularSensors.h @@ -18,8 +18,7 @@ * A pre-release version will always be indicated as slightly ahead of the * EnviroDIY branch that it is based on. */ -#define MODULAR_SENSORS_VERSION "0.34.1" - +#define MODULAR_SENSORS_VERSION "0.35.0" // To get all of the base classes for ModularSensors, include LoggerBase. // NOTE: Individual sensor definitions must be included separately. diff --git a/src/SensorBase.h b/src/SensorBase.h index 6d581e377..b411af2f5 100644 --- a/src/SensorBase.h +++ b/src/SensorBase.h @@ -544,6 +544,8 @@ class Sensor { * defined once for the whole class. */ Variable* variables[MAX_NUMBER_VARS]; + + virtual void set_sensorName(const char *sensorName) {_sensorName=sensorName;} }; #endif // SRC_SENSORBASE_H_ diff --git a/src/dataPublisherBase.h b/src/dataPublisherBase.h index d43ba5dde..78f3ddd14 100644 --- a/src/dataPublisherBase.h +++ b/src/dataPublisherBase.h @@ -332,6 +332,71 @@ class dataPublisher { * @brief the text "\r\nHost: " */ static const char* hostHeader; + + + /** + * @brief TimerPost (ms); How long to wait for a response to a POST before + * declaring a timeout + */ + // uint8_t _timerPost_mS; + + public: + + bool useQueueDataSource = false; + bool virtual setQueuedState(bool state, char uniqueId = '0') { + PRINTOUT(F("dataPublisherBase setQueued check"), useQueueDataSource); + return useQueueDataSource; // Default not updated. + } + bool virtual getQueuedStatus() { + PRINTOUT(F("dataPublisherBase gQS check"), useQueueDataSource); + return useQueueDataSource; // Default for not supported. + } + + //Required to implement the following + uint16_t virtual setTimerPostTimeout_mS(uint16_t tpt_ms) { + MS_DBG(F("setTPT rejected ")); + return 0; // Default not updated. + } + + uint16_t virtual getTimerPostTimeout_mS() { + MS_DBG(F("getTPT rejected")); + return 0; + } + + //Required to implement the following + uint16_t virtual setTimerPostPacing_mS(uint16_t tpt_ms) { + MS_DBG(F("setTPP rejected ")); + return 0; // Default not updated. + } + + uint16_t virtual getTimerPostPacing_mS() { + MS_DBG(F("setTPP rejected")); + return 0; + } + }; +/* atl_extension */ +/* + * HTTP STATUS Codes that are used by Modular Sensors + * Placed at the end of the file, to facilitate mergein code + * https://en.wikipedia.org/wiki/List_of_HTTP_status_codes + */ + +#define HTTPSTATUS_CREATED_201 201 +// Server Error indicating a Gateway Timeout. +// Server error that doesn't seem to receover +// https://github.com/ODM2/ODM2DataSharingPortal/issues/628 +#define HTTPSTATUS_GT_500 500 +// Also supplied if the server didn't respond to a POST +#define HTTPSTATUS_GT_504 504 +// This is an internaly created error, indicating No Connection with server +#define HTTPSTATUS_NC_901 901 +// internal error, not enough power to connect with server +#define HTTPSTATUS_NC_902 902 +// internal error, value dumped +#define HTTPSTATUS_NC_903 903 +// internal error, value queued +#define HTTPSTATUS_NC_904 904 + #endif // SRC_DATAPUBLISHERBASE_H_ diff --git a/src/ms_common.h b/src/ms_common.h new file mode 100644 index 000000000..e615b5bfd --- /dev/null +++ b/src/ms_common.h @@ -0,0 +1,322 @@ +#ifndef ms_common_h +#define ms_common_h +/***************************************************************************** +ms_common.h +Written By: Neil Hancock www.envirodiy.org/members/neilh20/ +Development Environment: PlatformIO +Hardware Platform: EnviroDIY Mayfly Arduino Datalogger +Software License: BSD-3. + Copyright (c) 2018, Neil Hancock - all rights assigned to Stroud Water +Research Center (SWRC) and they may change this title to Stroud Water Research +Center as required and the EnviroDIY Development Team + +This example sketch is written for ModularSensors library version 0.xx.0 + +DISCLAIMER: +THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. +*****************************************************************************/ +/***************************************************************************** + * Support ARDUINO_ARCH_AVR ARDUINO_ARCH_SAM ARDUINO_ARCH_SAMD + * Implementation: Due to ram limitations and of course Atmel Harvard (von + Neuman) Architecture and Princeton (eg ARM) + * some special macros a required. + * Atmel Harvard has seperate space for constants/program, where as Princeton is + all the same memory space + * C/Cpp only uses one space for all data and program + * For Mayfly Atmel ARDUINO_ARCH_AVR assume Harvard Architecture and copy of + CONSTs from PROGMEM + * For not defined PROGMEM assume Princeton Architecture + * examples + * \TinyGSM\src\TinyGsmCommon.h and subsequent useage eg TinyGsmClientXBee.h + .platformio\platforms\atmelsam\boards adafruit feather_m0 feather_m4 + ARDUINO_ARCH_SAMD .platformio\platforms\teensy\boards teensylc -DTEENSYLC + teensy36 -DTEENSY36 + + + ??.platformio\packages\framework-arduinoavr\libraries\WiFi\extras\wifiHD\src\SOFTWARE_FRAMEWORK\SERVICES\LWIP\lwip-1.3.2\src\core\pbuf.c + SYS_ARCH_; + * +*/ +#include // The base Arduino library +// Local routing defitions here +#if defined(__AVR__) +#define EDIY_PROGMEM PROGMEM +typedef const __FlashStringHelper* EdiyConstStr; +#define EFP(x) (reinterpret_cast(x)) +#define EF(x) F(x) +#else +#define EDIY_PROGMEM +typedef const char* EdiyConstStr; +#define EFP(x) x +#define EF(x) x +#endif +//#include "ms_cfg.h" + +#if defined USE_PS_EEPROM +#include +#endif // USE_PS_EEPROM + +/***************************************************************************** + * Persistent structures. + * Defines data structures for per software build & geographical location + customizations + * Needs to be extensible. If a change is made, ensure size increments for + strucuture. + * This is initially implemented as reading form .ini file on nicroSD. + * The implementation could change in the future for parts of initial + * Board level (including internal) Persistent Storage - + * eg Internal EEPROM, though any board level data EEPROM or data flash is + the same. + * This may be very useful for hw tracking (eg sn/rev) and essential boot + elements + * - Internal persistent storage presents special problems with upgrades + * + + * + */ +//#define USE_PLAN_FOR_UPGRADE 1 +#if defined(USE_PLAN_FOR_UPGRADE) +#define STRCT_SZ(parm1) uint16_t parm1; +#else +#define STRCT_SZ(parm1) sizof(parm1) +#endif // USE_PLAN_FOR_UPGRADE +//#define USE_PS_HW_BOOT 1 +//****** +#if defined(USE_PS_HW_BOOT) +// Hardware Boot Structure - rarely expected to change +#define HW_BOOT_BOARD_NAME_SZ 21 +#define HW_BOOT_SERIAL_NUM_SZ 21 +#define HW_BOOT_REV_SZ 11 +#define HW_BOOT_EXP 17 +typedef struct { + uint16_t crc16; // Across just the hw_boot_t except crc16 + uint8_t struct_ver; // 1-255 - increment for any changes in this structure + uint8_t board_name[HW_BOOT_BOARD_NAME_SZ]; // eg Mayfly + uint8_t serial_num[HW_BOOT_SERIAL_NUM_SZ]; // eg 1234 + uint8_t rev[HW_BOOT_REV_SZ]; // eg 0.5b + uint8_t exp[HW_BOOT_EXP]; +} hw_boot001_t; +#define HW_BOOT_STRUCT_VER_001 001 +#define HW_BOOT_STRUCT_VER HW_BOOT_STRUCT_VER_001 +#define mHw_boot_t(p1) hw_boot001_t p1 +#define sizeof_hw_boot sizeof(hw_boot001_t) +#else +#define mHw_boot_t(p1) +#endif // USE_PS_HW_BOOT +//****** +// For extensibility - each structures data size (not including sz) can be +// defined at the beginning of the structure this allows any parsing algorithims +// to step through the structures easly and also to invoke the right parm1: +// 1-65535 - struct size increase for any changes in this structure + +//****** [COMMON] +#define USE_PS_modularSensorsCommon 1 +// +#if defined(USE_PS_modularSensorsCommon) +#define MSC_LOGGER_ID_SZ 21 +#define MSC_GEOLOCATION_ID_SZ 61 +typedef struct { + // v01 Initial structure + uint16_t logging_interval_min; + int8_t time_zone; //-12,0 to +11? + uint8_t battery_type; + uint16_t battery_mAhr; //0-65,536 + // uint8_t colllectReadings; + // uint8_t sendOffset_min; + uint8_t logger_id[MSC_LOGGER_ID_SZ]; + uint8_t geolocation_id[MSC_GEOLOCATION_ID_SZ]; +} msc01_t; +#define MSC_ACTIVE msc01_t + +#define epc_logging_interval_min epc.app.msc.s.logging_interval_min +#define epc_battery_type epc.app.msc.s.battery_type +#define epc_battery_mAhr epc.app.msc.s.battery_mAhr +#define BATTERY_mAhr_DEF 4400 +#define BATTERY_mAhr_MAX 65501 + +#define epc_logger_id (char*)epc.app.msc.s.logger_id +#define epc_logger_id1st epc.app.msc.s.logger_id[0] + +typedef struct { + uint8_t sz; + MSC_ACTIVE s; +} modularSensorsCommon_t; +#define mModularSensorsCommon_t(p1) modularSensorsCommon_t p1 +#else +#define mModularSensorsCommon_t(p1) +#endif // USE_PS_modularSensorsCommon) + +#if defined UseModem_Module +#define USE_PS_modularSensorsNetwork 1 +#endif // UseModem_Module +#if defined(USE_PS_modularSensorsNetwork) +//MSCN_TYPE_XXX is the Network Type modem - modemTypesCurrent_t +/*#define MSCN_TYPE_NONE 0 +#define MSCN_TYPE_CELL 1 +#define MSCN_TYPE_WIFI 2 +#define MSCN_TYPE_LORA 3 */ +#define MSCN_APN_SZ 32 +#define MSCN_APN_DEF_STR "APN_NONE" +#define MSCN_WIFI_ID_SZ 32 +#define MSCN_WIFIID_DEF_STR "WIFIID_NONE" +#define MSCN_WIFI_PWD_SZ 32 +#define MSCN_WIFIPWD_DEF_STR "WIFIPWD_NONE" +#ifndef MNGI_COLLECT_READINGS_DEF +#define MNGI_COLLECT_READINGS_DEF 1 +#endif //MNGI_COLLECT_READINGS_DEF +#ifndef MNGI_SEND_OFFSET_MIN_DEF +#define MNGI_SEND_OFFSET_MIN_DEF 100 +#endif //MNGI_SEND_OFFSET_MIN_DEF +#ifndef MMWGI_POST_MAX_RECS_MUM_DEF +#define MMWGI_POST_MAX_RECS_MUM_DEF 100 +#endif //MMW_POST_MAX_RECS_MUM_DEF + +//This turns off all queue operation +#define MMWGI_SEND_QUE_SZ_NUM_NOP 0xFFFF +#ifndef MMWGI_SEND_QUE_SZ_NUM_DEF +#define MMWGI_SEND_QUE_SZ_NUM_DEF MMWGI_SEND_QUE_SZ_NUM_NOP +#endif //MMWGI_SEND_QUE_SZ_NUM_DEF + +typedef struct { + uint8_t network_type; // 0=apn ,1=wifi network - modem type modemTypesCurrent_t + uint8_t apn[MSCN_APN_SZ]; // 32 + uint8_t WiFiId[MSCN_WIFI_ID_SZ]; // 32? + uint8_t WiFiPwd[MSCN_WIFI_PWD_SZ]; // 32?? + uint8_t collectReadings_num; // 1-30 + uint8_t sendOffset_min; //0-30 + uint16_t postMax_num; //0 no limit,1~2000 + uint16_t sendQueSz_num; //0 none stored. FFFF no limit +} msn01_t; +#define MSN_ACTIVE msn01_t + +#define epc_network epc.app.msn.s.network_type +#define epc_apn (char*)epc.app.msn.s.apn +#define epc_apn1st epc.app.msn.s.apn[0] +#define epc_WiFiId (char*)epc.app.msn.s.WiFiId +#define epc_WiFiId1st epc.app.msn.s.WiFiId[0] +#define epc_WiFiPwd (char*)epc.app.msn.s.WiFiPwd +#define epc_WiFiPwd1st epc.app.msn.s.WiFiPwd[0] + +typedef struct { + uint8_t sz; + MSN_ACTIVE s; +} modularSensorsNetwork_t; + + +#define mModularSensorsNetwork_t(p1) modularSensorsNetwork_t p1 +#else +#define mModularSensorsNetwork_t(p1) +#endif // USE_PS_modularSensorsNetwork + +//****** +#if defined UseModem_Module +#define USE_PS_Provider 1 +#endif // UseModem_Module +//****** +//Provider types supported +#define PROVID_TYPE_NONE 0x00 +#define PROVID_TYPE_MMW 0x01 +#define PROVID_TYPE_TS 0x02 +#define PROVID_TYPE_UBIDOTS 0x04 + +#define PROVID_CLOUD_ID_SZ 38 +#define PROVID_DEF_STR "NONE" +#define PROVID_NULL_TERMINATOR 0 + +#define PROVID_MW_REGISTRATION_TOKEN_SZ 38 +#define PROVID_MW_SAMPLING_FEAUTRE_SZ 38 + +#define PROVID_TSMQTTKEY_SZ 17 +#define PROVID_TSCHANNELID_SZ 7 +#define PROVID_TSCHANNELKEY_SZ 17 + +#define PROVID_UB_AUTH_TOKEN_SZ 38 +#define PROVID_UB_DEVICEID_SZ 38 + +// If provider requires mapping, then use common mapping to whatever type +#define PROVID_UUID_SENSOR_NAME_SZ 40 +#define PROVID_UUID_SENSOR_VALUE_SZ 38 +#define PROVID_UUID_SENSOR_CNTMAX_SZ 12 + +#if defined(USE_PS_Provider) +typedef struct { + char name[PROVID_UUID_SENSOR_NAME_SZ]; + char value[PROVID_UUID_SENSOR_VALUE_SZ]; +} ini_name_value_t; + +typedef struct { + // v01 initial structure + // All are in ascii strings, with the first unused octet \0 + char cloudId[PROVID_CLOUD_ID_SZ]; // ASCII url + //FUT int cloudPort + char registration_token[PROVID_MW_REGISTRATION_TOKEN_SZ]; + char sampling_feature[PROVID_MW_SAMPLING_FEAUTRE_SZ]; + uint16_t timerPostTout_ms; // Gateway Timeout (ms) + uint16_t timerPostPace_ms; // Gateway Pacing (ms) + ini_name_value_t uuid[PROVID_UUID_SENSOR_CNTMAX_SZ]; +} provid_envirodiy01_t; +typedef struct { + // v01 initial structure + // All are in ascii strings, with the first unused octet \0 + /* +const char* thingSpeakMQTTKey = 17x "XXXXXXXXXXXXXXXX"; // Your MQTT API Key from Account > MyProfile. +const char* thingSpeakChannelID = 7x "######"; // The numeric channel id for your channel +const char* thingSpeakChannelKey = 17x "XXXXXXXXXXXXXXXX"; // The Write API Key for your channel + */ + char cloudId[PROVID_CLOUD_ID_SZ]; // ASCII url + char thingSpeakMQTTKey[PROVID_TSMQTTKEY_SZ]; + char thingSpeakChannelID[PROVID_TSCHANNELID_SZ]; + char thingSpeakChannelKey [PROVID_TSCHANNELKEY_SZ]; + uint16_t timerPostTout_ms; // Gateway Timeout (ms) + uint16_t timerPostPace_ms; // Gateway Pacing (ms) + //uuid[] not used sequential +} provid_thingspeak01_t; + +typedef struct { + // v01 initial structure + // All are in ascii strings, with the first unused octet \0 + char cloudId[PROVID_CLOUD_ID_SZ]; // ASCII url + char authentificationToken[PROVID_UB_AUTH_TOKEN_SZ]; + char deviceID[PROVID_UB_DEVICEID_SZ]; + uint16_t timerPostTout_ms; // Gateway Timeout (ms) + uint16_t timerPostPace_ms; // Gateway Pacing (ms) + ini_name_value_t uuid[PROVID_UUID_SENSOR_CNTMAX_SZ]; +} provid_ubidots01_t; +typedef struct { + //Providers meta data stored here. + // Only one provider using variables/uuid supported. + /// Fut : union or simulataneous? + provid_envirodiy01_t ed; + provid_thingspeak01_t ts; + provid_ubidots01_t ub; +} msp01_t; +#define MSP_ACTIVE msp01_t + +typedef struct { + uint8_t sz; + uint8_t provider_type; // Bit pos. 1=enviroDIY, 2=? + MSP_ACTIVE s; +} provider_t; +#define mProvider_t(p1) provider_t p1 +#else +#define mProvider_t(p1) +#endif // USE_PS_provider + +#define EP_HW_BOOT_ADDR 0 +#define EP_PERSISTENT_STORE_ADDR (sizeof_hw_boot) +typedef struct { + uint16_t crc16; // Across persistent_store_t + uint16_t struct_size; // Of struct including crc8, struct_ver struct_size + uint8_t struct_ver; // 1-255 - increment for any changes in this structure + mModularSensorsCommon_t(msc); + mModularSensorsNetwork_t(msn); + mProvider_t(provider); +} app_storage_t; +typedef struct { + mHw_boot_t(hw_boot); + app_storage_t app; +} persistent_store_t; + +//#define LOGGER_ID_ADDR ps.msc.s.logger_id + +#endif // ms_common_h diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index 5d017ea99..25f5417f1 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -30,7 +30,11 @@ const char* EnviroDIYPublisher::timestampTag = "\",\"timestamp\":\""; // Constructors -EnviroDIYPublisher::EnviroDIYPublisher() : dataPublisher() {} +EnviroDIYPublisher::EnviroDIYPublisher() : dataPublisher() { + // MS_DBG(F("dataPublisher object created")); + _registrationToken = NULL; + setDIYHost(enviroDIYHost); +} EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, uint8_t sendEveryX, uint8_t sendOffset) : dataPublisher(baseLogger, sendEveryX, sendOffset) {} @@ -43,6 +47,7 @@ EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, uint8_t sendEveryX, uint8_t sendOffset) : dataPublisher(baseLogger, sendEveryX, sendOffset) { setToken(registrationToken); + setDIYHost(enviroDIYHost); _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); } EnviroDIYPublisher::EnviroDIYPublisher(Logger& baseLogger, Client* inClient, @@ -73,11 +78,19 @@ uint16_t EnviroDIYPublisher::calculateJsonSize() { jsonLength += 1; // " jsonLength += 36; // variable UUID jsonLength += 2; // ": - jsonLength += _baseLogger->getValueStringAtI(i).length(); - if (i + 1 != _baseLogger->getArrayVarCount()) { - jsonLength += 1; // , + if (!useQueueDataSource) { + // Actively calculate the length + jsonLength += _baseLogger->getValueStringAtI(i).length(); + if (i + 1 != _baseLogger->getArrayVarCount()) { + jsonLength += 1; // , + } } } + if (useQueueDataSource) { + // Get precalculated length + jsonLength += _baseLogger->deszq_timeVariant_sz; + } + jsonLength += 1; // } return jsonLength; @@ -111,7 +124,7 @@ void EnviroDIYPublisher::printEnviroDIYRequest(Stream* stream) { stream->print(postEndpoint); stream->print(HTTPtag); stream->print(hostHeader); - stream->print(enviroDIYHost); + stream->print(_enviroDIYHost); stream->print(tokenHeader); stream->print(_registrationToken); stream->print(contentLengthHeader); @@ -147,107 +160,57 @@ void EnviroDIYPublisher::begin(Logger& baseLogger, // int16_t EnviroDIYPublisher::postDataEnviroDIY(void) int16_t EnviroDIYPublisher::publishData(Client* outClient) { // Create a buffer for the portions of the request and response - char tempBuffer[37] = ""; - uint16_t did_respond = 0; - - MS_DBG(F("Outgoing JSON size:"), calculateJsonSize()); +#define REQUIRED_MIN_RSP_SZ 12 +#define TEMP_BUFFER_SZ (REQUIRED_MIN_RSP_SZ + 25) +#define RESPONSE_UNINIT 0xFFFE + char tempBuffer[TEMP_BUFFER_SZ] = ""; + uint16_t did_respond = RESPONSE_UNINIT; + + // Following is record specific - start with space in buffer + uint32_t elapsed_ms = 0; + uint32_t gatewayStart_ms; + uint16_t bufferSz = bufferFree(); + if (bufferSz < (MS_SEND_BUFFER_SIZE - 50)) printTxBuffer(outClient); // Open a TCP/IP connection to the Enviro DIY Data Portal (WebSDL) - MS_DBG(F("Connecting client")); +const int32_t CONNECT_TIMEOUT_SEC =7; + MS_DBG(F("Connecting client. Timer(sec)"), CONNECT_TIMEOUT_SEC); MS_START_DEBUG_TIMER; - if (outClient->connect(enviroDIYHost, enviroDIYPort)) { - MS_DBG(F("Client connected after"), MS_PRINT_DEBUG_TIMER, F("ms\n")); - - // copy the initial post header into the tx buffer - snprintf(txBuffer, sizeof(txBuffer), "%s", postHeader); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", postEndpoint); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", HTTPtag); - - // add the rest of the HTTP POST headers to the outgoing buffer - // before adding each line/chunk to the outgoing buffer, we make sure - // there is space for that line, sending out buffer if not - if (bufferFree() < 28) printTxBuffer(outClient); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", hostHeader); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", enviroDIYHost); - - if (bufferFree() < 47) printTxBuffer(outClient); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", tokenHeader); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", _registrationToken); - - if (bufferFree() < 26) printTxBuffer(outClient); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", - contentLengthHeader); - itoa(calculateJsonSize(), tempBuffer, 10); // BASE 10 - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", tempBuffer); - - if (bufferFree() < 42) printTxBuffer(outClient); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", contentTypeHeader); - - // put the start of the JSON into the outgoing response_buffer - if (bufferFree() < 21) printTxBuffer(outClient); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", samplingFeatureTag); - - if (bufferFree() < 36) printTxBuffer(outClient); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", - _baseLogger->getSamplingFeatureUUID()); - - if (bufferFree() < 42) printTxBuffer(outClient); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", timestampTag); - Logger::formatDateTime_ISO8601(Logger::markedLocalEpochTime) - .toCharArray(tempBuffer, 37); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", tempBuffer); - txBuffer[strlen(txBuffer)] = '"'; - txBuffer[strlen(txBuffer)] = ','; - - for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { - // Once the buffer fills, send it out - if (bufferFree() < 47) printTxBuffer(outClient); - - txBuffer[strlen(txBuffer)] = '"'; - _baseLogger->getVarUUIDAtI(i).toCharArray(tempBuffer, 37); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", tempBuffer); - txBuffer[strlen(txBuffer)] = '"'; - txBuffer[strlen(txBuffer)] = ':'; - _baseLogger->getValueStringAtI(i).toCharArray(tempBuffer, 37); - snprintf(txBuffer + strlen(txBuffer), - sizeof(txBuffer) - strlen(txBuffer), "%s", tempBuffer); - if (i + 1 != _baseLogger->getArrayVarCount()) { - txBuffer[strlen(txBuffer)] = ','; - } else { - txBuffer[strlen(txBuffer)] = '}'; - } + outClient->setTimeout(CONNECT_TIMEOUT_SEC); + if (outClient->connect(_enviroDIYHost, (uint16_t)enviroDIYPort) ) { + MS_DBG(F("Client connected after"), MS_PRINT_DEBUG_TIMER, F("ms to "),_enviroDIYHost,':',enviroDIYPort); + + mmwPostHeader(tempBuffer); + if (useQueueDataSource) { + mmwPostDataQueued(tempBuffer); + } else { + mmwPostDataArray(tempBuffer); } - + MS_DEEP_DBG(F("SZEND "), bufferFree()); // Send out the finished request (or the last unsent section of it) printTxBuffer(outClient, true); - // Wait 10 seconds for a response from the server - uint32_t start = millis(); - while ((millis() - start) < 10000L && outClient->available() < 12) { - delay(10); - } + // Poll for a response from the server with timeout + gatewayStart_ms = millis(); + did_respond = 0; + while ((elapsed_ms < _timerPostTimeout_ms) && + (did_respond < REQUIRED_MIN_RSP_SZ)) { + delay(10); // mS delay to poll + did_respond = outClient->available(); + elapsed_ms = millis() - gatewayStart_ms; + } + // MS_DBG(F("Rsp avl,"), did_respond, F("bytes in"), elapsed_ms, + // F("mS")); // Read only the first 12 characters of the response // We're only reading as far as the http code, anything beyond that // we don't care about. - did_respond = outClient->readBytes(tempBuffer, 12); - + memset(tempBuffer,0,TEMP_BUFFER_SZ); + did_respond = outClient->readBytes(tempBuffer, REQUIRED_MIN_RSP_SZ); + // MS_DBG(F("Rsp read,"), did_respond, F("bytes in"), elapsed_ms, + // F("mS")); // Close the TCP/IP connection - MS_DBG(F("Stopping client")); + // MS_DBG(F("Stopping client")); MS_RESET_DEBUG_TIMER; outClient->stop(); MS_DBG(F("Client stopped after"), MS_PRINT_DEBUG_TIMER, F("ms")); @@ -258,18 +221,139 @@ int16_t EnviroDIYPublisher::publishData(Client* outClient) { // Process the HTTP response int16_t responseCode = 0; - if (did_respond > 0) { + if (RESPONSE_UNINIT == did_respond) { + // 901 Outside HTTP Status, No Connection to server + responseCode = HTTPSTATUS_NC_901; + } else if (did_respond >= REQUIRED_MIN_RSP_SZ) { char responseCode_char[4]; + // Put in monitor check on actual size received + if ((did_respond + 5) >= TEMP_BUFFER_SZ) { + PRINTOUT(F(" -- Gateway Timeout warning buffer sz small"), + did_respond, TEMP_BUFFER_SZ); + } for (uint8_t i = 0; i < 3; i++) { responseCode_char[i] = tempBuffer[i + 9]; } responseCode = atoi(responseCode_char); } else { - responseCode = 504; + // 504 Gateway Timeout + responseCode = HTTPSTATUS_GT_504; } - PRINTOUT(F("-- Response Code --")); - PRINTOUT(responseCode); + tempBuffer[TEMP_BUFFER_SZ - 1] = 0; + MS_DBG(F("Rsp:'"), tempBuffer, F("'")); + PRINTOUT(F("-- Response Code --"), responseCode, F("waited "), elapsed_ms, + F("mS Timeout"), _timerPostTimeout_ms); return responseCode; } + +void EnviroDIYPublisher::setDIYHost(const char* enviroDIYHost) { + _enviroDIYHost = enviroDIYHost; + //MS_DBG(F("DIY Host set to "), _enviroDIYHost); +} + +/* FUT: void EnviroDIYPublisher::setDIYPort(const int enviroDIYPort) { + _enviroDIYPort = enviroDIYPort; + MS_DBG(F("DIY Port set to "), _enviroDIYPort); +}*/ + +void EnviroDIYPublisher::mmwPostHeader(char* tempBuffer) { + // char tempBuffer[TEMP_BUFFER_SZ] = ""; + // copy the initial post header into the tx buffer + strcpy(txBuffer, postHeader); + strcat(txBuffer, postEndpoint); + strcat(txBuffer, HTTPtag); + + // add the rest of the HTTP POST headers to the outgoing buffer + // before adding each line/chunk to the outgoing buffer, we make sure + // there is space for that line, sending out buffer if not + // if (bufferFree() < 28) printTxBuffer(_outClient); + strcat(txBuffer, hostHeader); + strcat(txBuffer, _enviroDIYHost); + + // if (bufferFree() < 47) printTxBuffer(_outClient); + strcat(txBuffer, tokenHeader); + strcat(txBuffer, _registrationToken); + + // if (bufferFree() < 27) printTxBuffer(_outClient); + // strcat(txBuffer, cacheHeader); + + // if (bufferFree() < 21) printTxBuffer(_outClient); + // strcat(txBuffer, connectionHeader); + + // if (bufferFree() < 26) printTxBuffer(_outClient); + strcat(txBuffer, contentLengthHeader); + itoa(calculateJsonSize(), tempBuffer, 10); // BASE 10 + strcat(txBuffer, tempBuffer); + + // if (bufferFree() < 42) printTxBuffer(_outClient); + strcat(txBuffer, contentTypeHeader); + + // put the start of the JSON into the outgoing response_buffer + // if (bufferFree() < 21) printTxBuffer(_outClient); + strcat(txBuffer, samplingFeatureTag); + + // if (bufferFree() < 36) printTxBuffer(_outClient); + strcat(txBuffer, _baseLogger->getSamplingFeatureUUID()); +} + +void EnviroDIYPublisher::mmwPostDataArray(char* tempBuffer) { + // Fill the body + MS_DBG(F("Filling from Array")); + strcat(txBuffer, timestampTag); + _baseLogger->formatDateTime_ISO8601(Logger::markedLocalEpochTime) + .toCharArray(tempBuffer, TEMP_BUFFER_SZ); + strcat(txBuffer, tempBuffer); + txBuffer[strlen(txBuffer)] = '"'; + txBuffer[strlen(txBuffer)] = ','; + + for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + // Fill with variables + txBuffer[strlen(txBuffer)] = '"'; + _baseLogger->getVarUUIDAtI(i).toCharArray(tempBuffer, TEMP_BUFFER_SZ); + strcat(txBuffer, tempBuffer); + txBuffer[strlen(txBuffer)] = '"'; + txBuffer[strlen(txBuffer)] = ':'; + _baseLogger->getValueStringAtI(i).toCharArray(tempBuffer, + TEMP_BUFFER_SZ); + strcat(txBuffer, tempBuffer); + if (i + 1 != _baseLogger->getArrayVarCount()) { + txBuffer[strlen(txBuffer)] = ','; + } else { + txBuffer[strlen(txBuffer)] = '}'; + } + } +} +void EnviroDIYPublisher::mmwPostDataQueued(char* tempBuffer) { + // Fill the body - format is per MMW requirements + // MS_DBG(F("Filling from Queue")); + MS_START_DEBUG_TIMER; + strcat(txBuffer, timestampTag); + _baseLogger->formatDateTime_ISO8601(_baseLogger->deszq_epochTime) + .toCharArray(tempBuffer, TEMP_BUFFER_SZ); + strcat(txBuffer, tempBuffer); + txBuffer[strlen(txBuffer)] = '"'; + txBuffer[strlen(txBuffer)] = ','; + + for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + // Fill with variables + txBuffer[strlen(txBuffer)] = '"'; + _baseLogger->getVarUUIDAtI(i).toCharArray(tempBuffer, TEMP_BUFFER_SZ); + strcat(txBuffer, tempBuffer); + txBuffer[strlen(txBuffer)] = '"'; + txBuffer[strlen(txBuffer)] = ':'; + strncat(txBuffer, _baseLogger->deszq_nextChar, + _baseLogger->deszq_nextCharSz); + //_baseLogger-> getValueStringAtI(i).toCharArray(tempBuffer, 37); + // strcat(txBuffer, tempBuffer); + if (i + 1 != _baseLogger->getArrayVarCount()) { + txBuffer[strlen(txBuffer)] = ','; + } else { + txBuffer[strlen(txBuffer)] = '}'; + } + // Read the Next Stored character of SD + if (!_baseLogger->deszqNextCh()) break; + } + MS_DBG(F("Filled from SD QUE in "), MS_PRINT_DEBUG_TIMER, F("ms")); +} diff --git a/src/publishers/EnviroDIYPublisher.h b/src/publishers/EnviroDIYPublisher.h index 8a728dd1d..1b974b723 100644 --- a/src/publishers/EnviroDIYPublisher.h +++ b/src/publishers/EnviroDIYPublisher.h @@ -20,9 +20,16 @@ #define MS_DEBUGGING_STD "EnviroDIYPublisher" #endif +// #define MS_ENVIRODIYPUBLISHER_DEBUG_DEEP +#ifdef MS_ENVIRODIYPUBLISHER_DEBUG_DEEP +#define MS_DEBUGGING_DEEP "EnviroDIYPublisher" +#endif + // Included Dependencies #include "ModSensorDebugger.h" #undef MS_DEBUGGING_STD +#undef MS_DEBUGGING_DEEP +#undef MS_ENVIRODIYPUBLISHER_DEBUG_DEEP #include "dataPublisherBase.h" @@ -131,7 +138,7 @@ class EnviroDIYPublisher : public dataPublisher { // Returns the data destination String getEndpoint(void) override { - return String(enviroDIYHost); + return String(_enviroDIYHost)+':'+String(enviroDIYPort); } // Adds the site registration token @@ -240,6 +247,85 @@ class EnviroDIYPublisher : public dataPublisher { private: // Tokens and UUID's for EnviroDIY const char* _registrationToken = nullptr; + const char* _enviroDIYHost = enviroDIYHost; + //FUT: int _enviroDIYPort; + + public: + /** + * @brief Set the destination Host URL + * + * @param enviroDIYHost The Host URL for the site on the + * Monitor My Watershed data portal. + */ + void setDIYHost(const char* enviroDIYHost); + + /* FUT: + * @brief Set the destination Port + * + * @param enviroDIYPort The Port on Host URL for the site on the + * Monitor My Watershed data portal. + */ + //void setDIYPort(const int enviroDIYPort); + + protected: + /** + * @brief This constructs a POST header for MMW + * + * @param tempBuffer - place for the POST. + */ + void mmwPostHeader(char* tempBuffer); + + /** + * @brief This constructs a POST body EnviroDIY + * + * @param tempBuffer - place for the POST. + */ + void mmwPostDataArray(char* tempBuffer); + void mmwPostDataQueued(char* tempBuffer); + + public: + /** + * @brief This routes subsequent POST construction + * + * @param state - true for Queued, false for standard + */ + bool setQueuedState(bool state, char uniqueId = '0') override { + MS_DBG(F("EnvrDIYPub setQueued "), state); + return useQueueDataSource = state; + } + bool getQueuedStatus() override { + MS_DBG(F("EnvrDIYPub gQS "), useQueueDataSource); + return useQueueDataSource; + } + +#if !defined TIMER_EDP_POST_TIMEOUT_DEF_MSEC +#define TIMER_EDP_POST_TIMEOUT_DEF_MSEC 10000L +#endif // TIMER_EDP_POST_TIMEOUT_DEF_MSEC + uint16_t _timerPostTimeout_ms = TIMER_EDP_POST_TIMEOUT_DEF_MSEC; + uint16_t virtual setTimerPostTimeout_mS(uint16_t tpt_ms) { + MS_DBG(F("setTPT(mS)"), tpt_ms); + return _timerPostTimeout_ms = tpt_ms; // Default for not supported. + } + + uint16_t getTimerPostTimeout_mS() { + MS_DBG(F("getTPT(mS)"), _timerPostTimeout_ms); + return _timerPostTimeout_ms; // Default for not supported. + } + +#if !defined TIMER_EDP_POSTED_PACING_DEF_MSEC +#define TIMER_EDP_POSTED_PACING_DEF_MSEC 2000L +#endif // TIMER_EDP_POSTED_PACING_DEF_MSEC + uint16_t _timerPostPacing_ms = TIMER_EDP_POSTED_PACING_DEF_MSEC; + uint16_t virtual setTimerPostPacing_mS(uint16_t tpp_ms) { + MS_DBG(F("setTPP(mS)"), tpp_ms); + return _timerPostPacing_ms = tpp_ms; + } + + uint16_t getTimerPostPacing_mS() { + MS_DBG(F("getTPP(mS)"), _timerPostPacing_ms); + return _timerPostPacing_ms; + } + }; #endif // SRC_PUBLISHERS_ENVIRODIYPUBLISHER_H_