More extensive MQTT + OTA support (#266)

Extend existing MQTT demo code:
* Add debug info (IP and MQTT connection status) to the ESP32 display
* Adjust MQTT home assistant auto config, add availability topic and
last will
* Add support for ArduinoOTA uploads (only available via the MQTT task
for now; not available for the HTTP demo)

Author: scottbez1

firmware/esp32/core/splitflap_task.cpp

@@ -373,11 +373,12 @@ void SplitflapTask::log(const char* msg) {
     }
 }
 
-void SplitflapTask::showString(const char* str, uint8_t length, bool force_full_rotation) {
+void SplitflapTask::showString(const char* str, uint8_t length, bool force_full_rotation, bool default_unspecified_home) {
     Command command = {};
     command.command_type = CommandType::MODULES;
-    for (uint8_t i = 0; i < length && i < NUM_MODULES; i++) {
-        int8_t index = findFlapIndex(str[i]);
+    uint8_t num_to_update = default_unspecified_home ? NUM_MODULES : length;
+    for (uint8_t i = 0; i < num_to_update && i < NUM_MODULES; i++) {
+        int8_t index = i >= length ? 0 : findFlapIndex(str[i]);
         if (index != -1) {
             if (force_full_rotation || index != modules[i]->GetTargetFlapIndex()) {
                 command.data.module_command[i] = QCMD_FLAP + index;

firmware/esp32/core/splitflap_task.h

@@ -129,7 +129,7 @@ class SplitflapTask : public Task<SplitflapTask> {
 
         SplitflapState getState();
 
-        void showString(const char *str, uint8_t length, bool force_full_rotation = FORCE_FULL_ROTATION);
+        void showString(const char *str, uint8_t length, bool force_full_rotation = FORCE_FULL_ROTATION, bool default_unspecified_home = false);
         void resetAll();
         void disableAll();
         void setLed(uint8_t id, bool on);

firmware/esp32/splitflap/display_layouts.h

@@ -18,7 +18,7 @@
 #include <stdint.h>
 
 // Customize these settings and select a layout algorithm at the bottom if you have a different arrangement of modules:
-#define DISPLAY_COLUMNS 6
+#define DISPLAY_COLUMNS (NUM_MODULES)
 
 
 // EXAMPLE LAYOUT ALGORITHMS:

firmware/esp32/splitflap/main.cpp

@@ -42,7 +42,7 @@ BaseSupervisorTask baseSupervisorTask(splitflapTask, serialTask, 0);
 
 #if MQTT
 #include "mqtt_task.h"
-MQTTTask mqttTask(splitflapTask, serialTask, 0);
+MQTTTask mqttTask(splitflapTask, displayTask, serialTask, 0);
 #endif
 
 #if HTTP

firmware/esp32/splitflap/mqtt_task.cpp

@@ -14,13 +14,25 @@
    limitations under the License.
 */
 #if MQTT
+#include <ArduinoOTA.h>
+
 #include "mqtt_task.h"
 #include "secrets.h"
 
+using namespace json11;
+
+// This MQTT demo assumes a home assistant MQTT instance, and will do automatic registration
+// via the home assistant config topic when it comes online and connects to MQTT. You shouldn't
+// need to change any of this to get it to work with home assistant.
+#define MQTT_CONFIG_TOPIC "homeassistant/text/" DEVICE_INSTANCE_NAME "/config"
+#define MQTT_COMMAND_TOPIC "home/" DEVICE_INSTANCE_NAME "/command"
+#define MQTT_STATE_TOPIC "home/" DEVICE_INSTANCE_NAME "/state"
+#define MQTT_AVAILABILITY_TOPIC "home/" DEVICE_INSTANCE_NAME "/availability"
 
-MQTTTask::MQTTTask(SplitflapTask& splitflap_task, Logger& logger, const uint8_t task_core) :
+MQTTTask::MQTTTask(SplitflapTask& splitflap_task, DisplayTask& display_task, Logger& logger, const uint8_t task_core) :
         Task("MQTT", 8192, 1, task_core),
         splitflap_task_(splitflap_task),
+        display_task_(display_task),
         logger_(logger),
         wifi_client_(),
         mqtt_client_(wifi_client_) {
@@ -30,56 +42,173 @@ MQTTTask::MQTTTask(SplitflapTask& splitflap_task, Logger& logger, const uint8_t
 
 void MQTTTask::connectWifi() {
     WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
-
     // Disable WiFi sleep as it causes glitches on pin 39; see https://github.com/espressif/arduino-esp32/issues/4903#issuecomment-793187707
     WiFi.setSleep(WIFI_PS_NONE);
 
+    char buf[256];
+
+    snprintf(buf, sizeof(buf), "Wifi connecting to %s", WIFI_SSID);
+    display_task_.setMessage(0, String(buf));
+
     while (WiFi.status() != WL_CONNECTED) {
         delay(1000);
         logger_.log("Establishing connection to WiFi..");
     }
 
-    char buf[256];
     snprintf(buf, sizeof(buf), "Connected to network %s", WIFI_SSID);
     logger_.log(buf);
+
+    snprintf(buf, sizeof(buf), "Wifi IP: %s", WiFi.localIP().toString().c_str());
+    display_task_.setMessage(0, String(buf));
 }
 
 void MQTTTask::mqttCallback(char *topic, byte *payload, unsigned int length) {
     char buf[256];
     snprintf(buf, sizeof(buf), "Received mqtt callback for topic %s, length %u", topic, length);
     logger_.log(buf);
-    splitflap_task_.showString((const char *)payload, length);
+
+
+    splitflap_task_.showString((const char *)payload, length, false, true);
 }
 
 void MQTTTask::connectMQTT() {
-    char buf[256];
-    mqtt_client_.setServer(MQTT_SERVER, 1883);
+    char buf[400];
+    mqtt_client_.setServer(MQTT_SERVER, MQTT_PORT);
     logger_.log("Attempting MQTT connection...");
-    if (mqtt_client_.connect(HOSTNAME "-" MQTT_USER, MQTT_USER, MQTT_PASSWORD)) {
+    snprintf(buf, sizeof(buf), "MQTT connecting to %s:%d", MQTT_SERVER, MQTT_PORT);
+    display_task_.setMessage(1, String(buf));
+
+    if (mqtt_client_.connect(DEVICE_INSTANCE_NAME, MQTT_USER, MQTT_PASSWORD, MQTT_AVAILABILITY_TOPIC, 1, true, "offline")) {
         logger_.log("MQTT connected");
         mqtt_client_.subscribe(MQTT_COMMAND_TOPIC);
-        char buf[256];
-        snprintf(buf, sizeof(buf), "{\"name\": \"%s\", \"command_topic\": \"%s\", \"state_topic\": \"%s\", \"unique_id\": \"%s\"}", HOSTNAME, MQTT_COMMAND_TOPIC, MQTT_COMMAND_TOPIC, HOSTNAME);
-        mqtt_client_.publish("homeassistant/text/splitflap/config", buf);
+
+        // TODO: I believe it's possible to do more complex config to register as a device with multiple
+        // entities; it'd be great to explore additional entities like a display backlight control,
+        // detailed module status info, etc. Though at a certain point it may be preferable to make the
+        // splitflap library an ESPHome external component for even more options, easier programming,
+        // and more customization than MQTT offers...
+        Json config = Json::object {
+            { "name", DEVICE_INSTANCE_NAME },
+            { "command_topic", MQTT_COMMAND_TOPIC },
+            { "state_topic", MQTT_STATE_TOPIC },
+            { "availability_topic", MQTT_AVAILABILITY_TOPIC },
+            { "payload_available", "online" },
+            { "payload_not_available", "offline" },
+            { "unique_id", DEVICE_INSTANCE_NAME },
+            { "max", NUM_MODULES },
+        };
+        std::string json_str = config.dump();
+        boolean result = mqtt_client_.publish(MQTT_CONFIG_TOPIC, json_str.c_str());
+        snprintf(buf, sizeof(buf), "Result of publish: %d", result);
+        logger_.log(buf);
         logger_.log("Published MQTT discovery message");
+        logger_.log(json_str.c_str());
+
+        mqtt_client_.publish(MQTT_AVAILABILITY_TOPIC, "online", true);
+
+        snprintf(buf, sizeof(buf), "MQTT connected! (%s:%d)", MQTT_SERVER, MQTT_PORT);
+        display_task_.setMessage(1, String(buf));
     } else {
         snprintf(buf, sizeof(buf), "MQTT failed rc=%d will try again in 5 seconds", mqtt_client_.state());
         logger_.log(buf);
+
+        snprintf(buf, sizeof(buf), "MQTT failed rc=%d", mqtt_client_.state());
+        display_task_.setMessage(1, String(buf));
     }
 }
 
 void MQTTTask::run() {
+    char buf[256];
+    display_task_.setMessage(0, "");
+    display_task_.setMessage(1, "");
     connectWifi();
     connectMQTT();
 
+    ArduinoOTA
+        .onStart([this]() {
+            if (ArduinoOTA.getCommand() == U_FLASH) {
+                logger_.log("Start OTA (flash)");
+            } else { // U_SPIFFS
+                logger_.log("Start OTA (filesystem)");
+                // NOTE: if updating SPIFFS this would be the place to unmount SPIFFS using SPIFFS.end()
+            }
+        })
+        .onEnd([this]() {
+            logger_.log("OTA End");
+        })
+        .onProgress([this](unsigned int progress, unsigned int total) {
+            char buf2[256];
+            static uint32_t last_progress;
+            if (millis() - last_progress > 1000) {
+                snprintf(buf2, sizeof(buf2), "OTA Progress: %d%%", (int)(progress * 100 / total));
+                logger_.log(buf2);
+                last_progress = millis();
+            }
+        })
+        .onError([this](ota_error_t error) {
+            char buf2[256];
+            snprintf(buf2, sizeof(buf2), "OTA Error: %u", error);
+            logger_.log(buf2);
+            if (error == OTA_AUTH_ERROR) logger_.log("Auth Failed");
+            else if (error == OTA_BEGIN_ERROR) logger_.log("Begin Failed");
+            else if (error == OTA_CONNECT_ERROR) logger_.log("Connect Failed");
+            else if (error == OTA_RECEIVE_ERROR) logger_.log("Receive Failed");
+            else if (error == OTA_END_ERROR) logger_.log("End Failed");
+        })
+        .setHostname(DEVICE_INSTANCE_NAME)
+        .setPassword(OTA_PASSWORD)
+        .begin();
+
+    wl_status_t wifi_last_status = WL_DISCONNECTED;
+    uint32_t last_state_publish = 0;
+    SplitflapState last_state = {};
+    uint32_t last_availability_publish = 0;
     while(1) {
         long now = millis();
+        wl_status_t wifi_new_status = WiFi.status();
+        if (wifi_new_status != wifi_last_status) {
+            if (wifi_new_status == WL_CONNECTED) {
+                snprintf(buf, sizeof(buf), "Wifi IP: %s", WiFi.localIP().toString().c_str());
+                display_task_.setMessage(0, String(buf));
+            } else {
+                snprintf(buf, sizeof(buf), "Wifi connecting to %s", WIFI_SSID);
+                display_task_.setMessage(0, String(buf));
+            }
+            wifi_last_status = wifi_new_status;
+        }
         if (!mqtt_client_.connected() && (now - mqtt_last_connect_time_) > 5000) {
             logger_.log("Reconnecting MQTT");
             mqtt_last_connect_time_ = now;
             connectMQTT();
         }
+        if (mqtt_client_.connected()) {
+            SplitflapState state = splitflap_task_.getState();
+            if (state != last_state) {
+                char flap_buf[NUM_MODULES+1];
+                bool all_idle = true;
+                for (uint8_t i = 0; i < NUM_MODULES; i++) {
+                    flap_buf[i] = flaps[state.modules[i].flap_index];
+                    if (state.modules[i].moving) {
+                        all_idle = false;
+                    }
+                }
+                flap_buf[NUM_MODULES] = 0;
+
+                if (all_idle && (now - last_state_publish) > 200) {
+                    last_state = state;
+                    last_state_publish = now;
+                    snprintf(buf, sizeof(buf), "Publishing state: %s", flap_buf);
+                    logger_.log(buf);
+                    mqtt_client_.publish(MQTT_STATE_TOPIC, flap_buf);
+                }
+            }
+            if (now > last_availability_publish + 1800000) {
+                mqtt_client_.publish(MQTT_AVAILABILITY_TOPIC, "online", true);
+                last_availability_publish = now;
+            }
+        }
         mqtt_client_.loop();
+        ArduinoOTA.handle();
         delay(1);
     }
 }

firmware/esp32/splitflap/mqtt_task.h

@@ -16,26 +16,29 @@
 #pragma once
 
 #include <Arduino.h>
+#include <PubSubClient.h>
+#include <WiFi.h>
+
+#include <json11.hpp>
 
 #include "../core/logger.h"
 #include "../core/splitflap_task.h"
 #include "../core/task.h"
 
-#include <PubSubClient.h>
-#include <WiFi.h>
-
+#include "display_task.h"
 
 class MQTTTask : public Task<MQTTTask> {
     friend class Task<MQTTTask>; // Allow base Task to invoke protected run()
 
     public:
-        MQTTTask(SplitflapTask& splitflapTask, Logger& logger, const uint8_t taskCore);
+        MQTTTask(SplitflapTask& splitflapTask, DisplayTask& DisplayTask, Logger& logger, const uint8_t taskCore);
 
     protected:
         void run();
 
     private:
         SplitflapTask& splitflap_task_;
+        DisplayTask& display_task_;
         Logger& logger_;
         WiFiClient wifi_client_;
         PubSubClient mqtt_client_;

firmware/esp32/splitflap/secrets.h.example

@@ -4,9 +4,29 @@
 #define WIFI_SSID "myssid"
 #define WIFI_PASSWORD "supersecretpassword"
 
-#define HOSTNAME "splitflap" // e.g. splitflap.local
+// Hostname for this device, which is also used for MDNS, e.g. "splitflap" will result
+// in an mdns name of "splitflap.local"
+// If you plan to use OTA uploads, make sure to update the "upload_port" in the "env:chainlink_ota"
+// section of platformio.ini to match.
+#define DEVICE_INSTANCE_NAME "splitflap"
 
-#define MQTT_SERVER "10.0.0.2"
+#define MQTT_SERVER "192.168.0.242"
+#define MQTT_PORT (1883)
 #define MQTT_USER "mqttuser"
 #define MQTT_PASSWORD "megasecretpassword"
-#define MQTT_COMMAND_TOPIC "homeassistant/text/splitflap/state"
+
+// Password for wireless firmware uploads (only works with MQTT=1 in build_flags!).
+// To use OTA functionality:
+//   Setup:
+//     1. Change this value to something unique in your secrets.h file
+//     2. Upload the firmware via USB using the "chainlink" env in platformio - this must be
+//        done via USB first to install the initial firmware, including this password for
+//        future wifi uploads.
+//     3. If you've changed the DEVICE_INSTANCE_NAME, make sure to update the "upload_port" in
+//        the "env:chainlink_ota" section of platformio.ini to match, so the OTA uploads know
+//        who to talk to.
+//   OTA updates:
+//     1. Make sure the ESP32 is online
+//     2. Upload the firmware using the "chainlink_ota" environment instead of the "chainlink"
+//        environment
+#define OTA_PASSWORD "replace_me!"

platformio.ini

@@ -92,6 +92,7 @@ build_flags =
 
     ; Set to true to enable MQTT support (see secrets.h.example for configuration)
     -DMQTT=false
+    -DMQTT_MAX_PACKET_SIZE=512
 
     ; Set to true to enable HTTP support (see secrets.h.example for configuration)
     -DHTTP=false
@@ -122,6 +123,18 @@ build_flags =
     -DCHAINLINK
     -DNUM_MODULES=6
 
+[env:chainlink_ota]
+extends=env:chainlink
+upload_protocol = espota
+upload_port = splitflap.local  ; Replace with your device's IP address
+upload_flags =
+    --auth=replace_me!
+    --port=3232
+    --host_port=16106
+    --debug
+    --progress
+    --timeout=30
+
 [env:advanced_chainlinkBase]
 extends=esp32base
 build_src_filter = ${esp32base.build_src_filter} +<../esp32/base>