ArduinoJson serializing empty JSON data when printing serial output to a Python program

Question

I'm trying to create a two ways communication between an Arduino UNO board and a Python program. The format chosen for the messages exchanged is JSON.

As Arduino UNO has an input buffer of 64 bytes, and the JSON messages are considerably bigger than that, I implemented a way to break JSON data into 64 bytes messages in Python, and then, reassembling it on Arduino code.

To mark the end of a message, I'm using the string "<e>", informing both Arduino and Python that the message was completely delivered.

The Arduino code is working fine when inputting data via Arduino Serial Monitor. The JSON I'm using as input is below. Two lines, send via Serial Monitor, one at a time.

{"sequence": 0, "state": 0, "commands": [{"device_id": "1", "val
ue": 1.0, "result": 0}], "statuses": [{"device_id": "1"}]}<e>

The output for Serial Monitor is this one:

But when I run the same Arduino code with my Python code, the JSON generated and sent back to Python is empty ('{}').

This is my Arduino code:

#include <ArduinoJson.h>

String inputMessage = "";
bool processingRequest = false;

void serialEventRun(void) {
  if (Serial.available()) serialEvent();
}

void serialEvent() {

  if (!processingRequest) {

    String message = "";
    char c;

    while (Serial.available() > 0) {
      c = Serial.read();
      message.concat(c);
    }

    message.trim();

    inputMessage += message;

    if (inputMessage.endsWith("<e>")) {

      processingRequest = true;
      inputMessage = inputMessage.substring(0, inputMessage.indexOf("<e>"));

      const size_t bufferSize = 2 * JSON_ARRAY_SIZE(3) + JSON_OBJECT_SIZE(2) + 6 * JSON_OBJECT_SIZE(3) + 240;
      DynamicJsonBuffer jsonBuffer(bufferSize);

      JsonObject& root = jsonBuffer.parseObject(inputMessage);

      const int sequence = root["sequence"];
      const int state = root["state"];
      JsonArray& commands = root["commands"];
      JsonArray& statuses = root["statuses"];

      // TODO include real command/status call
      if (commands.size() > 0) {
        for (int i = 0; i < commands.size(); i++) {
          JsonObject& command = commands[i];
          command["result"] = 1;
        }
      }
      if (statuses.size() > 0) {
        for (int i = 0; i < statuses.size(); i++) {
          JsonObject& status = statuses[i];
          status["value"] = 1.1;
          status["health"] = 0;
        }
      }

      root["state"] = 2;

      root.printTo(Serial);
      Serial.print("<e>");

      processingRequest = false;

    }

  }

}

void setup() {
  Serial.begin(115200);
}

void loop() {}

And this is my Python code:

import time

from serial import Serial


def main():

    result = ''

    serial_conn = Serial(port='COM5', baudrate=115200, timeout=0.1)
    time.sleep(1)
    serial_conn.write(str.encode('{"sequence": 0, "state": 0, "commands": [{"device_id": "1", "val'))
    time.sleep(0.1)
    serial_conn.write(str.encode('ue": 1.0, "result": 0}], "statuses": [{"device_id": "1"}]}<e>'))
    time.sleep(0.1)

    # serial_conn.flushInput()

    while True:
        # bytes_to_read = serial_conn.inWaiting()
        # msg = serial_conn.read(bytes_to_read)
        msg = serial_conn.readline()
        time.sleep(0.1)
        result += msg.decode()
        print("<{}>".format(result))
        if result.endswith('<e>'):
            break
    result = result.strip('<e>')
    print("Message received: <{}>".format(result))

    serial_conn.close()


if __name__ == '__main__':
    main()

When running with Python, this is the console output:

<{}<e>>
Message received: <{}>

Process finished with exit code 0

Trying to figure it out what is going on, I changed the Arduino code to print to Serial port only the length of the JSON serialized data. When running on Serial Monitor, the length is 135, the same length of the outout in the screenshot above. But when running with Python, the length is 5, exactly the same length of "{}<e>" that we can see in Python's console output. So, apparently, when I'm running with Python, the JSON serialization generates empty data ("{}").

I already thought if I should not print to serial port (in Arduino code) also respecting the 64 bytes buffer, as I did when receiving data. But as the JSON serialized data is being generated empty, I'm not sure this is the problem.

Any ideas?

Thanks in advance!

Answer 1

After several tests, I was able to solve the problem.

As commented by me above, the cause of the empty JSON being returned by Arduino was the data from Python was not being sent correctly. My Python code sends two JSON strings to Arduino, but only the second one was arriving.

With only the second message, ArduinoJson were not able to parse the JSON data, generating an empty JsonObject that was generating an empty JSON serialized data ("{}").

And the cause for that is, apparently, the PySerial lib needs some time to open the serial port, and be able to send data. I could not find much about this situation, only this issue reported in PySerial GitHub page.

At the end of the day, the solution was to include a 2 second delay after the serial port was open.

During the pursuit for a solution, I changed a lot my Arduino code.

Arduino code main changes:

1 - Changed to do not use serialEvent().
2 - Included JsonObject.success() check.
3 - Returning specific error if JsonObject.success() fails.

#include <ArduinoJson.h>

String receivedMessage = "";

void processMessage(String message) {

  const size_t bufferSize = 2 * JSON_ARRAY_SIZE(3) + JSON_OBJECT_SIZE(2) + 6 * JSON_OBJECT_SIZE(3) + 240;
  DynamicJsonBuffer jsonBuffer(bufferSize);
  //StaticJsonBuffer<bufferSize> jsonBuffer;

  JsonObject& root = jsonBuffer.parseObject(message);

  if (root.success()) {

    const int sequence = root["sequence"];
    const int state = root["state"];
    JsonArray& commands = root["commands"];
    JsonArray& statuses = root["statuses"];

    // TODO include real command/status call
    if (commands.size() > 0) {
      for (int i = 0; i < commands.size(); i++) {
        JsonObject& command = commands[i];
        command["result"] = 1;
      }
    }
    if (statuses.size() > 0) {
      for (int i = 0; i < statuses.size(); i++) {
        JsonObject& status = statuses[i];
        status["value"] = 1.1;
        status["health"] = 0;
      }
    }

    root["state"] = 0;
    root.printTo(Serial);

  } else {
    jsonBuffer.clear();
    JsonObject& error = jsonBuffer.createObject();
    error["state"] = 3;
    error.printTo(Serial);
  }

  Serial.print("<e>");

}

void setup() {
  Serial.begin(115200);
  while (!Serial) {}
}

void loop() {

  while (!Serial.available()) {}
  receivedMessage = Serial.readString();

  receivedMessage.trim();
  receivedMessage.replace("\n", "");
  if (receivedMessage.endsWith("<e>")) {
    receivedMessage = receivedMessage.substring(0, receivedMessage.indexOf("<e>"));
    processMessage(receivedMessage);
    receivedMessage = "";
  }

}

But the solution was to increase the delay in line 11 to 2 seconds:

import time

from serial import Serial


def main():

    result = ''

    serial_conn = Serial(port='COM5', baudrate=115200, timeout=0.1)
    time.sleep(2)
    serial_conn.write(str.encode('{"sequence": 0, "state": 0, "commands": [{"device_id": "1", "val'))
    serial_conn.write(str.encode('ue": 1.0, "result": 0}], "statuses": [{"device_id": "1"}]}<e>'))

    # serial_conn.flushInput()

    while True:
        # bytes_to_read = serial_conn.inWaiting()
        # msg = serial_conn.read(bytes_to_read)
        msg = serial_conn.readline()
        time.sleep(0.1)
        result += msg.decode()
        print("<{}>".format(result))
        if result.endswith('<e>'):
            break
    result = result.strip('<e>')
    print("Message received: <{}>".format(result))

    serial_conn.close()


if __name__ == '__main__':
    main()