Fix lax parsing of `true`, `false`, and `null` (fixes #1781)

CHANGELOG.md

@@ -7,6 +7,7 @@ HEAD
 * Add `JsonVariant::shallowCopy()` (issue #1343)
 * Fix `9.22337e+18 is outside the range of representable values of type 'long'`
 * Fix comparison operators for `JsonArray`, `JsonArrayConst`, `JsonObject`, and `JsonObjectConst`
+* Fix lax parsing of `true`, `false`, and `null` (issue #1781)
 * Remove undocumented `accept()` functions
 * Rename `addElement()` to `add()`
 * Remove `getElement()`, `getOrAddElement()`, `getMember()`, and `getOrAddMember()`

extras/tests/JsonDeserializer/filter.cpp

@@ -71,6 +71,15 @@ TEST_CASE("Filtering") {
       "{\"example\":null}",
       JSON_OBJECT_SIZE(1) + 8
     },
+    {
+      // Member is a number, but filter wants an array
+      "{\"example\":42}",
+      "{\"example\":[true]}",
+      10,
+      DeserializationError::Ok,
+      "{\"example\":null}",
+      JSON_OBJECT_SIZE(1) + 8
+    },
     {
       // Input is an array, but filter wants an object
       "[\"hello\",\"world\"]",
@@ -117,7 +126,7 @@ TEST_CASE("Filtering") {
       JSON_OBJECT_SIZE(1) + 8
     },
     {
-      // can skip a boolean
+      // skip false
       "{\"a_bool\":false,example:42}",
       "{\"example\":true}",
       10,
@@ -125,6 +134,24 @@ TEST_CASE("Filtering") {
       "{\"example\":42}",
       JSON_OBJECT_SIZE(1) + 8
     },
+    {
+      // skip true
+      "{\"a_bool\":true,example:42}",
+      "{\"example\":true}",
+      10,
+      DeserializationError::Ok,
+      "{\"example\":42}",
+      JSON_OBJECT_SIZE(1) + 8
+    },
+    {
+      // skip null
+      "{\"a_bool\":null,example:42}",
+      "{\"example\":true}",
+      10,
+      DeserializationError::Ok,
+      "{\"example\":42}",
+      JSON_OBJECT_SIZE(1) + 8
+    },
     {
       // can skip a double-quoted string
       "{\"a_double_quoted_string\":\"hello\",example:42}",
@@ -618,7 +645,7 @@ TEST_CASE("Filtering") {
       0
     },
     {
-      // incomplete after after key of a skipped object
+      // incomplete comment after key of a skipped object
       "{\"example\"/*:2}",
       "false",
       10,
@@ -636,7 +663,7 @@ TEST_CASE("Filtering") {
       0
     },
     {
-      // incomplete after after value of a skipped object
+      // incomplete comment after value of a skipped object
       "{\"example\":2/*}",
       "false",
       10,
@@ -644,6 +671,15 @@ TEST_CASE("Filtering") {
       "null",
       0
     },
+     {
+      // incomplete comment after comma in skipped object
+      "{\"example\":2,/*}",
+      "false",
+      10,
+      DeserializationError::IncompleteInput,
+      "null",
+      0
+    },
   };  // clang-format on
 
   for (size_t i = 0; i < sizeof(testCases) / sizeof(testCases[0]); i++) {

extras/tests/JsonDeserializer/invalid_input.cpp

@@ -9,7 +9,8 @@
 TEST_CASE("Invalid JSON input") {
   const char* testCases[] = {"'\\u'",     "'\\u000g'", "'\\u000'", "'\\u000G'",
                              "'\\u000/'", "\\x1234",   "6a9",      "1,",
-                             "2]",        "3}"};
+                             "nulL",      "tru3",      "fals3",    "2]",
+                             "3}"};
   const size_t testCount = sizeof(testCases) / sizeof(testCases[0]);
 
   DynamicJsonDocument doc(4096);
@@ -23,9 +24,6 @@ TEST_CASE("Invalid JSON input") {
 
 TEST_CASE("Invalid JSON input that should pass") {
   const char* testCases[] = {
-      "nulL",
-      "tru3",
-      "fals3",
       "'\\ud83d'",         // leading surrogate without a trailing surrogate
       "'\\udda4'",         // trailing surrogate without a leading surrogate
       "'\\ud83d\\ud83d'",  // two leading surrogates

src/ArduinoJson/Json/JsonDeserializer.hpp

@@ -85,7 +85,22 @@ class JsonDeserializer {
         if (filter.allowValue())
           return parseStringValue(variant);
         else
-          return skipString();
+          return skipQuotedString();
+
+      case 't':
+        if (filter.allowValue())
+          variant.setBoolean(true);
+        return skipKeyword("true");
+
+      case 'f':
+        if (filter.allowValue())
+          variant.setBoolean(false);
+        return skipKeyword("false");
+
+      case 'n':
+        // the variant should already by null, except if the same object key was
+        // used twice, as in {"a":1,"a":null}
+        return skipKeyword("null");
 
       default:
         if (filter.allowValue())
@@ -111,7 +126,16 @@ class JsonDeserializer {
 
       case '\"':
       case '\'':
-        return skipString();
+        return skipQuotedString();
+
+      case 't':
+        return skipKeyword("true");
+
+      case 'f':
+        return skipKeyword("false");
+
+      case 'n':
+        return skipKeyword("null");
 
       default:
         return skipNumericValue();
@@ -310,7 +334,7 @@ class JsonDeserializer {
     // Read each key value pair
     for (;;) {
       // Skip key
-      err = skipVariant(nestingLimit.decrement());
+      err = skipKey();
       if (err)
         return err;
 
@@ -338,6 +362,10 @@ class JsonDeserializer {
         return DeserializationError::Ok;
       if (!eat(','))
         return DeserializationError::InvalidInput;
+
+      err = skipSpacesAndComments();
+      if (err)
+        return err;
     }
   }
 
@@ -438,7 +466,15 @@ class JsonDeserializer {
     return DeserializationError::Ok;
   }
 
-  DeserializationError::Code skipString() {
+  DeserializationError::Code skipKey() {
+    if (isQuote(current())) {
+      return skipQuotedString();
+    } else {
+      return skipNonQuotedString();
+    }
+  }
+
+  DeserializationError::Code skipQuotedString() {
     const char stopChar = current();
 
     move();
@@ -458,37 +494,26 @@ class JsonDeserializer {
     return DeserializationError::Ok;
   }
 
+  DeserializationError::Code skipNonQuotedString() {
+    char c = current();
+    while (canBeInNonQuotedString(c)) {
+      move();
+      c = current();
+    }
+    return DeserializationError::Ok;
+  }
+
   DeserializationError::Code parseNumericValue(VariantData &result) {
     uint8_t n = 0;
 
     char c = current();
-    while (canBeInNonQuotedString(c) && n < 63) {
+    while (canBeInNumber(c) && n < 63) {
       move();
       _buffer[n++] = c;
       c = current();
     }
     _buffer[n] = 0;
 
-    c = _buffer[0];
-    if (c == 't') {  // true
-      result.setBoolean(true);
-      if (n != 4)
-        return DeserializationError::IncompleteInput;
-      return DeserializationError::Ok;
-    }
-    if (c == 'f') {  // false
-      result.setBoolean(false);
-      if (n != 5)
-        return DeserializationError::IncompleteInput;
-      return DeserializationError::Ok;
-    }
-    if (c == 'n') {  // null
-      // the variant is already null
-      if (n != 4)
-        return DeserializationError::IncompleteInput;
-      return DeserializationError::Ok;
-    }
-
     if (!parseNumber(_buffer, result))
       return DeserializationError::InvalidInput;
 
@@ -497,7 +522,7 @@ class JsonDeserializer {
 
   DeserializationError::Code skipNumericValue() {
     char c = current();
-    while (canBeInNonQuotedString(c)) {
+    while (canBeInNumber(c)) {
       move();
       c = current();
     }
@@ -523,9 +548,18 @@ class JsonDeserializer {
     return min <= c && c <= max;
   }
 
+  static inline bool canBeInNumber(char c) {
+    return isBetween(c, '0', '9') || c == '+' || c == '-' || c == '.' ||
+#if ARDUINOJSON_ENABLE_NAN || ARDUINOJSON_ENABLE_INFINITY
+           isBetween(c, 'A', 'Z') || isBetween(c, 'a', 'z');
+#else
+           c == 'e' || c == 'E';
+#endif
+  }
+
   static inline bool canBeInNonQuotedString(char c) {
     return isBetween(c, '0', '9') || isBetween(c, '_', 'z') ||
-           isBetween(c, 'A', 'Z') || c == '+' || c == '-' || c == '.';
+           isBetween(c, 'A', 'Z');
   }
 
   static inline bool isQuote(char c) {
@@ -605,6 +639,19 @@ class JsonDeserializer {
     }
   }
 
+  DeserializationError::Code skipKeyword(const char *s) {
+    while (*s) {
+      char c = current();
+      if (c == '\0')
+        return DeserializationError::IncompleteInput;
+      if (*s != c)
+        return DeserializationError::InvalidInput;
+      ++s;
+      move();
+    }
+    return DeserializationError::Ok;
+  }
+
   TStringStorage _stringStorage;
   bool _foundSomething;
   Latch<TReader> _latch;