Store the strings in the heap

CHANGELOG.md

@@ -11,3 +11,4 @@ HEAD
 * Remove `JSON_ARRAY_SIZE()`, `JSON_OBJECT_SIZE()`, and `JSON_STRING_SIZE()`
 * Remove `ARDUINOJSON_ENABLE_STRING_DEDUPLICATION` (string deduplication cannot be enabled anymore)
 * Remove `JsonDocument::capacity()`
+* Store the strings in the heap

README.md

@@ -31,7 +31,6 @@ ArduinoJson is a C++ JSON library for Arduino and IoT (Internet Of Things).
     * [Twice smaller than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/)
     * [Almost 10% faster than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/)
     * [Consumes roughly 10% less RAM than the "official" Arduino_JSON library](https://arduinojson.org/2019/11/19/arduinojson-vs-arduino_json/)
-    * [Fixed memory allocation, no heap fragmentation](https://arduinojson.org/v6/api/jsondocument/)
     * [Deduplicates strings](https://arduinojson.org/news/2020/08/01/version-6-16-0/)
 * Versatile
     * Supports [custom allocators (to use external RAM chip, for example)](https://arduinojson.org/v6/how-to/use-external-ram-on-esp32/)

extras/tests/JsonDeserializer/string.cpp

@@ -6,6 +6,8 @@
 #include <ArduinoJson.h>
 #include <catch.hpp>
 
+#include "Allocators.hpp"
+
 using ArduinoJson::detail::sizeofArray;
 using ArduinoJson::detail::sizeofObject;
 using ArduinoJson::detail::sizeofString;
@@ -96,42 +98,67 @@ TEST_CASE("Invalid JSON string") {
   }
 }
 
-TEST_CASE("Not enough room to save the key") {
-  JsonDocument doc(sizeofObject(1) + sizeofString(7));
+TEST_CASE("Allocation of the key fails") {
+  TimebombAllocator timebombAllocator(1);
+  SpyingAllocator spyingAllocator(&timebombAllocator);
+  JsonDocument doc(1024, &spyingAllocator);
 
-  SECTION("Quoted string") {
+  SECTION("Quoted string, first member") {
     REQUIRE(deserializeJson(doc, "{\"example\":1}") ==
-            DeserializationError::Ok);
-    REQUIRE(deserializeJson(doc, "{\"accuracy\":1}") ==
             DeserializationError::NoMemory);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(1024)
+                           << AllocatorLog::AllocateFail(sizeofString(31)));
+  }
+
+  SECTION("Quoted string, second member") {
+    timebombAllocator.setCountdown(2);
     REQUIRE(deserializeJson(doc, "{\"hello\":1,\"world\"}") ==
-            DeserializationError::NoMemory);  // fails in the second string
+            DeserializationError::NoMemory);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(1024)
+                           << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Reallocate(sizeofString(31),
+                                                       sizeofString(5))
+                           << AllocatorLog::AllocateFail(sizeofString(31)));
   }
 
-  SECTION("Non-quoted string") {
-    REQUIRE(deserializeJson(doc, "{example:1}") == DeserializationError::Ok);
-    REQUIRE(deserializeJson(doc, "{accuracy:1}") ==
+  SECTION("Non-Quoted string, first member") {
+    REQUIRE(deserializeJson(doc, "{example:1}") ==
             DeserializationError::NoMemory);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(1024)
+                           << AllocatorLog::AllocateFail(sizeofString(31)));
+  }
+
+  SECTION("Non-Quoted string, second member") {
+    timebombAllocator.setCountdown(2);
     REQUIRE(deserializeJson(doc, "{hello:1,world}") ==
-            DeserializationError::NoMemory);  // fails in the second string
+            DeserializationError::NoMemory);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(1024)
+                           << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Reallocate(sizeofString(31),
+                                                       sizeofString(5))
+                           << AllocatorLog::AllocateFail(sizeofString(31)));
   }
 }
 
-TEST_CASE("Empty memory pool") {
-  // NOLINTNEXTLINE(clang-analyzer-optin.portability.UnixAPI)
-  JsonDocument doc(0);
+TEST_CASE("String allocation fails") {
+  SpyingAllocator spyingAllocator(FailingAllocator::instance());
+  JsonDocument doc(0, &spyingAllocator);
 
   SECTION("Input is const char*") {
     REQUIRE(deserializeJson(doc, "\"hello\"") ==
             DeserializationError::NoMemory);
-    REQUIRE(deserializeJson(doc, "\"\"") == DeserializationError::NoMemory);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31)));
   }
 
   SECTION("Input is const char*") {
     char hello[] = "\"hello\"";
     REQUIRE(deserializeJson(doc, hello) == DeserializationError::Ok);
-    char empty[] = "\"hello\"";
-    REQUIRE(deserializeJson(doc, empty) == DeserializationError::Ok);
+    REQUIRE(spyingAllocator.log() == AllocatorLog());
   }
 }
 

extras/tests/JsonDocument/assignment.cpp

@@ -9,63 +9,59 @@
 
 using ArduinoJson::detail::sizeofArray;
 using ArduinoJson::detail::sizeofObject;
+using ArduinoJson::detail::sizeofString;
 
 TEST_CASE("JsonDocument assignment") {
   SpyingAllocator spyingAllocator;
 
   SECTION("Copy assignment same capacity") {
-    {
-      JsonDocument doc1(1024, &spyingAllocator);
-      deserializeJson(doc1, "{\"hello\":\"world\"}");
-      JsonDocument doc2(1024, &spyingAllocator);
+    JsonDocument doc1(1024, &spyingAllocator);
+    deserializeJson(doc1, "{\"hello\":\"world\"}");
+    JsonDocument doc2(1024, &spyingAllocator);
+    spyingAllocator.clearLog();
 
-      doc2 = doc1;
+    doc2 = doc1;
 
-      REQUIRE(doc2.as<std::string>() == "{\"hello\":\"world\"}");
-    }
-    REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(1024)
-                                         << AllocatorLog::Allocate(1024)
-                                         << AllocatorLog::Deallocate(1024)
-                                         << AllocatorLog::Deallocate(1024));
+    REQUIRE(doc2.as<std::string>() == "{\"hello\":\"world\"}");
+
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(sizeofString(5))  // hello
+                           << AllocatorLog::Allocate(sizeofString(5))  // world
+    );
   }
 
   SECTION("Copy assignment reallocates when capacity is smaller") {
-    {
-      JsonDocument doc1(4096, &spyingAllocator);
-      deserializeJson(doc1, "{\"hello\":\"world\"}");
-      JsonDocument doc2(8, &spyingAllocator);
+    JsonDocument doc1(4096, &spyingAllocator);
+    deserializeJson(doc1, "{\"hello\":\"world\"}");
+    JsonDocument doc2(8, &spyingAllocator);
+    spyingAllocator.clearLog();
 
-      doc2 = doc1;
+    doc2 = doc1;
 
-      REQUIRE(doc2.as<std::string>() == "{\"hello\":\"world\"}");
-    }
-    REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Allocate(8)
-                                         << AllocatorLog::Deallocate(8)
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Deallocate(4096)
-                                         << AllocatorLog::Deallocate(4096));
+    REQUIRE(doc2.as<std::string>() == "{\"hello\":\"world\"}");
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Deallocate(8)
+                           << AllocatorLog::Allocate(4096)
+                           << AllocatorLog::Allocate(sizeofString(5))  // hello
+                           << AllocatorLog::Allocate(sizeofString(5))  // world
+    );
   }
 
   SECTION("Copy assignment reallocates when capacity is larger") {
-    {
-      JsonDocument doc1(1024, &spyingAllocator);
-      deserializeJson(doc1, "{\"hello\":\"world\"}");
-      JsonDocument doc2(4096, &spyingAllocator);
+    JsonDocument doc1(1024, &spyingAllocator);
+    deserializeJson(doc1, "{\"hello\":\"world\"}");
+    JsonDocument doc2(4096, &spyingAllocator);
+    spyingAllocator.clearLog();
 
-      doc2 = doc1;
+    doc2 = doc1;
 
-      REQUIRE(doc2.as<std::string>() == "{\"hello\":\"world\"}");
-    }
-    REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(1024)
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Deallocate(4096)
-                                         << AllocatorLog::Allocate(1024)
-                                         << AllocatorLog::Deallocate(1024)
-                                         << AllocatorLog::Deallocate(1024));
+    REQUIRE(doc2.as<std::string>() == "{\"hello\":\"world\"}");
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Deallocate(4096)
+                           << AllocatorLog::Allocate(1024)
+                           << AllocatorLog::Allocate(sizeofString(5))  // hello
+                           << AllocatorLog::Allocate(sizeofString(5))  // world
+    );
   }
 
   SECTION("Move assign") {
@@ -79,11 +75,13 @@ TEST_CASE("JsonDocument assignment") {
       REQUIRE(doc2.as<std::string>() == "The size of this string is 32!!");
       REQUIRE(doc1.as<std::string>() == "null");
     }
-    REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Allocate(8)
-                                         << AllocatorLog::Deallocate(8)
-                                         << AllocatorLog::Deallocate(4096));
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(4096)
+                           << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Allocate(8)
+                           << AllocatorLog::Deallocate(8)
+                           << AllocatorLog::Deallocate(sizeofString(31))
+                           << AllocatorLog::Deallocate(4096));
   }
 
   SECTION("Assign from JsonObject") {

extras/tests/JsonDocument/constructor.cpp

@@ -8,6 +8,7 @@
 #include "Allocators.hpp"
 
 using ArduinoJson::detail::addPadding;
+using ArduinoJson::detail::sizeofString;
 
 TEST_CASE("JsonDocument constructor") {
   SpyingAllocator spyingAllocator;
@@ -29,11 +30,15 @@ TEST_CASE("JsonDocument constructor") {
       REQUIRE(doc1.as<std::string>() == "The size of this string is 32!!");
       REQUIRE(doc2.as<std::string>() == "The size of this string is 32!!");
     }
-    REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Deallocate(4096)
-                                         << AllocatorLog::Deallocate(4096));
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(4096)
+                           << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Allocate(4096)
+                           << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Deallocate(sizeofString(31))
+                           << AllocatorLog::Deallocate(4096)
+                           << AllocatorLog::Deallocate(sizeofString(31))
+                           << AllocatorLog::Deallocate(4096));
   }
 
   SECTION("JsonDocument(JsonDocument&&)") {
@@ -46,9 +51,11 @@ TEST_CASE("JsonDocument constructor") {
       REQUIRE(doc2.as<std::string>() == "The size of this string is 32!!");
       REQUIRE(doc1.as<std::string>() == "null");
     }
-    REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Deallocate(4096));
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(4096)
+                           << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Deallocate(sizeofString(31))
+                           << AllocatorLog::Deallocate(4096));
   }
 
   SECTION("JsonDocument(JsonObject)") {
@@ -82,7 +89,9 @@ TEST_CASE("JsonDocument constructor") {
     JsonDocument doc2(doc1.as<JsonVariant>(), &spyingAllocator);
 
     REQUIRE(doc2.as<std::string>() == "hello");
-    REQUIRE(spyingAllocator.log() == AllocatorLog() << AllocatorLog::Allocate(
-                                         addPadding(doc1.memoryUsage())));
+    REQUIRE(
+        spyingAllocator.log() ==
+        AllocatorLog() << AllocatorLog::Allocate(addPadding(doc1.memoryUsage()))
+                       << AllocatorLog::Allocate(sizeofString(5)));
   }
 }

extras/tests/JsonDocument/garbageCollect.cpp

@@ -21,16 +21,19 @@ TEST_CASE("JsonDocument::garbageCollect()") {
     deserializeJson(doc, "{\"blanket\":1,\"dancing\":2}");
     REQUIRE(doc.memoryUsage() == sizeofObject(2) + 2 * sizeofString(7));
     doc.remove("blanket");
+    spyingAllocator.clearLog();
 
     bool result = doc.garbageCollect();
 
     REQUIRE(result == true);
     REQUIRE(doc.memoryUsage() == sizeofObject(1) + sizeofString(7));
     REQUIRE(doc.as<std::string>() == "{\"dancing\":2}");
-    REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Allocate(4096)
-                                         << AllocatorLog::Deallocate(4096));
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(4096)
+                           << AllocatorLog::Allocate(sizeofString(7))
+                           << AllocatorLog::Deallocate(sizeofString(7))
+                           << AllocatorLog::Deallocate(sizeofString(7))
+                           << AllocatorLog::Deallocate(4096));
   }
 
   SECTION("when allocation fails") {
@@ -38,6 +41,7 @@ TEST_CASE("JsonDocument::garbageCollect()") {
     REQUIRE(doc.memoryUsage() == sizeofObject(2) + 2 * sizeofString(7));
     doc.remove("blanket");
     controllableAllocator.disable();
+    spyingAllocator.clearLog();
 
     bool result = doc.garbageCollect();
 
@@ -46,7 +50,6 @@ TEST_CASE("JsonDocument::garbageCollect()") {
     REQUIRE(doc.as<std::string>() == "{\"dancing\":2}");
 
     REQUIRE(spyingAllocator.log() == AllocatorLog()
-                                         << AllocatorLog::Allocate(4096)
                                          << AllocatorLog::AllocateFail(4096));
   }
 }

extras/tests/JsonDocument/overflowed.cpp

@@ -5,8 +5,9 @@
 #include <ArduinoJson.h>
 #include <catch.hpp>
 
+#include "Allocators.hpp"
+
 using ArduinoJson::detail::sizeofArray;
-using ArduinoJson::detail::sizeofString;
 
 TEST_CASE("JsonDocument::overflowed()") {
   SECTION("returns false on a fresh object") {
@@ -27,13 +28,15 @@ TEST_CASE("JsonDocument::overflowed()") {
   }
 
   SECTION("returns true after a failed string copy") {
-    JsonDocument doc(sizeofArray(1));
+    ControllableAllocator allocator;
+    JsonDocument doc(sizeofArray(1), &allocator);
+    allocator.disable();
     doc.add(std::string("example"));
     CHECK(doc.overflowed() == true);
   }
 
   SECTION("returns false after a successful string copy") {
-    JsonDocument doc(sizeofArray(1) + sizeofString(7));
+    JsonDocument doc(sizeofArray(1));
     doc.add(std::string("example"));
     CHECK(doc.overflowed() == false);
   }
@@ -46,12 +49,12 @@ TEST_CASE("JsonDocument::overflowed()") {
 
   SECTION("returns true after a failed deserialization") {
     JsonDocument doc(sizeofArray(1));
-    deserializeJson(doc, "[\"example\"]");
+    deserializeJson(doc, "[1, 2]");
     CHECK(doc.overflowed() == true);
   }
 
   SECTION("returns false after a successful deserialization") {
-    JsonDocument doc(sizeofArray(1) + sizeofString(7));
+    JsonDocument doc(sizeofArray(1));
     deserializeJson(doc, "[\"example\"]");
     CHECK(doc.overflowed() == false);
   }

extras/tests/JsonDocument/shrinkToFit.cpp

@@ -86,13 +86,15 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
 
   SECTION("owned string") {
     doc.set(std::string("abcdefg"));
+    REQUIRE(doc.as<std::string>() == "abcdefg");
 
     doc.shrinkToFit();
 
     REQUIRE(doc.as<std::string>() == "abcdefg");
     REQUIRE(spyingAllocator.log() ==
             AllocatorLog() << AllocatorLog::Allocate(4096)
-                           << AllocatorLog::Reallocate(4096, sizeofString(7)));
+                           << AllocatorLog::Allocate(sizeofString(7))
+                           << AllocatorLog::Reallocate(4096, 0));
   }
 
   SECTION("linked raw") {
@@ -114,7 +116,8 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
     REQUIRE(doc.as<std::string>() == "[{},12]");
     REQUIRE(spyingAllocator.log() ==
             AllocatorLog() << AllocatorLog::Allocate(4096)
-                           << AllocatorLog::Reallocate(4096, sizeofString(7)));
+                           << AllocatorLog::Allocate(sizeofString(7))
+                           << AllocatorLog::Reallocate(4096, 0));
   }
 
   SECTION("linked key") {
@@ -136,8 +139,8 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
     REQUIRE(doc.as<std::string>() == "{\"abcdefg\":42}");
     REQUIRE(spyingAllocator.log() ==
             AllocatorLog() << AllocatorLog::Allocate(4096)
-                           << AllocatorLog::Reallocate(
-                                  4096, sizeofObject(1) + sizeofString(7)));
+                           << AllocatorLog::Allocate(sizeofString(7))
+                           << AllocatorLog::Reallocate(4096, sizeofObject(1)));
   }
 
   SECTION("linked string in array") {
@@ -159,8 +162,8 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
     REQUIRE(doc.as<std::string>() == "[\"abcdefg\"]");
     REQUIRE(spyingAllocator.log() ==
             AllocatorLog() << AllocatorLog::Allocate(4096)
-                           << AllocatorLog::Reallocate(
-                                  4096, sizeofArray(1) + sizeofString(7)));
+                           << AllocatorLog::Allocate(sizeofString(7))
+                           << AllocatorLog::Reallocate(4096, sizeofArray(1)));
   }
 
   SECTION("linked string in object") {
@@ -182,21 +185,7 @@ TEST_CASE("JsonDocument::shrinkToFit()") {
     REQUIRE(doc.as<std::string>() == "{\"key\":\"abcdefg\"}");
     REQUIRE(spyingAllocator.log() ==
             AllocatorLog() << AllocatorLog::Allocate(4096)
-                           << AllocatorLog::Reallocate(
-                                  4096, sizeofObject(1) + sizeofString(7)));
-  }
-
-  SECTION("unaligned") {
-    doc.add(std::string("?"));  // two bytes in the string pool
-    REQUIRE(doc.memoryUsage() == sizeofObject(1) + sizeofString(1));
-
-    doc.shrinkToFit();
-
-    // the new capacity should be padded to align the pointers
-    REQUIRE(doc[0] == "?");
-    REQUIRE(spyingAllocator.log() ==
-            AllocatorLog() << AllocatorLog::Allocate(4096)
-                           << AllocatorLog::Reallocate(
-                                  4096, sizeofArray(1) + sizeof(void*)));
+                           << AllocatorLog::Allocate(sizeofString(7))
+                           << AllocatorLog::Reallocate(4096, sizeofObject(1)));
   }
 }

extras/tests/JsonObject/copy.cpp

@@ -77,24 +77,25 @@ TEST_CASE("JsonObject::set()") {
     JsonDocument doc3(sizeofObject(1));
     JsonObject obj3 = doc3.to<JsonObject>();
 
-    obj1[std::string("hello")] = "world";
+    obj1["a"] = 1;
+    obj1["b"] = 2;
 
     bool success = obj3.set(obj1);
 
     REQUIRE(success == false);
-    REQUIRE(doc3.as<std::string>() == "{}");
+    REQUIRE(doc3.as<std::string>() == "{\"a\":1}");
   }
 
   SECTION("destination too small to store the value") {
     JsonDocument doc3(sizeofObject(1));
     JsonObject obj3 = doc3.to<JsonObject>();
 
-    obj1["hello"] = std::string("world");
+    obj1["hello"][1] = "world";
 
     bool success = obj3.set(obj1);
 
     REQUIRE(success == false);
-    REQUIRE(doc3.as<std::string>() == "{\"hello\":null}");
+    REQUIRE(doc3.as<std::string>() == "{\"hello\":[]}");
   }
 
   SECTION("destination is null") {

extras/tests/JsonVariant/misc.cpp

@@ -10,6 +10,11 @@ TEST_CASE("VariantData") {
           true);
 }
 
+TEST_CASE("StringNode") {
+  REQUIRE(std::is_standard_layout<ArduinoJson::detail::StringNode>::value ==
+          true);
+}
+
 TEST_CASE("JsonVariant from JsonArray") {
   SECTION("JsonArray is null") {
     JsonArray arr;

extras/tests/JsonVariant/set.cpp

@@ -5,6 +5,8 @@
 #include <ArduinoJson.h>
 #include <catch.hpp>
 
+#include "Allocators.hpp"
+
 enum ErrorCode { ERROR_01 = 1, ERROR_10 = 10 };
 
 TEST_CASE("JsonVariant::set() when there is enough memory") {
@@ -128,7 +130,7 @@ TEST_CASE("JsonVariant::set() when there is enough memory") {
 }
 
 TEST_CASE("JsonVariant::set() with not enough memory") {
-  JsonDocument doc(1);
+  JsonDocument doc(1, FailingAllocator::instance());
 
   JsonVariant v = doc.to<JsonVariant>();
 

extras/tests/MemoryPool/StringCopier.cpp

@@ -5,11 +5,30 @@
 #include <ArduinoJson/StringStorage/StringCopier.hpp>
 #include <catch.hpp>
 
+#include "Allocators.hpp"
+
 using namespace ArduinoJson::detail;
 
 TEST_CASE("StringCopier") {
-  SECTION("Works when buffer is big enough") {
-    MemoryPool pool(addPadding(sizeofString(5)));
+  ControllableAllocator controllableAllocator;
+  SpyingAllocator spyingAllocator(&controllableAllocator);
+  MemoryPool pool(0, &spyingAllocator);
+
+  SECTION("Empty string") {
+    StringCopier str(&pool);
+
+    str.startString();
+    str.save();
+
+    REQUIRE(pool.size() == sizeofString(0));
+    REQUIRE(pool.overflowed() == false);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Reallocate(sizeofString(31),
+                                                       sizeofString(0)));
+  }
+
+  SECTION("Short string fits in first allocation") {
     StringCopier str(&pool);
 
     str.startString();
@@ -18,37 +37,59 @@ TEST_CASE("StringCopier") {
     REQUIRE(str.isValid() == true);
     REQUIRE(str.str() == "hello");
     REQUIRE(pool.overflowed() == false);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(sizeofString(31)));
   }
 
-  SECTION("Returns null when too small") {
-    MemoryPool pool(sizeof(void*));
+  SECTION("Long string needs reallocation") {
     StringCopier str(&pool);
 
     str.startString();
-    str.append("hello world!");
+    str.append(
+        "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
+        "eiusmod tempor incididunt ut labore et dolore magna aliqua.");
 
-    REQUIRE(str.isValid() == false);
-    REQUIRE(pool.overflowed() == true);
+    REQUIRE(str.isValid() == true);
+    REQUIRE(str.str() ==
+            "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
+            "eiusmod tempor incididunt ut labore et dolore magna aliqua.");
+    REQUIRE(pool.overflowed() == false);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::Reallocate(sizeofString(31),
+                                                       sizeofString(63))
+                           << AllocatorLog::Reallocate(sizeofString(63),
+                                                       sizeofString(127)));
   }
 
-  SECTION("Increases size of memory pool") {
-    MemoryPool pool(addPadding(sizeofString(6)));
+  SECTION("Realloc fails") {
     StringCopier str(&pool);
 
     str.startString();
-    str.save();
-
-    REQUIRE(1 == pool.size());
-    REQUIRE(pool.overflowed() == false);
+    controllableAllocator.disable();
+    str.append(
+        "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
+        "eiusmod tempor incididunt ut labore et dolore magna aliqua.");
+
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::ReallocateFail(sizeofString(31),
+                                                           sizeofString(63))
+                           << AllocatorLog::Deallocate(sizeofString(31)));
+    REQUIRE(str.isValid() == false);
+    REQUIRE(pool.overflowed() == true);
   }
 
-  SECTION("Works when memory pool is 0 bytes") {
-    MemoryPool pool(0);
+  SECTION("Initial allocation fails") {
     StringCopier str(&pool);
 
+    controllableAllocator.disable();
     str.startString();
+
     REQUIRE(str.isValid() == false);
     REQUIRE(pool.overflowed() == true);
+    REQUIRE(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31)));
   }
 }
 

extras/tests/MemoryPool/saveString.cpp

@@ -39,13 +39,6 @@ TEST_CASE("MemoryPool::saveString()") {
     const char* a = saveString(pool, "hello\0world", 11);
     const char* b = saveString(pool, "hello\0world", 11);
     REQUIRE(a == b);
-  }
-
-  SECTION("Reuse part of a string if it ends with NUL") {
-    const char* a = saveString(pool, "hello\0world", 11);
-    const char* b = saveString(pool, "hello");
-    REQUIRE(a == b);
-    REQUIRE(pool.size() == 12);
     REQUIRE(pool.size() == sizeofString(11));
   }
 
@@ -56,52 +49,8 @@ TEST_CASE("MemoryPool::saveString()") {
     REQUIRE(pool.size() == sizeofString(5) + sizeofString(11));
   }
 
-  SECTION("Returns NULL when full") {
-    REQUIRE(pool.capacity() == 32);
-
-    const void* p1 = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
-    REQUIRE(p1 != 0);
-    REQUIRE(pool.size() == 32);
-
-    const void* p2 = saveString(pool, "b");
-    REQUIRE(p2 == 0);
-  }
-
-  SECTION("Returns NULL when pool is too small") {
-    const void* p = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
-    REQUIRE(0 == p);
-  }
-
-  SECTION("Returns NULL when buffer is NULL") {
+  SECTION("Returns NULL when allocation fails") {
     MemoryPool pool2(32, FailingAllocator::instance());
     REQUIRE(0 == saveString(pool2, "a"));
   }
-
-  SECTION("Returns NULL when capacity is 0") {
-    MemoryPool pool2(0);
-    REQUIRE(0 == saveString(pool2, "a"));
-  }
-
-  SECTION("Returns same address after clear()") {
-    const void* a = saveString(pool, "hello");
-    pool.clear();
-    const void* b = saveString(pool, "world");
-
-    REQUIRE(a == b);
-  }
-
-  SECTION("Can use full capacity when fresh") {
-    const void* a = saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
-
-    REQUIRE(a != 0);
-  }
-
-  SECTION("Can use full capacity after clear") {
-    saveString(pool, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
-    pool.clear();
-
-    const void* a = saveString(pool, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");
-
-    REQUIRE(a != 0);
-  }
 }

extras/tests/Misc/printable.cpp

@@ -8,6 +8,8 @@
 #define ARDUINOJSON_ENABLE_ARDUINO_STREAM 1
 #include <ArduinoJson.h>
 
+#include "Allocators.hpp"
+
 using ArduinoJson::detail::sizeofArray;
 using ArduinoJson::detail::sizeofString;
 
@@ -52,7 +54,7 @@ struct PrintableString : public Printable {
 TEST_CASE("Printable") {
   SECTION("Doesn't overflow") {
     JsonDocument doc(8);
-    const char* value = "example";  // == 7 chars
+    const char* value = "example";
 
     doc.set(666);  // to make sure we override the value
 
@@ -77,53 +79,82 @@ TEST_CASE("Printable") {
     }
   }
 
-  SECTION("Overflows early") {
-    JsonDocument doc(8);
-    const char* value = "hello world";  // > 8 chars
+  SECTION("First allocation fails") {
+    SpyingAllocator spyingAllocator(FailingAllocator::instance());
+    JsonDocument doc(0, &spyingAllocator);
+    const char* value = "hello world";
 
     doc.set(666);  // to make sure we override the value
 
     SECTION("Via Print::write(char)") {
       PrintableString<PrintOneCharacterAtATime> printable(value);
-      CHECK(doc.set(printable) == false);
+
+      bool success = doc.set(printable);
+
+      CHECK(success == false);
       CHECK(doc.isNull());
-      CHECK(printable.totalBytesWritten() == 8);
+      CHECK(printable.totalBytesWritten() == 0);
       CHECK(doc.overflowed() == true);
       CHECK(doc.memoryUsage() == 0);
+      CHECK(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31)));
     }
 
     SECTION("Via Print::write(const char*, size_t)") {
       PrintableString<PrintAllAtOnce> printable(value);
-      CHECK(doc.set(printable) == false);
+
+      bool success = doc.set(printable);
+
+      CHECK(success == false);
       CHECK(doc.isNull());
       CHECK(printable.totalBytesWritten() == 0);
       CHECK(doc.overflowed() == true);
       CHECK(doc.memoryUsage() == 0);
+      CHECK(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::AllocateFail(sizeofString(31)));
     }
   }
 
-  SECTION("Overflows adding terminator") {
-    JsonDocument doc(8);
-    const char* value = "overflow";  // == 8 chars
+  SECTION("Reallocation fails") {
+    TimebombAllocator timebombAllocator(1);
+    SpyingAllocator spyingAllocator(&timebombAllocator);
+    JsonDocument doc(0, &spyingAllocator);
+    const char* value = "Lorem ipsum dolor sit amet, cons";  // > 31 chars
 
     doc.set(666);  // to make sure we override the value
 
     SECTION("Via Print::write(char)") {
       PrintableString<PrintOneCharacterAtATime> printable(value);
-      CHECK(doc.set(printable) == false);
+
+      bool success = doc.set(printable);
+
+      CHECK(success == false);
       CHECK(doc.isNull());
-      CHECK(printable.totalBytesWritten() == 8);
+      CHECK(printable.totalBytesWritten() == 31);
       CHECK(doc.overflowed() == true);
       CHECK(doc.memoryUsage() == 0);
+      CHECK(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::ReallocateFail(sizeofString(31),
+                                                           sizeofString(63))
+                           << AllocatorLog::Deallocate(sizeofString(31)));
     }
 
     SECTION("Via Print::write(const char*, size_t)") {
       PrintableString<PrintAllAtOnce> printable(value);
-      CHECK(doc.set(printable) == false);
+
+      bool success = doc.set(printable);
+
+      CHECK(success == false);
       CHECK(doc.isNull());
-      CHECK(printable.totalBytesWritten() == 0);
+      CHECK(printable.totalBytesWritten() == 31);
       CHECK(doc.overflowed() == true);
       CHECK(doc.memoryUsage() == 0);
+      CHECK(spyingAllocator.log() ==
+            AllocatorLog() << AllocatorLog::Allocate(sizeofString(31))
+                           << AllocatorLog::ReallocateFail(sizeofString(31),
+                                                           sizeofString(63))
+                           << AllocatorLog::Deallocate(sizeofString(31)));
     }
   }
 

extras/tests/MsgPackDeserializer/deserializeVariant.cpp

@@ -5,6 +5,8 @@
 #include <ArduinoJson.h>
 #include <catch.hpp>
 
+#include "Allocators.hpp"
+
 using ArduinoJson::detail::sizeofArray;
 using ArduinoJson::detail::sizeofObject;
 using ArduinoJson::detail::sizeofString;
@@ -20,9 +22,10 @@ static void checkValue(const char* input, T expected) {
   REQUIRE(doc.as<T>() == expected);
 }
 
-static void checkError(size_t capacity, const char* input,
-                       DeserializationError expected) {
-  JsonDocument doc(capacity);
+static void checkError(size_t capacity, size_t timebombCountDown,
+                       const char* input, DeserializationError expected) {
+  TimebombAllocator timebombAllocator(timebombCountDown);
+  JsonDocument doc(capacity, &timebombAllocator);
 
   DeserializationError error = deserializeMsgPack(doc, input);
 
@@ -144,133 +147,120 @@ TEST_CASE("deserialize MsgPack value") {
 
 TEST_CASE("deserializeMsgPack() under memory constaints") {
   SECTION("single values always fit") {
-    checkError(0, "\xc0", DeserializationError::Ok);                  // nil
-    checkError(0, "\xc2", DeserializationError::Ok);                  // false
-    checkError(0, "\xc3", DeserializationError::Ok);                  // true
-    checkError(0, "\xcc\x00", DeserializationError::Ok);              // uint 8
-    checkError(0, "\xcd\x30\x39", DeserializationError::Ok);          // uint 16
-    checkError(0, "\xCE\x12\x34\x56\x78", DeserializationError::Ok);  // uint 32
+    checkError(0, 0, "\xc0", DeserializationError::Ok);          // nil
+    checkError(0, 0, "\xc2", DeserializationError::Ok);          // false
+    checkError(0, 0, "\xc3", DeserializationError::Ok);          // true
+    checkError(0, 0, "\xcc\x00", DeserializationError::Ok);      // uint 8
+    checkError(0, 0, "\xcd\x30\x39", DeserializationError::Ok);  // uint 16
+    checkError(0, 0, "\xCE\x12\x34\x56\x78",
+               DeserializationError::Ok);  // uint 32
   }
 
   SECTION("fixstr") {
-    checkError(8, "\xA0", DeserializationError::Ok);
-    checkError(8, "\xA7ZZZZZZZ", DeserializationError::Ok);
-    checkError(8, "\xA8ZZZZZZZZ", DeserializationError::NoMemory);
-    checkError(16, "\xAFZZZZZZZZZZZZZZZ", DeserializationError::Ok);
-    checkError(16, "\xB0ZZZZZZZZZZZZZZZZ", DeserializationError::NoMemory);
+    checkError(0, 2, "\xA7ZZZZZZZ", DeserializationError::Ok);
+    checkError(0, 0, "\xA7ZZZZZZZ", DeserializationError::NoMemory);
   }
 
   SECTION("str 8") {
-    checkError(8, "\xD9\x00", DeserializationError::Ok);
-    checkError(8, "\xD9\x07ZZZZZZZ", DeserializationError::Ok);
-    checkError(8, "\xD9\x08ZZZZZZZZ", DeserializationError::NoMemory);
-    checkError(16, "\xD9\x0FZZZZZZZZZZZZZZZ", DeserializationError::Ok);
-    checkError(16, "\xD9\x10ZZZZZZZZZZZZZZZZ", DeserializationError::NoMemory);
+    checkError(0, 2, "\xD9\x07ZZZZZZZ", DeserializationError::Ok);
+    checkError(0, 0, "\xD9\x07ZZZZZZZ", DeserializationError::NoMemory);
   }
 
   SECTION("str 16") {
-    checkError(8, "\xDA\x00\x00", DeserializationError::Ok);
-    checkError(8, "\xDA\x00\x07ZZZZZZZ", DeserializationError::Ok);
-    checkError(8, "\xDA\x00\x08ZZZZZZZZ", DeserializationError::NoMemory);
-    checkError(16, "\xDA\x00\x0FZZZZZZZZZZZZZZZ", DeserializationError::Ok);
-    checkError(16, "\xDA\x00\x10ZZZZZZZZZZZZZZZZ",
-               DeserializationError::NoMemory);
+    checkError(0, 2, "\xDA\x00\x07ZZZZZZZ", DeserializationError::Ok);
+    checkError(0, 0, "\xDA\x00\x07ZZZZZZZ", DeserializationError::NoMemory);
   }
 
   SECTION("str 32") {
-    checkError(8, "\xDB\x00\x00\x00\x00", DeserializationError::Ok);
-    checkError(8, "\xDB\x00\x00\x00\x07ZZZZZZZ", DeserializationError::Ok);
-    checkError(8, "\xDB\x00\x00\x00\x08ZZZZZZZZ",
-               DeserializationError::NoMemory);
-    checkError(16, "\xDB\x00\x00\x00\x0FZZZZZZZZZZZZZZZ",
-               DeserializationError::Ok);
-    checkError(16, "\xDB\x00\x00\x00\x10ZZZZZZZZZZZZZZZZ",
+    checkError(0, 2, "\xDB\x00\x00\x00\x07ZZZZZZZ", DeserializationError::Ok);
+    checkError(0, 0, "\xDB\x00\x00\x00\x07ZZZZZZZ",
                DeserializationError::NoMemory);
   }
 
   SECTION("fixarray") {
-    checkError(sizeofArray(0), "\x90", DeserializationError::Ok);  // []
-    checkError(sizeofArray(0), "\x91\x01",
+    checkError(sizeofArray(0), 1, "\x90", DeserializationError::Ok);  // []
+    checkError(sizeofArray(0), 1, "\x91\x01",
                DeserializationError::NoMemory);  // [1]
-    checkError(sizeofArray(1), "\x91\x01",
+    checkError(sizeofArray(1), 1, "\x91\x01",
                DeserializationError::Ok);  // [1]
-    checkError(sizeofArray(1), "\x92\x01\x02",
+    checkError(sizeofArray(1), 1, "\x92\x01\x02",
                DeserializationError::NoMemory);  // [1,2]
   }
 
   SECTION("array 16") {
-    checkError(sizeofArray(0), "\xDC\x00\x00", DeserializationError::Ok);
-    checkError(sizeofArray(0), "\xDC\x00\x01\x01",
+    checkError(sizeofArray(0), 1, "\xDC\x00\x00", DeserializationError::Ok);
+    checkError(sizeofArray(0), 1, "\xDC\x00\x01\x01",
                DeserializationError::NoMemory);
-    checkError(sizeofArray(1), "\xDC\x00\x01\x01", DeserializationError::Ok);
-    checkError(sizeofArray(1), "\xDC\x00\x02\x01\x02",
+    checkError(sizeofArray(1), 1, "\xDC\x00\x01\x01", DeserializationError::Ok);
+    checkError(sizeofArray(1), 1, "\xDC\x00\x02\x01\x02",
                DeserializationError::NoMemory);
   }
 
   SECTION("array 32") {
-    checkError(sizeofArray(0), "\xDD\x00\x00\x00\x00",
+    checkError(sizeofArray(0), 1, "\xDD\x00\x00\x00\x00",
                DeserializationError::Ok);
-    checkError(sizeofArray(0), "\xDD\x00\x00\x00\x01\x01",
+    checkError(sizeofArray(0), 1, "\xDD\x00\x00\x00\x01\x01",
                DeserializationError::NoMemory);
-    checkError(sizeofArray(1), "\xDD\x00\x00\x00\x01\x01",
+    checkError(sizeofArray(1), 1, "\xDD\x00\x00\x00\x01\x01",
                DeserializationError::Ok);
-    checkError(sizeofArray(1), "\xDD\x00\x00\x00\x02\x01\x02",
+    checkError(sizeofArray(1), 1, "\xDD\x00\x00\x00\x02\x01\x02",
                DeserializationError::NoMemory);
   }
 
   SECTION("fixmap") {
     SECTION("{}") {
-      checkError(sizeofObject(0), "\x80", DeserializationError::Ok);
+      checkError(sizeofObject(0), 0, "\x80", DeserializationError::Ok);
     }
     SECTION("{H:1}") {
-      checkError(sizeofObject(0), "\x81\xA1H\x01",
+      checkError(sizeofObject(0), 0, "\x81\xA1H\x01",
                  DeserializationError::NoMemory);
-      checkError(sizeofObject(1) + sizeofString(2), "\x81\xA1H\x01",
+      checkError(sizeofObject(1) + sizeofString(2), 3, "\x81\xA1H\x01",
                  DeserializationError::Ok);
     }
     SECTION("{H:1,W:2}") {
-      checkError(sizeofObject(1) + sizeofString(2), "\x82\xA1H\x01\xA1W\x02",
+      checkError(sizeofObject(1) + sizeofString(2), 3, "\x82\xA1H\x01\xA1W\x02",
                  DeserializationError::NoMemory);
-      checkError(sizeofObject(2) + 2 * sizeofString(2),
+      checkError(sizeofObject(2) + 2 * sizeofString(2), 5,
                  "\x82\xA1H\x01\xA1W\x02", DeserializationError::Ok);
     }
   }
 
   SECTION("map 16") {
     SECTION("{}") {
-      checkError(sizeofObject(0), "\xDE\x00\x00", DeserializationError::Ok);
+      checkError(sizeofObject(0), 0, "\xDE\x00\x00", DeserializationError::Ok);
     }
     SECTION("{H:1}") {
-      checkError(sizeofObject(0), "\xDE\x00\x01\xA1H\x01",
+      checkError(sizeofObject(1) + sizeofString(2), 1, "\xDE\x00\x01\xA1H\x01",
                  DeserializationError::NoMemory);
-      checkError(sizeofObject(1) + sizeofString(2), "\xDE\x00\x01\xA1H\x01",
+      checkError(sizeofObject(1) + sizeofString(2), 3, "\xDE\x00\x01\xA1H\x01",
                  DeserializationError::Ok);
     }
     SECTION("{H:1,W:2}") {
-      checkError(sizeofObject(1) + sizeofString(2),
+      checkError(sizeofObject(1) + sizeofString(2), 3,
                  "\xDE\x00\x02\xA1H\x01\xA1W\x02",
                  DeserializationError::NoMemory);
-      checkError(sizeofObject(2) + 2 * sizeofObject(1),
+      checkError(sizeofObject(2) + 2 * sizeofObject(1), 5,
                  "\xDE\x00\x02\xA1H\x01\xA1W\x02", DeserializationError::Ok);
     }
   }
 
   SECTION("map 32") {
     SECTION("{}") {
-      checkError(sizeofObject(0), "\xDF\x00\x00\x00\x00",
+      checkError(sizeofObject(0), 0, "\xDF\x00\x00\x00\x00",
                  DeserializationError::Ok);
     }
     SECTION("{H:1}") {
-      checkError(sizeofObject(0), "\xDF\x00\x00\x00\x01\xA1H\x01",
+      checkError(sizeofObject(1) + sizeofString(2), 1,
+                 "\xDF\x00\x00\x00\x01\xA1H\x01",
                  DeserializationError::NoMemory);
-      checkError(sizeofObject(1) + sizeofString(2),
+      checkError(sizeofObject(1) + sizeofString(2), 3,
                  "\xDF\x00\x00\x00\x01\xA1H\x01", DeserializationError::Ok);
     }
     SECTION("{H:1,W:2}") {
-      checkError(sizeofObject(1) + sizeofString(2),
+      checkError(sizeofObject(1) + 2 * sizeofString(2), 3,
                  "\xDF\x00\x00\x00\x02\xA1H\x01\xA1W\x02",
                  DeserializationError::NoMemory);
-      checkError(sizeofObject(2) + 2 * sizeofObject(1),
+      checkError(sizeofObject(2) + 2 * sizeofObject(1), 5,
                  "\xDF\x00\x00\x00\x02\xA1H\x01\xA1W\x02",
                  DeserializationError::Ok);
     }

idf_component.yml

@@ -1,7 +1,7 @@
 version: "7.0.0-alpha"
 description: >-
   A simple and efficient JSON library for embedded C++.
-  ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ fixed allocation, ✔ zero-copy, ✔ streams, ✔ filtering, and more.
+  ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ zero-copy, ✔ streams, ✔ filtering, and more.
   It is the most popular Arduino library on GitHub ❤❤❤❤❤.
   Check out arduinojson.org for a comprehensive documentation.
 url: https://arduinojson.org/

library.json

@@ -1,7 +1,7 @@
 {
   "name": "ArduinoJson",
   "keywords": "json, rest, http, web",
-  "description": "A simple and efficient JSON library for embedded C++. ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ fixed allocation, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation.",
+  "description": "A simple and efficient JSON library for embedded C++. ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation.",
   "homepage": "https://arduinojson.org/?utm_source=meta&utm_medium=library.json",
   "repository": {
     "type": "git",

library.properties

@@ -3,7 +3,7 @@ version=7.0.0-alpha
 author=Benoit Blanchon <blog.benoitblanchon.fr>
 maintainer=Benoit Blanchon <blog.benoitblanchon.fr>
 sentence=A simple and efficient JSON library for embedded C++.
-paragraph=ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ fixed allocation, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation.
+paragraph=ArduinoJson supports ✔ serialization, ✔ deserialization, ✔ MessagePack, ✔ zero-copy, ✔ streams, ✔ filtering, and more. It is the most popular Arduino library on GitHub ❤❤❤❤❤. Check out arduinojson.org for a comprehensive documentation.
 category=Data Processing
 url=https://arduinojson.org/?utm_source=meta&utm_medium=library.properties
 architectures=*

src/ArduinoJson/Collection/CollectionData.hpp

@@ -70,7 +70,7 @@ class CollectionData {
     return _head;
   }
 
-  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance);
+  void movePointers(ptrdiff_t variantDistance);
 
  private:
   VariantSlot* getSlot(size_t index) const;

src/ArduinoJson/Collection/CollectionImpl.hpp

@@ -168,7 +168,7 @@ inline size_t CollectionData::memoryUsage() const {
   for (VariantSlot* s = _head; s; s = s->next()) {
     total += sizeof(VariantSlot) + s->data()->memoryUsage();
     if (s->ownsKey())
-      total += strlen(s->key()) + 1;
+      total += sizeofString(strlen(s->key()));
   }
   return total;
 }
@@ -186,12 +186,11 @@ inline void movePointer(T*& p, ptrdiff_t offset) {
   ARDUINOJSON_ASSERT(isAligned(p));
 }
 
-inline void CollectionData::movePointers(ptrdiff_t stringDistance,
-                                         ptrdiff_t variantDistance) {
+inline void CollectionData::movePointers(ptrdiff_t variantDistance) {
   movePointer(_head, variantDistance);
   movePointer(_tail, variantDistance);
   for (VariantSlot* slot = _head; slot; slot = slot->next())
-    slot->movePointers(stringDistance, variantDistance);
+    slot->movePointers(variantDistance);
 }
 
 ARDUINOJSON_END_PRIVATE_NAMESPACE

src/ArduinoJson/Json/JsonDeserializer.hpp

@@ -277,7 +277,6 @@ class JsonDeserializer {
         VariantData* variant = object.getMember(adaptString(key.c_str()));
         if (!variant) {
           // Save key in memory pool.
-          // This MUST be done before adding the slot.
           key = _stringStorage.save();
 
           // Allocate slot in object

src/ArduinoJson/Memory/MemoryPool.hpp

@@ -25,9 +25,15 @@ constexpr size_t sizeofObject(size_t n) {
   return n * sizeof(VariantSlot);
 }
 
+struct StringNode {
+  struct StringNode* next;
+  uint16_t length;
+  char data[1];
+};
+
 // Returns the size (in bytes) of an string with n characters.
 constexpr size_t sizeofString(size_t n) {
-  return n + 1;
+  return n + 1 + offsetof(StringNode, data);
 }
 
 // _begin                                   _end
@@ -46,6 +52,7 @@ class MemoryPool {
   }
 
   ~MemoryPool() {
+    deallocAllStrings();
     deallocPool();
   }
 
@@ -53,6 +60,7 @@ class MemoryPool {
   MemoryPool& operator=(const MemoryPool& src) = delete;
 
   MemoryPool& operator=(MemoryPool&& src) {
+    deallocAllStrings();
     deallocPool();
     _allocator = src._allocator;
     _begin = src._begin;
@@ -61,6 +69,8 @@ class MemoryPool {
     _right = src._right;
     _overflowed = src._overflowed;
     src._begin = src._end = src._left = src._right = nullptr;
+    _strings = src._strings;
+    src._strings = nullptr;
     return *this;
   }
 
@@ -87,7 +97,10 @@ class MemoryPool {
   }
 
   size_t size() const {
-    return size_t(_left - _begin + _end - _right);
+    size_t total = size_t(_left - _begin + _end - _right);
+    for (auto node = _strings; node; node = node->next)
+      total += sizeofString(node->length);
+    return total;
   }
 
   bool overflowed() const {
@@ -103,45 +116,71 @@ class MemoryPool {
     if (str.isNull())
       return 0;
 
-    const char* existingCopy = findString(str);
-    if (existingCopy)
-      return existingCopy;
+    auto node = findString(str);
+    if (node) {
+      return node->data;
+    }
 
     size_t n = str.size();
 
-    char* newCopy = allocString(n + 1);
-    if (newCopy) {
-      stringGetChars(str, newCopy, n);
-      newCopy[n] = 0;  // force null-terminator
-    }
-    return newCopy;
+    node = allocString(n);
+    if (!node)
+      return nullptr;
+
+    stringGetChars(str, node->data, n);
+    node->data[n] = 0;  // force NUL terminator
+    addStringToList(node);
+    return node->data;
   }
 
-  void getFreeZone(char** zoneStart, size_t* zoneSize) const {
-    *zoneStart = _left;
-    *zoneSize = size_t(_right - _left);
+  void addStringToList(StringNode* node) {
+    ARDUINOJSON_ASSERT(node != nullptr);
+    node->next = _strings;
+    _strings = node;
   }
 
-  const char* saveStringFromFreeZone(size_t len) {
-    const char* dup = findString(adaptString(_left, len));
-    if (dup)
-      return dup;
+  template <typename TAdaptedString>
+  StringNode* findString(const TAdaptedString& str) const {
+    for (auto node = _strings; node; node = node->next) {
+      if (stringEquals(str, adaptString(node->data, node->length)))
+        return node;
+    }
+    return nullptr;
+  }
+
+  StringNode* allocString(size_t length) {
+    auto node = reinterpret_cast<StringNode*>(
+        _allocator->allocate(sizeofString(length)));
+    if (node) {
+      node->length = uint16_t(length);
+    } else {
+      _overflowed = true;
+    }
+    return node;
+  }
 
-    const char* str = _left;
-    _left += len;
-    *_left++ = 0;
-    checkInvariants();
-    return str;
+  StringNode* reallocString(StringNode* node, size_t length) {
+    ARDUINOJSON_ASSERT(node != nullptr);
+    auto newNode = reinterpret_cast<StringNode*>(
+        _allocator->reallocate(node, sizeofString(length)));
+    if (newNode) {
+      newNode->length = uint16_t(length);
+    } else {
+      _overflowed = true;
+      _allocator->deallocate(node);
+    }
+    return newNode;
   }
 
-  void markAsOverflowed() {
-    _overflowed = true;
+  void deallocString(StringNode* node) {
+    _allocator->deallocate(node);
   }
 
   void clear() {
     _left = _begin;
     _right = _end;
     _overflowed = false;
+    deallocAllStrings();
   }
 
   bool canAlloc(size_t bytes) const {
@@ -169,8 +208,8 @@ class MemoryPool {
         static_cast<char*>(new_ptr) - static_cast<char*>(old_ptr);
 
     movePointers(ptr_offset);
-    reinterpret_cast<VariantSlot&>(variant).movePointers(
-        ptr_offset, ptr_offset - bytes_reclaimed);
+    reinterpret_cast<VariantSlot&>(variant).movePointers(ptr_offset -
+                                                         bytes_reclaimed);
   }
 
  private:
@@ -215,29 +254,12 @@ class MemoryPool {
     ARDUINOJSON_ASSERT(isAligned(_right));
   }
 
-  template <typename TAdaptedString>
-  const char* findString(const TAdaptedString& str) const {
-    size_t n = str.size();
-    for (char* next = _begin; next + n < _left; ++next) {
-      if (next[n] == '\0' && stringEquals(str, adaptString(next, n)))
-        return next;
-
-      // jump to next terminator
-      while (*next)
-        ++next;
-    }
-    return 0;
-  }
-
-  char* allocString(size_t n) {
-    if (!canAlloc(n)) {
-      _overflowed = true;
-      return 0;
+  void deallocAllStrings() {
+    while (_strings) {
+      auto node = _strings;
+      _strings = node->next;
+      deallocString(node);
     }
-    char* s = _left;
-    _left += n;
-    checkInvariants();
-    return s;
   }
 
   template <typename T>
@@ -271,6 +293,7 @@ class MemoryPool {
   Allocator* _allocator;
   char *_begin, *_left, *_right, *_end;
   bool _overflowed;
+  StringNode* _strings = nullptr;
 };
 
 template <typename TAdaptedString, typename TCallback>

src/ArduinoJson/MsgPack/MsgPackDeserializer.hpp

@@ -494,7 +494,6 @@ class MsgPackDeserializer {
         ARDUINOJSON_ASSERT(object != 0);
 
         // Save key in memory pool.
-        // This MUST be done before adding the slot.
         key = _stringStorage.save();
 
         VariantSlot* slot = object->addSlot(_pool);

src/ArduinoJson/StringStorage/StringCopier.hpp

@@ -10,20 +10,31 @@ ARDUINOJSON_BEGIN_PRIVATE_NAMESPACE
 
 class StringCopier {
  public:
+  static const size_t initialCapacity = 31;
+
   StringCopier(MemoryPool* pool) : _pool(pool) {}
 
+  ~StringCopier() {
+    if (_node)
+      _pool->deallocString(_node);
+  }
+
   void startString() {
-    _pool->getFreeZone(&_ptr, &_capacity);
     _size = 0;
-    if (_capacity == 0)
-      _pool->markAsOverflowed();
+    if (!_node)
+      _node = _pool->allocString(initialCapacity);
   }
 
   JsonString save() {
-    ARDUINOJSON_ASSERT(_ptr);
-    ARDUINOJSON_ASSERT(_size < _capacity);  // needs room for the terminator
-    return JsonString(_pool->saveStringFromFreeZone(_size), _size,
-                      JsonString::Copied);
+    ARDUINOJSON_ASSERT(_node != nullptr);
+    _node->data[_size] = 0;
+    StringNode* node = _pool->findString(adaptString(_node->data, _size));
+    if (!node) {
+      node = _pool->reallocString(_node, _size);
+      _pool->addStringToList(node);
+      _node = nullptr;  // next time we need a new string
+    }
+    return JsonString(node->data, node->length, JsonString::Copied);
   }
 
   void append(const char* s) {
@@ -32,19 +43,19 @@ class StringCopier {
   }
 
   void append(const char* s, size_t n) {
-    while (n-- > 0)
+    while (n-- > 0)  // TODO: memcpy
       append(*s++);
   }
 
   void append(char c) {
-    if (_size + 1 < _capacity)
-      _ptr[_size++] = c;
-    else
-      _pool->markAsOverflowed();
+    if (_node && _size == _node->length)
+      _node = _pool->reallocString(_node, _size * 2U + 1);
+    if (_node)
+      _node->data[_size++] = c;
   }
 
   bool isValid() const {
-    return !_pool->overflowed();
+    return _node != nullptr;
   }
 
   size_t size() const {
@@ -52,21 +63,15 @@ class StringCopier {
   }
 
   JsonString str() const {
-    ARDUINOJSON_ASSERT(_ptr);
-    ARDUINOJSON_ASSERT(_size < _capacity);
-    _ptr[_size] = 0;
-    return JsonString(_ptr, _size, JsonString::Copied);
+    ARDUINOJSON_ASSERT(_node != nullptr);
+    _node->data[_size] = 0;
+    return JsonString(_node->data, _size, JsonString::Copied);
   }
 
  private:
   MemoryPool* _pool;
-
-  // These fields aren't initialized by the constructor but startString()
-  //
-  // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject)
-  char* _ptr;
-  // NOLINTNEXTLINE(clang-analyzer-optin.cplusplus.UninitializedObject)
-  size_t _size, _capacity;
+  StringNode* _node = nullptr;
+  size_t _size = 0;
 };
 
 ARDUINOJSON_END_PRIVATE_NAMESPACE

src/ArduinoJson/Variant/ConverterImpl.hpp

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <ArduinoJson/Json/JsonSerializer.hpp>
+#include <ArduinoJson/StringStorage/StringCopier.hpp>
 #include <ArduinoJson/Variant/JsonVariantConst.hpp>
 #include <ArduinoJson/Variant/VariantFunctions.hpp>
 
@@ -205,43 +206,35 @@ struct Converter<decltype(nullptr)> : private detail::VariantAttorney {
 namespace detail {
 class MemoryPoolPrint : public Print {
  public:
-  MemoryPoolPrint(MemoryPool* pool) : _pool(pool), _size(0) {
-    pool->getFreeZone(&_string, &_capacity);
+  MemoryPoolPrint(MemoryPool* pool) : _copier(pool) {
+    _copier.startString();
   }
 
   JsonString str() {
-    ARDUINOJSON_ASSERT(_size < _capacity);
-    return JsonString(_pool->saveStringFromFreeZone(_size), _size,
-                      JsonString::Copied);
+    ARDUINOJSON_ASSERT(!overflowed());
+    return _copier.save();
   }
 
   size_t write(uint8_t c) {
-    if (_size >= _capacity)
-      return 0;
-
-    _string[_size++] = char(c);
-    return 1;
+    _copier.append(char(c));
+    return _copier.isValid() ? 1 : 0;
   }
 
   size_t write(const uint8_t* buffer, size_t size) {
-    if (_size + size >= _capacity) {
-      _size = _capacity;  // mark as overflowed
-      return 0;
+    for (size_t i = 0; i < size; i++) {
+      _copier.append(char(buffer[i]));
+      if (!_copier.isValid())
+        return i;
     }
-    memcpy(&_string[_size], buffer, size);
-    _size += size;
     return size;
   }
 
   bool overflowed() const {
-    return _size >= _capacity;
+    return !_copier.isValid();
   }
 
  private:
-  MemoryPool* _pool;
-  size_t _size;
-  char* _string;
-  size_t _capacity;
+  StringCopier _copier;
 };
 }  // namespace detail
 
@@ -253,7 +246,6 @@ inline void convertToJson(const ::Printable& src, JsonVariant dst) {
   detail::MemoryPoolPrint print(pool);
   src.printTo(print);
   if (print.overflowed()) {
-    pool->markAsOverflowed();
     data->setNull();
     return;
   }

src/ArduinoJson/Variant/VariantData.hpp

@@ -226,9 +226,7 @@ class VariantData {
     switch (type()) {
       case VALUE_IS_OWNED_STRING:
       case VALUE_IS_OWNED_RAW:
-        // We always add a zero at the end: the deduplication function uses it
-        // to detect the beginning of the next string.
-        return _content.asString.size + 1;
+        return sizeofString(_content.asString.size);
       case VALUE_IS_OBJECT:
       case VALUE_IS_ARRAY:
         return _content.asCollection.memoryUsage();
@@ -277,11 +275,9 @@ class VariantData {
     return _content.asCollection.getOrAddMember(key, pool);
   }
 
-  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) {
-    if (_flags & OWNED_VALUE_BIT)
-      _content.asString.data += stringDistance;
+  void movePointers(ptrdiff_t variantDistance) {
     if (_flags & COLLECTION_MASK)
-      _content.asCollection.movePointers(stringDistance, variantDistance);
+      _content.asCollection.movePointers(variantDistance);
   }
 
   uint8_t type() const {

src/ArduinoJson/Variant/VariantSlot.hpp

@@ -99,13 +99,9 @@ class VariantSlot {
     _key = 0;
   }
 
-  void movePointers(ptrdiff_t stringDistance, ptrdiff_t variantDistance) {
-    if (_flags & OWNED_KEY_BIT)
-      _key += stringDistance;
-    if (_flags & OWNED_VALUE_BIT)
-      _content.asString.data += stringDistance;
+  void movePointers(ptrdiff_t variantDistance) {
     if (_flags & COLLECTION_MASK)
-      _content.asCollection.movePointers(stringDistance, variantDistance);
+      _content.asCollection.movePointers(variantDistance);
   }
 };