Store offset between slots to reduce memory usage

CHANGELOG.md

@@ -6,6 +6,7 @@ HEAD
 
 * Removed the automatic expansion of `DynamicJsonDocument`, it now has a fixed capacity.
 * Restored the monotonic allocator because the code was getting too big
+* Reduced the memory usage
 
 v6.6.0-beta (2018-11-13)
 -----------

fuzzing/CMakeLists.txt

@@ -2,6 +2,10 @@
 # Copyright Benoit Blanchon 2014-2018
 # MIT License
 
+if(MSVC)
+	add_compile_options(-D_CRT_SECURE_NO_WARNINGS)
+endif()
+
 add_executable(msgpack_fuzzer
 	msgpack_fuzzer.cpp
 	fuzzer_main.cpp

src/ArduinoJson/Data/ArrayFunctions.hpp

@@ -17,10 +17,10 @@ inline JsonVariantData* arrayAdd(JsonArrayData* arr, MemoryPool* pool) {
 
   slot->next = 0;
   slot->value.type = JSON_NULL;
+  slot->value.keyIsOwned = false;
 
   if (arr->tail) {
-    slot->prev = arr->tail;
-    arr->tail->next = slot;
+    slot->attachTo(arr->tail);
     arr->tail = slot;
   } else {
     slot->prev = 0;
@@ -28,13 +28,12 @@ inline JsonVariantData* arrayAdd(JsonArrayData* arr, MemoryPool* pool) {
     arr->tail = slot;
   }
 
-  slot->value.keyIsOwned = false;
   return &slot->value;
 }
 
 inline VariantSlot* arrayGetSlot(const JsonArrayData* arr, size_t index) {
   if (!arr) return 0;
-  return slotAdvance(arr->head, index);
+  return arr->head->getNext(index);
 }
 
 inline JsonVariantData* arrayGet(const JsonArrayData* arr, size_t index) {
@@ -46,13 +45,13 @@ inline void arrayRemove(JsonArrayData* arr, VariantSlot* slot) {
   if (!arr || !slot) return;
 
   if (slot->prev)
-    slot->prev->next = slot->next;
+    slot->getPrev()->setNext(slot->getNext());
   else
-    arr->head = slot->next;
+    arr->head = slot->getNext();
   if (slot->next)
-    slot->next->prev = slot->prev;
+    slot->getNext()->setPrev(slot->getPrev());
   else
-    arr->tail = slot->prev;
+    arr->tail = slot->getPrev();
 }
 
 inline void arrayRemove(JsonArrayData* arr, size_t index) {
@@ -71,7 +70,7 @@ inline bool arrayCopy(JsonArrayData* dst, const JsonArrayData* src,
                       MemoryPool* pool) {
   if (!dst || !src) return false;
   arrayClear(dst);
-  for (VariantSlot* s = src->head; s; s = s->next) {
+  for (VariantSlot* s = src->head; s; s = s->getNext()) {
     if (!variantCopy(arrayAdd(dst, pool), &s->value, pool)) return false;
   }
   return true;
@@ -88,8 +87,8 @@ inline bool arrayEquals(const JsonArrayData* a1, const JsonArrayData* a2) {
     if (s1 == s2) return true;
     if (!s1 || !s2) return false;
     if (!variantEquals(&s1->value, &s2->value)) return false;
-    s1 = s1->next;
-    s2 = s2->next;
+    s1 = s1->getNext();
+    s2 = s2->getNext();
   }
 }
 

src/ArduinoJson/Data/ObjectFunctions.hpp

@@ -16,7 +16,7 @@ inline VariantSlot* objectFindSlot(const JsonObjectData* obj, TKey key) {
   VariantSlot* slot = obj->head;
   while (slot) {
     if (key.equals(slotGetKey(slot))) break;
-    slot = slot->next;
+    slot = slot->getNext();
   }
   return slot;
 }
@@ -36,8 +36,7 @@ inline JsonVariantData* objectAdd(JsonObjectData* obj, TKey key,
   slot->value.type = JSON_NULL;
 
   if (obj->tail) {
-    slot->prev = obj->tail;
-    obj->tail->next = slot;
+    slot->attachTo(obj->tail);
     obj->tail = slot;
   } else {
     slot->prev = 0;
@@ -79,14 +78,16 @@ inline void objectClear(JsonObjectData* obj) {
 inline void objectRemove(JsonObjectData* obj, VariantSlot* slot) {
   if (!obj) return;
   if (!slot) return;
-  if (slot->prev)
-    slot->prev->next = slot->next;
+  VariantSlot* prev = slot->getPrev();
+  VariantSlot* next = slot->getNext();
+  if (prev)
+    prev->setNext(next);
   else
-    obj->head = slot->next;
-  if (slot->next)
-    slot->next->prev = slot->prev;
+    obj->head = next;
+  if (next)
+    next->setPrev(prev);
   else
-    obj->tail = slot->prev;
+    obj->tail = prev;
 }
 
 inline size_t objectSize(const JsonObjectData* obj) {
@@ -100,7 +101,7 @@ inline bool objectCopy(JsonObjectData* dst, const JsonObjectData* src,
                        MemoryPool* pool) {
   if (!dst || !src) return false;
   objectClear(dst);
-  for (VariantSlot* s = src->head; s; s = s->next) {
+  for (VariantSlot* s = src->head; s; s = s->getNext()) {
     JsonVariantData* var;
     if (s->value.keyIsOwned)
       var = objectAdd(dst, ZeroTerminatedRamString(s->key), pool);
@@ -115,7 +116,7 @@ inline bool objectEquals(const JsonObjectData* o1, const JsonObjectData* o2) {
   if (o1 == o2) return true;
   if (!o1 || !o2) return false;
 
-  for (VariantSlot* s = o1->head; s; s = s->next) {
+  for (VariantSlot* s = o1->head; s; s = s->getNext()) {
     JsonVariantData* v1 = &s->value;
     JsonVariantData* v2 = objectGet(o2, makeString(slotGetKey(s)));
     if (!variantEquals(v1, v2)) return false;

src/ArduinoJson/Data/SlotFunctions.hpp

@@ -37,27 +37,11 @@ inline const char* slotGetKey(const VariantSlot* var) {
   return var->key;
 }
 
-inline const VariantSlot* slotAdvance(const VariantSlot* var, size_t distance) {
-  while (distance && var) {
-    var = var->next;
-    distance--;
-  }
-  return var;
-}
-
-inline VariantSlot* slotAdvance(VariantSlot* var, size_t distance) {
-  while (distance && var) {
-    var = var->next;
-    distance--;
-  }
-  return var;
-}
-
 inline size_t slotSize(const VariantSlot* var) {
   size_t n = 0;
   while (var) {
     n++;
-    var = var->next;
+    var = var->getNext();
   }
   return n;
 }

src/ArduinoJson/JsonArrayIterator.hpp

@@ -48,12 +48,12 @@ class JsonArrayIterator {
   }
 
   JsonArrayIterator &operator++() {
-    _slot = _slot->next;
+    _slot = _slot->getNext();
     return *this;
   }
 
   JsonArrayIterator &operator+=(size_t distance) {
-    _slot = slotAdvance(_slot, distance);
+    _slot = _slot->getNext(distance);
     return *this;
   }
 
@@ -103,12 +103,12 @@ class JsonArrayConstIterator {
   }
 
   JsonArrayConstIterator &operator++() {
-    _slot = _slot->next;
+    _slot = _slot->getNext();
     return *this;
   }
 
   JsonArrayConstIterator &operator+=(size_t distance) {
-    _slot = slotAdvance(_slot, distance);
+    _slot = _slot->getNext(distance);
     return *this;
   }
 

src/ArduinoJson/JsonObjectIterator.hpp

@@ -49,12 +49,12 @@ class JsonObjectIterator {
   }
 
   JsonObjectIterator &operator++() {
-    if (_slot) _slot = _slot->next;
+    _slot = _slot->getNext();
     return *this;
   }
 
   JsonObjectIterator &operator+=(size_t distance) {
-    _slot = slotAdvance(_slot, distance);
+    _slot = _slot->getNext(distance);
     return *this;
   }
 
@@ -105,12 +105,12 @@ class JsonObjectConstIterator {
   }
 
   JsonObjectConstIterator &operator++() {
-    if (_slot) _slot = _slot->next;
+    _slot = _slot->getNext();
     return *this;
   }
 
   JsonObjectConstIterator &operator+=(size_t distance) {
-    _slot = slotAdvance(_slot, distance);
+    _slot = _slot->getNext(distance);
     return *this;
   }
 

src/ArduinoJson/Memory/VariantSlot.hpp

@@ -5,14 +5,58 @@
 #pragma once
 
 #include "../Data/JsonVariantData.hpp"
+#include "../Polyfills/type_traits.hpp"
 
 namespace ARDUINOJSON_NAMESPACE {
 
+typedef conditional<sizeof(void*) <= 2, int8_t, int16_t>::type VariantSlotDiff;
+
 struct VariantSlot {
   JsonVariantData value;
-  struct VariantSlot* next;
-  struct VariantSlot* prev;
+  VariantSlotDiff next;
+  VariantSlotDiff prev;
   const char* key;
+
+  // Must be a POD! so no constructor, nor destructor, nor virtual
+
+  VariantSlot* getNext() {
+    return next ? this + next : 0;
+  }
+
+  const VariantSlot* getNext() const {
+    return const_cast<VariantSlot*>(this)->getNext();
+  }
+
+  VariantSlot* getNext(size_t distance) {
+    VariantSlot* slot = this;
+    while (distance--) {
+      if (!slot->next) return 0;
+      slot += slot->next;
+    }
+    return slot;
+  }
+
+  const VariantSlot* getNext(size_t distance) const {
+    return const_cast<VariantSlot*>(this)->getNext(distance);
+  }
+
+  VariantSlot* getPrev() {
+    return prev ? this + prev : 0;
+  }
+
+  void setNext(VariantSlot* slot) {
+    this->next = VariantSlotDiff(slot ? slot - this : 0);
+  }
+
+  void setPrev(VariantSlot* slot) {
+    this->prev = VariantSlotDiff(slot ? slot - this : 0);
+  }
+
+  void attachTo(VariantSlot* tail) {
+    VariantSlotDiff offset = VariantSlotDiff(tail - this);
+    this->prev = offset;
+    tail->next = VariantSlotDiff(-offset);
+  }
 };
 
 }  // namespace ARDUINOJSON_NAMESPACE

src/ArduinoJson/Polyfills/type_traits.hpp

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include "type_traits/conditional.hpp"
 #include "type_traits/enable_if.hpp"
 #include "type_traits/integral_constant.hpp"
 #include "type_traits/is_array.hpp"

src/ArduinoJson/Polyfills/type_traits/conditional.hpp

@@ -0,0 +1,18 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2018
+// MIT License
+
+#pragma once
+
+namespace ARDUINOJSON_NAMESPACE {
+
+template <bool Condition, class TrueType, class FalseType>
+struct conditional {
+  typedef TrueType type;
+};
+
+template <class TrueType, class FalseType>
+struct conditional<false, TrueType, FalseType> {
+  typedef FalseType type;
+};
+}  // namespace ARDUINOJSON_NAMESPACE

test/JsonArray/CMakeLists.txt

@@ -8,6 +8,7 @@ add_executable(JsonArrayTests
 	copyTo.cpp
 	createNested.cpp
 	equals.cpp
+	get.cpp
 	isNull.cpp
 	iterator.cpp
 	remove.cpp

test/JsonArray/get.cpp

@@ -0,0 +1,16 @@
+// ArduinoJson - arduinojson.org
+// Copyright Benoit Blanchon 2014-2018
+// MIT License
+
+#include <ArduinoJson.h>
+#include <catch.hpp>
+
+TEST_CASE("JsonArray::get()") {
+  DynamicJsonDocument doc;
+  deserializeJson(doc, "[1,2,3]");
+  JsonArray array = doc.as<JsonArray>();
+
+  SECTION("Overflow") {
+    REQUIRE(array.get(3).isNull());
+  }
+}