Initial setup of the new generator script.

generator/eams/Field.py

@@ -0,0 +1,328 @@
+#
+# Copyright (C) 2020 Embedded AMS B.V. - All Rights Reserved
+#
+# This file is part of Embedded Proto.
+#
+# Embedded Proto is open source software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, version 3 of the license.
+#
+# Embedded Proto  is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Embedded Proto. If not, see <https://www.gnu.org/licenses/>.
+#
+# For commercial and closed source application please visit:
+# <https://EmbeddedProto.com/license/>.
+#
+# Embedded AMS B.V.
+# Info:
+#   info at EmbeddedProto dot com
+#
+# Postal address:
+#   Johan Huizingalaan 763a
+#   1066 VH, Amsterdam
+#   the Netherlands
+#
+
+from google.protobuf.descriptor_pb2 import FieldDescriptorProto
+
+
+# This class is the base class for any kind of field used in protobuf messages.
+class Field:
+    def __init__(self, proto_descriptor, parent_msg, oneof=None):
+        # A reference to the FieldDescriptorProto object which defines this field.
+        self.descriptor = proto_descriptor
+
+        # A reference to the parent message in which this field is defined.
+        self.parent = parent_msg
+
+        # If this field is part of a oneof this is the reference to it.
+        self.oneof = oneof
+
+        self.name = self.descriptor.name
+        self.variable_name = self.name + "_"
+        self.variable_id_name = self.name.upper()
+        self.variable_id = self.descriptor.number
+
+    @staticmethod
+    # This function create the appropriate field object for a variable defined in the message.
+    # The descriptor and parent message parameters are required parameters, all field need them to be created. The oneof
+    # parameter is only required for fields which are part of a oneof. The parameter is the reference to the oneof
+    # object.
+    # The last parameter is not to be used manually. It is set when we are already in a FieldNested class.
+    def factory(proto_descriptor, parent_msg, oneof=None, already_nested=False):
+        if (FieldDescriptorProto.LABEL_REPEATED == proto_descriptor.label) and not already_nested:
+            result = FieldRepeated(proto_descriptor, parent_msg, oneof)
+        elif FieldDescriptorProto.TYPE_MESSAGE == proto_descriptor.type:
+            result = FieldMessage(proto_descriptor, parent_msg, oneof)
+        elif FieldDescriptorProto.TYPE_ENUM == proto_descriptor.type:
+            result = FieldEnum(proto_descriptor, parent_msg, oneof)
+        elif FieldDescriptorProto.TYPE_STRING == proto_descriptor.type:
+            result = FieldString(proto_descriptor, parent_msg, oneof)
+        elif FieldDescriptorProto.TYPE_BYTES == proto_descriptor.type:
+            result = FieldBytes(proto_descriptor, parent_msg, oneof)
+        else:
+            result = FieldBasic(proto_descriptor, parent_msg, oneof)
+        return result
+
+    def get_wire_type_str(self):
+        return ""
+
+    def get_type(self):
+        return ""
+
+    def get_short_type(self):
+        return ""
+
+    def get_default_value(self):
+        return ""
+
+    # Returns a list with a dictionaries for each template parameter this field had. The dictionary holds the parameter
+    # name and its type.
+    def get_template_parameters(self):
+        # For the field that do not have any templates return an empty list.
+        return []
+
+    def match_field_with_definitions(self, all_types_definitions):
+        pass
+
+# -----------------------------------------------------------------------------
+
+
+# This class is used to define any type of basic field.
+class FieldBasic(Field):
+    # A dictionary to convert the wire type into a default value.
+    type_to_default_value = {FieldDescriptorProto.TYPE_DOUBLE:   "0.0",
+                             FieldDescriptorProto.TYPE_FLOAT:    "0.0",
+                             FieldDescriptorProto.TYPE_INT64:    "0",
+                             FieldDescriptorProto.TYPE_UINT64:   "0U",
+                             FieldDescriptorProto.TYPE_INT32:    "0",
+                             FieldDescriptorProto.TYPE_FIXED64:  "0U",
+                             FieldDescriptorProto.TYPE_FIXED32:  "0U",
+                             FieldDescriptorProto.TYPE_BOOL:     "false",
+                             FieldDescriptorProto.TYPE_UINT32:   "0U",
+                             FieldDescriptorProto.TYPE_SFIXED32: "0",
+                             FieldDescriptorProto.TYPE_SFIXED64: "0",
+                             FieldDescriptorProto.TYPE_SINT32:   "0",
+                             FieldDescriptorProto.TYPE_SINT64:   "0"}
+
+    # A dictionary to convert the protobuf wire type into a C++ type.
+    type_to_cpp_type = {FieldDescriptorProto.TYPE_DOUBLE:   "EmbeddedProto::doublefixed",
+                        FieldDescriptorProto.TYPE_FLOAT:    "EmbeddedProto::floatfixed",
+                        FieldDescriptorProto.TYPE_INT64:    "EmbeddedProto::int64",
+                        FieldDescriptorProto.TYPE_UINT64:   "EmbeddedProto::uint64",
+                        FieldDescriptorProto.TYPE_INT32:    "EmbeddedProto::int32",
+                        FieldDescriptorProto.TYPE_FIXED64:  "EmbeddedProto::fixed64",
+                        FieldDescriptorProto.TYPE_FIXED32:  "EmbeddedProto::fixed32",
+                        FieldDescriptorProto.TYPE_BOOL:     "EmbeddedProto::boolean",
+                        FieldDescriptorProto.TYPE_UINT32:   "EmbeddedProto::uint32",
+                        FieldDescriptorProto.TYPE_SFIXED32: "EmbeddedProto::sfixed32",
+                        FieldDescriptorProto.TYPE_SFIXED64: "EmbeddedProto::sfixed64",
+                        FieldDescriptorProto.TYPE_SINT32:   "EmbeddedProto::sint32",
+                        FieldDescriptorProto.TYPE_SINT64:   "EmbeddedProto::sint64"}
+
+    # A dictionary to convert the wire type number into a wire type string.
+    type_to_wire_type = {FieldDescriptorProto.TYPE_INT32:    "VARINT",
+                         FieldDescriptorProto.TYPE_INT64:    "VARINT",
+                         FieldDescriptorProto.TYPE_UINT32:   "VARINT",
+                         FieldDescriptorProto.TYPE_UINT64:   "VARINT",
+                         FieldDescriptorProto.TYPE_SINT32:   "VARINT",
+                         FieldDescriptorProto.TYPE_SINT64:   "VARINT",
+                         FieldDescriptorProto.TYPE_BOOL:     "VARINT",
+                         FieldDescriptorProto.TYPE_FIXED64:  "FIXED64",
+                         FieldDescriptorProto.TYPE_SFIXED64: "FIXED64",
+                         FieldDescriptorProto.TYPE_DOUBLE:   "FIXED64",
+                         FieldDescriptorProto.TYPE_FIXED32:  "FIXED32",
+                         FieldDescriptorProto.TYPE_FLOAT:    "FIXED32",
+                         FieldDescriptorProto.TYPE_SFIXED32: "FIXED32"}
+
+    def __init__(self, proto_descriptor, parent_msg, oneof=None):
+        super().__init__(proto_descriptor, parent_msg, oneof)
+
+    def get_wire_type_str(self):
+        return self.type_to_wire_type[self.descriptor.type]
+
+    def get_type(self):
+        return self.type_to_cpp_type[self.descriptor.type]
+
+    def get_short_type(self):
+        return self.get_type().split("::")[-1]
+
+    def get_default_value(self):
+        return self.type_to_default_value[self.descriptor.type]
+
+# -----------------------------------------------------------------------------
+
+
+# This class defines a string field
+class FieldString(Field):
+    def __init__(self, proto_descriptor, parent_msg, oneof=None):
+        super().__init__(proto_descriptor, parent_msg, oneof)
+
+        # This is the name given to the template parameter for the length.
+        self.template_param_str = self.variable_name + "LENGTH"
+
+    def get_wire_type_str(self):
+        return "LENGTH_DELIMITED"
+
+    def get_type(self):
+        return "::EmbeddedProto::FieldString"
+
+    def get_short_type(self):
+        return "FieldString"
+
+    def get_default_value(self):
+        return "''"
+
+    def get_template_parameters(self):
+        return [{"name": self.template_param_str, "type": "uint32_t"}]
+
+# -----------------------------------------------------------------------------
+
+
+# This class defines a bytes array field
+class FieldBytes(Field):
+    def __init__(self, proto_descriptor, parent_msg, oneof=None):
+        super().__init__(proto_descriptor, parent_msg, oneof)
+
+        # This is the name given to the template parameter for the length.
+        self.template_param_str = self.variable_name + "LENGTH"
+
+    def get_wire_type_str(self):
+        return "LENGTH_DELIMITED"
+
+    def get_type(self):
+        return "::EmbeddedProto::FieldBytes"
+
+    def get_short_type(self):
+        return "FieldBytes"
+
+    def get_default_value(self):
+        return "0U"
+
+    def get_template_parameters(self):
+        return [{"name": self.template_param_str, "type": "uint32_t"}]
+
+# -----------------------------------------------------------------------------
+
+
+# This class is used to wrap around any enum used as a field.
+class FieldEnum(Field):
+    def __init__(self, proto_descriptor, parent_msg, oneof=None):
+        super().__init__(proto_descriptor, parent_msg, oneof)
+
+        # Reserve a member variable for the reference to the enum definition used for this field.
+        self.definition = None
+
+    def get_wire_type_str(self):
+        return "VARINT"
+
+    def get_type(self):
+        if not self.definition:
+            # When the actual definition is unknown use the protobuf type.
+            type = self.descriptor.type_name if "." != self.descriptor.type_name[0] else self.descriptor.type_name[1:]
+            type = type.replace(".", "::")
+        else:
+            type = "TODO"
+        return type
+
+    def get_short_type(self):
+        return self.get_type().split("::")[-1]
+
+    def get_default_value(self):
+        return "static_cast<" + self.get_type() + ">(0)"
+
+    def match_field_with_definitions(self, all_types_definitions):
+        found = False
+        my_type = self.get_type()
+        for enum_defs in all_types_definitions["enums"]:
+            other_scope = enum_defs.scope.get_scope_str()
+            if my_type == other_scope:
+                found = True
+                break
+
+        if not found:
+            raise Exception("Unable to match enum type for: " + self.name)
+
+# -----------------------------------------------------------------------------
+
+
+# This class is used to wrap around any type of message used as a field.
+class FieldMessage(Field):
+    def __init__(self, proto_descriptor, parent_msg, oneof=None):
+        super().__init__(proto_descriptor, parent_msg, oneof)
+
+        # Reserve a member variable for the reference to the message definition used for this field.
+        self.definition = None
+
+    def get_wire_type_str(self):
+        return "LENGTH_DELIMITED"
+
+    def get_type(self):
+        if not self.definition:
+            # When the actual definition is unknown use the protobuf type.
+            type = self.descriptor.type_name if "." != self.descriptor.type_name[0] else self.descriptor.type_name[1:]
+            type = type.replace(".", "::")
+        else:
+            type = "TODO"
+        return type
+
+    def get_short_type(self):
+        return self.get_type().split("::")[-1]
+
+    def get_default_value(self):
+        return ""
+
+    def get_template_parameters(self):
+        # TODO
+        return []
+
+    def match_field_with_definitions(self, all_types_definitions):
+        found = False
+        my_type = self.get_type()
+        for msg_defs in all_types_definitions["messages"]:
+            other_scope = msg_defs.scope.get_scope_str()
+            if my_type == other_scope:
+                found = True
+                break
+
+        if not found:
+            raise Exception("Unable to match enum type for: " + self.name)
+
+# -----------------------------------------------------------------------------
+
+
+# This class wraps around any other type of field which is repeated.
+class FieldRepeated(Field):
+    def __init__(self, proto_descriptor, parent_msg, oneof=None):
+        super().__init__(proto_descriptor, parent_msg, oneof)
+
+        # To make use of the field object actual type create one of their objects.
+        self.actual_type = Field.factory(proto_descriptor, parent_msg, oneof, already_nested=True)
+
+        # This is the name given to the template parameter for the length.
+        self.template_param_str = self.variable_name + "LENGTH"
+
+    def get_wire_type_str(self):
+        return self.actual_type.get_wire_type_str()
+
+    def get_type(self):
+        return "::EmbeddedProto::RepeatedFieldFixedSize<" + self.actual_type.get_type() + ", " + \
+               self.template_param_str + ">"
+
+    def get_short_type(self):
+        return "::EmbeddedProto::RepeatedFieldFixedSize<" + self.actual_type.get_short_type() + ", " + \
+               self.template_param_str + ">"
+
+    def get_default_value(self):
+        return self.actual_type.get_default_value()
+
+    def get_template_parameters(self):
+        return [{"name": self.template_param_str, "type": "uint32_t"}]
+
+    def match_field_with_definitions(self, all_types_definitions):
+        self.actual_type.match_field_with_definitions(all_types_definitions)

generator/eams/Oneof.py

@@ -0,0 +1,61 @@
+#
+# Copyright (C) 2020 Embedded AMS B.V. - All Rights Reserved
+#
+# This file is part of Embedded Proto.
+#
+# Embedded Proto is open source software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, version 3 of the license.
+#
+# Embedded Proto  is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Embedded Proto. If not, see <https://www.gnu.org/licenses/>.
+#
+# For commercial and closed source application please visit:
+# <https://EmbeddedProto.com/license/>.
+#
+# Embedded AMS B.V.
+# Info:
+#   info at EmbeddedProto dot com
+#
+# Postal address:
+#   Johan Huizingalaan 763a
+#   1066 VH, Amsterdam
+#   the Netherlands
+#
+
+from google.protobuf.descriptor_pb2 import FieldDescriptorProto
+from .Field import Field
+
+
+class Oneof:
+    def __init__(self, oneof_proto_descriptor, index, msg_descriptor, parent_msg):
+        # A reference to the OneofDescriptorProto object which defines this field.
+        self.descriptor = oneof_proto_descriptor
+
+        # A reference to the parent message in which this oneof is defined.
+        self.parent = parent_msg
+
+        self.fields_array = []
+        # Loop over all the fields in this oneof
+        for f in msg_descriptor.field:
+            if f.HasField('oneof_index') and index == f.oneof_index:
+                new_field = Field.factor(f, self.parent, oneof=self)
+                self.fields_array.append(new_field)
+
+    def get_name(self):
+        return self.descriptor.name
+
+    def get_which_oneof(self):
+        return "which_" + self.get_name() + "_"
+
+    def fields(self):
+        return self.fields_array
+
+    def match_field_with_definitions(self, all_types_definitions):
+        for field in self.fields_array:
+            field.match_field_with_definitions(all_types_definitions)

generator/eams/ProtoFile.py

@@ -0,0 +1,82 @@
+#
+# Copyright (C) 2020 Embedded AMS B.V. - All Rights Reserved
+#
+# This file is part of Embedded Proto.
+#
+# Embedded Proto is open source software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, version 3 of the license.
+#
+# Embedded Proto  is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Embedded Proto. If not, see <https://www.gnu.org/licenses/>.
+#
+# For commercial and closed source application please visit:
+# <https://EmbeddedProto.com/license/>.
+#
+# Embedded AMS B.V.
+# Info:
+#   info at EmbeddedProto dot com
+#
+# Postal address:
+#   Johan Huizingalaan 763a
+#   1066 VH, Amsterdam
+#   the Netherlands
+#
+
+from .TypeDefinitions import *
+import os
+
+
+class ProtoFile:
+    def __init__(self, proto_descriptor):
+        self.descriptor = proto_descriptor
+
+        if "proto2" == proto_descriptor.syntax:
+            raise Exception(proto_descriptor.name + ": Sorry, proto2 is not supported, please use proto3.")
+
+        # These file names are the ones used for creating the C++ files.
+        self.filename_with_folder = os.path.splitext(proto_descriptor.name)[0]
+        self.filename_without_folder = os.path.basename(self.filename_with_folder)
+
+        # Construct the base scope used in this file.
+        self.scope = None
+        if self.descriptor.package:
+            package_list = self.descriptor.package.split(".")
+            # The first element is the base scope.
+            self.scope = Scope(package_list[0], None)
+            # Next add additional scopes nesting the previous one.
+            for package in package_list[1:]:
+                self.scope = Scope(package, self.scope)
+
+        self.enum_definitions = [EnumDefinition(enum, self.scope) for enum in self.descriptor.enum_type]
+        self.msg_definitions = [MessageDefinition(msg, self.scope) for msg in self.descriptor.message_type]
+
+    def get_dependencies(self):
+        imported_dependencies = []
+        if self.descriptor.dependency:
+            imported_dependencies = [os.path.splitext(dependency)[0] + ".h" for dependency in
+                                     self.descriptor.dependency]
+        return imported_dependencies
+
+    def get_header_guard(self):
+        return self.filename_with_folder.replace("/", "_").upper()
+
+    # Obtain a dictionary with references to all nested enums and messages
+    def get_all_nested_types(self):
+        nested_types = {"enums": self.enum_definitions, "messages": []}
+        for msg in self.msg_definitions:
+            nt = msg.get_all_nested_types()
+            nested_types["enums"].extend(nt["enums"])
+            nested_types["messages"].extend(nt["messages"])
+            nested_types["messages"].append(msg)
+
+        return nested_types
+
+    def match_fields_with_definitions(self, all_types_definitions):
+        for msg in self.msg_definitions:
+            msg.match_fields_with_definitions(all_types_definitions)

generator/TypeDefinitions.py → generator/eams/TypeDefinitions.py

@@ -28,6 +28,9 @@
 #   the Netherlands
 #
 
+from .Field import Field
+from .Oneof import Oneof
+
 
 # This class deal with the scope in which definitions and field are located. It is used to keep track of template
 # parameters required for a given scope.
@@ -50,6 +53,14 @@ class Scope:
     def add_child_scope(self, child_scope):
         self.child_scopes.append(child_scope)
 
+    def get_scope_str(self):
+        scope_str = ""
+        if self.parent:
+            scope_str = self.parent.get_scope_str() + "::" + self.name
+        else:
+            scope_str = self.name
+
+        return scope_str
 
 # -----------------------------------------------------------------------------
 
@@ -59,7 +70,6 @@ class TypeDefinition:
         self.name = proto_descriptor.name
         self.scope = Scope(self.name, parent_scope)
 
-
 # -----------------------------------------------------------------------------
 
 class EnumDefinition(TypeDefinition):
@@ -77,3 +87,51 @@ class EnumDefinition(TypeDefinition):
 class MessageDefinition(TypeDefinition):
     def __init__(self, proto_descriptor, parent_scope):
         super().__init__(proto_descriptor, parent_scope)
+
+        self.nested_enum_definitions = [EnumDefinition(enum, self.scope) for enum in self.descriptor.enum_type]
+        self.nested_msg_definitions = [MessageDefinition(msg, self.scope) for msg in self.descriptor.nested_type]
+
+        # Store the id numbers of all the fields to create the ID enum.
+        self.field_ids = []
+
+        # Store all the variable fields in this message.
+        self.fields_array = []
+        for f in self.descriptor.field:
+            if not f.HasField('oneof_index'):
+                new_field = Field.factory(f, self)
+                self.fields_array.append(new_field)
+                self.field_ids.append((new_field.variable_id, new_field.variable_id_name))
+
+        # Store all the oneof definitions in this message.
+        self.oneof_fields = []
+        for index, oneof in enumerate(self.descriptor.oneof_decl):
+            new_oneof = Oneof(oneof, index, proto_descriptor, self)
+            self.oneof_fields.append(new_oneof)
+            for oneof_field in oneof.fields():
+                self.field_ids.append((oneof_field.variable_id, oneof_field.variable_id_name))
+
+        # Sort the field id's such they will appear in order in the id enum.
+        self.field_ids.sort()
+
+    # Obtain a dictionary with references to all nested enums and messages
+    def get_all_nested_types(self):
+        nested_types = {"enums": self.nested_enum_definitions, "messages": []}
+        for msg in self.nested_msg_definitions:
+            nt = msg.get_all_nested_types()
+            nested_types["enums"].extend(nt["enums"])
+            nested_types["messages"].extend(nt["messages"])
+            nested_types["messages"].append(msg)
+
+        return nested_types
+
+    def match_fields_with_definitions(self, all_types_definitions):
+        # Resolve the types of the nested messages.
+        for msg in self.nested_msg_definitions:
+            msg.match_fields_with_definitions(all_types_definitions)
+
+        # Resolve the types of the fields defined in this message.
+        for field in self.fields_array:
+            field.match_field_with_definitions(all_types_definitions)
+
+        for oneof in self.oneof_fields:
+            oneof.match_field_with_definitions(all_types_definitions)

generator/protoc-gen_eams.py → generator/protoc-gen-eams.py

@@ -30,9 +30,30 @@
 
 import io
 import sys
-
+from generator.eams.ProtoFile import ProtoFile
 from google.protobuf.compiler import plugin_pb2 as plugin
 
+
+# -----------------------------------------------------------------------------
+
+
+def generate_code(request, respones):
+    # Create definitions for al proto files in the request.
+    file_definitions = [ProtoFile(proto_file) for proto_file in request.proto_file]
+
+    # Obtain all definitions made in all the files to properly link definitions with fields using them. This to properly
+    # create template parameters.
+    all_types_definitions = {"enums": [], "messages": []}
+    for fd in file_definitions:
+        nt = fd.get_all_nested_types()
+        all_types_definitions["enums"].extend(nt["enums"])
+        all_types_definitions["messages"].extend(nt["messages"])
+
+    # Match all fields with their respective type definition.
+    for fd in file_definitions:
+        fd.match_fields_with_definitions(all_types_definitions)
+
+
 # -----------------------------------------------------------------------------
 
 def main_plugin():