Parsing from proto should keep field ID. (fixes #7645) (#7655)

* Parsing from proto should keep field ID. (fixes #7645)

* Fix failed tests

* Fix windows warning

* Improve attribute generation in proto to fbs

* Check if id is used twice. fix Some clang-format problems

* Test if fake id can solve the test problem

* Validate proto file in proto -> fbs generation.

* Fix error messages

* Ignore id in union

* Add keep proto id for legacy and check gap flag have been added. Reserved id will be checked.

* Add needed flags

* unit tests

* fix fromat problem. fix comments and error messages.

* clear

* More unit tests

* Fix windows build

* Fix include problems

* Fake commit to invoke rebuild

* Fix buzel build

* Fix some issues

* Fix comments, fix return value and sort for android NDK

* Fix return type

* Break down big function

* Place todo

---------

Co-authored-by: Derek Bailey <derekbailey@google.com>

include/flatbuffers/idl.h

@@ -381,6 +381,7 @@ struct StructDef : public Definition {
   size_t bytesize;  // Size if fixed.
 
   flatbuffers::unique_ptr<std::string> original_location;
+  std::vector<voffset_t> reserved_ids;
 };
 
 struct EnumDef;
@@ -594,7 +595,7 @@ inline bool operator<(const IncludedFile &a, const IncludedFile &b) {
 struct IDLOptions {
   // field case style options for C++
   enum CaseStyle { CaseStyle_Unchanged = 0, CaseStyle_Upper, CaseStyle_Lower };
-
+  enum class ProtoIdGapAction { NO_OP, WARNING, ERROR };
   bool gen_jvmstatic;
   // Use flexbuffers instead for binary and text generation
   bool use_flexbuffers;
@@ -663,6 +664,8 @@ struct IDLOptions {
   bool ts_no_import_ext;
   bool no_leak_private_annotations;
   bool require_json_eof;
+  bool keep_proto_id;
+  ProtoIdGapAction proto_id_gap_action;
 
   // Possible options for the more general generator below.
   enum Language {
@@ -769,6 +772,8 @@ struct IDLOptions {
         ts_no_import_ext(false),
         no_leak_private_annotations(false),
         require_json_eof(true),
+        keep_proto_id(false),
+        proto_id_gap_action(ProtoIdGapAction::WARNING),
         mini_reflect(IDLOptions::kNone),
         require_explicit_ids(false),
         rust_serialize(false),

src/flatc.cpp

@@ -170,6 +170,15 @@ const static FlatCOption flatc_options[] = {
   { "", "proto-namespace-suffix", "SUFFIX",
     "Add this namespace to any flatbuffers generated from protobufs." },
   { "", "oneof-union", "", "Translate .proto oneofs to flatbuffer unions." },
+  { "", "keep-proto-id", "", "Keep protobuf field ids in generated fbs file." },
+  { "", "proto-id-gap", "",
+    "Action that should be taken when a gap between protobuf ids found. "
+    "Supported values: * "
+    "'nop' - do not care about gap * 'warn' - A warning message will be shown "
+    "about the gap in protobuf ids"
+    "(default) "
+    "* 'error' - An error message will be shown and the fbs generation will be "
+    "interrupted." },
   { "", "grpc", "", "Generate GRPC interfaces for the specified languages." },
   { "", "schema", "", "Serialize schemas instead of JSON (use with -b)." },
   { "", "bfbs-filenames", "PATH",
@@ -541,6 +550,18 @@ FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc,
         opts.proto_namespace_suffix = argv[argi];
       } else if (arg == "--oneof-union") {
         opts.proto_oneof_union = true;
+      } else if (arg == "--keep-proto-id") {
+        opts.keep_proto_id = true;
+      } else if (arg == "--proto-id-gap") {
+        if (++argi >= argc) Error("missing case style following: " + arg, true);
+        if (!strcmp(argv[argi], "nop"))
+          opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::NO_OP;
+        else if (!strcmp(argv[argi], "warn"))
+          opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::WARNING;
+        else if (!strcmp(argv[argi], "error"))
+          opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::ERROR;
+        else
+          Error("unknown case style: " + std::string(argv[argi]), true);
       } else if (arg == "--schema") {
         options.schema_binary = true;
       } else if (arg == "-M") {

src/idl_gen_fbs.cpp

@@ -15,6 +15,9 @@
  */
 
 // independent from idl_parser, since this code is not needed for most clients
+#include <unordered_map>
+#include <utility>
+#include <vector>
 
 #include "flatbuffers/code_generators.h"
 #include "flatbuffers/flatbuffers.h"
@@ -39,6 +42,184 @@ static std::string GenType(const Type &type, bool underlying = false) {
   }
 }
 
+static bool HasFieldWithId(const std::vector<FieldDef *> &fields) {
+  static const std::string ID = "id";
+
+  for (const auto *field : fields) {
+    const auto *id_attribute = field->attributes.Lookup(ID);
+    if (id_attribute != nullptr && !id_attribute->constant.empty()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static bool HasNonPositiveFieldId(const std::vector<FieldDef *> &fields) {
+  static const std::string ID = "id";
+
+  for (const auto *field : fields) {
+    const auto *id_attribute = field->attributes.Lookup(ID);
+    if (id_attribute != nullptr && !id_attribute->constant.empty()) {
+      voffset_t proto_id = 0;
+      bool done = StringToNumber(id_attribute->constant.c_str(), &proto_id);
+      if (!done) { return true; }
+    }
+  }
+  return false;
+}
+
+static bool HasFieldIdFromReservedIds(
+    const std::vector<FieldDef *> &fields,
+    const std::vector<voffset_t> &reserved_ids) {
+  static const std::string ID = "id";
+
+  for (const auto *field : fields) {
+    const auto *id_attribute = field->attributes.Lookup(ID);
+    if (id_attribute != nullptr && !id_attribute->constant.empty()) {
+      voffset_t proto_id = 0;
+      bool done = StringToNumber(id_attribute->constant.c_str(), &proto_id);
+      if (!done) { return true; }
+      auto id_it =
+          std::find(std::begin(reserved_ids), std::end(reserved_ids), proto_id);
+      if (id_it != reserved_ids.end()) { return true; }
+    }
+  }
+  return false;
+}
+
+static std::vector<voffset_t> ExtractProtobufIds(
+    const std::vector<FieldDef *> &fields) {
+  static const std::string ID = "id";
+  std::vector<voffset_t> used_proto_ids;
+  for (const auto *field : fields) {
+    const auto *id_attribute = field->attributes.Lookup(ID);
+    if (id_attribute != nullptr && !id_attribute->constant.empty()) {
+      voffset_t proto_id = 0;
+      bool done = StringToNumber(id_attribute->constant.c_str(), &proto_id);
+      if (done) { used_proto_ids.push_back(proto_id); }
+    }
+  }
+
+  return used_proto_ids;
+}
+
+static bool HasTwiceUsedId(const std::vector<FieldDef *> &fields) {
+  std::vector<voffset_t> used_proto_ids = ExtractProtobufIds(fields);
+  std::sort(std::begin(used_proto_ids), std::end(used_proto_ids));
+  for (auto it = std::next(std::begin(used_proto_ids));
+       it != std::end(used_proto_ids); it++) {
+    if (*it == *std::prev(it)) { return true; }
+  }
+
+  return false;
+}
+
+static bool HasGapInProtoId(const std::vector<FieldDef *> &fields) {
+  std::vector<voffset_t> used_proto_ids = ExtractProtobufIds(fields);
+  std::sort(std::begin(used_proto_ids), std::end(used_proto_ids));
+  for (auto it = std::next(std::begin(used_proto_ids));
+       it != std::end(used_proto_ids); it++) {
+    if (*it != *std::prev(it) + 1) { return true; }
+  }
+
+  return false;
+}
+
+static bool ProtobufIdSanityCheck(const StructDef &struct_def,
+                                  IDLOptions::ProtoIdGapAction gap_action) {
+  const auto &fields = struct_def.fields.vec;
+  if (HasNonPositiveFieldId(fields)) {
+    // TODO: Use LogCompilerWarn
+    fprintf(stderr,
+                    "Field id in struct %s has a non positive number value\n",
+                    struct_def.name.c_str());
+    return false;
+  }
+
+  if (HasTwiceUsedId(fields)) {
+    // TODO: Use LogCompilerWarn
+    fprintf(stderr, "Fields in struct %s have used an id twice\n", struct_def.name.c_str());
+    return false;
+  }
+
+  if (HasFieldIdFromReservedIds(fields, struct_def.reserved_ids)) {
+    // TODO: Use LogCompilerWarn
+    fprintf(stderr,
+    "Fields in struct %s use id from reserved ids\n", struct_def.name.c_str());
+    return false;
+  }
+
+  if (gap_action != IDLOptions::ProtoIdGapAction::NO_OP) {
+    if (HasGapInProtoId(fields)) {
+      // TODO: Use LogCompilerWarn
+      fprintf(stderr, "Fields in struct %s have gap between ids\n", struct_def.name.c_str());
+      if (gap_action == IDLOptions::ProtoIdGapAction::ERROR) { return false; }
+    }
+  }
+
+  return true;
+}
+
+struct ProtobufToFbsIdMap {
+  using FieldName = std::string;
+  using FieldID = voffset_t;
+  using FieldNameToIdMap = std::unordered_map<FieldName, FieldID>;
+
+  FieldNameToIdMap field_to_id;
+  bool successful = false;
+};
+
+static ProtobufToFbsIdMap MapProtoIdsToFieldsId(
+    const StructDef &struct_def, IDLOptions::ProtoIdGapAction gap_action) {
+  const auto &fields = struct_def.fields.vec;
+
+  if (!HasFieldWithId(fields)) {
+    ProtobufToFbsIdMap result;
+    result.successful = true;
+    return result;
+  }
+
+  if (!ProtobufIdSanityCheck(struct_def, gap_action)) { return {}; }
+
+  static constexpr int UNION_ID = -1;
+  using ProtoIdFieldNamePair = std::pair<int, std::string>;
+  std::vector<ProtoIdFieldNamePair> proto_ids;
+
+  for (const auto *field : fields) {
+    const auto *id_attribute = field->attributes.Lookup("id");
+    if (id_attribute != nullptr) {
+      // When we have union but do not use union flag to keep them
+      if (id_attribute->constant.empty() &&
+          field->value.type.base_type == BASE_TYPE_UNION) {
+        proto_ids.emplace_back(UNION_ID, field->name);
+      } else {
+        voffset_t proto_id = 0;
+        StringToNumber(id_attribute->constant.c_str(), &proto_id);
+        proto_ids.emplace_back(proto_id, field->name);
+      }
+    } else {
+      // TODO: Use LogCompilerWarn
+      fprintf(stderr, "Fields id in struct %s is missing\n", struct_def.name.c_str());
+      return {};
+    }
+  }
+
+  std::sort(
+      std::begin(proto_ids), std::end(proto_ids),
+      [](const ProtoIdFieldNamePair &rhs, const ProtoIdFieldNamePair &lhs) {
+        return rhs.first < lhs.first;
+      });
+  struct ProtobufToFbsIdMap proto_to_fbs;
+
+  voffset_t id = 0;
+  for (const auto &element : proto_ids) {
+    if (element.first == UNION_ID) { id++; }
+    proto_to_fbs.field_to_id.emplace(element.second, id++);
+  }
+  proto_to_fbs.successful = true;
+  return proto_to_fbs;
+}
+
 static void GenNameSpace(const Namespace &name_space, std::string *_schema,
                          const Namespace **last_namespace) {
   if (*last_namespace == &name_space) return;
@@ -79,8 +260,9 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) {
     int num_includes = 0;
     for (auto it = parser.included_files_.begin();
          it != parser.included_files_.end(); ++it) {
-      if (it->second.empty())
+      if (it->second.empty()) {
         continue;
+}
       std::string basename;
       if(parser.opts.keep_prefix) {
         basename = flatbuffers::StripExtension(it->second);
@@ -94,6 +276,7 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) {
     if (num_includes) schema += "\n";
     // clang-format on
   }
+
   // Generate code for all the enum declarations.
   const Namespace *last_namespace = nullptr;
   for (auto enum_def_it = parser.enums_.vec.begin();
@@ -104,28 +287,37 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) {
     }
     GenNameSpace(*enum_def.defined_namespace, &schema, &last_namespace);
     GenComment(enum_def.doc_comment, &schema, nullptr);
-    if (enum_def.is_union)
+    if (enum_def.is_union) {
       schema += "union " + enum_def.name;
-    else
+    } else {
       schema += "enum " + enum_def.name + " : ";
+    }
+
     schema += GenType(enum_def.underlying_type, true) + " {\n";
+
     for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) {
       auto &ev = **it;
       GenComment(ev.doc_comment, &schema, nullptr, "  ");
-      if (enum_def.is_union)
+      if (enum_def.is_union) {
         schema += "  " + GenType(ev.union_type) + ",\n";
-      else
+      } else {
         schema += "  " + ev.name + " = " + enum_def.ToString(ev) + ",\n";
       }
+    }
     schema += "}\n\n";
   }
   // Generate code for all structs/tables.
   for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end();
        ++it) {
     StructDef &struct_def = **it;
+    const auto proto_fbs_ids =
+        MapProtoIdsToFieldsId(struct_def, parser.opts.proto_id_gap_action);
+    if (!proto_fbs_ids.successful) { return {}; }
+
     if (parser.opts.include_dependence_headers && struct_def.generated) {
       continue;
     }
+
     GenNameSpace(*struct_def.defined_namespace, &schema, &last_namespace);
     GenComment(struct_def.doc_comment, &schema, nullptr);
     schema += "table " + struct_def.name + " {\n";
@@ -136,8 +328,26 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) {
         GenComment(field.doc_comment, &schema, nullptr, "  ");
         schema += "  " + field.name + ":" + GenType(field.value.type);
         if (field.value.constant != "0") schema += " = " + field.value.constant;
-        if (field.IsRequired()) schema += " (required)";
-        if (field.key) schema += " (key)";
+        std::vector<std::string> attributes;
+        if (field.IsRequired()) attributes.push_back("required");
+        if (field.key) attributes.push_back("key");
+
+        if (parser.opts.keep_proto_id) {
+          auto it = proto_fbs_ids.field_to_id.find(field.name);
+          if (it != proto_fbs_ids.field_to_id.end()) {
+            attributes.push_back("id: " + NumToString(it->second));
+          }  // If not found it means we do not have any ids
+        }
+
+        if (!attributes.empty()) {
+          schema += " (";
+          for (const auto &attribute : attributes) {
+            schema += attribute + ",";
+          }
+          schema.pop_back();
+          schema += ")";
+        }
+
         schema += ";\n";
       }
     }
@@ -148,8 +358,13 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) {
 
 bool GenerateFBS(const Parser &parser, const std::string &path,
                  const std::string &file_name) {
-  return SaveFile((path + file_name + ".fbs").c_str(),
-                  GenerateFBS(parser, file_name), false);
+  const std::string fbs = GenerateFBS(parser, file_name);
+  if (fbs.empty()) { return false; }
+  // TODO: Use LogCompilerWarn
+  fprintf(stderr,
+          "When you use --proto, that you should check for conformity "
+          "yourself, using the existing --conform");
+  return SaveFile((path + file_name + ".fbs").c_str(), fbs, false);
 }
 
 }  // namespace flatbuffers

src/idl_parser.cpp

@@ -920,8 +920,11 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
   if (struct_def.fixed) {
     if (IsIncompleteStruct(type) ||
         (IsArray(type) && IsIncompleteStruct(type.VectorType()))) {
-      std::string type_name = IsArray(type) ? type.VectorType().struct_def->name : type.struct_def->name;
-      return Error(std::string("Incomplete type in struct is not allowed, type name: ") + type_name);
+      std::string type_name = IsArray(type) ? type.VectorType().struct_def->name
+                                            : type.struct_def->name;
+      return Error(
+          std::string("Incomplete type in struct is not allowed, type name: ") +
+          type_name);
     }
 
     auto valid = IsScalar(type.base_type) || IsStruct(type);
@@ -2289,12 +2292,8 @@ template<typename T> void EnumDef::ChangeEnumValue(EnumVal *ev, T new_value) {
 }
 
 namespace EnumHelper {
-template<BaseType E> struct EnumValType {
-  typedef int64_t type;
-};
-template<> struct EnumValType<BASE_TYPE_ULONG> {
-  typedef uint64_t type;
-};
+template<BaseType E> struct EnumValType { typedef int64_t type; };
+template<> struct EnumValType<BASE_TYPE_ULONG> { typedef uint64_t type; };
 }  // namespace EnumHelper
 
 struct EnumValBuilder {
@@ -2698,6 +2697,7 @@ CheckedError Parser::ParseDecl(const char *filename) {
       for (voffset_t i = 0; i < static_cast<voffset_t>(fields.size()); i++) {
         auto &field = *fields[i];
         const auto &id_str = field.attributes.Lookup("id")->constant;
+
         // Metadata values have a dynamic type, they can be `float`, 'int', or
         // 'string`.
         // The FieldIndexToOffset(i) expects the voffset_t so `id` is limited by
@@ -2921,8 +2921,40 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
       ECHECK(ParseProtoOption());
       EXPECT(';');
     } else if (IsIdent("reserved")) {  // Skip these.
+      /**
+       * Reserved proto ids can be comma seperated (e.g. 1,2,4,5;)
+       * or range based (e.g. 9 to 11;)
+       * or combination of them (e.g. 1,2,9 to 11,4,5;)
+       * It will be ended by a semicolon.
+       */
       NEXT();
-      while (!Is(';')) { NEXT(); }  // A variety of formats, just skip.
+      bool range = false;
+      voffset_t from = 0;
+
+      while (!Is(';')) {
+        if (token_ == kTokenIntegerConstant) {
+          voffset_t attribute = 0;
+          bool done = StringToNumber(attribute_.c_str(), &attribute);
+          if (!done)
+            return Error("Protobuf has non positive number in reserved ids");
+
+          if (range) {
+            for (voffset_t id = from + 1; id <= attribute; id++)
+              struct_def->reserved_ids.push_back(id);
+
+            range = false;
+          } else {
+            struct_def->reserved_ids.push_back(attribute);
+          }
+
+          from = attribute;
+        }
+
+        if (attribute_ == "to") range = true;
+
+        NEXT();
+      }  // A variety of formats, just skip.
+
       NEXT();
     } else if (IsIdent("map")) {
       ECHECK(ParseProtoMapField(struct_def));
@@ -2980,11 +3012,13 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
       }
       std::string name = attribute_;
       EXPECT(kTokenIdentifier);
+      std::string proto_field_id;
       if (!oneof) {
         // Parse the field id. Since we're just translating schemas, not
         // any kind of binary compatibility, we can safely ignore these, and
         // assign our own.
         EXPECT('=');
+        proto_field_id = attribute_;
         EXPECT(kTokenIntegerConstant);
       }
       FieldDef *field = nullptr;
@@ -2995,6 +3029,11 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
       }
       if (!field) ECHECK(AddField(*struct_def, name, type, &field));
       field->doc_comment = field_comment;
+      if (!proto_field_id.empty() || oneof) {
+        auto val = new Value();
+        val->constant = proto_field_id;
+        field->attributes.Add("id", val);
+      }
       if (!IsScalar(type.base_type) && required) {
         field->presence = FieldDef::kRequired;
       }
@@ -3072,6 +3111,7 @@ CheckedError Parser::ParseProtoMapField(StructDef *struct_def) {
   auto field_name = attribute_;
   NEXT();
   EXPECT('=');
+  std::string proto_field_id = attribute_;
   EXPECT(kTokenIntegerConstant);
   EXPECT(';');
 
@@ -3091,6 +3131,11 @@ CheckedError Parser::ParseProtoMapField(StructDef *struct_def) {
   field_type.struct_def = entry_table;
   FieldDef *field;
   ECHECK(AddField(*struct_def, field_name, field_type, &field));
+  if (!proto_field_id.empty()) {
+    auto val = new Value();
+    val->constant = proto_field_id;
+    field->attributes.Add("id", val);
+  }
 
   return NoError();
 }

tests/BUILD.bazel

@@ -82,13 +82,22 @@ cc_test(
         ":optional_scalars.json",
         ":optional_scalars_defaults.json",
         ":prototest/imported.proto",
-        ":prototest/test.golden",
+        ":prototest/non-positive-id.proto",
+        ":prototest/test.golden.fbs",
         ":prototest/test.proto",
-        ":prototest/test_include.golden",
-        ":prototest/test_suffix.golden",
-        ":prototest/test_union.golden",
-        ":prototest/test_union_include.golden",
-        ":prototest/test_union_suffix.golden",
+        ":prototest/test_id.golden.fbs",
+        ":prototest/test_include.golden.fbs",
+        ":prototest/test_include_id.golden.fbs",
+        ":prototest/test_suffix.golden.fbs",
+        ":prototest/test_suffix_id.golden.fbs",
+        ":prototest/test_union.golden.fbs",
+        ":prototest/test_union_id.golden.fbs",
+        ":prototest/test_union_include.golden.fbs",
+        ":prototest/test_union_include_id.golden.fbs",
+        ":prototest/test_union_suffix.golden.fbs",
+        ":prototest/test_union_suffix_id.golden.fbs",
+        ":prototest/twice-id.proto",
+        ":prototest/use-reserved-id.proto",
         ":unicode_test.json",
         ":union_vector/union_vector.fbs",
         ":union_vector/union_vector.json",

tests/proto_test.cpp

@@ -1,183 +1,292 @@
 #include "proto_test.h"
 
-#include "flatbuffers/idl.h"
 #include "test_assert.h"
 
 namespace flatbuffers {
 namespace tests {
 
-// Parse a .proto schema, output as .fbs
-void ParseProtoTest(const std::string &tests_data_path) {
-  // load the .proto and the golden file from disk
-  std::string protofile;
-  std::string goldenfile;
-  std::string goldenunionfile;
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
-                            false, &protofile),
-      true);
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.golden").c_str(),
-                            false, &goldenfile),
-      true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_union.golden").c_str(), false,
-              &goldenunionfile),
-          true);
-
-  flatbuffers::IDLOptions opts;
-  opts.include_dependence_headers = false;
-  opts.proto_mode = true;
+void RunTest(const flatbuffers::IDLOptions &opts, const std::string &proto_path,
+             const std::string &proto_file, const std::string &golden_file,
+             const std::string import_proto_file) {
+  const char *include_directories[] = { proto_path.c_str(), nullptr };
 
   // Parse proto.
   flatbuffers::Parser parser(opts);
-  auto protopath = tests_data_path + "prototest/";
-  const char *include_directories[] = { protopath.c_str(), nullptr };
-  TEST_EQ(parser.Parse(protofile.c_str(), include_directories), true);
+  TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
 
   // Generate fbs.
   auto fbs = flatbuffers::GenerateFBS(parser, "test");
 
   // Ensure generated file is parsable.
   flatbuffers::Parser parser2;
+
+  if (!import_proto_file.empty()) {
+    // Generate fbs from import.proto
+    flatbuffers::Parser import_parser(opts);
+    TEST_EQ(import_parser.Parse(import_proto_file.c_str(), include_directories),
+            true);
+    auto import_fbs = flatbuffers::GenerateFBS(import_parser, "test");
+    // Since `imported.fbs` isn't in the filesystem AbsolutePath can't figure it
+    // out by itself. We manually construct it so Parser works.
+    std::string imported_fbs = flatbuffers::PosixPath(
+        flatbuffers::AbsolutePath(proto_path) + "/imported.fbs");
+    TEST_EQ(parser2.Parse(import_fbs.c_str(), include_directories,
+                          imported_fbs.c_str()),
+            true);
+  }
+
   TEST_EQ(parser2.Parse(fbs.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs.c_str(), goldenfile.c_str());
-
-  // Parse proto with --oneof-union option.
-  opts.proto_oneof_union = true;
-  flatbuffers::Parser parser3(opts);
-  TEST_EQ(parser3.Parse(protofile.c_str(), include_directories), true);
-
-  // Generate fbs.
-  auto fbs_union = flatbuffers::GenerateFBS(parser3, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser4;
-  TEST_EQ(parser4.Parse(fbs_union.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs_union.c_str(), goldenunionfile.c_str());
+  TEST_EQ_STR(fbs.c_str(), golden_file.c_str());
 }
 
-// Parse a .proto schema, output as .fbs
-void ParseProtoTestWithSuffix(const std::string &tests_data_path) {
+void proto_test(const std::string &proto_path, const std::string &proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+
   // load the .proto and the golden file from disk
-  std::string protofile;
-  std::string goldenfile;
-  std::string goldenunionfile;
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
-                            false, &protofile),
-      true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_suffix.golden").c_str(), false,
-              &goldenfile),
-          true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_union_suffix.golden").c_str(),
-              false, &goldenunionfile),
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile((proto_path + "test.golden.fbs").c_str(), false,
+                                &golden_file),
           true);
 
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_id(const std::string &proto_path,
+                   const std::string &proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.keep_proto_id = true;
+
+  // load the .proto and the golden file from disk
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile((proto_path + "test_id.golden.fbs").c_str(),
+                                false, &golden_file),
+          true);
+
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_union(const std::string &proto_path,
+                      const std::string &proto_file) {
+  // Parse proto with --oneof-union option.
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile((proto_path + "test_union.golden.fbs").c_str(),
+                                false, &golden_file),
+          true);
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_union_id(const std::string &proto_path,
+                         const std::string &proto_file) {
+  // Parse proto with --oneof-union option.
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+  opts.keep_proto_id = true;
+
+  std::string golden_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((proto_path + "test_union_id.golden.fbs").c_str(),
+                            false, &golden_file),
+      true);
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_union_suffix(const std::string &proto_path,
+                             const std::string &proto_file) {
   flatbuffers::IDLOptions opts;
   opts.include_dependence_headers = false;
   opts.proto_mode = true;
   opts.proto_namespace_suffix = "test_namespace_suffix";
-
-  // Parse proto.
-  flatbuffers::Parser parser(opts);
-  auto protopath = tests_data_path + "prototest/";
-  const char *include_directories[] = { protopath.c_str(), nullptr };
-  TEST_EQ(parser.Parse(protofile.c_str(), include_directories), true);
-
-  // Generate fbs.
-  auto fbs = flatbuffers::GenerateFBS(parser, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser2;
-  TEST_EQ(parser2.Parse(fbs.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs.c_str(), goldenfile.c_str());
-
-  // Parse proto with --oneof-union option.
   opts.proto_oneof_union = true;
-  flatbuffers::Parser parser3(opts);
-  TEST_EQ(parser3.Parse(protofile.c_str(), include_directories), true);
 
-  // Generate fbs.
-  auto fbs_union = flatbuffers::GenerateFBS(parser3, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser4;
-  TEST_EQ(parser4.Parse(fbs_union.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs_union.c_str(), goldenunionfile.c_str());
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_suffix.golden.fbs").c_str(), false,
+              &golden_file),
+          true);
+  RunTest(opts, proto_path, proto_file, golden_file);
 }
 
-// Parse a .proto schema, output as .fbs
-void ParseProtoTestWithIncludes(const std::string &tests_data_path) {
-  // load the .proto and the golden file from disk
-  std::string protofile;
-  std::string goldenfile;
-  std::string goldenunionfile;
-  std::string importprotofile;
-  TEST_EQ(
-      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
-                            false, &protofile),
-      true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/imported.proto").c_str(), false,
-              &importprotofile),
-          true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_include.golden").c_str(),
-              false, &goldenfile),
-          true);
-  TEST_EQ(flatbuffers::LoadFile(
-              (tests_data_path + "prototest/test_union_include.golden").c_str(),
-              false, &goldenunionfile),
-          true);
+void proto_test_union_suffix_id(const std::string &proto_path,
+                                const std::string &proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = false;
+  opts.proto_mode = true;
+  opts.proto_namespace_suffix = "test_namespace_suffix";
+  opts.proto_oneof_union = true;
+  opts.keep_proto_id = true;
 
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_suffix_id.golden.fbs").c_str(), false,
+              &golden_file),
+          true);
+  RunTest(opts, proto_path, proto_file, golden_file);
+}
+
+void proto_test_include(const std::string &proto_path,
+                        const std::string &proto_file,
+                        const std::string &import_proto_file) {
   flatbuffers::IDLOptions opts;
   opts.include_dependence_headers = true;
   opts.proto_mode = true;
 
-  // Parse proto.
-  flatbuffers::Parser parser(opts);
-  auto protopath = tests_data_path + "prototest/";
-  const char *include_directories[] = { protopath.c_str(), nullptr };
-  TEST_EQ(parser.Parse(protofile.c_str(), include_directories), true);
-
-  // Generate fbs.
-  auto fbs = flatbuffers::GenerateFBS(parser, "test");
-
-  // Generate fbs from import.proto
-  flatbuffers::Parser import_parser(opts);
-  TEST_EQ(import_parser.Parse(importprotofile.c_str(), include_directories),
+  std::string golden_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((proto_path + "test_include.golden.fbs").c_str(),
+                            false, &golden_file),
       true);
-  auto import_fbs = flatbuffers::GenerateFBS(import_parser, "test");
 
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser2;
-  // Since `imported.fbs` isn't in the filesystem AbsolutePath can't figure it
-  // out by itself. We manually construct it so Parser works.
-  std::string imported_fbs = flatbuffers::PosixPath(
-      flatbuffers::AbsolutePath(protopath) + "/imported.fbs");
-  TEST_EQ(parser2.Parse(import_fbs.c_str(), include_directories,
-                        imported_fbs.c_str()),
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
+
+void proto_test_include_id(const std::string &proto_path,
+                           const std::string &proto_file,
+                           const std::string &import_proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
+  opts.keep_proto_id = true;
+
+  std::string golden_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((proto_path + "test_include_id.golden.fbs").c_str(),
+                            false, &golden_file),
       true);
-  TEST_EQ(parser2.Parse(fbs.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs.c_str(), goldenfile.c_str());
 
-  // Parse proto with --oneof-union option.
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
+
+void proto_test_include_union(const std::string &proto_path,
+                              const std::string &proto_file,
+                              const std::string &import_proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
   opts.proto_oneof_union = true;
-  flatbuffers::Parser parser3(opts);
-  TEST_EQ(parser3.Parse(protofile.c_str(), include_directories), true);
 
-  // Generate fbs.
-  auto fbs_union = flatbuffers::GenerateFBS(parser3, "test");
-
-  // Ensure generated file is parsable.
-  flatbuffers::Parser parser4;
-  TEST_EQ(parser4.Parse(import_fbs.c_str(), nullptr, imported_fbs.c_str()),
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_include.golden.fbs").c_str(), false,
+              &golden_file),
           true);
-  TEST_EQ(parser4.Parse(fbs_union.c_str(), nullptr), true);
-  TEST_EQ_STR(fbs_union.c_str(), goldenunionfile.c_str());
+
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
+
+void proto_test_include_union_id(const std::string &proto_path,
+                                 const std::string &proto_file,
+                                 const std::string &import_proto_file) {
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+  opts.keep_proto_id = true;
+
+  std::string golden_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (proto_path + "test_union_include_id.golden.fbs").c_str(), false,
+              &golden_file),
+          true);
+
+  RunTest(opts, proto_path, proto_file, golden_file, import_proto_file);
+}
+
+void ParseCorruptedProto(const std::string &proto_path) {
+  const char *include_directories[] = { proto_path.c_str(), nullptr };
+
+  flatbuffers::IDLOptions opts;
+  opts.include_dependence_headers = true;
+  opts.proto_mode = true;
+  opts.proto_oneof_union = true;
+
+  std::string proto_file;
+
+  // Parse proto with non positive id.
+  {
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(
+        flatbuffers::LoadFile((proto_path + "non-positive-id.proto").c_str(),
+                              false, &proto_file),
+        true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+
+  // Parse proto with twice id.
+  {
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(flatbuffers::LoadFile((proto_path + "twice-id.proto").c_str(),
+                                  false, &proto_file),
+            true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+
+  // Parse proto with using reserved id.
+  {
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(flatbuffers::LoadFile((proto_path + "twice-id.proto").c_str(),
+                                  false, &proto_file),
+            true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+
+  // Parse proto with error on gap.
+  {
+    opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::ERROR;
+    flatbuffers::Parser parser(opts);
+    TEST_EQ(flatbuffers::LoadFile((proto_path + "test.proto").c_str(), false,
+                                  &proto_file),
+            true);
+    TEST_EQ(parser.Parse(proto_file.c_str(), include_directories), true);
+    auto fbs = flatbuffers::GenerateFBS(parser, "test");
+    TEST_EQ(fbs.empty(), true);
+  }
+}
+
+// Parse a .proto schema, output as .fbs
+void ParseProtoTest(const std::string &tests_data_path) {
+  auto proto_path = tests_data_path + "prototest/";
+  std::string proto_file;
+  TEST_EQ(
+      flatbuffers::LoadFile((tests_data_path + "prototest/test.proto").c_str(),
+                            false, &proto_file),
+      true);
+
+  std::string import_proto_file;
+  TEST_EQ(flatbuffers::LoadFile(
+              (tests_data_path + "prototest/imported.proto").c_str(), false,
+              &import_proto_file),
+          true);
+
+  proto_test(proto_path, proto_file);
+  proto_test_union(proto_path, proto_file);
+  proto_test_union_suffix(proto_path, proto_file);
+  proto_test_include(proto_path, proto_file, import_proto_file);
+  proto_test_include_union(proto_path, proto_file, import_proto_file);
+
+  proto_test_id(proto_path, proto_file);
+  proto_test_union_id(proto_path, proto_file);
+  proto_test_union_suffix_id(proto_path, proto_file);
+  proto_test_include_id(proto_path, proto_file, import_proto_file);
+  proto_test_include_union_id(proto_path, proto_file, import_proto_file);
+
+  ParseCorruptedProto(proto_path);
 }
 
 void ParseProtoBufAsciiTest() {

tests/proto_test.h

@@ -1,16 +1,30 @@
 #ifndef TESTS_PROTO_TEST_H
 #define TESTS_PROTO_TEST_H
 
+#include "flatbuffers/idl.h"
+
 #include <string>
 
 namespace flatbuffers {
 namespace tests {
 
-void ParseProtoTest(const std::string& tests_data_path);
-void ParseProtoTestWithSuffix(const std::string& tests_data_path);
-void ParseProtoTestWithIncludes(const std::string& tests_data_path);
-void ParseProtoBufAsciiTest();
+void RunTest(const flatbuffers::IDLOptions &opts, const std::string &proto_path, const std::string &proto_file,
+        const std::string &golden_file, const std::string import_proto_file = {});
+void proto_test(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union_suffix(const std::string &proto_path, const std::string &proto_file);
+void proto_test_include(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
+void proto_test_include_union(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
 
+void proto_test_id(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union_id(const std::string &proto_path, const std::string &proto_file);
+void proto_test_union_suffix_id(const std::string &proto_path, const std::string &proto_file);
+void proto_test_include_id(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
+void proto_test_include_union_id(const std::string &proto_path, const std::string &proto_file, const std::string &import_proto_file);
+
+void ParseCorruptedProto(const std::string &proto_path);
+void ParseProtoTest(const std::string& tests_data_path);
+void ParseProtoBufAsciiTest();
 
 }  // namespace tests
 }  // namespace flatbuffers

tests/prototest/GenerateProtoGoldens.sh

@@ -16,9 +16,16 @@
 
 pushd "$(dirname $0)" >/dev/null
 
-./../../flatc --proto test.proto && mv test.fbs test_include.golden
-./../../flatc --proto --gen-all test.proto && mv test.fbs test.golden
-./../../flatc --proto --oneof-union test.proto && mv test.fbs test_union_include.golden
-./../../flatc --proto --gen-all --oneof-union test.proto && mv test.fbs test_union.golden
-./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix.golden
-./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix.golden
+./../../flatc --proto test.proto && mv test.fbs test_include.golden.fbs
+./../../flatc --proto --gen-all test.proto && mv test.fbs test.golden.fbs
+./../../flatc --proto --oneof-union test.proto && mv test.fbs test_union_include.golden.fbs
+./../../flatc --proto --gen-all --oneof-union test.proto && mv test.fbs test_union.golden.fbs
+./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix.golden.fbs
+./../../flatc --proto --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix.golden.fbs
+
+./../../flatc --proto --keep-proto-id test.proto && mv test.fbs test_include_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all test.proto && mv test.fbs test_id.golden.fbs
+./../../flatc --proto --keep-proto-id --oneof-union test.proto && mv test.fbs test_union_include_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --oneof-union test.proto && mv test.fbs test_union_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --proto-namespace-suffix test_namespace_suffix test.proto && mv test.fbs test_suffix_id.golden.fbs
+./../../flatc --proto --keep-proto-id --gen-all --proto-namespace-suffix test_namespace_suffix --oneof-union test.proto && mv test.fbs test_union_suffix_id.golden.fbs

tests/prototest/non-positive-id.proto

@@ -0,0 +1,9 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has non-positive ids.
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  optional uint64 NonPositiveId = -1;
+}

tests/prototest/test.proto

@@ -24,6 +24,7 @@ message ProtoMessage {
   // Ignored non-doc comment.
   // A nested message declaration, will be moved to top level in .fbs
   message OtherMessage {
+    reserved 2, 9 to 11, 15;
     optional double a = 26;
     /// doc comment for b.
     optional float b = 32 [default = 3.14149];
@@ -40,7 +41,7 @@ message ProtoMessage {
   }
   optional int32 c = 12 [default = 16];
   optional int64 d = 1 [default = 0];
-  optional uint32 p = 1;
+  optional uint32 p = 40;
   optional uint64 e = 2;
   /// doc comment for f.
   optional sint32 f = 3 [default = -1];
@@ -55,7 +56,7 @@ message ProtoMessage {
   /// lines
   required string l = 10;
   optional bytes m = 11;
-  optional OtherMessage n = 12;
+  optional OtherMessage n = 41;
   repeated string o = 14;
   optional ImportedMessage z = 16;
   /// doc comment for r.

tests/prototest/test_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_include_id.golden.fbs

@@ -0,0 +1,85 @@
+// Generated from test.proto
+
+include "imported.fbs";
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test;
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_suffix_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+namespace proto.test.test_namespace_suffix;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.test_namespace_suffix;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 12);
+  d:long (id: 1);
+  p:uint (id: 21);
+  e:ulong (id: 2);
+  /// doc comment for f.
+  f:int = -1 (id: 3);
+  g:long (id: 4);
+  h:uint (id: 5);
+  q:ulong (id: 6);
+  i:int (id: 7);
+  j:long (id: 8);
+  /// doc comment for k.
+  k:bool (id: 9);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 10);
+  m:[ubyte] (id: 11);
+  n:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 22);
+  o:[string] (id: 13);
+  z:proto.test.test_namespace_suffix.ImportedMessage (id: 14);
+  /// doc comment for r.
+  r:proto.test.test_namespace_suffix.ProtoMessage_.Anonymous0 (id: 0);
+  outer_enum:proto.test.test_namespace_suffix.ProtoEnum (id: 15);
+  u:float = +inf (id: 16);
+  v:float = +inf (id: 17);
+  w:float = -inf (id: 18);
+  grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry] (id: 19);
+  other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry] (id: 20);
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table Anonymous0 {
+  /// doc comment for s.
+  s:proto.test.test_namespace_suffix.ImportedMessage (id: 0);
+  /// doc comment for t on 2
+  /// lines.
+  t:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 1);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_id.golden.fbs

@@ -0,0 +1,89 @@
+// Generated from test.proto
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_include_id.golden.fbs

@@ -0,0 +1,87 @@
+// Generated from test.proto
+
+include "imported.fbs";
+
+namespace proto.test;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test;
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/test_union_suffix_id.golden.fbs

@@ -0,0 +1,89 @@
+// Generated from test.proto
+
+namespace proto.test.test_namespace_suffix;
+
+/// Enum doc comment.
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  /// Enum 2nd value doc comment misaligned.
+  BAR = 5,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_;
+
+enum ProtoEnum : int {
+  NUL = 0,
+  FOO = 1,
+  BAR = 2,
+  BAZ = 3,
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+union RUnion {
+  /// doc comment for s.
+  proto.test.test_namespace_suffix.ImportedMessage,
+  /// doc comment for t on 2
+  /// lines.
+  proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage,
+}
+
+namespace proto.test.test_namespace_suffix;
+
+table ImportedMessage {
+  a:int (id: 0);
+}
+
+/// 2nd table doc comment with
+/// many lines.
+table ProtoMessage {
+  c:int = 16 (id: 13);
+  d:long (id: 2);
+  p:uint (id: 22);
+  e:ulong (id: 3);
+  /// doc comment for f.
+  f:int = -1 (id: 4);
+  g:long (id: 5);
+  h:uint (id: 6);
+  q:ulong (id: 7);
+  i:int (id: 8);
+  j:long (id: 9);
+  /// doc comment for k.
+  k:bool (id: 10);
+  /// doc comment for l on 2
+  /// lines
+  l:string (required,id: 11);
+  m:[ubyte] (id: 12);
+  n:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage (id: 23);
+  o:[string] (id: 14);
+  z:proto.test.test_namespace_suffix.ImportedMessage (id: 15);
+  /// doc comment for r.
+  r:proto.test.test_namespace_suffix.ProtoMessage_.RUnion (id: 1);
+  outer_enum:proto.test.test_namespace_suffix.ProtoEnum (id: 16);
+  u:float = +inf (id: 17);
+  v:float = +inf (id: 18);
+  w:float = -inf (id: 19);
+  grades:[proto.test.test_namespace_suffix.ProtoMessage_.GradesEntry] (id: 20);
+  other_message_map:[proto.test.test_namespace_suffix.ProtoMessage_.OtherMessageMapEntry] (id: 21);
+}
+
+namespace proto.test.test_namespace_suffix.ProtoMessage_;
+
+table OtherMessage {
+  a:double (id: 0);
+  /// doc comment for b.
+  b:float = 3.14149 (id: 1);
+  foo_bar_baz:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage_.ProtoEnum (id: 2);
+}
+
+table GradesEntry {
+  key:string (key);
+  value:float;
+}
+
+table OtherMessageMapEntry {
+  key:string (key);
+  value:proto.test.test_namespace_suffix.ProtoMessage_.OtherMessage;
+}
+

tests/prototest/twice-id.proto

@@ -0,0 +1,10 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has used an id twice
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  optional sint32 e = 2;
+  optional uint64 twice = 2;
+}

tests/prototest/use-reserved-id.proto

@@ -0,0 +1,10 @@
+// Sample .proto file that we can not translate to the corresponding .fbs because it has used ids from reserved ids.
+
+option some_option = is_ignored;
+
+package proto.test;
+
+message ProtoMessage {
+  reserved 200, 9 to 11, 1500;
+  optional sint32 reserved_id_usage = 10;
+}

tests/test.cpp

@@ -1542,8 +1542,6 @@ int FlatBufferTests(const std::string &tests_data_path) {
   FixedLengthArrayJsonTest(tests_data_path, true);
   ReflectionTest(tests_data_path, flatbuf.data(), flatbuf.size());
   ParseProtoTest(tests_data_path);
-  ParseProtoTestWithSuffix(tests_data_path);
-  ParseProtoTestWithIncludes(tests_data_path);
   EvolutionTest(tests_data_path);
   UnionDeprecationTest(tests_data_path);
   UnionVectorTest(tests_data_path);