Replace filenames in reflection with filenames+includes. (#6703)

* Replace filenames in reflection with filenames+includes.

This is needed for some use cases and may be just useful metadata.

* deser files_included_per_file_

* check project_root

* fix bazel

* git clang format

Co-authored-by: Casper Neo <cneo@google.com>

include/flatbuffers/reflection_generated.h

@@ -32,6 +32,9 @@ struct RPCCallBuilder;
 struct Service;
 struct ServiceBuilder;
 
+struct SchemaFile;
+struct SchemaFileBuilder;
+
 struct Schema;
 struct SchemaBuilder;
 
@@ -1126,6 +1129,84 @@ inline flatbuffers::Offset<Service> CreateServiceDirect(
       declaration_file__);
 }
 
+/// File specific information.
+/// Symbols declared within a file may be recovered by iterating over all
+/// symbols and examining the `declaration_file` field.
+struct SchemaFile FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef SchemaFileBuilder Builder;
+  enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
+    VT_FILENAME = 4,
+    VT_INCLUDED_FILENAMES = 6
+  };
+  /// Filename, relative to project root.
+  const flatbuffers::String *filename() const {
+    return GetPointer<const flatbuffers::String *>(VT_FILENAME);
+  }
+  bool KeyCompareLessThan(const SchemaFile *o) const {
+    return *filename() < *o->filename();
+  }
+  int KeyCompareWithValue(const char *val) const {
+    return strcmp(filename()->c_str(), val);
+  }
+  /// Names of included files, relative to project root.
+  const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> *included_filenames() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> *>(VT_INCLUDED_FILENAMES);
+  }
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           VerifyOffsetRequired(verifier, VT_FILENAME) &&
+           verifier.VerifyString(filename()) &&
+           VerifyOffset(verifier, VT_INCLUDED_FILENAMES) &&
+           verifier.VerifyVector(included_filenames()) &&
+           verifier.VerifyVectorOfStrings(included_filenames()) &&
+           verifier.EndTable();
+  }
+};
+
+struct SchemaFileBuilder {
+  typedef SchemaFile Table;
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  void add_filename(flatbuffers::Offset<flatbuffers::String> filename) {
+    fbb_.AddOffset(SchemaFile::VT_FILENAME, filename);
+  }
+  void add_included_filenames(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>> included_filenames) {
+    fbb_.AddOffset(SchemaFile::VT_INCLUDED_FILENAMES, included_filenames);
+  }
+  explicit SchemaFileBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  flatbuffers::Offset<SchemaFile> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<SchemaFile>(end);
+    fbb_.Required(o, SchemaFile::VT_FILENAME);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<SchemaFile> CreateSchemaFile(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    flatbuffers::Offset<flatbuffers::String> filename = 0,
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>> included_filenames = 0) {
+  SchemaFileBuilder builder_(_fbb);
+  builder_.add_included_filenames(included_filenames);
+  builder_.add_filename(filename);
+  return builder_.Finish();
+}
+
+inline flatbuffers::Offset<SchemaFile> CreateSchemaFileDirect(
+    flatbuffers::FlatBufferBuilder &_fbb,
+    const char *filename = nullptr,
+    const std::vector<flatbuffers::Offset<flatbuffers::String>> *included_filenames = nullptr) {
+  auto filename__ = filename ? _fbb.CreateString(filename) : 0;
+  auto included_filenames__ = included_filenames ? _fbb.CreateVector<flatbuffers::Offset<flatbuffers::String>>(*included_filenames) : 0;
+  return reflection::CreateSchemaFile(
+      _fbb,
+      filename__,
+      included_filenames__);
+}
+
 struct Schema FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   typedef SchemaBuilder Builder;
   enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {
@@ -1161,8 +1242,8 @@ struct Schema FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   }
   /// All the files used in this compilation. Files are relative to where
   /// flatc was invoked.
-  const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> *fbs_files() const {
-    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> *>(VT_FBS_FILES);
+  const flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>> *fbs_files() const {
+    return GetPointer<const flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>> *>(VT_FBS_FILES);
   }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
@@ -1184,7 +1265,7 @@ struct Schema FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
            VerifyField<uint64_t>(verifier, VT_ADVANCED_FEATURES) &&
            VerifyOffset(verifier, VT_FBS_FILES) &&
            verifier.VerifyVector(fbs_files()) &&
-           verifier.VerifyVectorOfStrings(fbs_files()) &&
+           verifier.VerifyVectorOfTables(fbs_files()) &&
            verifier.EndTable();
   }
 };
@@ -1214,7 +1295,7 @@ struct SchemaBuilder {
   void add_advanced_features(reflection::AdvancedFeatures advanced_features) {
     fbb_.AddElement<uint64_t>(Schema::VT_ADVANCED_FEATURES, static_cast<uint64_t>(advanced_features), 0);
   }
-  void add_fbs_files(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>> fbs_files) {
+  void add_fbs_files(flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>>> fbs_files) {
     fbb_.AddOffset(Schema::VT_FBS_FILES, fbs_files);
   }
   explicit SchemaBuilder(flatbuffers::FlatBufferBuilder &_fbb)
@@ -1239,7 +1320,7 @@ inline flatbuffers::Offset<Schema> CreateSchema(
     flatbuffers::Offset<reflection::Object> root_table = 0,
     flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<reflection::Service>>> services = 0,
     reflection::AdvancedFeatures advanced_features = static_cast<reflection::AdvancedFeatures>(0),
-    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>> fbs_files = 0) {
+    flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>>> fbs_files = 0) {
   SchemaBuilder builder_(_fbb);
   builder_.add_advanced_features(advanced_features);
   builder_.add_fbs_files(fbs_files);
@@ -1261,13 +1342,13 @@ inline flatbuffers::Offset<Schema> CreateSchemaDirect(
     flatbuffers::Offset<reflection::Object> root_table = 0,
     std::vector<flatbuffers::Offset<reflection::Service>> *services = nullptr,
     reflection::AdvancedFeatures advanced_features = static_cast<reflection::AdvancedFeatures>(0),
-    const std::vector<flatbuffers::Offset<flatbuffers::String>> *fbs_files = nullptr) {
+    std::vector<flatbuffers::Offset<reflection::SchemaFile>> *fbs_files = nullptr) {
   auto objects__ = objects ? _fbb.CreateVectorOfSortedTables<reflection::Object>(objects) : 0;
   auto enums__ = enums ? _fbb.CreateVectorOfSortedTables<reflection::Enum>(enums) : 0;
   auto file_ident__ = file_ident ? _fbb.CreateString(file_ident) : 0;
   auto file_ext__ = file_ext ? _fbb.CreateString(file_ext) : 0;
   auto services__ = services ? _fbb.CreateVectorOfSortedTables<reflection::Service>(services) : 0;
-  auto fbs_files__ = fbs_files ? _fbb.CreateVector<flatbuffers::Offset<flatbuffers::String>>(*fbs_files) : 0;
+  auto fbs_files__ = fbs_files ? _fbb.CreateVectorOfSortedTables<reflection::SchemaFile>(fbs_files) : 0;
   return reflection::CreateSchema(
       _fbb,
       objects__,

reflection/reflection.fbs

@@ -116,6 +116,16 @@ enum AdvancedFeatures : ulong (bit_flags) {
     DefaultVectorsAndStrings,
 }
 
+/// File specific information.
+/// Symbols declared within a file may be recovered by iterating over all
+/// symbols and examining the `declaration_file` field.
+table SchemaFile {
+  /// Filename, relative to project root.
+  filename:string (required, key);
+  /// Names of included files, relative to project root.
+  included_filenames:[string];
+}
+
 table Schema {
     objects:[Object] (required);    // Sorted.
     enums:[Enum] (required);        // Sorted.
@@ -126,7 +136,7 @@ table Schema {
     advanced_features:AdvancedFeatures;
     /// All the files used in this compilation. Files are relative to where
     /// flatc was invoked.
-    fbs_files:[string];
+    fbs_files:[SchemaFile];         // Sorted.
 }
 
 root_type Schema;

src/idl_parser.cpp

@@ -3530,10 +3530,29 @@ void Parser::Serialize() {
     const std::string *file = (*it)->declaration_file;
     if (file) files.insert(*file);
   }
-  // TODO(caspern): CreateVectorOfSharedStrings
-  std::vector<Offset<flatbuffers::String>> file_offsets;
-  for (auto it = files.begin(); it != files.end(); it++) {
-    file_offsets.push_back(builder_.CreateSharedString(*it));
+
+  // Create Schemafiles vector of tables.
+  flatbuffers::Offset<
+      flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>>>
+      schema_files__;
+  if (!opts.project_root.empty()) {
+    std::vector<Offset<reflection::SchemaFile>> schema_files;
+    std::vector<Offset<flatbuffers::String>> included_files;
+    for (auto f = files_included_per_file_.begin();
+         f != files_included_per_file_.end(); f++) {
+      const auto filename__ = builder_.CreateSharedString(
+          RelativeToRootPath(opts.project_root, f->first));
+      for (auto i = f->second.begin(); i != f->second.end(); i++) {
+        included_files.push_back(builder_.CreateSharedString(
+            RelativeToRootPath(opts.project_root, *i)));
+      }
+      const auto included_files__ = builder_.CreateVector(included_files);
+      included_files.clear();
+
+      schema_files.push_back(
+          reflection::CreateSchemaFile(builder_, filename__, included_files__));
+    }
+    schema_files__ = builder_.CreateVectorOfSortedTables(&schema_files);
   }
 
   const auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets);
@@ -3541,11 +3560,11 @@ void Parser::Serialize() {
   const auto fiid__ = builder_.CreateString(file_identifier_);
   const auto fext__ = builder_.CreateString(file_extension_);
   const auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets);
-  const auto files__ = builder_.CreateVector(file_offsets);
   const auto schema_offset = reflection::CreateSchema(
       builder_, objs__, enum__, fiid__, fext__,
       (root_struct_def_ ? root_struct_def_->serialized_location : 0), serv__,
-      static_cast<reflection::AdvancedFeatures>(advanced_features_), files__);
+      static_cast<reflection::AdvancedFeatures>(advanced_features_),
+      schema_files__);
   if (opts.size_prefixed) {
     builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier());
   } else {
@@ -3967,6 +3986,16 @@ bool Parser::Deserialize(const reflection::Schema *schema) {
     }
   }
   advanced_features_ = schema->advanced_features();
+
+  if (schema->fbs_files())
+    for (auto s = schema->fbs_files()->begin(); s != schema->fbs_files()->end();
+         ++s) {
+      for (auto f = s->included_filenames()->begin();
+           f != s->included_filenames()->end(); ++f) {
+        files_included_per_file_[s->filename()->str()].insert(f->str());
+      }
+    }
+
   return true;
 }
 

tests/test.cpp

@@ -919,6 +919,7 @@ void ReflectionTest(uint8_t *flatbuf, size_t length) {
   // Make sure the schema is what we expect it to be.
   auto &schema = *reflection::GetSchema(bfbsfile.c_str());
   auto root_table = schema.root_table();
+
   // Check the declaration files.
   TEST_EQ_STR(root_table->name()->c_str(), "MyGame.Example.Monster");
   TEST_EQ_STR(root_table->declaration_file()->c_str(), "//monster_test.fbs");
@@ -935,12 +936,37 @@ void ReflectionTest(uint8_t *flatbuf, size_t length) {
                   ->declaration_file()
                   ->c_str(),
               "//include_test/sub/include_test2.fbs");
+
+  // Check scheam filenames and their includes.
   TEST_EQ(schema.fbs_files()->size(), 3);
-  TEST_EQ_STR(schema.fbs_files()->Get(0)->c_str(),
+
+  const auto fbs0 = schema.fbs_files()->Get(0);
+  TEST_EQ_STR(fbs0->filename()->c_str(), "//include_test/include_test1.fbs");
+  const auto fbs0_includes = fbs0->included_filenames();
+  TEST_EQ(fbs0_includes->size(), 2);
+
+  // TODO(caspern): Should we force or disallow inclusion of self?
+  TEST_EQ_STR(fbs0_includes->Get(0)->c_str(),
               "//include_test/include_test1.fbs");
-  TEST_EQ_STR(schema.fbs_files()->Get(1)->c_str(),
+  TEST_EQ_STR(fbs0_includes->Get(1)->c_str(),
               "//include_test/sub/include_test2.fbs");
-  TEST_EQ_STR(schema.fbs_files()->Get(2)->c_str(), "//monster_test.fbs");
+
+  const auto fbs1 = schema.fbs_files()->Get(1);
+  TEST_EQ_STR(fbs1->filename()->c_str(),
+              "//include_test/sub/include_test2.fbs");
+  const auto fbs1_includes = fbs1->included_filenames();
+  TEST_EQ(fbs1_includes->size(), 2);
+  TEST_EQ_STR(fbs1_includes->Get(0)->c_str(),
+              "//include_test/include_test1.fbs");
+  TEST_EQ_STR(fbs1_includes->Get(1)->c_str(),
+              "//include_test/sub/include_test2.fbs");
+
+  const auto fbs2 = schema.fbs_files()->Get(2);
+  TEST_EQ_STR(fbs2->filename()->c_str(), "//monster_test.fbs");
+  const auto fbs2_includes = fbs2->included_filenames();
+  TEST_EQ(fbs2_includes->size(), 1);
+  TEST_EQ_STR(fbs2_includes->Get(0)->c_str(),
+              "//include_test/include_test1.fbs");
 
   // Check Root table fields
   auto fields = root_table->fields();