From d47cd10d773bff7efcee6d7b9f13bf2356b707b3 Mon Sep 17 00:00:00 2001 From: razvanalex Date: Mon, 18 Dec 2023 08:18:11 +0200 Subject: [PATCH] Add Clear() for python Builder (#8186) Co-authored-by: Derek Bailey --- python/flatbuffers/builder.py | 14 ++++++++ tests/PythonTest.sh | 16 ++++----- tests/py_test.py | 68 ++++++++++++++++++++++++++++++----- 3 files changed, 82 insertions(+), 16 deletions(-) diff --git a/python/flatbuffers/builder.py b/python/flatbuffers/builder.py index f3b901b1e..c39cd283a 100644 --- a/python/flatbuffers/builder.py +++ b/python/flatbuffers/builder.py @@ -146,6 +146,20 @@ class Builder(object): ## @endcond self.finished = False + def Clear(self) -> None: + ## @cond FLATBUFFERS_INTERNAL + self.current_vtable = None + self.head = UOffsetTFlags.py_type(len(self.Bytes)) + self.minalign = 1 + self.objectEnd = None + self.vtables = {} + self.nested = False + self.forceDefaults = False + self.sharedStrings = {} + self.vectorNumElems = None + ## @endcond + self.finished = False + def Output(self): """Return the portion of the buffer that has been used for writing data. diff --git a/tests/PythonTest.sh b/tests/PythonTest.sh index fc64e9f5a..84e82019d 100755 --- a/tests/PythonTest.sh +++ b/tests/PythonTest.sh @@ -40,7 +40,7 @@ function run_tests() { JYTHONPATH=${runtime_library_dir}:${gen_code_path} \ COMPARE_GENERATED_TO_GO=0 \ COMPARE_GENERATED_TO_JAVA=0 \ - $1 py_test.py $2 $3 $4 $5 + $1 py_test.py $2 $3 $4 $5 $6 if [ $1 = python3 ]; then PYTHONDONTWRITEBYTECODE=1 \ PYTHONPATH=${runtime_library_dir}:${gen_code_path} \ @@ -52,12 +52,12 @@ function run_tests() { } # Run test suite with these interpreters. The arguments are benchmark counts. -run_tests python2.6 100 100 100 false -run_tests python2.7 100 100 100 false -run_tests python2.7 100 100 100 true -run_tests python3 100 100 100 false -run_tests python3 100 100 100 true -run_tests pypy 100 100 100 false +run_tests python2.6 100 100 100 100 false +run_tests python2.7 100 100 100 100 false +run_tests python2.7 100 100 100 100 true +run_tests python3 100 100 100 100 false +run_tests python3 100 100 100 100 true +run_tests pypy 100 100 100 100 false # NOTE: We'd like to support python2.5 in the future. @@ -77,7 +77,7 @@ if $(which coverage >/dev/null); then PYTHONDONTWRITEBYTECODE=1 \ PYTHONPATH=${runtime_library_dir}:${gen_code_path} \ - coverage run --source=flatbuffers,MyGame py_test.py 0 0 0 false > /dev/null + coverage run --source=flatbuffers,MyGame py_test.py 0 0 0 0 false > /dev/null echo cov_result=`coverage report --omit="*flatbuffers/vendor*,*py_test*" \ diff --git a/tests/py_test.py b/tests/py_test.py index dc7b734b5..7d6c0a379 100644 --- a/tests/py_test.py +++ b/tests/py_test.py @@ -2017,10 +2017,10 @@ class TestByteLayout(unittest.TestCase): ]) -def make_monster_from_generated_code(sizePrefix=False, file_identifier=None): +def make_monster_from_generated_code(b=None, sizePrefix=False, file_identifier=None): """ Use generated code to build the example Monster. """ - - b = flatbuffers.Builder(0) + if b is None: + b = flatbuffers.Builder(0) string = b.CreateString('MyMonster') test1 = b.CreateString('test1') test2 = b.CreateString('test2') @@ -2767,6 +2767,43 @@ class TestNestedUnionTables(unittest.TestCase): self.assertEqual(nestUnionDecodeTFromBuf2.data.test3.b, nestUnion.data.test3.b) +class TestBuilderClear(unittest.TestCase): + + def test_consistency(self): + """ Checks if clear resets the state of the builder. """ + b = flatbuffers.Builder(0) + + # Add some data to the buffer + off1 = b.CreateString('a' * 1024) + want = b.Bytes[b.Head():] + + # Reset the builder + b.Clear() + + # Readd the same data into the buffer + off2 = b.CreateString('a' * 1024) + got = b.Bytes[b.Head():] + + # Expect to get the same data into the buffer at the same offset + self.assertEqual(off1, off2) + self.assertEqual(want, got) + + def test_repeated_clear_after_builder_reuse(self): + init_buf = None + init_off = None + b = flatbuffers.Builder(0) + + for i in range(5): + buf, off = make_monster_from_generated_code(b) + b.Clear() + + if i > 0: + self.assertEqual(init_buf, buf) + self.assertEqual(init_off, off) + else: + init_buf = buf + init_off = off + def CheckAgainstGoldDataGo(): try: gen_buf, gen_off = make_monster_from_generated_code() @@ -2912,6 +2949,18 @@ def BenchmarkMakeMonsterFromGeneratedCode(count, length): (count, length, duration, rate, data_rate))) +def BenchmarkBuilderClear(count, length): + b = flatbuffers.Builder(length) + duration = timeit.timeit(stmt=lambda: make_monster_from_generated_code(b), + number=count) + rate = float(count) / duration + data = float(length * count) / float(1024 * 1024) + data_rate = data / float(duration) + + print(('built %d %d-byte flatbuffers (reused buffer) in %.2fsec:' + ' %.2f/sec, %.2fMB/sec' % (count, length, duration, rate, data_rate))) + + def backward_compatible_run_tests(**kwargs): if PY_VERSION < (2, 6): sys.stderr.write('Python version less than 2.6 are not supported') @@ -2940,10 +2989,10 @@ def backward_compatible_run_tests(**kwargs): def main(): import os import sys - if not len(sys.argv) == 5: + if not len(sys.argv) == 6: sys.stderr.write('Usage: %s ' ' ' - '\n' % sys.argv[0]) + ' \n' % sys.argv[0]) sys.stderr.write(' Provide COMPARE_GENERATED_TO_GO=1 to check' 'for bytewise comparison to Go data.\n') sys.stderr.write(' Provide COMPARE_GENERATED_TO_JAVA=1 to check' @@ -2951,9 +3000,9 @@ def main(): sys.stderr.flush() sys.exit(1) - kwargs = dict(argv=sys.argv[:-4]) + kwargs = dict(argv=sys.argv[:-5]) - create_namespace_shortcut(sys.argv[4].lower() == 'true') + create_namespace_shortcut(sys.argv[5].lower() == 'true') # show whether numpy is present, as it changes the test logic: try: @@ -2978,6 +3027,7 @@ def main(): bench_vtable = int(sys.argv[1]) bench_traverse = int(sys.argv[2]) bench_build = int(sys.argv[3]) + bench_clear = int(sys.argv[4]) if bench_vtable: BenchmarkVtableDeduplication(bench_vtable) if bench_traverse: @@ -2986,7 +3036,9 @@ def main(): if bench_build: buf, off = make_monster_from_generated_code() BenchmarkMakeMonsterFromGeneratedCode(bench_build, len(buf)) - + if bench_clear: + buf, off = make_monster_from_generated_code() + BenchmarkBuilderClear(bench_build, len(buf)) if __name__ == '__main__': main()