From 11cdc38145e72fa17adb1673390a1777e57988fc Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Mon, 1 Aug 2022 12:12:02 -0700 Subject: [PATCH 001/119] refactor versioning tests into their own class Signed-off-by: ssteinbach --- tests/test_serializable_object.py | 90 ++++++++++++++++--------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index d9079bcd2..8d9889449 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -78,48 +78,6 @@ class Foo(otio.core.SerializableObjectWithMetadata): self.assertEqual(Foo, type(foo_copy)) - def test_schema_versioning(self): - @otio.core.register_type - class FakeThing(otio.core.SerializableObject): - _serializable_label = "Stuff.1" - foo_two = otio.core.serializable_field("foo_2", doc="test") - ft = FakeThing() - - self.assertEqual(ft.schema_name(), "Stuff") - self.assertEqual(ft.schema_version(), 1) - - with self.assertRaises(otio.exceptions.UnsupportedSchemaError): - otio.core.instance_from_schema( - "Stuff", - 2, - {"foo": "bar"} - ) - - ft = otio.core.instance_from_schema("Stuff", 1, {"foo": "bar"}) - self.assertEqual(ft._dynamic_fields['foo'], "bar") - - @otio.core.register_type - class FakeThing(otio.core.SerializableObject): - _serializable_label = "NewStuff.4" - foo_two = otio.core.serializable_field("foo_2") - - @otio.core.upgrade_function_for(FakeThing, 2) - def upgrade_one_to_two(_data_dict): - return {"foo_2": _data_dict["foo"]} - - @otio.core.upgrade_function_for(FakeThing, 3) - def upgrade_one_to_two_three(_data_dict): - return {"foo_3": _data_dict["foo_2"]} - - ft = otio.core.instance_from_schema("NewStuff", 1, {"foo": "bar"}) - self.assertEqual(ft._dynamic_fields['foo_3'], "bar") - - ft = otio.core.instance_from_schema("NewStuff", 3, {"foo_2": "bar"}) - self.assertEqual(ft._dynamic_fields['foo_3'], "bar") - - ft = otio.core.instance_from_schema("NewStuff", 4, {"foo_3": "bar"}) - self.assertEqual(ft._dynamic_fields['foo_3'], "bar") - def test_equality(self): o1 = otio.core.SerializableObject() o2 = otio.core.SerializableObject() @@ -177,5 +135,53 @@ def test_cycle_detection(self): o.clone() +class VersioningTests(unittest.TestCase, otio_test_utils.OTIOAssertions): + def test_schema_versioning(self): + """ test basic upgrade function and unsupported schema error """ + @otio.core.register_type + class FakeThing(otio.core.SerializableObject): + _serializable_label = "Stuff.1" + foo_two = otio.core.serializable_field("foo_2", doc="test") + ft = FakeThing() + + self.assertEqual(ft.schema_name(), "Stuff") + self.assertEqual(ft.schema_version(), 1) + + with self.assertRaises(otio.exceptions.UnsupportedSchemaError): + otio.core.instance_from_schema( + "Stuff", + 2, + {"foo": "bar"} + ) + + ft = otio.core.instance_from_schema("Stuff", 1, {"foo": "bar"}) + self.assertEqual(ft._dynamic_fields['foo'], "bar") + + def test_upgrading_skips_versions(self): + """ test that the upgrading system skips versions that don't have + upgrade functions""" + + @otio.core.register_type + class FakeThing(otio.core.SerializableObject): + _serializable_label = "NewStuff.4" + foo_two = otio.core.serializable_field("foo_2") + + @otio.core.upgrade_function_for(FakeThing, 2) + def upgrade_one_to_two(_data_dict): + return {"foo_2": _data_dict["foo"]} + + @otio.core.upgrade_function_for(FakeThing, 3) + def upgrade_one_to_two_three(_data_dict): + return {"foo_3": _data_dict["foo_2"]} + + ft = otio.core.instance_from_schema("NewStuff", 1, {"foo": "bar"}) + self.assertEqual(ft._dynamic_fields['foo_3'], "bar") + + ft = otio.core.instance_from_schema("NewStuff", 3, {"foo_2": "bar"}) + self.assertEqual(ft._dynamic_fields['foo_3'], "bar") + + ft = otio.core.instance_from_schema("NewStuff", 4, {"foo_3": "bar"}) + self.assertEqual(ft._dynamic_fields['foo_3'], "bar") + if __name__ == '__main__': unittest.main() From 5cb6143cea39fa9688a07710311290d327d195c0 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 12 Aug 2022 11:51:32 -0700 Subject: [PATCH 002/119] DRY cleanup in the serializer before other stuff Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 46 +++++++++------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 0c9ad8c68..0d81e7220 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -930,41 +930,25 @@ serialize_json_to_string( { OTIO_rapidjson::StringBuffer s; - if (indent < 0) - { - OTIO_rapidjson::Writer< - decltype(s), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(s); - JSONEncoder json_encoder(json_writer); + OTIO_rapidjson::PrettyWriter< + decltype(s), + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::CrtAllocator, + OTIO_rapidjson::kWriteNanAndInfFlag> + json_writer(s); - if (!SerializableObject::Writer::write_root( - value, json_encoder, error_status)) - { - return std::string(); - } - } - else - { - OTIO_rapidjson::PrettyWriter< - decltype(s), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(s); - - JSONEncoder json_encoder(json_writer); + JSONEncoder json_encoder(json_writer); + if (indent >= 0) + { json_writer.SetIndent(' ', indent); - if (!SerializableObject::Writer::write_root( + } + + if (!SerializableObject::Writer::write_root( value, json_encoder, error_status)) - { - return std::string(); - } + { + return std::string(); } return std::string(s.GetString()); From c52aedeaf461d1d38dab00d11496c2e1839e78f1 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 12 Aug 2022 11:56:54 -0700 Subject: [PATCH 003/119] DRY reduction in the json FILE serializer Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 38 ++++++++++------------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 0d81e7220..af938585d 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -983,35 +983,23 @@ serialize_json_to_file( OTIO_rapidjson::OStreamWrapper osw(os); bool status; - if (indent < 0) - { - OTIO_rapidjson::Writer< - decltype(osw), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(osw); - JSONEncoder json_encoder(json_writer); - status = SerializableObject::Writer::write_root( - value, json_encoder, error_status); - } - else - { - OTIO_rapidjson::PrettyWriter< - decltype(osw), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(osw); - JSONEncoder json_encoder(json_writer); + OTIO_rapidjson::PrettyWriter< + decltype(osw), + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::CrtAllocator, + OTIO_rapidjson::kWriteNanAndInfFlag> + json_writer(osw); + JSONEncoder json_encoder(json_writer); + if (indent >= 0) + { json_writer.SetIndent(' ', indent); - status = SerializableObject::Writer::write_root( - value, json_encoder, error_status); } + status = SerializableObject::Writer::write_root( + value, json_encoder, error_status); + return status; } From 9151fc0896551121093a23a192622f6b65e34181 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sat, 13 Aug 2022 21:28:15 -0700 Subject: [PATCH 004/119] Add io_perf_test to repo Signed-off-by: ssteinbach --- examples/CMakeLists.txt | 1 + examples/io_perf_test.cpp | 67 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 examples/io_perf_test.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index de02cade0..d42ac1375 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -8,6 +8,7 @@ include_directories(${PROJECT_SOURCE_DIR}/src list(APPEND examples conform) list(APPEND examples flatten_video_tracks) list(APPEND examples summarize_timing) +list(APPEND examples io_perf_test) if(OTIO_PYTHON_INSTALL) list(APPEND examples python_adapters_child_process) list(APPEND examples python_adapters_embed) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp new file mode 100644 index 000000000..2e97c90a3 --- /dev/null +++ b/examples/io_perf_test.cpp @@ -0,0 +1,67 @@ +#include + +#include "util.h" +#include +#include +#include +#include + +namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; + +using chrono_time_point = std::chrono::steady_clock::time_point; + +const void +print_elapsed_time( + const std::string& message, + const chrono_time_point& begin, + const chrono_time_point& end +) +{ + const auto dur = ( + std::chrono::duration_cast( + end - begin + ).count() + ); + std::cout << message << ": " << dur/1000000.0 << " [s]" << std::endl; +} + +int +main(int argc, char *argv[]) +{ + if (argc < 2) { + std::cerr << "usage: otio_io_perf_test path/to/timeline.otio"; + std::cerr << std::endl; + return 1; + } + + otio::ErrorStatus err; + otio::any tl; + std::string fname = std::string(argv[1]); + + chrono_time_point begin = std::chrono::steady_clock::now(); + otio::SerializableObject::Retainer timeline( + dynamic_cast( + otio::Timeline::from_json_file(argv[1], &err) + ) + ); + chrono_time_point end = std::chrono::steady_clock::now(); + if (!timeline) + { + examples::print_error(err); + return 1; + } + + print_elapsed_time("deserialize_json_from_file", begin, end); + + begin = std::chrono::steady_clock::now(); + const std::string result = timeline.value->to_json_string(&err); + if (otio::is_error(err)) + { + examples::print_error(err); + return 1; + } + end = std::chrono::steady_clock::now(); + print_elapsed_time("serialize_json_to_string", begin, end); + + return 0; +} From 05d5b4a40aa4b63a878e4be98272ec4a316c3092 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 17 Aug 2022 11:04:45 -0700 Subject: [PATCH 005/119] Add a call w/ downgrade manifest to io_perf_test Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 2e97c90a3..3ff8c6238 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -54,7 +54,14 @@ main(int argc, char *argv[]) print_elapsed_time("deserialize_json_from_file", begin, end); begin = std::chrono::steady_clock::now(); - const std::string result = timeline.value->to_json_string(&err); + otio::schema_version_map downgrade_version_manifest = { + {"Clip", 1} + }; + const std::string result = timeline.value->to_json_string( + &err, + &downgrade_version_manifest + ); + if (otio::is_error(err)) { examples::print_error(err); @@ -63,5 +70,11 @@ main(int argc, char *argv[]) end = std::chrono::steady_clock::now(); print_elapsed_time("serialize_json_to_string", begin, end); + timeline.value->to_json_file( + "/var/tmp/test.otio", + &err, + &downgrade_version_manifest + ); + return 0; } From df297aac8d89ea69be5a5046a6e06b7a48ab9b9a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 17 Aug 2022 11:05:22 -0700 Subject: [PATCH 006/119] add anydictionary convienence functions - get_default, set_default, has_key Signed-off-by: ssteinbach --- src/opentimelineio/anyDictionary.h | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/opentimelineio/anyDictionary.h b/src/opentimelineio/anyDictionary.h index 3d9be59a4..3533e4532 100644 --- a/src/opentimelineio/anyDictionary.h +++ b/src/opentimelineio/anyDictionary.h @@ -8,6 +8,7 @@ #include #include +#include #include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { @@ -122,6 +123,46 @@ class AnyDictionary : private std::map map::swap(other); } + /// @TODO: need the right policy for accessing/returning membors + template + containedType + get_default( + const std::string& key, + const containedType& default_value + ) + { + any value_any = (*this)[key]; + if (value_any.type() == typeid(containedType)) { + return any_cast(value_any); + } else { + return default_value; + } + } + + inline bool + has_key( + const std::string& key + ) + { + return (this->find(key) != this->end()); + } + + template + containedType + set_default( + const std::string& key, + const containedType& default_value + ) + { + if (this->has_key(key)) { + return this->get_default(key, default_value); + } else { + (*this)[key] = default_value; + return this->get_default(key, default_value); + } + } + + using map::empty; using map::max_size; using map::size; From 426c1282d14438f321f565e8dd8ad9cf206ec19f Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 17 Aug 2022 11:27:23 -0700 Subject: [PATCH 007/119] Checkpoint broken Writer approach - this version intercepts the Writer::Write call and converts that to an any dictionary to be downgraded. I think I need to convert that + all children recursively to any dictionaries (maybe using the encoder approach again?) so that children are also changed. This version doesn't work but does pass quite a few tests. Signed-off-by: ssteinbach --- src/opentimelineio/serializableObject.cpp | 24 +- src/opentimelineio/serializableObject.h | 24 +- src/opentimelineio/serialization.cpp | 410 +++++++++++++++--- src/opentimelineio/serialization.h | 23 +- src/opentimelineio/typeRegistry.cpp | 63 ++- src/opentimelineio/typeRegistry.h | 32 ++ .../opentimelineio-bindings/otio_bindings.cpp | 61 ++- .../otio_serializableObjects.cpp | 4 +- .../opentimelineio/adapters/otio_json.py | 19 +- .../opentimelineio/core/__init__.py | 56 ++- .../opentimelineio/plugins/manifest.py | 8 + 11 files changed, 636 insertions(+), 88 deletions(-) diff --git a/src/opentimelineio/serializableObject.cpp b/src/opentimelineio/serializableObject.cpp index fb749bd1e..0e583ea4b 100644 --- a/src/opentimelineio/serializableObject.cpp +++ b/src/opentimelineio/serializableObject.cpp @@ -112,18 +112,34 @@ SerializableObject::is_unknown_schema() const } std::string -SerializableObject::to_json_string(ErrorStatus* error_status, int indent) const +SerializableObject::to_json_string( + ErrorStatus* error_status, + optionaldowngrade_version_manifest, + int indent +) const { return serialize_json_to_string( - any(Retainer<>(this)), error_status, indent); + any(Retainer<>(this)), + downgrade_version_manifest, + error_status, + indent + ); } bool SerializableObject::to_json_file( - std::string const& file_name, ErrorStatus* error_status, int indent) const + std::string const& file_name, + ErrorStatus* error_status, + optionaldowngrade_version_manifest, + int indent) const { return serialize_json_to_file( - any(Retainer<>(this)), file_name, error_status, indent); + any(Retainer<>(this)), + file_name, + downgrade_version_manifest, + error_status, + indent + ); } SerializableObject* diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index 2df315859..6c3618381 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -17,9 +17,12 @@ #include #include +#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { +using schema_version_map = std::unordered_map; + class SerializableObject { public: @@ -42,12 +45,18 @@ class SerializableObject */ bool possibly_delete(); - bool to_json_file( + bool + to_json_file( std::string const& file_name, ErrorStatus* error_status = nullptr, + optionaldowngrade_version_manifest = {}, int indent = 4) const; + std::string - to_json_string(ErrorStatus* error_status = nullptr, int indent = 4) const; + to_json_string( + ErrorStatus* error_status = nullptr, + optionaldowngrade_version_manifest = {}, + int indent = 4) const; static SerializableObject* from_json_file( std::string const& file_name, ErrorStatus* error_status = nullptr); @@ -394,6 +403,7 @@ class SerializableObject static bool write_root( any const& value, class Encoder& encoder, + optional downgrade_version_manifest= {}, ErrorStatus* error_status = nullptr); void write(std::string const& key, bool value); @@ -502,8 +512,13 @@ class SerializableObject } ///@} - Writer(class Encoder& encoder) - : _encoder(encoder) + Writer( + class Encoder& encoder, + optional downgrade_version_manifest + ) + : _encoder(encoder), + _downgrade_version_manifest(downgrade_version_manifest) + { _build_dispatch_tables(); } @@ -533,6 +548,7 @@ class SerializableObject std::map _next_id_for_type; class Encoder& _encoder; + optional _downgrade_version_manifest; friend class SerializableObject; }; diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index af938585d..921a12797 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -2,6 +2,8 @@ // Copyright Contributors to the OpenTimelineIO project #include "opentimelineio/serializableObject.h" +#include "opentimelineio/serialization.h" +#include "opentimelineio/anyDictionary.h" #include "opentimelineio/unknownSchema.h" #include "stringUtils.h" @@ -90,11 +92,21 @@ class Encoder class CloningEncoder : public Encoder { public: - CloningEncoder(bool actually_clone) + enum class ResultObjectPolicy { + CloneBackToSerializableObject = 0, + MathTypesConcreteAnyDictionaryResult, + OnlyAnyDictionary, + }; + + CloningEncoder( + CloningEncoder::ResultObjectPolicy result_object_policy, + optional downgrade_version_manifest = {} + ) : + _result_object_policy(result_object_policy), + _downgrade_version_manifest(downgrade_version_manifest) { using namespace std::placeholders; _error_function = std::bind(&CloningEncoder::_error, this, _1); - _actually_clone = actually_clone; } virtual ~CloningEncoder() {} @@ -155,20 +167,85 @@ class CloningEncoder : public Encoder void write_value(double value) { _store(any(value)); } - void write_value(RationalTime const& value) { _store(any(value)); } - - void write_value(TimeRange const& value) { _store(any(value)); } - - void write_value(TimeTransform const& value) { _store(any(value)); } + // @{ @TODO: these three the json serializer knows how to dereference them + // ... probably better to scooch that into this class... EXCEPT + // the equivalence test wants to use their == methods + void + write_value(RationalTime const& value) + { + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { + AnyDictionary result; + result["OTIO_SCHEMA"] = "RationalTime.1"; + result["value"] = value.value(); + result["rate"] = value.rate(); + _store(any(std::move(result))); + } else { + _store(any(value)); + } + } + void + write_value(TimeRange const& value) { + + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { + AnyDictionary result; + result["OTIO_SCHEMA"] = "TimeRange.1"; + result["duration"] = value.duration(); + result["start_time"] = value.start_time(); + _store(any(std::move(result))); + } else { + _store(any(value)); + } + } + void write_value(TimeTransform const& value) { + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { + AnyDictionary result; + result["OTIO_SCHEMA"] = "TimeTransform.1"; + result["offset"] = value.offset(); + result["rate"] = value.rate(); + result["scale"] = value.scale(); + _store(any(std::move(result))); + } else { + _store(any(value)); + } + } void write_value(SerializableObject::ReferenceId value) { - _store(any(value)); + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { + AnyDictionary result; + result["OTIO_SCHEMA"] = "SerializableObjectRef.1"; + result["id"] = value.id.c_str(); + _store(any(std::move(result))); + } else { + _store(any(value)); + } + _store(any(value)); } + void write_value(Imath::V2d const& value) { + + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { + AnyDictionary result; + result["OTIO_SCHEMA"] = "V2d.1"; + result["x"] = value.x; + result["y"] = value.y; + _store(any(std::move(result))); + } else { + _store(any(value)); + } - void write_value(Imath::V2d const& value) { _store(any(value)); } - - void write_value(Imath::Box2d const& value) { _store(any(value)); } + } + void write_value(Imath::Box2d const& value) { + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { + AnyDictionary result; + result["OTIO_SCHEMA"] = "Box2d.1"; + result["min"] = value.min; + result["max"] = value.max; + _store(any(std::move(result))); + } else { + _store(any(value)); + } + } + // @} void start_array(size_t /* n */) { @@ -232,37 +309,133 @@ class CloningEncoder : public Encoder { _internal_error( "Encoder::end_object() called without matching start_object()"); + return; } - else + + auto& top = _stack.back(); + if (!top.is_dict) { - auto& top = _stack.back(); - if (!top.is_dict) - { - _internal_error( - "Encoder::end_object() called without matching start_object()"); - _stack.pop_back(); - } - else + _internal_error( + "Encoder::end_object() called without matching start_object()"); + _stack.pop_back(); + + return; + } + + /* + * Convert back to SerializableObject* right here. + */ + if ( + _result_object_policy + == ResultObjectPolicy::CloneBackToSerializableObject + ) + { + SerializableObject::Reader reader( + top.dict, + _error_function, + nullptr + ); + _stack.pop_back(); + _store(reader._decode(_resolver)); + + return; + } + + // otherwise, build out as an anydictionary & downgrade + AnyDictionary m; + m.swap(top.dict); + + const std::string schema_string = m.get_default( + "OTIO_SCHEMA", + std::string("") + ); + + const int sep = schema_string.rfind("."); + const std::string schema_name = schema_string.substr(0, sep); + const std::string schema_vers = schema_string.substr(sep+1); + + // @TODO: need to pull the version number from the schema string rather + // than the type record - in case there are some kind of weird + // mixed types in the document that don't match the desired + // target -- OR it'll only downgrade from latest/current down + // to target and assume that you've intentionally done something + // weird. Need to consider this I guess. + // int vers = -1; + + if (!schema_vers.empty()) + { + + } + + const auto dg_version_it = ( + (*_downgrade_version_manifest)->find(schema_name) + ); + + // @TODO: should this check to also make sure its in the + // schema table? + if ( + !schema_name.empty() + && !schema_vers.empty() + && dg_version_it != (*_downgrade_version_manifest)->end() + ) + { + const int target_version = (dg_version_it->second); + + const auto type_rec = ( + TypeRegistry::instance()._find_type_record( + schema_name + ) + ); + + int current_version = type_rec->schema_version; + + while (target_version < current_version) { - /* - * Convert back to SerializableObject* right here. - */ - if (_actually_clone) + auto next_dg_fn = ( + type_rec->downgrade_functions.find( + current_version + ) + ); + + if ( + next_dg_fn + == type_rec->downgrade_functions.end() + ) { - SerializableObject::Reader reader( - top.dict, _error_function, nullptr); - _stack.pop_back(); - _store(reader._decode(_resolver)); - } - else - { - AnyDictionary m; - m.swap(top.dict); - _stack.pop_back(); - _store(any(std::move(m))); + _internal_error( + string_printf( + "No downgrader function available for " + "going from version %d to version %d.", + current_version, + target_version + ) + ); + return; } + + // apply it + next_dg_fn->second(&m); + + current_version --; } } + + _stack.pop_back(); + _store(any(std::move(m))); + } + + // @TODO: what kind of ownership policy here? + AnyDictionary + root() + { + if (_root.type() == typeid(AnyDictionary)) + { + return any_cast(_root); + } + else + { + return AnyDictionary(); + } } private: @@ -287,7 +460,8 @@ class CloningEncoder : public Encoder friend class SerializableObject; std::vector<_DictOrArray> _stack; - bool _actually_clone; + ResultObjectPolicy _result_object_policy; + optional _downgrade_version_manifest = {}; }; template @@ -604,9 +778,13 @@ SerializableObject::Writer::_any_equals(any const& lhs, any const& rhs) bool SerializableObject::Writer::write_root( - any const& value, Encoder& encoder, ErrorStatus* error_status) + any const& value, + Encoder& encoder, + optional downgrade_version_manifest, + ErrorStatus* error_status +) { - Writer w(encoder); + Writer w(encoder, downgrade_version_manifest); w.write(w._no_key, value); return !encoder.has_errored(error_status); } @@ -705,6 +883,7 @@ SerializableObject::Writer::write( return; } + // look for the multiple instances of the same thing in the id map auto e = _id_for_object.find(value); if (e != _id_for_object.end()) { @@ -734,32 +913,124 @@ SerializableObject::Writer::write( _next_id_for_type[schema_type_name] = 0; } + // build a unique id for this instance of this type, to store in the id map + // table std::string next_id = schema_type_name + "-" + std::to_string(++_next_id_for_type[schema_type_name]); _id_for_object[value] = next_id; _encoder.start_object(); - _encoder.write_key("OTIO_SCHEMA"); + std::string schema_str = ""; + // if its an unknown schema, the schema name is computed from the + // _original_schema_name and _original_schema_version attributes if (UnknownSchema const* us = dynamic_cast(value)) { - _encoder.write_value(string_printf( + schema_str = string_printf( "%s.%d", us->_original_schema_name.c_str(), - us->_original_schema_version)); + us->_original_schema_version + ); } else { - _encoder.write_value(string_printf( - "%s.%d", value->schema_name().c_str(), value->schema_version())); + // otherwise, use the schema_name and schema_version attributes + // @TODO: this needs to be deferred until downgrading is complete + schema_str = string_printf( + "%s.%d", + value->schema_name().c_str(), + value->schema_version() + ); + } + + // cache in case transformed + const std::string schema_name = value->schema_name(); + const int schema_version = value->schema_version(); + + optional downgraded = {}; + + // if there is a valid schema string + if (!schema_name.empty()) + { + if (_downgrade_version_manifest.has_value()) + { + const auto& target_version_it = ( + (*_downgrade_version_manifest)->find(schema_name) + ); + + // down the manifest specify a target version + if (target_version_it != (*_downgrade_version_manifest)->end()) + { + const int target_version = target_version_it->second; + + const auto& tr = TypeRegistry::instance()._find_type_record(schema_name); + const auto& downgrade_fn_map = tr->downgrade_functions; + + int current_version = schema_version; + + downgraded = { AnyDictionary() }; + + // build the dictionary to downgrade + for (auto kv: value->_dynamic_fields) + { + (*downgraded)[kv.first] = kv.second; + } + + while (current_version > target_version) + { + // downgrade in place + auto dg_fn = downgrade_fn_map.find(current_version); + if (dg_fn == downgrade_fn_map.end()) + { + _encoder._error( + ErrorStatus( + ErrorStatus::SCHEMA_VERSION_UNSUPPORTED, + string_printf( + "Could not find a downgrade function" + "from %d to %d for schema %s", + current_version, + current_version - 1, + schema_name.c_str() + ) + ) + ); + return; + } + dg_fn->second(&(*downgraded)); + + current_version--; + } + + schema_str = string_printf( + "%s.%d", + schema_name.c_str(), + target_version + ); + } + } + + // @TODO: probably this is why metadata is getting a null schema_str + _encoder.write_key("OTIO_SCHEMA"); + _encoder.write_value(schema_str); } #ifdef OTIO_INSTANCING_SUPPORT _encoder.write_key("OTIO_REF_ID"); _encoder.write_value(next_id); #endif - value->write_to(*this); + + if (!downgraded) + { + value->write_to(*this); + } + else + { + for (auto kv : *downgraded) + { + this->write(kv.first, kv.second); + } + } _encoder.end_object(); @@ -885,9 +1156,13 @@ SerializableObject::is_equivalent_to(SerializableObject const& other) const return false; } - CloningEncoder e1(false), e2(false); - SerializableObject::Writer w1(e1); - SerializableObject::Writer w2(e2); + const auto policy = ( + CloningEncoder::ResultObjectPolicy::MathTypesConcreteAnyDictionaryResult + ); + + CloningEncoder e1(policy), e2(policy); + SerializableObject::Writer w1(e1, {}); + SerializableObject::Writer w2(e2, {}); w1.write(w1._no_key, any(Retainer<>(this))); w2.write(w2._no_key, any(Retainer<>(&other))); @@ -900,8 +1175,10 @@ SerializableObject::is_equivalent_to(SerializableObject const& other) const SerializableObject* SerializableObject::clone(ErrorStatus* error_status) const { - CloningEncoder e(true /* actually_clone*/); - SerializableObject::Writer w(e); + CloningEncoder e( + CloningEncoder::ResultObjectPolicy::CloneBackToSerializableObject + ); + SerializableObject::Writer w(e, {}); w.write(w._no_key, any(Retainer<>(this))); if (e.has_errored(error_status)) @@ -926,38 +1203,50 @@ SerializableObject::clone(ErrorStatus* error_status) const std::string serialize_json_to_string( - any const& value, ErrorStatus* error_status, int indent) + const any& value, + optional downgrade_version_manifest, + ErrorStatus* error_status, + int indent +) { - OTIO_rapidjson::StringBuffer s; + OTIO_rapidjson::StringBuffer output_string_buffer; OTIO_rapidjson::PrettyWriter< - decltype(s), + decltype(output_string_buffer), OTIO_rapidjson::UTF8<>, OTIO_rapidjson::UTF8<>, OTIO_rapidjson::CrtAllocator, OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(s); - - JSONEncoder json_encoder(json_writer); + json_writer(output_string_buffer); if (indent >= 0) { json_writer.SetIndent(' ', indent); } - if (!SerializableObject::Writer::write_root( - value, json_encoder, error_status)) + JSONEncoder json_encoder(json_writer); + + if ( + !SerializableObject::Writer::write_root( + value, + json_encoder, + downgrade_version_manifest, + error_status + ) + ) { return std::string(); } - return std::string(s.GetString()); + return std::string(output_string_buffer.GetString()); + } bool serialize_json_to_file( any const& value, std::string const& file_name, + optionaldowngrade_version_manifest, ErrorStatus* error_status, int indent) { @@ -998,7 +1287,10 @@ serialize_json_to_file( } status = SerializableObject::Writer::write_root( - value, json_encoder, error_status); + value, + json_encoder, + downgrade_version_manifest, + error_status); return status; } diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index e716740a3..635af32ef 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -6,18 +6,33 @@ #include "opentimelineio/any.h" #include "opentimelineio/errorStatus.h" #include "opentimelineio/version.h" +#include #include +#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -std::string serialize_json_to_string( - const any& value, ErrorStatus* error_status = nullptr, int indent = 4); -bool serialize_json_to_file( +// a mapping of schema name : schema version +using schema_version_map = std::unordered_map; +const static schema_version_map EMPTY_VERSION_MAP = schema_version_map(); + +std::string +serialize_json_to_string( + const any& value, + optional downgrade_version_manifest = {}, + ErrorStatus* error_status = nullptr, + int indent = 4 +); + +bool +serialize_json_to_file( const any& value, std::string const& file_name, + optional downgrade_version_manifest = {}, ErrorStatus* error_status = nullptr, - int indent = 4); + int indent = 4 +); }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 4aca5a9fa..63ed45014 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -30,8 +30,6 @@ #include #include -//#include -//#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { @@ -113,6 +111,33 @@ TypeRegistry::TypeRegistry() d->erase("media_reference"); }); + + // 2->1 + register_downgrade_function(Clip::Schema::name, 2, [](AnyDictionary* d) { + AnyDictionary media_refs = d->get_default("media_references", AnyDictionary()); + const std::string active_reference_key = ( + d->get_default("active_media_reference_key", std::string("")) + ); + AnyDictionary active_ref = AnyDictionary(); + if (active_reference_key != "") { + active_ref = media_refs.get_default(active_reference_key, AnyDictionary()); + std::cerr << "found reference: "; + } + + (*d)["media_reference"] = active_ref; + + auto downgrade_md = AnyDictionary(); + downgrade_md["media_references"] = media_refs; + downgrade_md["active_media_reference_key"] = active_reference_key; + + d->set_default( + "metadata", + AnyDictionary() + )["downgrade_Clip.2_to_Clip.1"] = downgrade_md; + + d->erase("media_references"); + d->erase("active_reference_key"); + }); } bool @@ -198,6 +223,28 @@ TypeRegistry::register_upgrade_function( return false; } +bool +TypeRegistry::register_downgrade_function( + std::string const& schema_name, + int version_to_downgrade_from, + std::function downgrade_function) +{ + std::lock_guard lock(_registry_mutex); + if (auto r = _find_type_record(schema_name)) + { + if (r->downgrade_functions.find(version_to_downgrade_from) == + r->downgrade_functions.end()) + { + r->downgrade_functions[version_to_downgrade_from] = ( + downgrade_function + ); + return true; + } + } + + return false; +} + SerializableObject* TypeRegistry::_instance_from_schema( std::string schema_name, @@ -327,4 +374,16 @@ TypeRegistry::set_type_record( return false; } +void +TypeRegistry::type_version_map( + std::map& result) +{ + std::lock_guard lock(_registry_mutex); + + for (const auto& pair: _type_records) { + const auto record_ptr = pair.second; + result[record_ptr->schema_name] = record_ptr->schema_version; + } +} + }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 20cf90e8f..195620d91 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -15,6 +15,7 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { class SerializableObject; +class Encoder; class AnyDictionary; class TypeRegistry @@ -93,6 +94,24 @@ class TypeRegistry CLASS::schema_name, version_to_upgrade_to, upgrade_function); } + /// Downgrade function from version_to_downgrade_from to + /// version_to_downgrade_from - 1 + bool register_downgrade_function( + std::string const& schema_name, + int version_to_downgrade_from, + std::function downgrade_function); + + /// Convenience API for C++ developers. See the documentation of the + /// non-templated register_downgrade_function() for details. + template + bool register_downgrade_function( + int version_to_upgrade_to, + std::function upgrade_function) + { + return register_downgrade_function( + CLASS::schema_name, version_to_upgrade_to, upgrade_function); + } + SerializableObject* instance_from_schema( std::string const& schema_name, int schema_version, @@ -113,6 +132,16 @@ class TypeRegistry std::string const& schema_name, ErrorStatus* error_status = nullptr); + int + version_number_of_schema( + const std::string& schema_name) + { + return _find_type_record(schema_name)->schema_version; + } + + // for inspecting the type registry, build a map of schema name to version + void type_version_map(std::map& result); + private: TypeRegistry(); @@ -127,6 +156,7 @@ class TypeRegistry std::function create; std::map> upgrade_functions; + std::map> downgrade_functions; _TypeRecord( std::string _schema_name, @@ -144,6 +174,7 @@ class TypeRegistry friend class TypeRegistry; friend class SerializableObject; + friend class CloningEncoder; }; // helper functions for lookup @@ -170,6 +201,7 @@ class TypeRegistry std::map _type_records_by_type_name; friend class SerializableObject; + friend class CloningEncoder; }; }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index ae8472c7f..73a58c1f1 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -55,6 +55,30 @@ static bool register_upgrade_function(std::string const& schema_name, upgrade_function); } +static bool +register_downgrade_function( + std::string const& schema_name, + int version_to_downgrade_from, + py::object const& downgrade_function_obj) +{ + std::function downgrade_function = ( + [downgrade_function_obj](AnyDictionary* d) + { + py::gil_scoped_acquire acquire; + + auto ptr = d->get_or_create_mutation_stamp(); + py::object dobj = py::cast((AnyDictionaryProxy*)ptr); + downgrade_function_obj(dobj); + } + ); + + return TypeRegistry::instance().register_downgrade_function( + schema_name, + version_to_downgrade_from, + downgrade_function + ); +} + static void set_type_record(SerializableObject* so, std::string schema_name) { TypeRegistry::instance().set_type_record(so, schema_name, ErrorStatusHandler()); } @@ -76,13 +100,30 @@ PYBIND11_MODULE(_otio, m) { otio_serializable_object_bindings(m); otio_tests_bindings(m); - m.def("_serialize_json_to_string", - [](PyAny* pyAny, int indent) { - return serialize_json_to_string(pyAny->a, ErrorStatusHandler(), indent); - }, "value"_a, "indent"_a) + m.def( + "_serialize_json_to_string", + []( + PyAny* pyAny, + const std::unordered_map& downgrade_version_manifest, + int indent + ) + { + auto result = serialize_json_to_string( + pyAny->a, + {&downgrade_version_manifest}, + ErrorStatusHandler(), + indent + ); + + return result; + }, + "value"_a, + "downgrade_version_manifest"_a, + "indent"_a + ) .def("_serialize_json_to_file", [](PyAny* pyAny, std::string filename, int indent) { - return serialize_json_to_file(pyAny->a, filename, ErrorStatusHandler(), indent); + return serialize_json_to_file(pyAny->a, filename, {}, ErrorStatusHandler(), indent); }, "value"_a, "filename"_a, "indent"_a) .def("deserialize_json_from_string", [](std::string input) { @@ -133,10 +174,20 @@ Return an instance of the schema from data in the data_dict. :raises UnsupportedSchemaError: when the requested schema version is greater than the registered schema version. )docstring"); + m.def("type_version_map", + []() { + std::map tmp; + TypeRegistry::instance().type_version_map(tmp); + return tmp; + }); m.def("register_upgrade_function", ®ister_upgrade_function, "schema_name"_a, "version_to_upgrade_to"_a, "upgrade_function"_a); + m.def("register_downgrade_function", ®ister_downgrade_function, + "schema_name"_a, + "version_to_downgrade_from"_a, + "downgrade_function"_a); m.def("flatten_stack", [](Stack* s) { return flatten_stack(s, ErrorStatusHandler()); }, "in_stack"_a); diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp index ffa8616d5..76b496e5a 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp @@ -163,10 +163,10 @@ static void define_bases1(py::module m) { .def("clone", [](SerializableObject* so) { return so->clone(ErrorStatusHandler()); }) .def("to_json_string", [](SerializableObject* so, int indent) { - return so->to_json_string(ErrorStatusHandler(), indent); }, + return so->to_json_string(ErrorStatusHandler(), {}, indent); }, "indent"_a = 4) .def("to_json_file", [](SerializableObject* so, std::string file_name, int indent) { - return so->to_json_file(file_name, ErrorStatusHandler(), indent); }, + return so->to_json_file(file_name, ErrorStatusHandler(), {}, indent); }, "file_name"_a, "indent"_a = 4) .def_static("from_json_file", [](std::string file_name) { diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 3db34c108..4390df974 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -37,7 +37,7 @@ def read_from_string(input_str): return core.deserialize_json_from_string(input_str) -def write_to_string(input_otio, indent=4): +def write_to_string(input_otio, downgrade_version_manifest=None, indent=4): """ Serializes an OpenTimelineIO object into a string @@ -49,10 +49,19 @@ def write_to_string(input_otio, indent=4): Returns: str: A json serialized string representation """ - return core.serialize_json_to_string(input_otio, indent) - - -def write_to_file(input_otio, filepath, indent=4): + return core.serialize_json_to_string( + input_otio, + downgrade_version_manifest or {}, + indent + ) + + +def write_to_file( + input_otio, + filepath, + downgrade_version_manifest=None, + indent=4 +): """ Serializes an OpenTimelineIO object into a file diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 4aa455bba..1abb71b58 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -22,9 +22,11 @@ instance_from_schema, register_serializable_object_type, register_upgrade_function, + register_downgrade_function, set_type_record, _serialize_json_to_string, _serialize_json_to_file, + type_version_map, ) from . _core_utils import ( # noqa @@ -56,19 +58,26 @@ 'instance_from_schema', 'register_serializable_object_type', 'register_upgrade_function', + 'register_downgrade_function', 'set_type_record', 'add_method', 'upgrade_function_for', + 'downgrade_function_for', 'serializable_field', 'deprecated_field', 'serialize_json_to_string', 'serialize_json_to_file', - 'register_type' + 'register_type', + 'type_version_map', ] -def serialize_json_to_string(root, indent=4): - return _serialize_json_to_string(_value_to_any(root), indent) +def serialize_json_to_string(root, downgrade_version_manifest=None, indent=4): + return _serialize_json_to_string( + _value_to_any(root), + downgrade_version_manifest or {}, + indent + ) def serialize_json_to_file(root, filename, indent=4): @@ -134,6 +143,47 @@ def wrapped_update(data): return decorator_func +def downgrade_function_for(cls, version_to_upgrade_to): + """ + @TODO <- fix docs + Decorator for identifying schema class downgrade functions. + + Example: + + .. code-block:: python + + @upgrade_function_for(MyClass, 5) + def upgrade_to_version_five(data): + pass + + This will get called to upgrade a schema of MyClass to version 5. MyClass + must be a class deriving from :class:`~SerializableObject`. + + The upgrade function should take a single argument - the dictionary to + upgrade, and return a dictionary with the fields upgraded. + + Remember that you don't need to provide an upgrade function for upgrades + that add or remove fields, only for schema versions that change the field + names. + + :param type cls: class to upgrade + :param int version_to_upgrade_to: the version to upgrade to + """ + + def decorator_func(func): + """ Decorator for marking upgrade functions """ + def wrapped_update(data): + modified = func(data) + data.clear() + data.update(modified) + + register_downgrade_function(cls._serializable_label.split(".")[0], + version_to_upgrade_to, wrapped_update) + return func + + return decorator_func + + def serializable_field(name, required_type=None, doc=None): """ Convienence function for adding attributes to child classes of diff --git a/src/py-opentimelineio/opentimelineio/plugins/manifest.py b/src/py-opentimelineio/opentimelineio/plugins/manifest.py index 6ef6e9bd1..603618db2 100644 --- a/src/py-opentimelineio/opentimelineio/plugins/manifest.py +++ b/src/py-opentimelineio/opentimelineio/plugins/manifest.py @@ -32,6 +32,7 @@ 'schemadefs', 'hook_scripts', 'hooks', + 'version_manifests', ] @@ -90,6 +91,8 @@ def __init__(self): self.hooks = {} self.hook_scripts = [] + self.version_manifests = {} + adapters = core.serializable_field( "adapters", type([]), @@ -115,6 +118,11 @@ def __init__(self): type([]), "Scripts that can be attached to hooks." ) + version_manifests = core.serializable_field( + "version_manifests", + type({}), + "Sets of versions to downgrade schemas to." + ) def extend(self, another_manifest): """ From 5eaf20ad8e306a7020266a7a8261e4bb6a4bc3d3 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Mon, 22 Aug 2022 16:09:27 -0700 Subject: [PATCH 008/119] add .cache to gitignore Signed-off-by: ssteinbach --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 053fa5a82..d1dfcb6a7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ htmlcov .vscode/ xcuserdata/ .venv/ +.cache # Pycharm metadata .idea/ From 81582a30659af33363fd284b5e67a74ed71e360c Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 19:55:38 -0700 Subject: [PATCH 009/119] Test runs, @TODO: check for correctness Signed-off-by: ssteinbach --- src/opentimelineio/serializableObject.h | 1 + src/opentimelineio/serialization.cpp | 354 ++++++++++-------- src/opentimelineio/typeRegistry.cpp | 1 - .../builtin_adapters.plugin_manifest.json | 5 + .../opentimelineio/core/__init__.py | 2 + tests/test_builtin_adapters.py | 8 + tests/test_serializable_object.py | 53 +++ 7 files changed, 274 insertions(+), 150 deletions(-) diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index 6c3618381..c2289917c 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -535,6 +535,7 @@ class SerializableObject bool _any_equals(any const& lhs, any const& rhs); std::string _no_key; + // @TODO: should probably be unordered maps std::map> _write_dispatch_table; std::map< diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 921a12797..beff02eb6 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -53,6 +53,8 @@ class Encoder bool has_errored() { return is_error(_error_status); } + virtual bool encoding_to_anydict() { return false; } + virtual void start_object() = 0; virtual void end_object() = 0; @@ -111,6 +113,20 @@ class CloningEncoder : public Encoder virtual ~CloningEncoder() {} + /** + * 1. + * e = JSONEncoder(_downgrade) + * SerializableObject::Writer::write_root(e, _downgrade) + * [recurse until downgrade] + * ce = CloningEncoder(_downgrade) + * SerializableObject::Writer::write_root(e), + */ + virtual bool + encoding_to_anydict() override + { + return (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary); + } + void write_key(std::string const& key) { if (has_errored()) @@ -308,7 +324,8 @@ class CloningEncoder : public Encoder if (_stack.empty()) { _internal_error( - "Encoder::end_object() called without matching start_object()"); + "Encoder::end_object() called without matching start_object()" + ); return; } @@ -345,78 +362,101 @@ class CloningEncoder : public Encoder AnyDictionary m; m.swap(top.dict); - const std::string schema_string = m.get_default( - "OTIO_SCHEMA", - std::string("") - ); - - const int sep = schema_string.rfind("."); - const std::string schema_name = schema_string.substr(0, sep); - const std::string schema_vers = schema_string.substr(sep+1); - - // @TODO: need to pull the version number from the schema string rather - // than the type record - in case there are some kind of weird - // mixed types in the document that don't match the desired - // target -- OR it'll only downgrade from latest/current down - // to target and assume that you've intentionally done something - // weird. Need to consider this I guess. - // int vers = -1; - - if (!schema_vers.empty()) + if (_downgrade_version_manifest.has_value()) { - - } - - const auto dg_version_it = ( - (*_downgrade_version_manifest)->find(schema_name) - ); - - // @TODO: should this check to also make sure its in the - // schema table? - if ( - !schema_name.empty() - && !schema_vers.empty() - && dg_version_it != (*_downgrade_version_manifest)->end() - ) - { - const int target_version = (dg_version_it->second); - - const auto type_rec = ( - TypeRegistry::instance()._find_type_record( - schema_name - ) + const std::string schema_string = m.get_default( + "OTIO_SCHEMA", + std::string("") ); - int current_version = type_rec->schema_version; - - while (target_version < current_version) + if (!schema_string.empty()) { - auto next_dg_fn = ( - type_rec->downgrade_functions.find( - current_version - ) - ); + const int sep = schema_string.rfind("."); + const std::string schema_name = schema_string.substr(0, sep); + const std::string schema_vers = schema_string.substr(sep+1); + + // @TODO: need to pull the version number from the schema string rather + // than the type record - in case there are some kind of weird + // mixed types in the document that don't match the desired + // target -- OR it'll only downgrade from latest/current down + // to target and assume that you've intentionally done something + // weird. Need to consider this I guess. + int vers = -1; + if (!schema_vers.empty()) + { + vers = std::stoi(schema_vers); + } - if ( - next_dg_fn - == type_rec->downgrade_functions.end() - ) + if (vers < 0) { _internal_error( - string_printf( - "No downgrader function available for " - "going from version %d to version %d.", - current_version, - target_version - ) - ); + string_printf( + "Could not parse version number from Schema string: %s", + schema_string.c_str() + ) + ); return; } - // apply it - next_dg_fn->second(&m); + if (_downgrade_version_manifest.has_value()) + { + // @TODO: fill this in + const auto dg_version_it = ( + (*_downgrade_version_manifest)->find(schema_name) + ); + + // @TODO: should this check to also make sure its in the + // schema table? + if (dg_version_it != (*_downgrade_version_manifest)->end()) + { + const int target_version = (dg_version_it->second); + + int current_version = vers; - current_version --; + const auto type_rec = ( + TypeRegistry::instance()._find_type_record( + schema_name + ) + ); + + while (target_version < current_version) + { + auto next_dg_fn = ( + type_rec->downgrade_functions.find( + current_version + ) + ); + + if ( + next_dg_fn + == type_rec->downgrade_functions.end() + ) + { + _internal_error( + string_printf( + "No downgrader function available for " + "going from version %d to version %d.", + current_version, + target_version + ) + ); + return; + } + + // apply it + next_dg_fn->second(&m); + + current_version --; + } + + vers = target_version; + } + } + + if (!schema_name.empty() && vers > 0) + { + m["OTIO_SCHEMA"] = string_printf("%s.%d", schema_name.c_str(), vers); + } } } @@ -453,6 +493,7 @@ class CloningEncoder : public Encoder std::string cur_key; }; + void _internal_error(std::string const& err_msg) { _error(ErrorStatus(ErrorStatus::INTERNAL_ERROR, err_msg)); @@ -628,6 +669,7 @@ SerializableObject::Writer::_build_dispatch_tables() /* * These are basically atomic writes to the encoder: */ + auto& wt = _write_dispatch_table; wt[&typeid(void)] = [this](any const&) { _encoder.write_null_value(); }; wt[&typeid(bool)] = [this](any const& value) { @@ -866,7 +908,10 @@ SerializableObject::Writer::write( } void -SerializableObject::Writer::write(std::string const& key, TimeTransform value) +SerializableObject::Writer::write( + std::string const& key, + TimeTransform value +) { _encoder_write_key(key); _encoder.write_value(value); @@ -874,8 +919,10 @@ SerializableObject::Writer::write(std::string const& key, TimeTransform value) void SerializableObject::Writer::write( - std::string const& key, SerializableObject const* value) -{ + std::string const& key, + SerializableObject const* value +) +{ _encoder_write_key(key); if (!value) { @@ -883,7 +930,6 @@ SerializableObject::Writer::write( return; } - // look for the multiple instances of the same thing in the id map auto e = _id_for_object.find(value); if (e != _id_for_object.end()) { @@ -913,16 +959,57 @@ SerializableObject::Writer::write( _next_id_for_type[schema_type_name] = 0; } - // build a unique id for this instance of this type, to store in the id map - // table std::string next_id = schema_type_name + "-" + std::to_string(++_next_id_for_type[schema_type_name]); _id_for_object[value] = next_id; - _encoder.start_object(); - std::string schema_str = ""; + // detect if downgrading needs to happen + const std::string& schema_name = value->schema_name(); + int schema_version = value->schema_version(); + + optional downgraded = {}; + + // if there is a manifest & the encoder is not already converting to + // AnyDictionary + if ( + _downgrade_version_manifest.has_value() + && _encoder.encoding_to_anydict() + ) + { + const auto& target_version_it = ( + (*_downgrade_version_manifest)->find(schema_name) + ); + + // ...and if that downgrade manifest specifies a target version for + // this schema + if (target_version_it != (*_downgrade_version_manifest)->end()) + { + const int target_version = target_version_it->second; + + // and the current_version is greater than the target version + if (schema_version > target_version) + { + CloningEncoder e( + CloningEncoder::ResultObjectPolicy::OnlyAnyDictionary, + { *_downgrade_version_manifest } + ); + + Writer w(e, {} ); + w.write(w._no_key, value); + + if (e.has_errored(&_encoder._error_status)) + { + return; + } + + downgraded = { e.root() }; + schema_version = target_version; + } + } + } + // if its an unknown schema, the schema name is computed from the // _original_schema_name and _original_schema_version attributes if (UnknownSchema const* us = dynamic_cast(value)) @@ -940,97 +1027,34 @@ SerializableObject::Writer::write( schema_str = string_printf( "%s.%d", value->schema_name().c_str(), - value->schema_version() + schema_version ); } - // cache in case transformed - const std::string schema_name = value->schema_name(); - const int schema_version = value->schema_version(); - - optional downgraded = {}; - - // if there is a valid schema string - if (!schema_name.empty()) - { - if (_downgrade_version_manifest.has_value()) - { - const auto& target_version_it = ( - (*_downgrade_version_manifest)->find(schema_name) - ); - - // down the manifest specify a target version - if (target_version_it != (*_downgrade_version_manifest)->end()) - { - const int target_version = target_version_it->second; - - const auto& tr = TypeRegistry::instance()._find_type_record(schema_name); - const auto& downgrade_fn_map = tr->downgrade_functions; - - int current_version = schema_version; - - downgraded = { AnyDictionary() }; - - // build the dictionary to downgrade - for (auto kv: value->_dynamic_fields) - { - (*downgraded)[kv.first] = kv.second; - } - - while (current_version > target_version) - { - // downgrade in place - auto dg_fn = downgrade_fn_map.find(current_version); - if (dg_fn == downgrade_fn_map.end()) - { - _encoder._error( - ErrorStatus( - ErrorStatus::SCHEMA_VERSION_UNSUPPORTED, - string_printf( - "Could not find a downgrade function" - "from %d to %d for schema %s", - current_version, - current_version - 1, - schema_name.c_str() - ) - ) - ); - return; - } - dg_fn->second(&(*downgraded)); - - current_version--; - } - - schema_str = string_printf( - "%s.%d", - schema_name.c_str(), - target_version - ); - } - } + _encoder.start_object(); - // @TODO: probably this is why metadata is getting a null schema_str - _encoder.write_key("OTIO_SCHEMA"); - _encoder.write_value(schema_str); - } + _encoder.write_key("OTIO_SCHEMA"); + _encoder.write_value(schema_str); #ifdef OTIO_INSTANCING_SUPPORT _encoder.write_key("OTIO_REF_ID"); _encoder.write_value(next_id); #endif - if (!downgraded) + // write the contents of the object to the encoder, either the downgraded + // anydictionary or the SerializableObject + if (downgraded.has_value()) { - value->write_to(*this); - } - else - { - for (auto kv : *downgraded) + // the inner loop of write( + for (const auto& kv : (*downgraded)) { this->write(kv.first, kv.second); } } + else + { + value->write_to(*this); + } _encoder.end_object(); @@ -1090,8 +1114,16 @@ SerializableObject::Writer::write( } void -SerializableObject::Writer::write(std::string const& key, any const& value) +SerializableObject::Writer::write( + std::string const& key, + any const& value) { + // if (value.empty()) + // { + // _encoder.write_key(key); + // _encoder.write_null_value(); + // return; + // } std::type_info const& type = value.type(); _encoder_write_key(key); @@ -1196,11 +1228,16 @@ SerializableObject::clone(ErrorStatus* error_status) const e._resolver.finalize(error_function); + const auto t = &e._root.type(); + const auto tid_retainer = &typeid(SerializableObject::Retainer<>); + const auto equi = (t == tid_retainer); + return e._root.type() == typeid(SerializableObject::Retainer<>) ? any_cast&>(e._root).take_value() : nullptr; } +// to json_string std::string serialize_json_to_string( const any& value, @@ -1209,6 +1246,22 @@ serialize_json_to_string( int indent ) { + + CloningEncoder e( + CloningEncoder::ResultObjectPolicy::OnlyAnyDictionary, + downgrade_version_manifest + ); + if (!SerializableObject::Writer::write_root( + value, + e, + downgrade_version_manifest, + error_status + ) + ) + { + return std::string(); + } + OTIO_rapidjson::StringBuffer output_string_buffer; OTIO_rapidjson::PrettyWriter< @@ -1228,7 +1281,8 @@ serialize_json_to_string( if ( !SerializableObject::Writer::write_root( - value, + // value, + e.root(), json_encoder, downgrade_version_manifest, error_status @@ -1259,6 +1313,7 @@ serialize_json_to_file( #else // _WINDOWS std::ofstream os(file_name); #endif // _WINDOWS + if (!os.is_open()) { if (error_status) @@ -1290,7 +1345,8 @@ serialize_json_to_file( value, json_encoder, downgrade_version_manifest, - error_status); + error_status + ); return status; } diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 63ed45014..f5a9c7f1a 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -121,7 +121,6 @@ TypeRegistry::TypeRegistry() AnyDictionary active_ref = AnyDictionary(); if (active_reference_key != "") { active_ref = media_refs.get_default(active_reference_key, AnyDictionary()); - std::cerr << "found reference: "; } (*d)["media_reference"] = active_ref; diff --git a/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json b/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json index 0662e7ba2..f84afc98d 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json +++ b/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json @@ -49,5 +49,10 @@ "post_media_linker" : [], "pre_adapter_write" : [], "post_adapter_write" : [] + }, + "version_manifests": { + "test": { + "Clip": 1 + } } } diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 1abb71b58..4e9e437a9 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project +print("this file is being read") + from .. _otio import ( # noqa # errors CannotComputeAvailableRangeError, diff --git a/tests/test_builtin_adapters.py b/tests/test_builtin_adapters.py index 4fa707a27..a3f3771a6 100755 --- a/tests/test_builtin_adapters.py +++ b/tests/test_builtin_adapters.py @@ -68,6 +68,14 @@ def test_disk_vs_string(self): with open(temp_file, 'r') as f: on_disk = f.read() + self.maxDiff = None + + with open("/var/tmp/in_memory.otio", "w") as fo: + fo.write(in_memory) + + with open("/var/tmp/on_disk.otio", "w") as fo: + fo.write(on_disk) + self.assertEqual(in_memory, on_disk) def test_adapters_fetch(self): diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index 8d9889449..fd87a71db 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -55,7 +55,9 @@ def test_copy_lib(self): so_cp = copy.copy(so) # deep copy + import ipdb; ipdb.set_trace() so_cp = copy.deepcopy(so) + self.assertIsNotNone(so_cp) self.assertIsOTIOEquivalentTo(so, so_cp) so_cp.metadata["foo"] = "bar" @@ -138,6 +140,10 @@ def test_cycle_detection(self): class VersioningTests(unittest.TestCase, otio_test_utils.OTIOAssertions): def test_schema_versioning(self): """ test basic upgrade function and unsupported schema error """ + + # ensure that the type hasn't already been registered + self.assertNotIn("Stuff", otio.core.type_version_map()) + @otio.core.register_type class FakeThing(otio.core.SerializableObject): _serializable_label = "Stuff.1" @@ -154,6 +160,9 @@ class FakeThing(otio.core.SerializableObject): {"foo": "bar"} ) + version_map = otio.core.type_version_map() + self.assertEqual(version_map["Stuff"], 1) + ft = otio.core.instance_from_schema("Stuff", 1, {"foo": "bar"}) self.assertEqual(ft._dynamic_fields['foo'], "bar") @@ -161,6 +170,7 @@ def test_upgrading_skips_versions(self): """ test that the upgrading system skips versions that don't have upgrade functions""" + @otio.core.register_type class FakeThing(otio.core.SerializableObject): _serializable_label = "NewStuff.4" @@ -183,5 +193,48 @@ def upgrade_one_to_two_three(_data_dict): ft = otio.core.instance_from_schema("NewStuff", 4, {"foo_3": "bar"}) self.assertEqual(ft._dynamic_fields['foo_3'], "bar") + def test_upgrade_rename(self): + """test that upgrading system handles schema renames correctly""" + + @otio.core.register_type + class FakeThingToRename(otio.core.SerializableObject): + _serializable_label = "ThingToRename.2" + my_field = otio.core.serializable_field("my_field", doc="example") + + thing = otio.core.type_version_map() + self.assertTrue(thing) + + def test_downgrade_version(self): + """ test a python defined downgrade function""" + + @otio.core.register_type + class FakeThing(otio.core.SerializableObject): + _serializable_label = "FakeThingToDowngrade.2" + foo_two = otio.core.serializable_field("foo_2") + + print("running otio from:") + print(otio.__file__) + + @otio.core.downgrade_function_for(FakeThing, 2) + def downgrade_2_to_1(_data_dict): + return {"foo": _data_dict["foo_2"]} + + f = FakeThing() + f.foo_two = "a thing here" + + self.assertDictEqual( + eval( + otio.adapters.otio_json.write_to_string( + f, + downgrade_version_manifest={"FakeThingToDowngrade": 1} + ) + ), + { + "OTIO_SCHEMA": "FakeThingToDowngrade.1", + "foo": "a thing here", + } + ) + + if __name__ == '__main__': unittest.main() From 9848c8422e342a69577c483fb02efd6b8e201175 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 19:55:53 -0700 Subject: [PATCH 010/119] cleanup to the test Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 68 ++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 3ff8c6238..fc41c8ca7 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -1,5 +1,6 @@ #include +#include "opentimelineio/clip.h" #include "util.h" #include #include @@ -10,6 +11,13 @@ namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; using chrono_time_point = std::chrono::steady_clock::time_point; +constexpr struct { + bool TO_JSON_STRING = false; + bool TO_JSON_FILE = true; + bool CLONE_TEST = false; +} RUN_STRUCT ; + +/// utility function for printing std::chrono elapsed time const void print_elapsed_time( const std::string& message, @@ -26,7 +34,10 @@ print_elapsed_time( } int -main(int argc, char *argv[]) +main( + int argc, + char *argv[] +) { if (argc < 2) { std::cerr << "usage: otio_io_perf_test path/to/timeline.otio"; @@ -34,7 +45,21 @@ main(int argc, char *argv[]) return 1; } + // unit test of clone otio::ErrorStatus err; + + if (RUN_STRUCT.CLONE_TEST) + { + otio::SerializableObject::Retainer cl = new otio::Clip("test"); + cl->metadata()["example thing"] = "banana"; + const auto intermediate = cl->clone(&err); + assert(intermediate != nullptr); + const auto cl_clone = dynamic_cast(intermediate); + assert(cl_clone != nullptr); + assert(!otio::is_error(err)); + assert(cl->name() == cl_clone->name()); + } + otio::any tl; std::string fname = std::string(argv[1]); @@ -53,28 +78,39 @@ main(int argc, char *argv[]) print_elapsed_time("deserialize_json_from_file", begin, end); - begin = std::chrono::steady_clock::now(); otio::schema_version_map downgrade_version_manifest = { {"Clip", 1} }; - const std::string result = timeline.value->to_json_string( - &err, - &downgrade_version_manifest - ); - if (otio::is_error(err)) + if (RUN_STRUCT.TO_JSON_STRING) { - examples::print_error(err); - return 1; + begin = std::chrono::steady_clock::now(); + const std::string result = timeline.value->to_json_string( + &err, + // {} + &downgrade_version_manifest + ); + end = std::chrono::steady_clock::now(); + + if (otio::is_error(err)) + { + examples::print_error(err); + return 1; + } + print_elapsed_time("serialize_json_to_string", begin, end); } - end = std::chrono::steady_clock::now(); - print_elapsed_time("serialize_json_to_string", begin, end); - timeline.value->to_json_file( - "/var/tmp/test.otio", - &err, - &downgrade_version_manifest - ); + if (RUN_STRUCT.TO_JSON_FILE) + { + begin = std::chrono::steady_clock::now(); + timeline.value->to_json_file( + "/var/tmp/test.otio", + &err, + &downgrade_version_manifest + ); + end = std::chrono::steady_clock::now(); + print_elapsed_time("serialize_json_to_file", begin, end); + } return 0; } From 440b5dd165e0c2208b47f67571b0f55db96d0a6c Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 20:41:46 -0700 Subject: [PATCH 011/119] formatting tweaks Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 32 ++++++++++++++++------------ 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index beff02eb6..1395dfee7 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -391,10 +391,11 @@ class CloningEncoder : public Encoder { _internal_error( string_printf( - "Could not parse version number from Schema string: %s", + "Could not parse version number from Schema" + " string: %s", schema_string.c_str() - ) - ); + ) + ); return; } @@ -434,8 +435,9 @@ class CloningEncoder : public Encoder { _internal_error( string_printf( - "No downgrader function available for " - "going from version %d to version %d.", + "No downgrader function available" + " for going from version %d to " + "version %d.", current_version, target_version ) @@ -455,7 +457,11 @@ class CloningEncoder : public Encoder if (!schema_name.empty() && vers > 0) { - m["OTIO_SCHEMA"] = string_printf("%s.%d", schema_name.c_str(), vers); + m["OTIO_SCHEMA"] = string_printf( + "%s.%d", + schema_name.c_str(), + vers + ); } } } @@ -971,11 +977,10 @@ SerializableObject::Writer::write( optional downgraded = {}; - // if there is a manifest & the encoder is not already converting to - // AnyDictionary + // if there is a manifest & the encoder is not converting to AnyDictionary if ( _downgrade_version_manifest.has_value() - && _encoder.encoding_to_anydict() + && !_encoder.encoding_to_anydict() ) { const auto& target_version_it = ( @@ -1033,9 +1038,6 @@ SerializableObject::Writer::write( _encoder.start_object(); - _encoder.write_key("OTIO_SCHEMA"); - _encoder.write_value(schema_str); - #ifdef OTIO_INSTANCING_SUPPORT _encoder.write_key("OTIO_REF_ID"); _encoder.write_value(next_id); @@ -1053,6 +1055,8 @@ SerializableObject::Writer::write( } else { + _encoder.write_key("OTIO_SCHEMA"); + _encoder.write_value(schema_str); value->write_to(*this); } @@ -1257,7 +1261,7 @@ serialize_json_to_string( downgrade_version_manifest, error_status ) - ) + ) { return std::string(); } @@ -1344,7 +1348,7 @@ serialize_json_to_file( status = SerializableObject::Writer::write_root( value, json_encoder, - downgrade_version_manifest, + *downgrade_version_manifest, error_status ); From 319382269850a1d0baad6ffbda4ce45f10291093 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 20:41:52 -0700 Subject: [PATCH 012/119] turn on all the tests Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index fc41c8ca7..c08ea2afd 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -12,9 +12,10 @@ namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; using chrono_time_point = std::chrono::steady_clock::time_point; constexpr struct { - bool TO_JSON_STRING = false; + bool TO_JSON_STRING = true; bool TO_JSON_FILE = true; - bool CLONE_TEST = false; + bool CLONE_TEST = true; + bool SINGLE_CLIP_DOWNGRADE_TEST = true; } RUN_STRUCT ; /// utility function for printing std::chrono elapsed time @@ -60,6 +61,20 @@ main( assert(cl->name() == cl_clone->name()); } + otio::schema_version_map downgrade_version_manifest = { + {"Clip", 1} + }; + + if (RUN_STRUCT.SINGLE_CLIP_DOWNGRADE_TEST) + { + otio::SerializableObject::Retainer cl = new otio::Clip("test"); + cl->metadata()["example thing"] = "banana"; + chrono_time_point begin = std::chrono::steady_clock::now(); + cl->to_json_file("/var/tmp/clip.otio", &err, &downgrade_version_manifest); + chrono_time_point end = std::chrono::steady_clock::now(); + print_elapsed_time("downgrade clip", begin, end); + } + otio::any tl; std::string fname = std::string(argv[1]); @@ -78,9 +93,6 @@ main( print_elapsed_time("deserialize_json_from_file", begin, end); - otio::schema_version_map downgrade_version_manifest = { - {"Clip", 1} - }; if (RUN_STRUCT.TO_JSON_STRING) { From 2f9e77dc74bd886de7bd4764d55749be3e9e33bd Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 22:06:00 -0700 Subject: [PATCH 013/119] remove dead code Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 1395dfee7..f11d67349 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -1232,10 +1232,6 @@ SerializableObject::clone(ErrorStatus* error_status) const e._resolver.finalize(error_function); - const auto t = &e._root.type(); - const auto tid_retainer = &typeid(SerializableObject::Retainer<>); - const auto equi = (t == tid_retainer); - return e._root.type() == typeid(SerializableObject::Retainer<>) ? any_cast&>(e._root).take_value() : nullptr; From 33f698e46f4490bbd96cd2d3b5c96aa99dd3949a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 22:06:16 -0700 Subject: [PATCH 014/119] tightening the AnyDictionary code up a bit Signed-off-by: ssteinbach --- src/opentimelineio/anyDictionary.h | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/opentimelineio/anyDictionary.h b/src/opentimelineio/anyDictionary.h index 3533e4532..0ead3b167 100644 --- a/src/opentimelineio/anyDictionary.h +++ b/src/opentimelineio/anyDictionary.h @@ -131,12 +131,15 @@ class AnyDictionary : private std::map const containedType& default_value ) { - any value_any = (*this)[key]; - if (value_any.type() == typeid(containedType)) { - return any_cast(value_any); - } else { - return default_value; - } + const auto& it = this->find(key); + + return ( + it != this->end() + && it->second.type().hash_code() == typeid(containedType).hash_code() + ) ? + any_cast(it->second) + : default_value + ; } inline bool @@ -154,11 +157,15 @@ class AnyDictionary : private std::map const containedType& default_value ) { - if (this->has_key(key)) { - return this->get_default(key, default_value); - } else { - (*this)[key] = default_value; - return this->get_default(key, default_value); + const auto& d_it = this->find(key); + if (d_it != this->end()) + { + return any_cast(d_it->second); + } + else + { + this->insert({key, default_value}); + return default_value; } } From 769cbc32af18001369b0a3dc79d3f3dc46152f60 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 22:12:51 -0700 Subject: [PATCH 015/119] more odds and ends Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index f11d67349..e1e472605 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -983,13 +983,12 @@ SerializableObject::Writer::write( && !_encoder.encoding_to_anydict() ) { - const auto& target_version_it = ( - (*_downgrade_version_manifest)->find(schema_name) - ); + const auto& dg_man = *(*_downgrade_version_manifest); + const auto& target_version_it = dg_man.find(schema_name); // ...and if that downgrade manifest specifies a target version for // this schema - if (target_version_it != (*_downgrade_version_manifest)->end()) + if (target_version_it != dg_man.end()) { const int target_version = target_version_it->second; @@ -1001,7 +1000,7 @@ SerializableObject::Writer::write( { *_downgrade_version_manifest } ); - Writer w(e, {} ); + Writer w(e, {}); w.write(w._no_key, value); if (e.has_errored(&_encoder._error_status)) @@ -1028,10 +1027,9 @@ SerializableObject::Writer::write( else { // otherwise, use the schema_name and schema_version attributes - // @TODO: this needs to be deferred until downgrading is complete schema_str = string_printf( "%s.%d", - value->schema_name().c_str(), + schema_name.c_str(), schema_version ); } From dd128b04713c3939394c6d0979ac8daa70993761 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 22:16:32 -0700 Subject: [PATCH 016/119] add override tags Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 44 +++++++++++++--------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index e1e472605..17e3ee627 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -127,7 +127,7 @@ class CloningEncoder : public Encoder return (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary); } - void write_key(std::string const& key) + void write_key(std::string const& key) override { if (has_errored()) { @@ -169,25 +169,19 @@ class CloningEncoder : public Encoder } } - void write_null_value() { _store(any()); } - - void write_value(bool value) { _store(any(value)); } - - void write_value(int value) { _store(any(value)); } - - void write_value(int64_t value) { _store(any(value)); } - - void write_value(uint64_t value) { _store(any(value)); } - - void write_value(std::string const& value) { _store(any(value)); } - - void write_value(double value) { _store(any(value)); } + void write_null_value() override { _store(any()); } + void write_value(bool value) override { _store(any(value)); } + void write_value(int value) override { _store(any(value)); } + void write_value(int64_t value) override { _store(any(value)); } + void write_value(uint64_t value) override { _store(any(value)); } + void write_value(std::string const& value) override { _store(any(value)); } + void write_value(double value) override { _store(any(value)); } // @{ @TODO: these three the json serializer knows how to dereference them // ... probably better to scooch that into this class... EXCEPT // the equivalence test wants to use their == methods void - write_value(RationalTime const& value) + write_value(RationalTime const& value) override { if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { AnyDictionary result; @@ -200,7 +194,8 @@ class CloningEncoder : public Encoder } } void - write_value(TimeRange const& value) { + write_value(TimeRange const& value) override + { if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { AnyDictionary result; @@ -213,7 +208,9 @@ class CloningEncoder : public Encoder } } - void write_value(TimeTransform const& value) { + void + write_value(TimeTransform const& value) override + { if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { AnyDictionary result; result["OTIO_SCHEMA"] = "TimeTransform.1"; @@ -225,7 +222,8 @@ class CloningEncoder : public Encoder _store(any(value)); } } - void write_value(SerializableObject::ReferenceId value) + void + write_value(SerializableObject::ReferenceId value) override { if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { AnyDictionary result; @@ -250,7 +248,7 @@ class CloningEncoder : public Encoder } } - void write_value(Imath::Box2d const& value) { + void write_value(Imath::Box2d const& value) override { if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { AnyDictionary result; result["OTIO_SCHEMA"] = "Box2d.1"; @@ -263,7 +261,7 @@ class CloningEncoder : public Encoder } // @} - void start_array(size_t /* n */) + void start_array(size_t /* n */) override { if (has_errored()) { @@ -273,7 +271,7 @@ class CloningEncoder : public Encoder _stack.emplace_back(_DictOrArray{ false /* is_dict*/ }); } - void start_object() + void start_object() override { if (has_errored()) { @@ -283,7 +281,7 @@ class CloningEncoder : public Encoder _stack.emplace_back(_DictOrArray{ true /* is_dict*/ }); } - void end_array() + void end_array() override { if (has_errored()) { @@ -314,7 +312,7 @@ class CloningEncoder : public Encoder } } - void end_object() + void end_object() override { if (has_errored()) { From 5ac78c9a8c72c28b03e749d7d6c61d14b3f54760 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 22:40:35 -0700 Subject: [PATCH 017/119] noodling Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 188 +++++++++++++++------------ 1 file changed, 104 insertions(+), 84 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 17e3ee627..5d25b0822 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -183,13 +183,17 @@ class CloningEncoder : public Encoder void write_value(RationalTime const& value) override { - if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { - AnyDictionary result; - result["OTIO_SCHEMA"] = "RationalTime.1"; - result["value"] = value.value(); - result["rate"] = value.rate(); + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result = { + {"OTIO_SCHEMA", "RationalTime.1"}, + {"value", value.value()}, + {"rate", value.rate()}, + }; _store(any(std::move(result))); - } else { + } + else + { _store(any(value)); } } @@ -197,13 +201,17 @@ class CloningEncoder : public Encoder write_value(TimeRange const& value) override { - if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { - AnyDictionary result; - result["OTIO_SCHEMA"] = "TimeRange.1"; - result["duration"] = value.duration(); - result["start_time"] = value.start_time(); + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result = { + {"OTIO_SCHEMA", "TimeRange.1"}, + {"duration", value.duration()}, + {"start_time", value.start_time()}, + }; _store(any(std::move(result))); - } else { + } + else + { _store(any(value)); } @@ -211,49 +219,69 @@ class CloningEncoder : public Encoder void write_value(TimeTransform const& value) override { - if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { - AnyDictionary result; - result["OTIO_SCHEMA"] = "TimeTransform.1"; - result["offset"] = value.offset(); - result["rate"] = value.rate(); - result["scale"] = value.scale(); + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "TimeTransform.1"}, + {"offset", value.offset()}, + {"rate", value.rate()}, + {"scale", value.scale()}, + }; _store(any(std::move(result))); - } else { + } + else + { _store(any(value)); } } void write_value(SerializableObject::ReferenceId value) override { - if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { - AnyDictionary result; - result["OTIO_SCHEMA"] = "SerializableObjectRef.1"; - result["id"] = value.id.c_str(); + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "SerializableObjectRef.1"}, + {"id", value.id.c_str()}, + }; _store(any(std::move(result))); - } else { + } + else + { _store(any(value)); } _store(any(value)); } - void write_value(Imath::V2d const& value) { + + void + write_value(Imath::V2d const& value) + { - if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { - AnyDictionary result; - result["OTIO_SCHEMA"] = "V2d.1"; - result["x"] = value.x; - result["y"] = value.y; + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "V2d.1"}, + {"x", value.x}, + {"y", value.y}, + }; _store(any(std::move(result))); - } else { + } + else + { _store(any(value)); } } - void write_value(Imath::Box2d const& value) override { - if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) { - AnyDictionary result; - result["OTIO_SCHEMA"] = "Box2d.1"; - result["min"] = value.min; - result["max"] = value.max; + + void + write_value(Imath::Box2d const& value) override + { + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "Box2d.1"}, + {"min", value.min}, + {"max", value.max}, + }; _store(any(std::move(result))); } else { _store(any(value)); @@ -362,7 +390,7 @@ class CloningEncoder : public Encoder if (_downgrade_version_manifest.has_value()) { - const std::string schema_string = m.get_default( + const std::string& schema_string = m.get_default( "OTIO_SCHEMA", std::string("") ); @@ -370,8 +398,8 @@ class CloningEncoder : public Encoder if (!schema_string.empty()) { const int sep = schema_string.rfind("."); - const std::string schema_name = schema_string.substr(0, sep); - const std::string schema_vers = schema_string.substr(sep+1); + const std::string& schema_name = schema_string.substr(0, sep); + const std::string& schema_vers = schema_string.substr(sep+1); // @TODO: need to pull the version number from the schema string rather // than the type record - in case there are some kind of weird @@ -392,68 +420,60 @@ class CloningEncoder : public Encoder "Could not parse version number from Schema" " string: %s", schema_string.c_str() - ) - ); + ) + ); return; } - if (_downgrade_version_manifest.has_value()) + const auto& dg_man = *(*_downgrade_version_manifest); + + const auto dg_version_it = dg_man.find(schema_name); + + // @TODO: should this check to also make sure its in the + // schema table? + if (dg_version_it != dg_man.end()) { - // @TODO: fill this in - const auto dg_version_it = ( - (*_downgrade_version_manifest)->find(schema_name) - ); + const int target_version = (dg_version_it->second); - // @TODO: should this check to also make sure its in the - // schema table? - if (dg_version_it != (*_downgrade_version_manifest)->end()) - { - const int target_version = (dg_version_it->second); + int current_version = vers; - int current_version = vers; + const auto& type_rec = ( + TypeRegistry::instance()._find_type_record( + schema_name + ) + ); - const auto type_rec = ( - TypeRegistry::instance()._find_type_record( - schema_name + while (target_version < current_version) + { + const auto& next_dg_fn = ( + type_rec->downgrade_functions.find( + current_version ) ); - while (target_version < current_version) + if (next_dg_fn == type_rec->downgrade_functions.end()) { - auto next_dg_fn = ( - type_rec->downgrade_functions.find( - current_version + _internal_error( + string_printf( + "No downgrader function available for " + "going from version %d to version %d.", + current_version, + target_version ) ); - - if ( - next_dg_fn - == type_rec->downgrade_functions.end() - ) - { - _internal_error( - string_printf( - "No downgrader function available" - " for going from version %d to " - "version %d.", - current_version, - target_version - ) - ); - return; - } - - // apply it - next_dg_fn->second(&m); - - current_version --; + return; } - vers = target_version; + // apply it + next_dg_fn->second(&m); + + current_version --; } + + vers = target_version; } - if (!schema_name.empty() && vers > 0) + if (vers > 0) { m["OTIO_SCHEMA"] = string_printf( "%s.%d", From eaf2801de1a5d3d6c4f8c50aa987627e4290170a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 24 Aug 2022 23:14:45 -0700 Subject: [PATCH 018/119] refactored into function Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 186 ++++++++++++++------------- 1 file changed, 94 insertions(+), 92 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 5d25b0822..dc0bead85 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -390,98 +390,7 @@ class CloningEncoder : public Encoder if (_downgrade_version_manifest.has_value()) { - const std::string& schema_string = m.get_default( - "OTIO_SCHEMA", - std::string("") - ); - - if (!schema_string.empty()) - { - const int sep = schema_string.rfind("."); - const std::string& schema_name = schema_string.substr(0, sep); - const std::string& schema_vers = schema_string.substr(sep+1); - - // @TODO: need to pull the version number from the schema string rather - // than the type record - in case there are some kind of weird - // mixed types in the document that don't match the desired - // target -- OR it'll only downgrade from latest/current down - // to target and assume that you've intentionally done something - // weird. Need to consider this I guess. - int vers = -1; - if (!schema_vers.empty()) - { - vers = std::stoi(schema_vers); - } - - if (vers < 0) - { - _internal_error( - string_printf( - "Could not parse version number from Schema" - " string: %s", - schema_string.c_str() - ) - ); - return; - } - - const auto& dg_man = *(*_downgrade_version_manifest); - - const auto dg_version_it = dg_man.find(schema_name); - - // @TODO: should this check to also make sure its in the - // schema table? - if (dg_version_it != dg_man.end()) - { - const int target_version = (dg_version_it->second); - - int current_version = vers; - - const auto& type_rec = ( - TypeRegistry::instance()._find_type_record( - schema_name - ) - ); - - while (target_version < current_version) - { - const auto& next_dg_fn = ( - type_rec->downgrade_functions.find( - current_version - ) - ); - - if (next_dg_fn == type_rec->downgrade_functions.end()) - { - _internal_error( - string_printf( - "No downgrader function available for " - "going from version %d to version %d.", - current_version, - target_version - ) - ); - return; - } - - // apply it - next_dg_fn->second(&m); - - current_version --; - } - - vers = target_version; - } - - if (vers > 0) - { - m["OTIO_SCHEMA"] = string_printf( - "%s.%d", - schema_name.c_str(), - vers - ); - } - } + _downgrade_dictionary(m); } _stack.pop_back(); @@ -527,6 +436,98 @@ class CloningEncoder : public Encoder std::vector<_DictOrArray> _stack; ResultObjectPolicy _result_object_policy; optional _downgrade_version_manifest = {}; + + void + _downgrade_dictionary( + AnyDictionary& m + ) + { + const std::string& schema_string = m.get_default( + "OTIO_SCHEMA", + std::string("") + ); + + if (schema_string.empty()) + { + return; + } + + const int sep = schema_string.rfind('.'); + const std::string& schema_name = schema_string.substr(0, sep); + + const auto& dg_man = *(*_downgrade_version_manifest); + const auto dg_version_it = dg_man.find(schema_name); + + if (dg_version_it == dg_man.end()) + { + return; + } + + const std::string& schema_vers = schema_string.substr(sep+1); + int current_version = -1; + + // @TODO: need to pull the version number from the schema string rather + // than the type record - in case there are some kind of weird + // mixed types in the document that don't match the desired + // target -- OR it'll only downgrade from latest/current down + // to target and assume that you've intentionally done something + // weird. Need to consider this I guess. + if (!schema_vers.empty()) + { + current_version = std::stoi(schema_vers); + } + + if (current_version < 0) + { + _internal_error( + string_printf( + "Could not parse version number from Schema" + " string: %s", + schema_string.c_str() + ) + ); + return; + } + + const int target_version = (dg_version_it->second); + + const auto& type_rec = ( + TypeRegistry::instance()._find_type_record(schema_name) + ); + + while (current_version > target_version ) + { + const auto& next_dg_fn = ( + type_rec->downgrade_functions.find( + current_version + ) + ); + + if (next_dg_fn == type_rec->downgrade_functions.end()) + { + _internal_error( + string_printf( + "No downgrader function available for " + "going from version %d to version %d.", + current_version, + target_version + ) + ); + return; + } + + // apply it + next_dg_fn->second(&m); + + current_version --; + } + + m["OTIO_SCHEMA"] = string_printf( + "%s.%d", + schema_name.c_str(), + current_version + ); + } }; template @@ -1367,4 +1368,5 @@ serialize_json_to_file( return status; } + }} // namespace opentimelineio::OPENTIMELINEIO_VERSION From 991febb059d322d62c030251b57ab3f6db332338 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 25 Aug 2022 11:48:22 -0700 Subject: [PATCH 019/119] only conver to AnyDictionary if downgrade detected Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index dc0bead85..3c741041f 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -1263,22 +1263,6 @@ serialize_json_to_string( int indent ) { - - CloningEncoder e( - CloningEncoder::ResultObjectPolicy::OnlyAnyDictionary, - downgrade_version_manifest - ); - if (!SerializableObject::Writer::write_root( - value, - e, - downgrade_version_manifest, - error_status - ) - ) - { - return std::string(); - } - OTIO_rapidjson::StringBuffer output_string_buffer; OTIO_rapidjson::PrettyWriter< @@ -1298,8 +1282,7 @@ serialize_json_to_string( if ( !SerializableObject::Writer::write_root( - // value, - e.root(), + value, json_encoder, downgrade_version_manifest, error_status @@ -1361,7 +1344,7 @@ serialize_json_to_file( status = SerializableObject::Writer::write_root( value, json_encoder, - *downgrade_version_manifest, + downgrade_version_manifest, error_status ); From e92389ce218cc70ef5e68bc67a4d79a063337163 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 25 Aug 2022 11:48:54 -0700 Subject: [PATCH 020/119] add perf tests for no-downgrade scenarios Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index c08ea2afd..2e6ba66d7 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -13,7 +13,9 @@ using chrono_time_point = std::chrono::steady_clock::time_point; constexpr struct { bool TO_JSON_STRING = true; + bool TO_JSON_STRING_NO_DOWNGRADE = true; bool TO_JSON_FILE = true; + bool TO_JSON_FILE_NO_DOWNGRADE = true; bool CLONE_TEST = true; bool SINGLE_CLIP_DOWNGRADE_TEST = true; } RUN_STRUCT ; @@ -112,11 +114,25 @@ main( print_elapsed_time("serialize_json_to_string", begin, end); } + if (RUN_STRUCT.TO_JSON_STRING_NO_DOWNGRADE) + { + begin = std::chrono::steady_clock::now(); + const std::string result = timeline.value->to_json_string(&err, {}); + end = std::chrono::steady_clock::now(); + + if (otio::is_error(err)) + { + examples::print_error(err); + return 1; + } + print_elapsed_time("serialize_json_to_string [no downgrade]", begin, end); + } + if (RUN_STRUCT.TO_JSON_FILE) { begin = std::chrono::steady_clock::now(); timeline.value->to_json_file( - "/var/tmp/test.otio", + "/var/tmp/io_perf_test.otio", &err, &downgrade_version_manifest ); @@ -124,5 +140,13 @@ main( print_elapsed_time("serialize_json_to_file", begin, end); } + if (RUN_STRUCT.TO_JSON_FILE_NO_DOWNGRADE) + { + begin = std::chrono::steady_clock::now(); + timeline.value->to_json_file("/var/tmp/io_perf_test.nodowngrade.otio", &err, {}); + end = std::chrono::steady_clock::now(); + print_elapsed_time("serialize_json_to_file [no downgrade]", begin, end); + } + return 0; } From 28f566928833a036f99190a4aaee3ae266aa4ca9 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 25 Aug 2022 11:49:08 -0700 Subject: [PATCH 021/119] use const auto& where possible in range-for Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 8 ++++---- src/opentimelineio/track.cpp | 6 +++--- src/opentimelineio/typeRegistry.cpp | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 3c741041f..aa84745b4 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -747,7 +747,7 @@ SerializableObject::Writer::_build_dispatch_tables() * Install a backup table, using the actual type name as a key. * This is to deal with type aliasing across compilation units. */ - for (auto e: wt) + for (const auto& e: wt) { _write_dispatch_table_by_name[e.first->name()] = e.second; } @@ -792,7 +792,7 @@ SerializableObject::Writer::_any_dict_equals(any const& lhs, any const& rhs) auto r_it = rd.begin(); - for (auto l_it: ld) + for (const auto& l_it: ld) { if (r_it == rd.end()) { @@ -1110,7 +1110,7 @@ SerializableObject::Writer::write( _encoder.start_object(); - for (auto e: value) + for (const auto& e: value) { write(e.first, e.second); } @@ -1126,7 +1126,7 @@ SerializableObject::Writer::write( _encoder.start_array(value.size()); - for (auto e: value) + for (const auto& e: value) { write(_no_key, e); } diff --git a/src/opentimelineio/track.cpp b/src/opentimelineio/track.cpp index 9118340a5..44e027666 100644 --- a/src/opentimelineio/track.cpp +++ b/src/opentimelineio/track.cpp @@ -135,7 +135,7 @@ TimeRange Track::available_range(ErrorStatus* error_status) const { RationalTime duration; - for (auto child: children()) + for (const auto& child: children()) { if (auto item = dynamic_retainer_cast(child)) { @@ -261,7 +261,7 @@ Track::range_of_all_children(ErrorStatus* error_status) const } RationalTime last_end_time(0, rate); - for (auto child: children()) + for (const auto& child: children()) { if (auto transition = dynamic_retainer_cast(child)) { @@ -300,7 +300,7 @@ Track::available_image_bounds(ErrorStatus* error_status) const { optional box; bool found_first_clip = false; - for (auto child: children()) + for (const auto& child: children()) { if (auto clip = dynamic_cast(child.value)) { diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index f5a9c7f1a..c4dded289 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -296,7 +296,7 @@ TypeRegistry::_instance_from_schema( } else if (schema_version < type_record->schema_version) { - for (auto e: type_record->upgrade_functions) + for (const auto& e: type_record->upgrade_functions) { if (schema_version <= e.first && e.first <= type_record->schema_version) From 97cce91e77bc77e06234a8895edd214b2302efe9 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 25 Aug 2022 15:26:52 -0700 Subject: [PATCH 022/119] in this case, faster to std::string concatenation Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index aa84745b4..6b64c5638 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -6,6 +6,7 @@ #include "opentimelineio/anyDictionary.h" #include "opentimelineio/unknownSchema.h" #include "stringUtils.h" +#include #define RAPIDJSON_NAMESPACE OTIO_rapidjson #include @@ -522,11 +523,7 @@ class CloningEncoder : public Encoder current_version --; } - m["OTIO_SCHEMA"] = string_printf( - "%s.%d", - schema_name.c_str(), - current_version - ); + m["OTIO_SCHEMA"] = schema_name + "." + std::to_string(current_version); } }; @@ -1016,7 +1013,7 @@ SerializableObject::Writer::write( { CloningEncoder e( CloningEncoder::ResultObjectPolicy::OnlyAnyDictionary, - { *_downgrade_version_manifest } + _downgrade_version_manifest ); Writer w(e, {}); @@ -1037,20 +1034,16 @@ SerializableObject::Writer::write( // _original_schema_name and _original_schema_version attributes if (UnknownSchema const* us = dynamic_cast(value)) { - schema_str = string_printf( - "%s.%d", - us->_original_schema_name.c_str(), - us->_original_schema_version + schema_str = ( + us->_original_schema_name + + "." + + std::to_string(us->_original_schema_version) ); } else { // otherwise, use the schema_name and schema_version attributes - schema_str = string_printf( - "%s.%d", - schema_name.c_str(), - schema_version - ); + schema_str = schema_name + "." + std::to_string(schema_version); } _encoder.start_object(); From 34434c1251f1323c5be45ed9bdfdb54e75d82ead Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 25 Aug 2022 16:14:09 -0700 Subject: [PATCH 023/119] checkpointing work with all tests passing Signed-off-by: ssteinbach --- src/py-opentimelineio/opentimelineio/core/__init__.py | 2 -- tests/test_serializable_object.py | 4 ---- 2 files changed, 6 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 4e9e437a9..1abb71b58 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project -print("this file is being read") - from .. _otio import ( # noqa # errors CannotComputeAvailableRangeError, diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index fd87a71db..df4b448ee 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -55,7 +55,6 @@ def test_copy_lib(self): so_cp = copy.copy(so) # deep copy - import ipdb; ipdb.set_trace() so_cp = copy.deepcopy(so) self.assertIsNotNone(so_cp) self.assertIsOTIOEquivalentTo(so, so_cp) @@ -212,9 +211,6 @@ class FakeThing(otio.core.SerializableObject): _serializable_label = "FakeThingToDowngrade.2" foo_two = otio.core.serializable_field("foo_2") - print("running otio from:") - print(otio.__file__) - @otio.core.downgrade_function_for(FakeThing, 2) def downgrade_2_to_1(_data_dict): return {"foo": _data_dict["foo_2"]} From 6b2842c79e0bb479518844f58bfb0f83838da0ad Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 25 Aug 2022 16:05:57 -0700 Subject: [PATCH 024/119] check point design before pushing arguments through Signed-off-by: ssteinbach --- src/opentimelineio/serialization.h | 38 ++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index 635af32ef..c4e7888c3 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -13,10 +13,44 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { +// +// { +// "OTIO_CORE": { +// ^ family +// "0.14.0": { +// ^ family_label +// "Clip": 1, +// ^ schema ^ schema_version +// ... +// }, +// "0.15.0": { +// ... +// }, +// ... +// }, +// "MY_COMPANY_PLUGIN_SETS": {} +// } -// a mapping of schema name : schema version + +// typedefs for the schema downgrading system using schema_version_map = std::unordered_map; -const static schema_version_map EMPTY_VERSION_MAP = schema_version_map(); +using label_to_schema_version_map = std::unordered_map; +using family_to_label_map = std::unordered_map; +using family_label_spec = std::pair; + +static family_to_label_map FAMILY_LABEL_MAP { + { + "OTIO_CORE", + { + { "test", + { + { "Clip", 1 } + } + } + } + } +}; + std::string serialize_json_to_string( From 5eb7ed18a2f7ca260bdf494cdf38c12154dee4d5 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 26 Aug 2022 14:03:07 -0700 Subject: [PATCH 025/119] fix a test bug - if the exception was raised _before_ setting the env variable, this test would fail to _delete_ the variable and the error would be masked. Signed-off-by: ssteinbach --- tests/test_adapter_plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_adapter_plugin.py b/tests/test_adapter_plugin.py index 6e92b9583..5477fe2a1 100755 --- a/tests/test_adapter_plugin.py +++ b/tests/test_adapter_plugin.py @@ -222,7 +222,8 @@ def test_deduplicate_env_variable_paths(self): if bak_env is not None: os.environ['OTIO_PLUGIN_MANIFEST_PATH'] = bak_env else: - del os.environ['OTIO_PLUGIN_MANIFEST_PATH'] + if "OTIO_PLUGIN_MANIFEST_PATH" in os.environ: + del os.environ['OTIO_PLUGIN_MANIFEST_PATH'] def test_find_manifest_by_environment_variable(self): basename = "unittest.plugin_manifest.json" From e3cab5be4d375f61dc47f59b27260f87f6614ab7 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 26 Aug 2022 14:06:48 -0700 Subject: [PATCH 026/119] plumb target_family_label_spec through Signed-off-by: ssteinbach --- docs/tutorials/otio-plugins.md | 2 ++ src/opentimelineio/serializableObject.cpp | 8 ++--- src/opentimelineio/serializableObject.h | 5 +-- src/opentimelineio/serialization.cpp | 34 +++++++++++++++++-- src/opentimelineio/serialization.h | 8 +++-- .../opentimelineio-bindings/otio_bindings.cpp | 27 +++++++++++---- .../opentimelineio/adapters/otio_json.py | 13 ++++--- .../opentimelineio/core/__init__.py | 18 +++++++--- 8 files changed, 91 insertions(+), 24 deletions(-) diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index a3c538e9a..0933098db 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -155,6 +155,7 @@ Serializes an OpenTimelineIO object into a file ``` - input_otio - filepath + - target_family_label_spec - indent - write_to_string: ``` @@ -169,6 +170,7 @@ Serializes an OpenTimelineIO object into a string str: A json serialized string representation ``` - input_otio + - target_family_label_spec - indent diff --git a/src/opentimelineio/serializableObject.cpp b/src/opentimelineio/serializableObject.cpp index 0e583ea4b..f10146393 100644 --- a/src/opentimelineio/serializableObject.cpp +++ b/src/opentimelineio/serializableObject.cpp @@ -114,13 +114,13 @@ SerializableObject::is_unknown_schema() const std::string SerializableObject::to_json_string( ErrorStatus* error_status, - optionaldowngrade_version_manifest, + optional target_family_label_spec, int indent ) const { return serialize_json_to_string( any(Retainer<>(this)), - downgrade_version_manifest, + target_family_label_spec, error_status, indent ); @@ -130,13 +130,13 @@ bool SerializableObject::to_json_file( std::string const& file_name, ErrorStatus* error_status, - optionaldowngrade_version_manifest, + optional target_family_label_spec, int indent) const { return serialize_json_to_file( any(Retainer<>(this)), file_name, - downgrade_version_manifest, + target_family_label_spec, error_status, indent ); diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index c2289917c..1aac2e59b 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -14,6 +14,7 @@ #include "opentimelineio/version.h" #include "ImathBox.h" +#include "serialization.h" #include #include @@ -49,13 +50,13 @@ class SerializableObject to_json_file( std::string const& file_name, ErrorStatus* error_status = nullptr, - optionaldowngrade_version_manifest = {}, + optional target_family_label_spec = {}, int indent = 4) const; std::string to_json_string( ErrorStatus* error_status = nullptr, - optionaldowngrade_version_manifest = {}, + optional target_family_label_spec = {}, int indent = 4) const; static SerializableObject* from_json_file( diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 6b64c5638..986104f79 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Contributors to the OpenTimelineIO project +#include "nonstd/optional.hpp" #include "opentimelineio/serializableObject.h" #include "opentimelineio/serialization.h" #include "opentimelineio/anyDictionary.h" @@ -1247,15 +1248,35 @@ SerializableObject::clone(ErrorStatus* error_status) const : nullptr; } +schema_version_map +_fetch_downgrade_manifest( + family_label_spec target_family_label_spec +) +{ + // @TODO: error handling + return (FAMILY_LABEL_MAP[ + target_family_label_spec.first + ][target_family_label_spec.second]); +} + // to json_string std::string serialize_json_to_string( const any& value, - optional downgrade_version_manifest, + optional target_family_label_spec, ErrorStatus* error_status, int indent ) { + // @TODO: gross + optional downgrade_version_manifest = {}; + schema_version_map mp; + if (target_family_label_spec.has_value()) + { + mp = _fetch_downgrade_manifest(*target_family_label_spec); + downgrade_version_manifest = { &mp }; + } + OTIO_rapidjson::StringBuffer output_string_buffer; OTIO_rapidjson::PrettyWriter< @@ -1293,10 +1314,19 @@ bool serialize_json_to_file( any const& value, std::string const& file_name, - optionaldowngrade_version_manifest, + optional target_family_label_spec, ErrorStatus* error_status, int indent) { + // @TODO: gross + optional downgrade_version_manifest = {}; + schema_version_map mp; + if (target_family_label_spec.has_value()) + { + mp = _fetch_downgrade_manifest(*target_family_label_spec); + downgrade_version_manifest = { &mp }; + } + #if defined(_WINDOWS) const int wlen = MultiByteToWideChar(CP_UTF8, 0, file_name.c_str(), -1, NULL, 0); diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index c4e7888c3..72812c14e 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -55,7 +55,7 @@ static family_to_label_map FAMILY_LABEL_MAP { std::string serialize_json_to_string( const any& value, - optional downgrade_version_manifest = {}, + optional target_family_label_spec = {}, ErrorStatus* error_status = nullptr, int indent = 4 ); @@ -64,7 +64,11 @@ bool serialize_json_to_file( const any& value, std::string const& file_name, - optional downgrade_version_manifest = {}, + // @TODO: I think this wants to be an optional, + // but that isn't allowed, so maybe a const family_label_spec*? + // (to avoid the copy). + // these aren't inner loop functions, so isn't *that* crucial anyway. + optional target_family_label_spec = {}, ErrorStatus* error_status = nullptr, int indent = 4 ); diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 73a58c1f1..41ab1a804 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -104,13 +104,13 @@ PYBIND11_MODULE(_otio, m) { "_serialize_json_to_string", []( PyAny* pyAny, - const std::unordered_map& downgrade_version_manifest, + optional target_family_label_spec, int indent ) { auto result = serialize_json_to_string( pyAny->a, - {&downgrade_version_manifest}, + target_family_label_spec, ErrorStatusHandler(), indent ); @@ -118,13 +118,28 @@ PYBIND11_MODULE(_otio, m) { return result; }, "value"_a, - "downgrade_version_manifest"_a, + py::arg("target_family_label_spec") = nullopt, "indent"_a ) .def("_serialize_json_to_file", - [](PyAny* pyAny, std::string filename, int indent) { - return serialize_json_to_file(pyAny->a, filename, {}, ErrorStatusHandler(), indent); - }, "value"_a, "filename"_a, "indent"_a) + []( + PyAny* pyAny, + std::string filename, + optional target_family_label_spec, + int indent + ) { + return serialize_json_to_file( + pyAny->a, + filename, + target_family_label_spec, + ErrorStatusHandler(), + indent + ); + }, + "value"_a, + "filename"_a, + py::arg("target_family_label_spec") = nullopt, + "indent"_a) .def("deserialize_json_from_string", [](std::string input) { any result; diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 4390df974..70a119875 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -37,7 +37,7 @@ def read_from_string(input_str): return core.deserialize_json_from_string(input_str) -def write_to_string(input_otio, downgrade_version_manifest=None, indent=4): +def write_to_string(input_otio, target_family_label_spec=None, indent=4): """ Serializes an OpenTimelineIO object into a string @@ -51,7 +51,7 @@ def write_to_string(input_otio, downgrade_version_manifest=None, indent=4): """ return core.serialize_json_to_string( input_otio, - downgrade_version_manifest or {}, + target_family_label_spec, indent ) @@ -59,7 +59,7 @@ def write_to_string(input_otio, downgrade_version_manifest=None, indent=4): def write_to_file( input_otio, filepath, - downgrade_version_manifest=None, + target_family_label_spec=None, indent=4 ): """ @@ -78,4 +78,9 @@ def write_to_file( Raises: ValueError: on write error """ - return core.serialize_json_to_file(input_otio, filepath, indent) + return core.serialize_json_to_file( + input_otio, + filepath, + target_family_label_spec, + indent + ) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 1abb71b58..b3cca7a8d 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -72,16 +72,26 @@ ] -def serialize_json_to_string(root, downgrade_version_manifest=None, indent=4): +def serialize_json_to_string(root, target_family_label_spec=None, indent=4): return _serialize_json_to_string( _value_to_any(root), - downgrade_version_manifest or {}, + target_family_label_spec, indent ) -def serialize_json_to_file(root, filename, indent=4): - return _serialize_json_to_file(_value_to_any(root), filename, indent) +def serialize_json_to_file( + root, + filename, + target_family_label_spec=None, + indent=4 +): + return _serialize_json_to_file( + _value_to_any(root), + filename, + target_family_label_spec, + indent + ) def register_type(classobj, schemaname=None): From 67c0412558e9b0fc1c4a7242279bc4c3ac758963 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 26 Aug 2022 14:10:51 -0700 Subject: [PATCH 027/119] pass target_family_label_spec into io_perf_test Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 2e6ba66d7..82057b485 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -63,16 +63,14 @@ main( assert(cl->name() == cl_clone->name()); } - otio::schema_version_map downgrade_version_manifest = { - {"Clip", 1} - }; + otio::family_label_spec target_version = {"OTIO_CORE", "test"}; if (RUN_STRUCT.SINGLE_CLIP_DOWNGRADE_TEST) { otio::SerializableObject::Retainer cl = new otio::Clip("test"); cl->metadata()["example thing"] = "banana"; chrono_time_point begin = std::chrono::steady_clock::now(); - cl->to_json_file("/var/tmp/clip.otio", &err, &downgrade_version_manifest); + cl->to_json_file("/var/tmp/clip.otio", &err, target_version); chrono_time_point end = std::chrono::steady_clock::now(); print_elapsed_time("downgrade clip", begin, end); } @@ -101,8 +99,7 @@ main( begin = std::chrono::steady_clock::now(); const std::string result = timeline.value->to_json_string( &err, - // {} - &downgrade_version_manifest + target_version ); end = std::chrono::steady_clock::now(); @@ -134,7 +131,7 @@ main( timeline.value->to_json_file( "/var/tmp/io_perf_test.otio", &err, - &downgrade_version_manifest + target_version ); end = std::chrono::steady_clock::now(); print_elapsed_time("serialize_json_to_file", begin, end); From d45f6bdc90e70337f3c084a41d1542ec3076ffba Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 26 Aug 2022 18:38:41 -0700 Subject: [PATCH 028/119] add downgrade version family sets (name is cbb) - + function for querying the map - + function for adding new sets Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 31 +++++++++- src/opentimelineio/serialization.cpp | 58 +++++++++++++++++++ src/opentimelineio/serialization.h | 22 ++++--- .../opentimelineio-bindings/otio_bindings.cpp | 14 +++++ .../opentimelineio/core/__init__.py | 4 ++ tests/test_serializable_object.py | 34 +++++++---- 6 files changed, 140 insertions(+), 23 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 82057b485..02da564a9 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -48,9 +48,38 @@ main( return 1; } - // unit test of clone otio::ErrorStatus err; + // inject a test version + otio::add_family_label_version( + "io_perf_test", + "label", + { + {"FakeSchema", 3}, + {"OtherThing", 12000} + }, + &err + ); + + std::cerr << "current version map: " << std::endl; + auto fammap = otio::family_label_version_map(); + for (auto kv_fam : fammap) + { + std::cerr << kv_fam.first << std::endl; + for (auto kv_lbl: kv_fam.second) + { + std::cerr << " " << kv_lbl.first << std::endl; + for (auto kv_schema_version : kv_lbl.second) + { + std::cerr << " \"" << kv_schema_version.first << "\": "; + std::cerr << kv_schema_version.second << std::endl; + } + } + } + + + // unit test of clone + if (RUN_STRUCT.CLONE_TEST) { otio::SerializableObject::Retainer cl = new otio::Clip("test"); diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 986104f79..0bc06f59f 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Contributors to the OpenTimelineIO project +#include "errorStatus.h" #include "nonstd/optional.hpp" #include "opentimelineio/serializableObject.h" #include "opentimelineio/serialization.h" @@ -29,6 +30,63 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { +// built in version labels (@TODO: move this into a generated .h/.cpp file) +static family_to_label_map FAMILY_LABEL_MAP { + { + "OTIO_CORE", + { + { "test", + { + { "Clip", 1 } + } + } + } + } +}; + +bool +add_family_label_version( + const std::string& family, + const std::string& label, + const schema_version_map& new_map, + ErrorStatus* err +) +{ + auto fam_map_it = FAMILY_LABEL_MAP.find(family); + + if (fam_map_it == FAMILY_LABEL_MAP.end()) + { + FAMILY_LABEL_MAP[family] = label_to_schema_version_map {}; + } + + auto& fam_map = FAMILY_LABEL_MAP[family]; + + // check to see if label already exists, not allowed to overwrite + auto label_map_it = fam_map.find(label); + if (label_map_it != fam_map.end()) + { + *err = ErrorStatus( + ErrorStatus::SCHEMA_VERSION_UNSUPPORTED, + ( + "version label: " + label + + " already exists in version family: " + family + + ", cannot add." + ) + ); + return false; + } + + fam_map[label] = new_map; + + return true; +} + +const family_to_label_map +family_label_version_map() +{ + return FAMILY_LABEL_MAP; +} + /** * Base class for encoders. Since rapidjson is templated (no virtual functions) * we need to do our dynamically classed hierarchy to abstract away which writer diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index 72812c14e..a4cc4062c 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -38,19 +38,17 @@ using label_to_schema_version_map = std::unordered_map; using family_label_spec = std::pair; -static family_to_label_map FAMILY_LABEL_MAP { - { - "OTIO_CORE", - { - { "test", - { - { "Clip", 1 } - } - } - } - } -}; +/// add a new family:version:schema_version_map for downgrading +bool +add_family_label_version( + const std::string& family, + const std::string& label, + const schema_version_map& new_map, + ErrorStatus* err +); +const family_to_label_map +family_label_version_map(); std::string serialize_json_to_string( diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 41ab1a804..f075aca2c 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -203,6 +203,20 @@ Return an instance of the schema from data in the data_dict. "schema_name"_a, "version_to_downgrade_from"_a, "downgrade_function"_a); + m.def( + "add_family_label_version", + []( + const std::string& fam, + const std::string& lbl, + const schema_version_map& new_map + ) { + return add_family_label_version(fam, lbl, new_map, ErrorStatusHandler()); + }, + "family"_a, + "label"_a, + "new_map"_a + ); + m.def("family_label_version_map", &family_label_version_map); m.def("flatten_stack", [](Stack* s) { return flatten_stack(s, ErrorStatusHandler()); }, "in_stack"_a); diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index b3cca7a8d..fa0c30d0d 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -27,6 +27,8 @@ _serialize_json_to_string, _serialize_json_to_file, type_version_map, + add_family_label_version, + family_label_version_map, ) from . _core_utils import ( # noqa @@ -63,6 +65,8 @@ 'add_method', 'upgrade_function_for', 'downgrade_function_for', + 'add_family_label_version', + 'family_label_version_map', 'serializable_field', 'deprecated_field', 'serialize_json_to_string', diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index df4b448ee..b234075dc 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -218,17 +218,31 @@ def downgrade_2_to_1(_data_dict): f = FakeThing() f.foo_two = "a thing here" + # @TODO: detect when conflicting version requests are made for schemas + fam = "OTIO_UNIT_TESTS" + lbl = "FakeThingDowngradeTest" + svm = {"FakeThingToDowngrade": 1} + otio.core.add_family_label_version(fam, lbl, svm) + + family_map = otio.core.family_label_version_map() + + self.assertIn(fam, family_map) + self.assertIn(lbl, family_map[fam]) + self.assertEqual(family_map[fam][lbl], svm) + + result = eval( + otio.adapters.otio_json.write_to_string( + f, + ("OTIO_UNIT_TESTS", "FakeThingDowngradeTest") + ) + ) + self.assertDictEqual( - eval( - otio.adapters.otio_json.write_to_string( - f, - downgrade_version_manifest={"FakeThingToDowngrade": 1} - ) - ), - { - "OTIO_SCHEMA": "FakeThingToDowngrade.1", - "foo": "a thing here", - } + result, + { + "OTIO_SCHEMA": "FakeThingToDowngrade.1", + "foo": "a thing here", + } ) From 1c857ceb9570eb3cef9929d170eb5eff9b2c713a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 26 Aug 2022 18:51:08 -0700 Subject: [PATCH 029/119] add some error checking on inserting families Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 12 ++++++++++++ tests/test_serializable_object.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 0bc06f59f..01d38b742 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -52,6 +52,18 @@ add_family_label_version( ErrorStatus* err ) { + if (family == "OTIO_CORE") + { + *err = ErrorStatus( + ErrorStatus::SCHEMA_VERSION_UNSUPPORTED, + ( + "Not allowed to insert new version maps into the OTIO_CORE" + " version family." + ) + ); + return false; + } + auto fam_map_it = FAMILY_LABEL_MAP.find(family); if (fam_map_it == FAMILY_LABEL_MAP.end()) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index b234075dc..ef43c9fc0 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -169,7 +169,6 @@ def test_upgrading_skips_versions(self): """ test that the upgrading system skips versions that don't have upgrade functions""" - @otio.core.register_type class FakeThing(otio.core.SerializableObject): _serializable_label = "NewStuff.4" @@ -218,7 +217,6 @@ def downgrade_2_to_1(_data_dict): f = FakeThing() f.foo_two = "a thing here" - # @TODO: detect when conflicting version requests are made for schemas fam = "OTIO_UNIT_TESTS" lbl = "FakeThingDowngradeTest" svm = {"FakeThingToDowngrade": 1} @@ -226,6 +224,32 @@ def downgrade_2_to_1(_data_dict): family_map = otio.core.family_label_version_map() + # @TODO: probably not the right exception to raise here + with self.assertRaises(otio.exceptions.UnsupportedSchemaError): + otio.core.add_family_label_version( + "OTIO_UNIT_TESTS", + "test", + {"Clip": 3} + ) + otio.core.add_family_label_version( + "OTIO_UNIT_TESTS", + "test2", + {"Clip": 2} + ) + otio.core.add_family_label_version( + "OTIO_UNIT_TESTS", + "test", + {"Clip": 1} + ) + + # never allowed to insert mappings into the OTIO_CORE family + with self.assertRaises(otio.exceptions.UnsupportedSchemaError): + otio.core.add_family_label_version( + "OTIO_CORE", + "not_allowed", + {"Clip": 3} + ) + self.assertIn(fam, family_map) self.assertIn(lbl, family_map[fam]) self.assertEqual(family_map[fam][lbl], svm) From 0b0ec22bc8e75c536434a651da6d165b8736a299 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 26 Aug 2022 21:22:03 -0700 Subject: [PATCH 030/119] minor cleanup of dead comments and stuff Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 01d38b742..b1316fb4d 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -249,9 +249,6 @@ class CloningEncoder : public Encoder void write_value(std::string const& value) override { _store(any(value)); } void write_value(double value) override { _store(any(value)); } - // @{ @TODO: these three the json serializer knows how to dereference them - // ... probably better to scooch that into this class... EXCEPT - // the equivalence test wants to use their == methods void write_value(RationalTime const& value) override { @@ -538,12 +535,6 @@ class CloningEncoder : public Encoder const std::string& schema_vers = schema_string.substr(sep+1); int current_version = -1; - // @TODO: need to pull the version number from the schema string rather - // than the type record - in case there are some kind of weird - // mixed types in the document that don't match the desired - // target -- OR it'll only downgrade from latest/current down - // to target and assume that you've intentionally done something - // weird. Need to consider this I guess. if (!schema_vers.empty()) { current_version = std::stoi(schema_vers); @@ -1323,10 +1314,11 @@ _fetch_downgrade_manifest( family_label_spec target_family_label_spec ) { + const auto& family = target_family_label_spec.first; + const auto& label = target_family_label_spec.second; + // @TODO: error handling - return (FAMILY_LABEL_MAP[ - target_family_label_spec.first - ][target_family_label_spec.second]); + return FAMILY_LABEL_MAP[family][label]; } // to json_string From a221c55b67fab49f8e95e89b93f52fea1c526b19 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 30 Aug 2022 16:58:31 -0700 Subject: [PATCH 031/119] clean up io_perf_test Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 40 +++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 02da564a9..478042be5 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -36,12 +36,35 @@ print_elapsed_time( std::cout << message << ": " << dur/1000000.0 << " [s]" << std::endl; } +void +print_version_map() +{ + std::cerr << "current version map: " << std::endl; + auto fammap = otio::family_label_version_map(); + for (auto kv_fam : fammap) + { + std::cerr << kv_fam.first << std::endl; + for (auto kv_lbl: kv_fam.second) + { + std::cerr << " " << kv_lbl.first << std::endl; + for (auto kv_schema_version : kv_lbl.second) + { + std::cerr << " \"" << kv_schema_version.first << "\": "; + std::cerr << kv_schema_version.second << std::endl; + } + } + } + +} + int main( int argc, char *argv[] ) { + print_version_map(); + if (argc < 2) { std::cerr << "usage: otio_io_perf_test path/to/timeline.otio"; std::cerr << std::endl; @@ -61,22 +84,7 @@ main( &err ); - std::cerr << "current version map: " << std::endl; - auto fammap = otio::family_label_version_map(); - for (auto kv_fam : fammap) - { - std::cerr << kv_fam.first << std::endl; - for (auto kv_lbl: kv_fam.second) - { - std::cerr << " " << kv_lbl.first << std::endl; - for (auto kv_schema_version : kv_lbl.second) - { - std::cerr << " \"" << kv_schema_version.first << "\": "; - std::cerr << kv_schema_version.second << std::endl; - } - } - } - + print_version_map(); // unit test of clone From 5a8ab28c1d34cf3d231719c089017597a51de99d Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 30 Aug 2022 17:00:10 -0700 Subject: [PATCH 032/119] autogen version info struct Signed-off-by: ssteinbach --- src/opentimelineio/CMakeLists.txt | 1 + src/opentimelineio/FAMILY_LABEL_MAP.cpp | 58 +++++++++ src/opentimelineio/serialization.cpp | 14 --- src/opentimelineio/serialization.h | 2 + .../console/autogen_version_info.py | 115 ++++++++++++++++++ 5 files changed, 176 insertions(+), 14 deletions(-) create mode 100644 src/opentimelineio/FAMILY_LABEL_MAP.cpp create mode 100644 src/py-opentimelineio/opentimelineio/console/autogen_version_info.py diff --git a/src/opentimelineio/CMakeLists.txt b/src/opentimelineio/CMakeLists.txt index a5b484b47..f0cd811ee 100644 --- a/src/opentimelineio/CMakeLists.txt +++ b/src/opentimelineio/CMakeLists.txt @@ -72,6 +72,7 @@ add_library(opentimelineio ${OTIO_SHARED_OR_STATIC_LIB} transition.cpp typeRegistry.cpp unknownSchema.cpp + FAMILY_LABEL_MAP.cpp ${OPENTIMELINEIO_HEADER_FILES}) add_library(OTIO::opentimelineio ALIAS opentimelineio) diff --git a/src/opentimelineio/FAMILY_LABEL_MAP.cpp b/src/opentimelineio/FAMILY_LABEL_MAP.cpp new file mode 100644 index 000000000..6544741d6 --- /dev/null +++ b/src/opentimelineio/FAMILY_LABEL_MAP.cpp @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project +// +// This document is automatically generated by running the +// `autogen_version_info` command, or by running `make version-info`. It is +// part of the unit tests suite and should be updated whenever schema versions +// change. If it needs to be updated, run: `make version-info-update` and this +// file should be regenerated. +// +// This is a mapping of "family" to "label", where each label is then mapped to +// a "schema-version map", a mapping of schema name to schema version for that +// label. +#include "opentimelineio/serialization.h" + +namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { + +family_to_label_map FAMILY_LABEL_MAP { + { "OTIO_CORE", + { + { "0.15.0.dev1", + { + { "Adapter", 1 }, + { "Clip", 2 }, + { "Composable", 1 }, + { "Composition", 1 }, + { "Effect", 1 }, + { "ExternalReference", 1 }, + { "FreezeFrame", 1 }, + { "Gap", 1 }, + { "GeneratorReference", 1 }, + { "HookScript", 1 }, + { "ImageSequenceReference", 1 }, + { "Item", 1 }, + { "LinearTimeWarp", 1 }, + { "Marker", 2 }, + { "MediaLinker", 1 }, + { "MediaReference", 1 }, + { "MissingReference", 1 }, + { "PluginManifest", 1 }, + { "SchemaDef", 1 }, + { "SerializableCollection", 1 }, + { "SerializableObject", 1 }, + { "SerializableObjectWithMetadata", 1 }, + { "Stack", 1 }, + { "Test", 1 }, + { "TimeEffect", 1 }, + { "Timeline", 1 }, + { "Track", 1 }, + { "Transition", 1 }, + { "UnknownSchema", 1 }, + } + }, + // {next} + }, + }, +}; + +} } // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index b1316fb4d..c5d861754 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -30,20 +30,6 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -// built in version labels (@TODO: move this into a generated .h/.cpp file) -static family_to_label_map FAMILY_LABEL_MAP { - { - "OTIO_CORE", - { - { "test", - { - { "Clip", 1 } - } - } - } - } -}; - bool add_family_label_version( const std::string& family, diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index a4cc4062c..ad5fd218c 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -38,6 +38,8 @@ using label_to_schema_version_map = std::unordered_map; using family_label_spec = std::pair; +extern family_to_label_map FAMILY_LABEL_MAP; + /// add a new family:version:schema_version_map for downgrading bool add_family_label_version( diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py new file mode 100644 index 000000000..cbd001f65 --- /dev/null +++ b/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the OpenTimelineIO project + +"""Generate the schema-version map for this version of OTIO""" + +import os +import argparse +import json +import tempfile + +import opentimelineio as otio + + +FAMILY = "OTIO_CORE" +LABEL_MAP_TEMPLATE = """{{ "{label}", + {{ +{sv_map} + }} + }}, + // {{next}}""" +MAP_ITEM_TEMPLATE = '{indent}{{ "{key}", {value} }},' +INDENT = 20 + + +def _parsed_args(): + """ parse commandline arguments with argparse """ + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "-d", + "--dryrun", + default=False, + action="store_true", + help="write to stdout instead of printing to file." + ) + parser.add_argument( + "-l", + "--label", + default=otio.__version__, + # @TODO - should we strip the .dev1 label? that would probably be + # more consistent since we don't do sub-beta releases + help="Version label to assign this schema map to." + ) + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help="Filepath to write result to." + ) + + return parser.parse_args() + + +def _insert_current_schema_version_map(src_text, label, version_map): + src_text = src_text.replace("{", "{{").replace("}", "}}") + src_text = src_text.replace("// {{next}}", "{next}") + + map_text = [] + for key, value in version_map.items(): + map_text.append( + MAP_ITEM_TEMPLATE.format( + indent=' ' * INDENT, + key=key, + value=value + ) + ) + map_text = '\n'.join(map_text) + + next_text = LABEL_MAP_TEMPLATE.format(label=label, sv_map=map_text) + return src_text.format(label=label, next=next_text) + + +def main(): + """ main entry point """ + + args = _parsed_args() + + dirname = os.path.dirname(__file__) + + with open(os.path.join(dirname, "built_in_version_header.cpp"), 'r') as fi: + input = fi.read() + + result = _insert_current_schema_version_map( + input, + args.label, + otio.core.type_version_map() + ) + + # print it out somewhere + if args.dryrun: + print(result) + return + + output = args.output + if not output: + output = tempfile.NamedTemporaryFile( + 'w', + suffix="built_in_version_header.cpp", + delete=False + ).name + + with open(output, 'w') as fo: + fo.write(result) + + print("Wrote schema-version map to: '{}'.".format(output)) + + +if __name__ == '__main__': + main() From 95075df6d7fb56b30f77602a0c7311b290f06e5a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 30 Aug 2022 21:29:45 -0700 Subject: [PATCH 033/119] tweak the io_perf_test to downgrade again Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 3 ++- src/opentimelineio/typeRegistry.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 478042be5..fdef19ad8 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -79,6 +79,7 @@ main( "label", { {"FakeSchema", 3}, + {"Clip", 1}, {"OtherThing", 12000} }, &err @@ -100,7 +101,7 @@ main( assert(cl->name() == cl_clone->name()); } - otio::family_label_spec target_version = {"OTIO_CORE", "test"}; + otio::family_label_spec target_version = {"io_perf_test", "label"}; if (RUN_STRUCT.SINGLE_CLIP_DOWNGRADE_TEST) { diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index c4dded289..08d5eafc3 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -135,7 +135,8 @@ TypeRegistry::TypeRegistry() )["downgrade_Clip.2_to_Clip.1"] = downgrade_md; d->erase("media_references"); - d->erase("active_reference_key"); + d->erase("active_media_reference_key"); + d->erase("enabled"); }); } From e8143810a132f89562a6590c6eb8e63a41e8c3ff Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 13:33:01 -0700 Subject: [PATCH 034/119] add exceptions for overwriting up/downgrade fn Signed-off-by: ssteinbach --- .../opentimelineio-bindings/otio_bindings.cpp | 41 +++++++++++++++---- tests/test_serializable_object.py | 21 +++++++--- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index f075aca2c..955cbb578 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -51,8 +51,23 @@ static bool register_upgrade_function(std::string const& schema_name, upgrade_function_obj(dobj); }; - return TypeRegistry::instance().register_upgrade_function(schema_name, version_to_upgrade_to, - upgrade_function); + if ( + !TypeRegistry::instance().register_upgrade_function( + schema_name, + version_to_upgrade_to, + upgrade_function + ) + ) + { + auto err = ErrorStatusHandler(); + err.error_status = ErrorStatus( + ErrorStatus::INTERNAL_ERROR, + "Upgrade function already exists for " + schema_name + ); + return false; + } + + return true; } static bool @@ -71,12 +86,24 @@ register_downgrade_function( downgrade_function_obj(dobj); } ); + + if ( + !TypeRegistry::instance().register_downgrade_function( + schema_name, + version_to_downgrade_from, + downgrade_function + ) + ) + { + auto err = ErrorStatusHandler(); + err.error_status = ErrorStatus( + ErrorStatus::INTERNAL_ERROR, + "Downgrade function already exists for " + schema_name + ); + return false; + } - return TypeRegistry::instance().register_downgrade_function( - schema_name, - version_to_downgrade_from, - downgrade_function - ); + return true; } static void set_type_record(SerializableObject* so, std::string schema_name) { diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index ef43c9fc0..6ef40d9ab 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -137,8 +137,8 @@ def test_cycle_detection(self): class VersioningTests(unittest.TestCase, otio_test_utils.OTIOAssertions): - def test_schema_versioning(self): - """ test basic upgrade function and unsupported schema error """ + def test_schema_definition(self): + """define a schema and instantiate it from python""" # ensure that the type hasn't already been registered self.assertNotIn("Stuff", otio.core.type_version_map()) @@ -165,9 +165,8 @@ class FakeThing(otio.core.SerializableObject): ft = otio.core.instance_from_schema("Stuff", 1, {"foo": "bar"}) self.assertEqual(ft._dynamic_fields['foo'], "bar") - def test_upgrading_skips_versions(self): - """ test that the upgrading system skips versions that don't have - upgrade functions""" + def test_upgrade_versions(self): + """Test adding an upgrade functions for a type""" @otio.core.register_type class FakeThing(otio.core.SerializableObject): @@ -182,6 +181,12 @@ def upgrade_one_to_two(_data_dict): def upgrade_one_to_two_three(_data_dict): return {"foo_3": _data_dict["foo_2"]} + # not allowed to overwrite registered functions + with self.assertRaises(ValueError): + @otio.core.upgrade_function_for(FakeThing, 3) + def upgrade_one_to_two_three_again(_data_dict): + raise RuntimeError("shouldn't see this ever") + ft = otio.core.instance_from_schema("NewStuff", 1, {"foo": "bar"}) self.assertEqual(ft._dynamic_fields['foo_3'], "bar") @@ -214,6 +219,12 @@ class FakeThing(otio.core.SerializableObject): def downgrade_2_to_1(_data_dict): return {"foo": _data_dict["foo_2"]} + # not allowed to overwrite registered functions + with self.assertRaises(ValueError): + @otio.core.downgrade_function_for(FakeThing, 2) + def downgrade_2_to_1_again(_data_dict): + raise RuntimeError("shouldn't see this ever") + f = FakeThing() f.foo_two = "a thing here" From f7c7aa280e11a8cb01e0738c5b2939620aaee43c Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 13:45:40 -0700 Subject: [PATCH 035/119] add exception when double registering a type Signed-off-by: ssteinbach --- .../opentimelineio-bindings/otio_bindings.cpp | 17 +++++++++++++++-- tests/test_serializable_object.py | 7 +++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 955cbb578..93fee391b 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -36,8 +36,21 @@ static void register_python_type(py::object class_object, return r.take_value(); }; - TypeRegistry::instance().register_type(schema_name, schema_version, - nullptr, create, schema_name); + if ( + !TypeRegistry::instance().register_type( + schema_name, + schema_version, + nullptr, + create, + schema_name + ) + ) { + auto err = ErrorStatusHandler(); + err.error_status = ErrorStatus( + ErrorStatus::INTERNAL_ERROR, + "Schema: " + schema_name + " has already been registered." + ); + } } static bool register_upgrade_function(std::string const& schema_name, diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index 6ef40d9ab..a4a985a1d 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -149,6 +149,13 @@ class FakeThing(otio.core.SerializableObject): foo_two = otio.core.serializable_field("foo_2", doc="test") ft = FakeThing() + # not allowed to register a type twice + with self.assertRaises(ValueError): + @otio.core.register_type + class FakeThing(otio.core.SerializableObject): + _serializable_label = "Stuff.1" + foo_two = otio.core.serializable_field("foo_2", doc="test") + self.assertEqual(ft.schema_name(), "Stuff") self.assertEqual(ft.schema_version(), 1) From f0d148d73609f392583aba388540f641d33f27f3 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 13:52:55 -0700 Subject: [PATCH 036/119] move schema version types into typeRegistry Signed-off-by: ssteinbach --- src/opentimelineio/serialization.h | 22 +--------------------- src/opentimelineio/typeRegistry.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index ad5fd218c..c64de2c5d 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -6,6 +6,7 @@ #include "opentimelineio/any.h" #include "opentimelineio/errorStatus.h" #include "opentimelineio/version.h" +#include "opentimelineio/typeRegistry.h" #include #include @@ -31,27 +32,6 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { // "MY_COMPANY_PLUGIN_SETS": {} // } - -// typedefs for the schema downgrading system -using schema_version_map = std::unordered_map; -using label_to_schema_version_map = std::unordered_map; -using family_to_label_map = std::unordered_map; -using family_label_spec = std::pair; - -extern family_to_label_map FAMILY_LABEL_MAP; - -/// add a new family:version:schema_version_map for downgrading -bool -add_family_label_version( - const std::string& family, - const std::string& label, - const schema_version_map& new_map, - ErrorStatus* err -); - -const family_to_label_map -family_label_version_map(); - std::string serialize_json_to_string( const any& value, diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 195620d91..54b4b7d47 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -9,8 +9,10 @@ #include #include #include +#include #include #include +#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { @@ -18,6 +20,14 @@ class SerializableObject; class Encoder; class AnyDictionary; +// typedefs for the schema downgrading system +using schema_version_map = std::unordered_map; +using label_to_schema_version_map = std::unordered_map; +using family_to_label_map = std::unordered_map; +using family_label_spec = std::pair; + +extern family_to_label_map FAMILY_LABEL_MAP; + class TypeRegistry { public: @@ -204,4 +214,20 @@ class TypeRegistry friend class CloningEncoder; }; +// Functions for dealing with schema versioning + +/// add a new family:version:schema_version_map for downgrading +bool +add_family_label_version( + const std::string& family, + const std::string& label, + const schema_version_map& new_map, + ErrorStatus* err +); + +const family_to_label_map +family_label_version_map(); + + + }} // namespace opentimelineio::OPENTIMELINEIO_VERSION From f09782746448bab2e3c22262a5ff4a3ca1c537f9 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 14:29:57 -0700 Subject: [PATCH 037/119] remove "family" level from internal maps Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 42 ++++------ src/opentimelineio/FAMILY_LABEL_MAP.cpp | 74 ++++++++--------- src/opentimelineio/serializableObject.cpp | 9 ++- src/opentimelineio/serializableObject.h | 4 +- src/opentimelineio/serialization.cpp | 99 +++-------------------- src/opentimelineio/serialization.h | 4 +- src/opentimelineio/typeRegistry.cpp | 6 ++ src/opentimelineio/typeRegistry.h | 9 +-- 8 files changed, 79 insertions(+), 168 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index fdef19ad8..108b767b6 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -1,6 +1,7 @@ #include #include "opentimelineio/clip.h" +#include "opentimelineio/typeRegistry.h" #include "util.h" #include #include @@ -40,18 +41,13 @@ void print_version_map() { std::cerr << "current version map: " << std::endl; - auto fammap = otio::family_label_version_map(); - for (auto kv_fam : fammap) + for (auto kv_lbl: opentimelineio::v1_0::core_release_to_schema_version_map()) { - std::cerr << kv_fam.first << std::endl; - for (auto kv_lbl: kv_fam.second) + std::cerr << " " << kv_lbl.first << std::endl; + for (auto kv_schema_version : kv_lbl.second) { - std::cerr << " " << kv_lbl.first << std::endl; - for (auto kv_schema_version : kv_lbl.second) - { - std::cerr << " \"" << kv_schema_version.first << "\": "; - std::cerr << kv_schema_version.second << std::endl; - } + std::cerr << " \"" << kv_schema_version.first << "\": "; + std::cerr << kv_schema_version.second << std::endl; } } @@ -73,22 +69,14 @@ main( otio::ErrorStatus err; - // inject a test version - otio::add_family_label_version( - "io_perf_test", - "label", - { - {"FakeSchema", 3}, - {"Clip", 1}, - {"OtherThing", 12000} - }, - &err - ); + otio::schema_version_map downgrade_manifest = { + {"FakeSchema", 3}, + {"Clip", 1}, + {"OtherThing", 12000} + }; print_version_map(); - // unit test of clone - if (RUN_STRUCT.CLONE_TEST) { otio::SerializableObject::Retainer cl = new otio::Clip("test"); @@ -101,14 +89,12 @@ main( assert(cl->name() == cl_clone->name()); } - otio::family_label_spec target_version = {"io_perf_test", "label"}; - if (RUN_STRUCT.SINGLE_CLIP_DOWNGRADE_TEST) { otio::SerializableObject::Retainer cl = new otio::Clip("test"); cl->metadata()["example thing"] = "banana"; chrono_time_point begin = std::chrono::steady_clock::now(); - cl->to_json_file("/var/tmp/clip.otio", &err, target_version); + cl->to_json_file("/var/tmp/clip.otio", &err, &downgrade_manifest); chrono_time_point end = std::chrono::steady_clock::now(); print_elapsed_time("downgrade clip", begin, end); } @@ -137,7 +123,7 @@ main( begin = std::chrono::steady_clock::now(); const std::string result = timeline.value->to_json_string( &err, - target_version + &downgrade_manifest ); end = std::chrono::steady_clock::now(); @@ -169,7 +155,7 @@ main( timeline.value->to_json_file( "/var/tmp/io_perf_test.otio", &err, - target_version + &downgrade_manifest ); end = std::chrono::steady_clock::now(); print_elapsed_time("serialize_json_to_file", begin, end); diff --git a/src/opentimelineio/FAMILY_LABEL_MAP.cpp b/src/opentimelineio/FAMILY_LABEL_MAP.cpp index 6544741d6..2142abeb0 100644 --- a/src/opentimelineio/FAMILY_LABEL_MAP.cpp +++ b/src/opentimelineio/FAMILY_LABEL_MAP.cpp @@ -10,49 +10,45 @@ // This is a mapping of "family" to "label", where each label is then mapped to // a "schema-version map", a mapping of schema name to schema version for that // label. -#include "opentimelineio/serialization.h" +#include "opentimelineio/typeRegistry.h" namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -family_to_label_map FAMILY_LABEL_MAP { - { "OTIO_CORE", - { - { "0.15.0.dev1", - { - { "Adapter", 1 }, - { "Clip", 2 }, - { "Composable", 1 }, - { "Composition", 1 }, - { "Effect", 1 }, - { "ExternalReference", 1 }, - { "FreezeFrame", 1 }, - { "Gap", 1 }, - { "GeneratorReference", 1 }, - { "HookScript", 1 }, - { "ImageSequenceReference", 1 }, - { "Item", 1 }, - { "LinearTimeWarp", 1 }, - { "Marker", 2 }, - { "MediaLinker", 1 }, - { "MediaReference", 1 }, - { "MissingReference", 1 }, - { "PluginManifest", 1 }, - { "SchemaDef", 1 }, - { "SerializableCollection", 1 }, - { "SerializableObject", 1 }, - { "SerializableObjectWithMetadata", 1 }, - { "Stack", 1 }, - { "Test", 1 }, - { "TimeEffect", 1 }, - { "Timeline", 1 }, - { "Track", 1 }, - { "Transition", 1 }, - { "UnknownSchema", 1 }, - } - }, - // {next} - }, +label_to_schema_version_map CORE_VERSION_MAP { + { "0.15.0.dev1", + { + { "Adapter", 1 }, + { "Clip", 2 }, + { "Composable", 1 }, + { "Composition", 1 }, + { "Effect", 1 }, + { "ExternalReference", 1 }, + { "FreezeFrame", 1 }, + { "Gap", 1 }, + { "GeneratorReference", 1 }, + { "HookScript", 1 }, + { "ImageSequenceReference", 1 }, + { "Item", 1 }, + { "LinearTimeWarp", 1 }, + { "Marker", 2 }, + { "MediaLinker", 1 }, + { "MediaReference", 1 }, + { "MissingReference", 1 }, + { "PluginManifest", 1 }, + { "SchemaDef", 1 }, + { "SerializableCollection", 1 }, + { "SerializableObject", 1 }, + { "SerializableObjectWithMetadata", 1 }, + { "Stack", 1 }, + { "Test", 1 }, + { "TimeEffect", 1 }, + { "Timeline", 1 }, + { "Track", 1 }, + { "Transition", 1 }, + { "UnknownSchema", 1 }, + } }, + // {next} }; } } // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/serializableObject.cpp b/src/opentimelineio/serializableObject.cpp index f10146393..2e98bc8d6 100644 --- a/src/opentimelineio/serializableObject.cpp +++ b/src/opentimelineio/serializableObject.cpp @@ -5,6 +5,7 @@ #include "opentimelineio/deserialization.h" #include "opentimelineio/serialization.h" #include "stringUtils.h" +#include "typeRegistry.h" namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { @@ -114,13 +115,13 @@ SerializableObject::is_unknown_schema() const std::string SerializableObject::to_json_string( ErrorStatus* error_status, - optional target_family_label_spec, + optional schema_version_targets, int indent ) const { return serialize_json_to_string( any(Retainer<>(this)), - target_family_label_spec, + schema_version_targets, error_status, indent ); @@ -130,13 +131,13 @@ bool SerializableObject::to_json_file( std::string const& file_name, ErrorStatus* error_status, - optional target_family_label_spec, + optional schema_version_targets, int indent) const { return serialize_json_to_file( any(Retainer<>(this)), file_name, - target_family_label_spec, + schema_version_targets, error_status, indent ); diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index 1aac2e59b..a38e54090 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -50,13 +50,13 @@ class SerializableObject to_json_file( std::string const& file_name, ErrorStatus* error_status = nullptr, - optional target_family_label_spec = {}, + optional target_family_label_spec = {}, int indent = 4) const; std::string to_json_string( ErrorStatus* error_status = nullptr, - optional target_family_label_spec = {}, + optional target_family_label_spec = {}, int indent = 4) const; static SerializableObject* from_json_file( diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index c5d861754..8acb21c2f 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -30,59 +30,10 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -bool -add_family_label_version( - const std::string& family, - const std::string& label, - const schema_version_map& new_map, - ErrorStatus* err -) +const label_to_schema_version_map +core_label_to_schema_version_map() { - if (family == "OTIO_CORE") - { - *err = ErrorStatus( - ErrorStatus::SCHEMA_VERSION_UNSUPPORTED, - ( - "Not allowed to insert new version maps into the OTIO_CORE" - " version family." - ) - ); - return false; - } - - auto fam_map_it = FAMILY_LABEL_MAP.find(family); - - if (fam_map_it == FAMILY_LABEL_MAP.end()) - { - FAMILY_LABEL_MAP[family] = label_to_schema_version_map {}; - } - - auto& fam_map = FAMILY_LABEL_MAP[family]; - - // check to see if label already exists, not allowed to overwrite - auto label_map_it = fam_map.find(label); - if (label_map_it != fam_map.end()) - { - *err = ErrorStatus( - ErrorStatus::SCHEMA_VERSION_UNSUPPORTED, - ( - "version label: " + label - + " already exists in version family: " + family - + ", cannot add." - ) - ); - return false; - } - - fam_map[label] = new_map; - - return true; -} - -const family_to_label_map -family_label_version_map() -{ - return FAMILY_LABEL_MAP; + return CORE_VERSION_MAP; } /** @@ -160,10 +111,10 @@ class CloningEncoder : public Encoder CloningEncoder( CloningEncoder::ResultObjectPolicy result_object_policy, - optional downgrade_version_manifest = {} + optional schema_version_targets = {} ) : _result_object_policy(result_object_policy), - _downgrade_version_manifest(downgrade_version_manifest) + _downgrade_version_manifest(schema_version_targets) { using namespace std::placeholders; _error_function = std::bind(&CloningEncoder::_error, this, _1); @@ -892,11 +843,11 @@ bool SerializableObject::Writer::write_root( any const& value, Encoder& encoder, - optional downgrade_version_manifest, + optional schema_version_targets, ErrorStatus* error_status ) { - Writer w(encoder, downgrade_version_manifest); + Writer w(encoder, schema_version_targets); w.write(w._no_key, value); return !encoder.has_errored(error_status); } @@ -1295,36 +1246,16 @@ SerializableObject::clone(ErrorStatus* error_status) const : nullptr; } -schema_version_map -_fetch_downgrade_manifest( - family_label_spec target_family_label_spec -) -{ - const auto& family = target_family_label_spec.first; - const auto& label = target_family_label_spec.second; - - // @TODO: error handling - return FAMILY_LABEL_MAP[family][label]; -} // to json_string std::string serialize_json_to_string( const any& value, - optional target_family_label_spec, + optional schema_version_targets, ErrorStatus* error_status, int indent ) { - // @TODO: gross - optional downgrade_version_manifest = {}; - schema_version_map mp; - if (target_family_label_spec.has_value()) - { - mp = _fetch_downgrade_manifest(*target_family_label_spec); - downgrade_version_manifest = { &mp }; - } - OTIO_rapidjson::StringBuffer output_string_buffer; OTIO_rapidjson::PrettyWriter< @@ -1346,7 +1277,7 @@ serialize_json_to_string( !SerializableObject::Writer::write_root( value, json_encoder, - downgrade_version_manifest, + schema_version_targets, error_status ) ) @@ -1362,18 +1293,10 @@ bool serialize_json_to_file( any const& value, std::string const& file_name, - optional target_family_label_spec, + optional schema_version_targets, ErrorStatus* error_status, int indent) { - // @TODO: gross - optional downgrade_version_manifest = {}; - schema_version_map mp; - if (target_family_label_spec.has_value()) - { - mp = _fetch_downgrade_manifest(*target_family_label_spec); - downgrade_version_manifest = { &mp }; - } #if defined(_WINDOWS) const int wlen = @@ -1415,7 +1338,7 @@ serialize_json_to_file( status = SerializableObject::Writer::write_root( value, json_encoder, - downgrade_version_manifest, + schema_version_targets, error_status ); diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index c64de2c5d..5d9fa43d7 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -35,7 +35,7 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { std::string serialize_json_to_string( const any& value, - optional target_family_label_spec = {}, + optional schema_version_targets = {}, ErrorStatus* error_status = nullptr, int indent = 4 ); @@ -48,7 +48,7 @@ serialize_json_to_file( // but that isn't allowed, so maybe a const family_label_spec*? // (to avoid the copy). // these aren't inner loop functions, so isn't *that* crucial anyway. - optional target_family_label_spec = {}, + optional schema_version_targets = {}, ErrorStatus* error_status = nullptr, int indent = 4 ); diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 08d5eafc3..0650ba07a 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -386,4 +386,10 @@ TypeRegistry::type_version_map( } } +const label_to_schema_version_map +core_release_to_schema_version_map() +{ + return CORE_VERSION_MAP; +} + }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 54b4b7d47..81d692f6c 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -23,10 +23,8 @@ class AnyDictionary; // typedefs for the schema downgrading system using schema_version_map = std::unordered_map; using label_to_schema_version_map = std::unordered_map; -using family_to_label_map = std::unordered_map; -using family_label_spec = std::pair; -extern family_to_label_map FAMILY_LABEL_MAP; +extern label_to_schema_version_map CORE_VERSION_MAP; class TypeRegistry { @@ -225,8 +223,9 @@ add_family_label_version( ErrorStatus* err ); -const family_to_label_map -family_label_version_map(); +// fetch the map of core release:schema version maps +const label_to_schema_version_map +core_release_to_schema_version_map(); From ad41fd6b31e7b994db048150f43253f4f8b1e648 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 14:31:57 -0700 Subject: [PATCH 038/119] mark the internal map as const Signed-off-by: ssteinbach --- src/opentimelineio/FAMILY_LABEL_MAP.cpp | 2 +- src/opentimelineio/typeRegistry.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/opentimelineio/FAMILY_LABEL_MAP.cpp b/src/opentimelineio/FAMILY_LABEL_MAP.cpp index 2142abeb0..096cdd831 100644 --- a/src/opentimelineio/FAMILY_LABEL_MAP.cpp +++ b/src/opentimelineio/FAMILY_LABEL_MAP.cpp @@ -14,7 +14,7 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -label_to_schema_version_map CORE_VERSION_MAP { +const label_to_schema_version_map CORE_VERSION_MAP { { "0.15.0.dev1", { { "Adapter", 1 }, diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 81d692f6c..5b5f7d0af 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -24,7 +24,7 @@ class AnyDictionary; using schema_version_map = std::unordered_map; using label_to_schema_version_map = std::unordered_map; -extern label_to_schema_version_map CORE_VERSION_MAP; +extern const label_to_schema_version_map CORE_VERSION_MAP; class TypeRegistry { From 7fc01e1135f64d48878c881bf84561fb855a501a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 14:34:11 -0700 Subject: [PATCH 039/119] move FAMILY_VERSION_MAP file to CORE_VERSION_MAP Signed-off-by: ssteinbach --- src/opentimelineio/CMakeLists.txt | 2 +- .../{FAMILY_LABEL_MAP.cpp => CORE_VERSION_MAP.cpp} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/opentimelineio/{FAMILY_LABEL_MAP.cpp => CORE_VERSION_MAP.cpp} (100%) diff --git a/src/opentimelineio/CMakeLists.txt b/src/opentimelineio/CMakeLists.txt index f0cd811ee..29cc3a8a8 100644 --- a/src/opentimelineio/CMakeLists.txt +++ b/src/opentimelineio/CMakeLists.txt @@ -72,7 +72,7 @@ add_library(opentimelineio ${OTIO_SHARED_OR_STATIC_LIB} transition.cpp typeRegistry.cpp unknownSchema.cpp - FAMILY_LABEL_MAP.cpp + CORE_VERSION_MAP.cpp ${OPENTIMELINEIO_HEADER_FILES}) add_library(OTIO::opentimelineio ALIAS opentimelineio) diff --git a/src/opentimelineio/FAMILY_LABEL_MAP.cpp b/src/opentimelineio/CORE_VERSION_MAP.cpp similarity index 100% rename from src/opentimelineio/FAMILY_LABEL_MAP.cpp rename to src/opentimelineio/CORE_VERSION_MAP.cpp From e0cf66c2b9dee421c51d24c21d9d5822b1c16a83 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 14:37:46 -0700 Subject: [PATCH 040/119] update autogen_version_info to match new files Signed-off-by: ssteinbach --- .../opentimelineio/console/autogen_version_info.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py index cbd001f65..9c25769b8 100644 --- a/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py +++ b/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py @@ -7,21 +7,19 @@ import os import argparse -import json import tempfile import opentimelineio as otio -FAMILY = "OTIO_CORE" LABEL_MAP_TEMPLATE = """{{ "{label}", - {{ + {{ {sv_map} - }} - }}, - // {{next}}""" + }} + }}, + // {{next}}""" MAP_ITEM_TEMPLATE = '{indent}{{ "{key}", {value} }},' -INDENT = 20 +INDENT = 12 def _parsed_args(): From 36e8bf15c97ef57b26e695411d6fbf7310b6de90 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 21:37:29 -0700 Subject: [PATCH 041/119] Autogen_version_info -> CORE_VERSION_MAP term Signed-off-by: ssteinbach --- .../console/autogen_version_info.py | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py index 9c25769b8..6ecdcdea8 100644 --- a/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py +++ b/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py @@ -3,9 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project -"""Generate the schema-version map for this version of OTIO""" +"""Generate the CORE_VERSION_MAP for this version of OTIO""" -import os import argparse import tempfile @@ -23,8 +22,6 @@ def _parsed_args(): - """ parse commandline arguments with argparse """ - parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -44,21 +41,32 @@ def _parsed_args(): # more consistent since we don't do sub-beta releases help="Version label to assign this schema map to." ) + parser.add_argument( + "-i", + "--input", + type=str, + default=None, + required=True, + help="Path to CORE_VERSION_MAP.last.cpp" + ) parser.add_argument( "-o", "--output", type=str, default=None, - help="Filepath to write result to." + help="Path to where CORE_VERSION_MAP.cpp should be written to." ) return parser.parse_args() -def _insert_current_schema_version_map(src_text, label, version_map): +def generate_core_version_map(src_text, label, version_map): + # turn the braces in the .cpp file into python-format template compatible + # form ({{ }} where needed) src_text = src_text.replace("{", "{{").replace("}", "}}") src_text = src_text.replace("// {{next}}", "{next}") + # iterate over the map and print the template out map_text = [] for key, value in version_map.items(): map_text.append( @@ -70,27 +78,23 @@ def _insert_current_schema_version_map(src_text, label, version_map): ) map_text = '\n'.join(map_text) + # assemble the result next_text = LABEL_MAP_TEMPLATE.format(label=label, sv_map=map_text) return src_text.format(label=label, next=next_text) def main(): - """ main entry point """ - args = _parsed_args() - dirname = os.path.dirname(__file__) - - with open(os.path.join(dirname, "built_in_version_header.cpp"), 'r') as fi: + with open(args.input, 'r') as fi: input = fi.read() - result = _insert_current_schema_version_map( + result = generate_core_version_map( input, args.label, otio.core.type_version_map() ) - # print it out somewhere if args.dryrun: print(result) return @@ -99,14 +103,14 @@ def main(): if not output: output = tempfile.NamedTemporaryFile( 'w', - suffix="built_in_version_header.cpp", + suffix="CORE_VERSION_MAP.cpp", delete=False ).name with open(output, 'w') as fo: fo.write(result) - print("Wrote schema-version map to: '{}'.".format(output)) + print("Wrote CORE_VERSION_MAP to: '{}'.".format(output)) if __name__ == '__main__': From f0a7871d847507799644b9bfe5c064c9da8321ce Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 21:39:48 -0700 Subject: [PATCH 042/119] build out CORE_VERSION_MAP.last.cpp - adds autogen_version_map which uses the .last file to generate a new one. There is a unit test to test this as well. Signed-off-by: ssteinbach --- Makefile | 7 ++++ src/opentimelineio/CORE_VERSION_MAP.cpp | 1 + src/opentimelineio/CORE_VERSION_MAP.last.cpp | 21 ++++++++++ ...version_info.py => autogen_version_map.py} | 0 tests/test_serialized_schema.py | 41 ++++++++++++++++++- 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/opentimelineio/CORE_VERSION_MAP.last.cpp rename src/py-opentimelineio/opentimelineio/console/{autogen_version_info.py => autogen_version_map.py} (100%) diff --git a/Makefile b/Makefile index c89d0f4db..b64fd167c 100644 --- a/Makefile +++ b/Makefile @@ -182,6 +182,13 @@ doc-plugins: doc-plugins-update: @python src/py-opentimelineio/opentimelineio/console/autogen_plugin_documentation.py -o docs/tutorials/otio-plugins.md --public-only --sanitized-paths +# build the CORE_VERSION_MAP cpp file +version-map: + @python src/py-opentimelineio/opentimelineio/console/autogen_version_map.py -i src/opentimelineio/CORE_VERSION_MAP.last.cpp --dryrun + +version-map-update: + @python src/py-opentimelineio/opentimelineio/console/autogen_version_map.py -i src/opentimelineio/CORE_VERSION_MAP.last.cpp -o src/opentimelineio/CORE_VERSION_MAP.cpp + # generate documentation in html doc-html: @# if you just want to build the docs yourself outside of RTD diff --git a/src/opentimelineio/CORE_VERSION_MAP.cpp b/src/opentimelineio/CORE_VERSION_MAP.cpp index 096cdd831..d5d6b08e0 100644 --- a/src/opentimelineio/CORE_VERSION_MAP.cpp +++ b/src/opentimelineio/CORE_VERSION_MAP.cpp @@ -14,6 +14,7 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { +// @TODO: could slit this into "current" and "last" files, that get swizzled const label_to_schema_version_map CORE_VERSION_MAP { { "0.15.0.dev1", { diff --git a/src/opentimelineio/CORE_VERSION_MAP.last.cpp b/src/opentimelineio/CORE_VERSION_MAP.last.cpp new file mode 100644 index 000000000..8276293a5 --- /dev/null +++ b/src/opentimelineio/CORE_VERSION_MAP.last.cpp @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project +// +// This document is automatically generated by running the +// `autogen_version_info` command, or by running `make version-info`. It is +// part of the unit tests suite and should be updated whenever schema versions +// change. If it needs to be updated, run: `make version-info-update` and this +// file should be regenerated. +// +// This is a mapping of "family" to "label", where each label is then mapped to +// a "schema-version map", a mapping of schema name to schema version for that +// label. +#include "opentimelineio/typeRegistry.h" + +namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { + +const label_to_schema_version_map CORE_VERSION_MAP { + // {next} +}; + +} } // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_info.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py similarity index 100% rename from src/py-opentimelineio/opentimelineio/console/autogen_version_info.py rename to src/py-opentimelineio/opentimelineio/console/autogen_version_map.py diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index 3dc76a917..a13f2d3a4 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -4,9 +4,12 @@ import unittest import os +import opentimelineio as otio + from opentimelineio.console import ( autogen_serialized_datamodel as asd, autogen_plugin_documentation as apd, + autogen_version_map as avm ) @@ -40,7 +43,7 @@ def test_serialized_schema(self): @unittest.skipIf( os.environ.get("OTIO_DISABLE_SERIALIZED_SCHEMA_TEST"), - "Serialized schema test disabled because " + "Plugin documentation test disabled because " "$OTIO_DISABLE_SERIALIZED_SCHEMA_TEST is set to something other than ''" ) class PluginDocumentationTester(unittest.TestCase): @@ -69,5 +72,41 @@ def test_plugin_documentation(self): ) +@unittest +@unittest.skipIf( + os.environ.get("OTIO_DISABLE_SERIALIZED_SCHEMA_TEST"), + "CORE_VERSION_MAP generation test disabled because " + "$OTIO_DISABLE_SERIALIZED_SCHEMA_TEST is set to something other than ''" +) +class CoreVersionMapGenerationTester(unittest.TestCase): + def test_core_version_map_generator(self): + """Verify the current CORE_VERSION_MAP matches the checked in one.""" + + pt = os.path.dirname(os.path.dirname(__file__)) + root = os.path.join(pt, "src", "opentimelineio") + template_fp = os.path.join(root, "CORE_VERSION_MAP.last.cpp") + target_fp = os.path.join(root, "CORE_VERSION_MAP.cpp") + + with open(target_fp) as fi: + baseline_text = fi.read() + + test_text = avm.generate_core_version_map( + template_fp, + otio.__version__, + otio.core.type_version_map() + ) + + self.maxDiff = None + self.longMessage = True + self.assertMultiLineEqual( + baseline_text, + test_text, + "\n The CORE_VERSION_MAP has changed and the autogenerated one in " + " {} needs to be updated. run: `make version-map-update`".format( + template_fp + ) + ) + + if __name__ == '__main__': unittest.main() From 1d01b4d678bbc5a5270fc8b084de810301b6547b Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 23:11:01 -0700 Subject: [PATCH 043/119] type registry allows mutliple-regsiter if same - helps for unit tests and manifest testing Signed-off-by: ssteinbach --- src/opentimelineio/typeRegistry.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 0650ba07a..6c4e773dc 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -150,6 +150,25 @@ TypeRegistry::register_type( { std::lock_guard lock(_registry_mutex); + auto existing_tr = _find_type_record(schema_name); + + // if the exact type record has already been added (happens in unit tests + // and re-setting manifest stuff) + if (existing_tr) + { + if ( + existing_tr->schema_name == schema_name + && existing_tr->schema_version == schema_version + && existing_tr->class_name == class_name + && ( + existing_tr->create.target() + == create.target() + ) + ) { + return true; + } + } + if (!_find_type_record(schema_name)) { _TypeRecord* r = From feddbda3c14f57863428bdf5fe5ac6bcf7976863 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 23:12:16 -0700 Subject: [PATCH 044/119] plumb through bindings (schema_version_map) Signed-off-by: ssteinbach --- .../opentimelineio-bindings/otio_bindings.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 93fee391b..5ee4b23a5 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -144,13 +144,13 @@ PYBIND11_MODULE(_otio, m) { "_serialize_json_to_string", []( PyAny* pyAny, - optional target_family_label_spec, + optional schema_version_targets, int indent ) { auto result = serialize_json_to_string( pyAny->a, - target_family_label_spec, + schema_version_targets, ErrorStatusHandler(), indent ); @@ -158,27 +158,27 @@ PYBIND11_MODULE(_otio, m) { return result; }, "value"_a, - py::arg("target_family_label_spec") = nullopt, + py::arg("schema_version_targets") = nullopt, "indent"_a ) .def("_serialize_json_to_file", []( PyAny* pyAny, std::string filename, - optional target_family_label_spec, + optional schema_version_targets, int indent ) { return serialize_json_to_file( pyAny->a, filename, - target_family_label_spec, + schema_version_targets, ErrorStatusHandler(), indent ); }, "value"_a, "filename"_a, - py::arg("target_family_label_spec") = nullopt, + py::arg("schema_version_targets") = nullopt, "indent"_a) .def("deserialize_json_from_string", [](std::string input) { From 47501ecb4fc3a20c4712912ca793fe9d38228815 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 31 Aug 2022 23:14:10 -0700 Subject: [PATCH 045/119] truing more names up in bindings Signed-off-by: ssteinbach --- .../opentimelineio-bindings/otio_bindings.cpp | 15 +-------------- .../opentimelineio/core/__init__.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 5ee4b23a5..1e0d5f994 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -243,20 +243,7 @@ Return an instance of the schema from data in the data_dict. "schema_name"_a, "version_to_downgrade_from"_a, "downgrade_function"_a); - m.def( - "add_family_label_version", - []( - const std::string& fam, - const std::string& lbl, - const schema_version_map& new_map - ) { - return add_family_label_version(fam, lbl, new_map, ErrorStatusHandler()); - }, - "family"_a, - "label"_a, - "new_map"_a - ); - m.def("family_label_version_map", &family_label_version_map); + m.def("release_to_schema_version_map", &core_release_to_schema_version_map); m.def("flatten_stack", [](Stack* s) { return flatten_stack(s, ErrorStatusHandler()); }, "in_stack"_a); diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index fa0c30d0d..2d3f27f44 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -27,8 +27,6 @@ _serialize_json_to_string, _serialize_json_to_file, type_version_map, - add_family_label_version, - family_label_version_map, ) from . _core_utils import ( # noqa @@ -65,8 +63,7 @@ 'add_method', 'upgrade_function_for', 'downgrade_function_for', - 'add_family_label_version', - 'family_label_version_map', + 'fetch_version_map', 'serializable_field', 'deprecated_field', 'serialize_json_to_string', @@ -76,10 +73,14 @@ ] -def serialize_json_to_string(root, target_family_label_spec=None, indent=4): +def fetch_version_map(family, label): + raise NotImplementedError() + + +def serialize_json_to_string(root, schema_version_targets=None, indent=4): return _serialize_json_to_string( _value_to_any(root), - target_family_label_spec, + schema_version_targets, indent ) @@ -87,13 +88,13 @@ def serialize_json_to_string(root, target_family_label_spec=None, indent=4): def serialize_json_to_file( root, filename, - target_family_label_spec=None, + schema_version_targets=None, indent=4 ): return _serialize_json_to_file( _value_to_any(root), filename, - target_family_label_spec, + schema_version_targets, indent ) From 1078a8d37de8212b9cc4f74d5e282db7d51ccc7d Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 1 Sep 2022 14:39:46 -0700 Subject: [PATCH 046/119] update tests for CORE_VERSION_MAP generation Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 4 +-- src/opentimelineio/CORE_VERSION_MAP.cpp | 1 - src/opentimelineio/serialization.cpp | 6 ---- src/opentimelineio/typeRegistry.cpp | 6 ---- src/opentimelineio/typeRegistry.h | 6 ---- tests/test_serializable_object.py | 45 ++++--------------------- tests/test_serialized_schema.py | 24 +++++++++---- 7 files changed, 24 insertions(+), 68 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 108b767b6..ea839adbc 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -41,7 +41,7 @@ void print_version_map() { std::cerr << "current version map: " << std::endl; - for (auto kv_lbl: opentimelineio::v1_0::core_release_to_schema_version_map()) + for (auto kv_lbl: otio::CORE_VERSION_MAP) { std::cerr << " " << kv_lbl.first << std::endl; for (auto kv_schema_version : kv_lbl.second) @@ -75,8 +75,6 @@ main( {"OtherThing", 12000} }; - print_version_map(); - if (RUN_STRUCT.CLONE_TEST) { otio::SerializableObject::Retainer cl = new otio::Clip("test"); diff --git a/src/opentimelineio/CORE_VERSION_MAP.cpp b/src/opentimelineio/CORE_VERSION_MAP.cpp index d5d6b08e0..096cdd831 100644 --- a/src/opentimelineio/CORE_VERSION_MAP.cpp +++ b/src/opentimelineio/CORE_VERSION_MAP.cpp @@ -14,7 +14,6 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -// @TODO: could slit this into "current" and "last" files, that get swizzled const label_to_schema_version_map CORE_VERSION_MAP { { "0.15.0.dev1", { diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 8acb21c2f..cf52e5b81 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -30,12 +30,6 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -const label_to_schema_version_map -core_label_to_schema_version_map() -{ - return CORE_VERSION_MAP; -} - /** * Base class for encoders. Since rapidjson is templated (no virtual functions) * we need to do our dynamically classed hierarchy to abstract away which writer diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 6c4e773dc..3cb9b448e 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -405,10 +405,4 @@ TypeRegistry::type_version_map( } } -const label_to_schema_version_map -core_release_to_schema_version_map() -{ - return CORE_VERSION_MAP; -} - }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 5b5f7d0af..1129df3e0 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -223,10 +223,4 @@ add_family_label_version( ErrorStatus* err ); -// fetch the map of core release:schema version maps -const label_to_schema_version_map -core_release_to_schema_version_map(); - - - }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index a4a985a1d..ba85bcfe8 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -235,48 +235,15 @@ def downgrade_2_to_1_again(_data_dict): f = FakeThing() f.foo_two = "a thing here" - fam = "OTIO_UNIT_TESTS" - lbl = "FakeThingDowngradeTest" - svm = {"FakeThingToDowngrade": 1} - otio.core.add_family_label_version(fam, lbl, svm) + # downgrade_target = otio.core.fetch_version_map( + # "OTIO_CORE", + # otio.__version__ + # ) - family_map = otio.core.family_label_version_map() - - # @TODO: probably not the right exception to raise here - with self.assertRaises(otio.exceptions.UnsupportedSchemaError): - otio.core.add_family_label_version( - "OTIO_UNIT_TESTS", - "test", - {"Clip": 3} - ) - otio.core.add_family_label_version( - "OTIO_UNIT_TESTS", - "test2", - {"Clip": 2} - ) - otio.core.add_family_label_version( - "OTIO_UNIT_TESTS", - "test", - {"Clip": 1} - ) - - # never allowed to insert mappings into the OTIO_CORE family - with self.assertRaises(otio.exceptions.UnsupportedSchemaError): - otio.core.add_family_label_version( - "OTIO_CORE", - "not_allowed", - {"Clip": 3} - ) - - self.assertIn(fam, family_map) - self.assertIn(lbl, family_map[fam]) - self.assertEqual(family_map[fam][lbl], svm) + downgrade_target = {"FakeThingToDowngrade": 1} result = eval( - otio.adapters.otio_json.write_to_string( - f, - ("OTIO_UNIT_TESTS", "FakeThingDowngradeTest") - ) + otio.adapters.otio_json.write_to_string(f, downgrade_target) ) self.assertDictEqual( diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index a13f2d3a4..143a9b451 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -3,6 +3,8 @@ import unittest import os +import sys +import subprocess import opentimelineio as otio @@ -72,7 +74,6 @@ def test_plugin_documentation(self): ) -@unittest @unittest.skipIf( os.environ.get("OTIO_DISABLE_SERIALIZED_SCHEMA_TEST"), "CORE_VERSION_MAP generation test disabled because " @@ -90,20 +91,29 @@ def test_core_version_map_generator(self): with open(target_fp) as fi: baseline_text = fi.read() - test_text = avm.generate_core_version_map( - template_fp, - otio.__version__, - otio.core.type_version_map() + proc = subprocess.Popen( + [ + sys.executable, + avm.__file__, + "-i", + template_fp, + "-d", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) + stdout, _ = proc.communicate() + + test_text = stdout.decode("utf-8")[:-1] self.maxDiff = None self.longMessage = True self.assertMultiLineEqual( baseline_text, test_text, - "\n The CORE_VERSION_MAP has changed and the autogenerated one in " + "\n The CORE_VERSION_MAP has changed and the autogenerated one in" " {} needs to be updated. run: `make version-map-update`".format( - template_fp + target_fp ) ) From 1fa6542292025336a7c5a089c571de5406d3e4c1 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 1 Sep 2022 14:40:57 -0700 Subject: [PATCH 047/119] back out of double-register exception Signed-off-by: ssteinbach --- src/opentimelineio/typeRegistry.cpp | 36 +++++++++---------- .../opentimelineio-bindings/otio_bindings.cpp | 35 +++++++++++++----- .../opentimelineio/core/__init__.py | 18 ++++++++-- tests/test_serializable_object.py | 20 +++++++---- 4 files changed, 72 insertions(+), 37 deletions(-) diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 3cb9b448e..90254796d 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -150,24 +150,24 @@ TypeRegistry::register_type( { std::lock_guard lock(_registry_mutex); - auto existing_tr = _find_type_record(schema_name); - - // if the exact type record has already been added (happens in unit tests - // and re-setting manifest stuff) - if (existing_tr) - { - if ( - existing_tr->schema_name == schema_name - && existing_tr->schema_version == schema_version - && existing_tr->class_name == class_name - && ( - existing_tr->create.target() - == create.target() - ) - ) { - return true; - } - } + // auto existing_tr = _find_type_record(schema_name); + // + // // if the exact type record has already been added (happens in unit tests + // // and re-setting manifest stuff) + // if (existing_tr) + // { + // if ( + // existing_tr->schema_name == schema_name + // && existing_tr->schema_version == schema_version + // && existing_tr->class_name == class_name + // && ( + // existing_tr->create.target() + // == create.target() + // ) + // ) { + // return true; + // } + // } if (!_find_type_record(schema_name)) { diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 1e0d5f994..1dab6efe5 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -16,6 +16,9 @@ namespace py = pybind11; using namespace pybind11::literals; +// temporarily disabling this feature while I chew on it +const static bool EXCEPTION_ON_DOUBLE_REGISTER = false; + static void register_python_type(py::object class_object, std::string schema_name, int schema_version) { @@ -44,6 +47,7 @@ static void register_python_type(py::object class_object, create, schema_name ) + && EXCEPTION_ON_DOUBLE_REGISTER ) { auto err = ErrorStatusHandler(); err.error_status = ErrorStatus( @@ -69,7 +73,7 @@ static bool register_upgrade_function(std::string const& schema_name, schema_name, version_to_upgrade_to, upgrade_function - ) + ) //&& EXCEPTION_ON_DOUBLE_REGISTER ) { auto err = ErrorStatusHandler(); @@ -105,7 +109,7 @@ register_downgrade_function( schema_name, version_to_downgrade_from, downgrade_function - ) + ) //&& EXCEPTION_ON_DOUBLE_REGISTER ) { auto err = ErrorStatusHandler(); @@ -144,13 +148,18 @@ PYBIND11_MODULE(_otio, m) { "_serialize_json_to_string", []( PyAny* pyAny, - optional schema_version_targets, + const schema_version_map& schema_version_targets, int indent ) { + optional pass_through = {}; + if (!schema_version_targets.empty()) + { + pass_through = {&schema_version_targets}; + } auto result = serialize_json_to_string( pyAny->a, - schema_version_targets, + pass_through, ErrorStatusHandler(), indent ); @@ -158,27 +167,32 @@ PYBIND11_MODULE(_otio, m) { return result; }, "value"_a, - py::arg("schema_version_targets") = nullopt, + "schema_version_targets"_a, "indent"_a ) .def("_serialize_json_to_file", []( PyAny* pyAny, std::string filename, - optional schema_version_targets, + const schema_version_map& schema_version_targets, int indent ) { + optional pass_through = {}; + if (!schema_version_targets.empty()) + { + pass_through = {&schema_version_targets}; + } return serialize_json_to_file( pyAny->a, filename, - schema_version_targets, + pass_through, ErrorStatusHandler(), indent ); }, "value"_a, "filename"_a, - py::arg("schema_version_targets") = nullopt, + "schema_version_targets"_a, "indent"_a) .def("deserialize_json_from_string", [](std::string input) { @@ -243,7 +257,10 @@ Return an instance of the schema from data in the data_dict. "schema_name"_a, "version_to_downgrade_from"_a, "downgrade_function"_a); - m.def("release_to_schema_version_map", &core_release_to_schema_version_map); + m.def( + "release_to_schema_version_map", + [](){ return label_to_schema_version_map(CORE_VERSION_MAP);} + ); m.def("flatten_stack", [](Stack* s) { return flatten_stack(s, ErrorStatusHandler()); }, "in_stack"_a); diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 2d3f27f44..a521d12b8 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -27,6 +27,7 @@ _serialize_json_to_string, _serialize_json_to_file, type_version_map, + release_to_schema_version_map, ) from . _core_utils import ( # noqa @@ -73,14 +74,25 @@ ] +def full_version_map(): + return { + "OTIO_CORE": release_to_schema_version_map() + } + + def fetch_version_map(family, label): - raise NotImplementedError() + if family != "OTIO_CORE": + raise NotImplementedError + + src = release_to_schema_version_map() + + return src[label] def serialize_json_to_string(root, schema_version_targets=None, indent=4): return _serialize_json_to_string( _value_to_any(root), - schema_version_targets, + schema_version_targets or {}, indent ) @@ -94,7 +106,7 @@ def serialize_json_to_file( return _serialize_json_to_file( _value_to_any(root), filename, - schema_version_targets, + schema_version_targets or {}, indent ) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index ba85bcfe8..b564cba15 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -149,13 +149,6 @@ class FakeThing(otio.core.SerializableObject): foo_two = otio.core.serializable_field("foo_2", doc="test") ft = FakeThing() - # not allowed to register a type twice - with self.assertRaises(ValueError): - @otio.core.register_type - class FakeThing(otio.core.SerializableObject): - _serializable_label = "Stuff.1" - foo_two = otio.core.serializable_field("foo_2", doc="test") - self.assertEqual(ft.schema_name(), "Stuff") self.assertEqual(ft.schema_version(), 1) @@ -172,6 +165,19 @@ class FakeThing(otio.core.SerializableObject): ft = otio.core.instance_from_schema("Stuff", 1, {"foo": "bar"}) self.assertEqual(ft._dynamic_fields['foo'], "bar") + @unittest.skip("@TODO: disabled pending discussion") + def test_double_register_schema(self): + @otio.core.register_type + class DoubleReg(otio.core.SerializableObject): + _serializable_label = "Stuff.1" + foo_two = otio.core.serializable_field("foo_2", doc="test") + + # not allowed to register a type twice + with self.assertRaises(ValueError): + @otio.core.register_type + class DoubleReg(otio.core.SerializableObject): + _serializable_label = "Stuff.1" + def test_upgrade_versions(self): """Test adding an upgrade functions for a type""" From 342f20759a3a4ff8b2cbccb7fd3ba431d54da958 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 1 Sep 2022 16:11:04 -0700 Subject: [PATCH 048/119] add version manifest plugin Signed-off-by: ssteinbach --- .../opentimelineio/__init__.py | 3 +- .../opentimelineio/core/__init__.py | 15 ------ .../opentimelineio/versioning.py | 51 +++++++++++++++++++ ...apter_plugin_manifest.plugin_manifest.json | 8 +++ tests/test_version_manifest.py | 45 ++++++++++++++++ 5 files changed, 106 insertions(+), 16 deletions(-) create mode 100644 src/py-opentimelineio/opentimelineio/versioning.py create mode 100644 tests/test_version_manifest.py diff --git a/src/py-opentimelineio/opentimelineio/__init__.py b/src/py-opentimelineio/opentimelineio/__init__.py index 3d3bab5fb..036907f4f 100644 --- a/src/py-opentimelineio/opentimelineio/__init__.py +++ b/src/py-opentimelineio/opentimelineio/__init__.py @@ -22,5 +22,6 @@ adapters, hooks, algorithms, - url_utils + url_utils, + versioning, ) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index a521d12b8..715a7a227 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -74,21 +74,6 @@ ] -def full_version_map(): - return { - "OTIO_CORE": release_to_schema_version_map() - } - - -def fetch_version_map(family, label): - if family != "OTIO_CORE": - raise NotImplementedError - - src = release_to_schema_version_map() - - return src[label] - - def serialize_json_to_string(root, schema_version_targets=None, indent=4): return _serialize_json_to_string( _value_to_any(root), diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py new file mode 100644 index 000000000..dddec453e --- /dev/null +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -0,0 +1,51 @@ +import copy + +from . import ( + core, + plugins +) + + +def full_map(): + """Return the full map of schema version sets, including core and plugins. + + Organized as follows: + { + "FAMILY_NAME": { + "LABEL": { + "SchemaName": schemaversion, + "Clip": 2, + "Timeline": 3, + ... + } + } + } + + The "OTIO_CORE" family is always provided and represents the built in + schemas defined in the C++ core. + IE: + { + "OTIO_CORE": { + "0.15.0": { + "Clip": 2, + ... + } + } + } + """ + result = copy.deepcopy(plugins.ActiveManifest().version_manifests) + result.update( + { + "OTIO_CORE": core.release_to_schema_version_map(), + } + ) + return result + + +def fetch_map(family, label): + if family == "OTIO_CORE": + src = core.release_to_schema_version_map() + else: + src = plugins.ActiveManifest().version_manifests[family] + + return copy.deepcopy(src[label]) diff --git a/tests/baselines/adapter_plugin_manifest.plugin_manifest.json b/tests/baselines/adapter_plugin_manifest.plugin_manifest.json index f3daafe0d..2d2be4bc9 100644 --- a/tests/baselines/adapter_plugin_manifest.plugin_manifest.json +++ b/tests/baselines/adapter_plugin_manifest.plugin_manifest.json @@ -23,5 +23,13 @@ "post_adapter_read" : [], "post_adapter_write" : ["post write example hook"], "post_media_linker" : ["example hook"] + }, + "version_manifests" : { + "TEST_FAMILY_NAME": { + "TEST_LABEL": { + "ExampleSchema":1, + "Clip": 1 + } + } } } diff --git a/tests/test_version_manifest.py b/tests/test_version_manifest.py new file mode 100644 index 000000000..b6c1677ee --- /dev/null +++ b/tests/test_version_manifest.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the OpenTimelineIO project + +"""unit tests for the version manifest plugin system""" + +import unittest + +import opentimelineio as otio +from tests import utils + + +class TestPlugin_VersionManifest(unittest.TestCase): + def setUp(self): + self.bak = otio.plugins.ActiveManifest() + self.man = utils.create_manifest() + otio.plugins.manifest._MANIFEST = self.man + + def tearDown(self): + otio.plugins.manifest._MANIFEST = self.bak + utils.remove_manifest(self.man) + + def test_read_in_manifest(self): + self.assertIn("TEST_FAMILY_NAME", self.man.version_manifests) + self.assertIn( + "TEST_LABEL", + self.man.version_manifests["TEST_FAMILY_NAME"] + ) + + def test_full_map(self): + d = otio.versioning.full_map() + self.assertIn("TEST_FAMILY_NAME", d) + self.assertIn( + "TEST_LABEL", + d["TEST_FAMILY_NAME"] + ) + + def test_fetch_map(self): + self.assertEquals( + otio.versioning.fetch_map("TEST_FAMILY_NAME", "TEST_LABEL"), + {"ExampleSchema": 1, "Clip": 1} + ) + + +if __name__ == '__main__': + unittest.main() From 3726e4f2c14d3efd120a950f9ede09657ecf5863 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 1 Sep 2022 20:19:49 -0700 Subject: [PATCH 049/119] add version manifest plugin testing Signed-off-by: ssteinbach --- .../builtin_adapters.plugin_manifest.json | 3 -- .../opentimelineio/adapters/otio_json.py | 47 +++++++++++++++---- ...apter_plugin_manifest.plugin_manifest.json | 3 +- tests/test_version_manifest.py | 37 ++++++++++++++- 4 files changed, 76 insertions(+), 14 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json b/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json index f84afc98d..4525f5392 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json +++ b/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json @@ -51,8 +51,5 @@ "post_adapter_write" : [] }, "version_manifests": { - "test": { - "Clip": 1 - } } } diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 70a119875..0f31a090c 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -4,11 +4,13 @@ """This adapter lets you read and write native .otio files""" from .. import ( - core + core, + versioning ) +import os -# @TODO: Implement out of process plugins that hand around JSON +DEFAULT_VERSION_ENVVAR = "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" def read_from_file(filepath): @@ -37,7 +39,7 @@ def read_from_string(input_str): return core.deserialize_json_from_string(input_str) -def write_to_string(input_otio, target_family_label_spec=None, indent=4): +def write_to_string(input_otio, target_schema_versions=None, indent=4): """ Serializes an OpenTimelineIO object into a string @@ -46,12 +48,30 @@ def write_to_string(input_otio, target_family_label_spec=None, indent=4): indent (int): number of spaces for each json indentation level. Use\ -1 for no indentation or newlines. + If target_schema_versions is None and the environment variable + "{}" is set, will read a map out of + that for downgrade target. The variable should be of the form FAMILY:LABEL, + for example "MYSTUDIO:JUNE2022". + Returns: str: A json serialized string representation - """ + """.format(DEFAULT_VERSION_ENVVAR) + + if target_schema_versions is None and DEFAULT_VERSION_ENVVAR in os.environ: + version_envvar = os.environ[DEFAULT_VERSION_ENVVAR] + family, label = version_envvar.split(":") + # @TODO: something isn't right, I shouldn't need to do this extra hop... + # if I don't, I end up with an AnyDictionary instead of a python + # {}, which pybind doesn't quite map into the std::map the way + # I'd hope when calling serialize_json_to_string() + target_schema_versions = versioning.fetch_map(family, label) + d = {} + d.update(target_schema_versions) + target_schema_versions = d + return core.serialize_json_to_string( input_otio, - target_family_label_spec, + target_schema_versions, indent ) @@ -59,7 +79,7 @@ def write_to_string(input_otio, target_family_label_spec=None, indent=4): def write_to_file( input_otio, filepath, - target_family_label_spec=None, + target_schema_versions=None, indent=4 ): """ @@ -72,15 +92,26 @@ def write_to_file( indent (int): number of spaces for each json indentation level.\ Use -1 for no indentation or newlines. + If target_schema_versions is None and the environment variable + "{}" is set, will read a map out of + that for downgrade target. The variable should be of the form FAMILY:LABEL, + for example "MYSTUDIO:JUNE2022". + Returns: bool: Write success Raises: ValueError: on write error - """ + """.format(DEFAULT_VERSION_ENVVAR) + + if target_schema_versions is None and DEFAULT_VERSION_ENVVAR in os.environ: + version_envvar = os.environ[DEFAULT_VERSION_ENVVAR] + family, label = version_envvar.split(":") + target_schema_versions = versioning.fetch_map(family, label) + return core.serialize_json_to_file( input_otio, filepath, - target_family_label_spec, + target_schema_versions, indent ) diff --git a/tests/baselines/adapter_plugin_manifest.plugin_manifest.json b/tests/baselines/adapter_plugin_manifest.plugin_manifest.json index 2d2be4bc9..839dcadcc 100644 --- a/tests/baselines/adapter_plugin_manifest.plugin_manifest.json +++ b/tests/baselines/adapter_plugin_manifest.plugin_manifest.json @@ -27,7 +27,8 @@ "version_manifests" : { "TEST_FAMILY_NAME": { "TEST_LABEL": { - "ExampleSchema":1, + "ExampleSchema":2, + "EnvVarTestSchema":1, "Clip": 1 } } diff --git a/tests/test_version_manifest.py b/tests/test_version_manifest.py index b6c1677ee..8b2ffedad 100644 --- a/tests/test_version_manifest.py +++ b/tests/test_version_manifest.py @@ -4,6 +4,7 @@ """unit tests for the version manifest plugin system""" import unittest +import os import opentimelineio as otio from tests import utils @@ -15,9 +16,14 @@ def setUp(self): self.man = utils.create_manifest() otio.plugins.manifest._MANIFEST = self.man + if "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" in os.environ: + del os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] + def tearDown(self): otio.plugins.manifest._MANIFEST = self.bak utils.remove_manifest(self.man) + if "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" in os.environ: + del os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] def test_read_in_manifest(self): self.assertIn("TEST_FAMILY_NAME", self.man.version_manifests) @@ -35,10 +41,37 @@ def test_full_map(self): ) def test_fetch_map(self): - self.assertEquals( + self.assertEqual( otio.versioning.fetch_map("TEST_FAMILY_NAME", "TEST_LABEL"), - {"ExampleSchema": 1, "Clip": 1} + {"ExampleSchema": 2, "EnvVarTestSchema":1, "Clip": 1} + ) + + def test_env_variable_downgrade(self): + @otio.core.register_type + class EnvVarTestSchema(otio.core.SerializableObject): + _serializable_label = "EnvVarTestSchema.2" + foo_two = otio.core.serializable_field("foo_2") + + @otio.core.downgrade_function_for(EnvVarTestSchema, 2) + def downgrade_2_to_1(_data_dict): + return {"foo": _data_dict["foo_2"]} + + evt = EnvVarTestSchema() + evt.foo_two = "asdf" + + result = eval(otio.adapters.otio_json.write_to_string(evt)) + self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.2") + + # env variable should make a downgrade by default... + os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( + "TEST_FAMILY_NAME:TEST_LABEL" ) + result = eval(otio.adapters.otio_json.write_to_string(evt)) + self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.1") + + # ...but can still be overridden by passing in an argument + result = eval(otio.adapters.otio_json.write_to_string(evt, {})) + self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.2") if __name__ == '__main__': From 82b2781b92706ba1db14cfd48c4954c16aad0b08 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 09:39:01 -0700 Subject: [PATCH 050/119] lint pass Signed-off-by: ssteinbach --- .../opentimelineio/adapters/otio_json.py | 24 ++++++------ .../console/autogen_version_map.py | 38 +++++++++---------- .../opentimelineio/core/__init__.py | 21 +++++----- .../opentimelineio/versioning.py | 6 +-- tests/test_serializable_object.py | 2 +- tests/test_serialized_schema.py | 2 - tests/test_version_manifest.py | 14 +++---- 7 files changed, 54 insertions(+), 53 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 0f31a090c..685886e29 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -50,8 +50,8 @@ def write_to_string(input_otio, target_schema_versions=None, indent=4): If target_schema_versions is None and the environment variable "{}" is set, will read a map out of - that for downgrade target. The variable should be of the form FAMILY:LABEL, - for example "MYSTUDIO:JUNE2022". + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". Returns: str: A json serialized string representation @@ -60,7 +60,7 @@ def write_to_string(input_otio, target_schema_versions=None, indent=4): if target_schema_versions is None and DEFAULT_VERSION_ENVVAR in os.environ: version_envvar = os.environ[DEFAULT_VERSION_ENVVAR] family, label = version_envvar.split(":") - # @TODO: something isn't right, I shouldn't need to do this extra hop... + # @TODO: something isn't right, I shouldn't need to do this extra hop- # if I don't, I end up with an AnyDictionary instead of a python # {}, which pybind doesn't quite map into the std::map the way # I'd hope when calling serialize_json_to_string() @@ -70,9 +70,9 @@ def write_to_string(input_otio, target_schema_versions=None, indent=4): target_schema_versions = d return core.serialize_json_to_string( - input_otio, - target_schema_versions, - indent + input_otio, + target_schema_versions, + indent ) @@ -94,8 +94,8 @@ def write_to_file( If target_schema_versions is None and the environment variable "{}" is set, will read a map out of - that for downgrade target. The variable should be of the form FAMILY:LABEL, - for example "MYSTUDIO:JUNE2022". + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". Returns: bool: Write success @@ -110,8 +110,8 @@ def write_to_file( target_schema_versions = versioning.fetch_map(family, label) return core.serialize_json_to_file( - input_otio, - filepath, - target_schema_versions, - indent + input_otio, + filepath, + target_schema_versions, + indent ) diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py index 6ecdcdea8..9820074a6 100644 --- a/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py +++ b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py @@ -27,19 +27,19 @@ def _parsed_args(): formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( - "-d", - "--dryrun", - default=False, - action="store_true", - help="write to stdout instead of printing to file." + "-d", + "--dryrun", + default=False, + action="store_true", + help="write to stdout instead of printing to file." ) parser.add_argument( - "-l", - "--label", - default=otio.__version__, - # @TODO - should we strip the .dev1 label? that would probably be - # more consistent since we don't do sub-beta releases - help="Version label to assign this schema map to." + "-l", + "--label", + default=otio.__version__, + # @TODO - should we strip the .dev1 label? that would probably be + # more consistent since we don't do sub-beta releases + help="Version label to assign this schema map to." ) parser.add_argument( "-i", @@ -70,11 +70,11 @@ def generate_core_version_map(src_text, label, version_map): map_text = [] for key, value in version_map.items(): map_text.append( - MAP_ITEM_TEMPLATE.format( - indent=' ' * INDENT, - key=key, - value=value - ) + MAP_ITEM_TEMPLATE.format( + indent=' ' * INDENT, + key=key, + value=value + ) ) map_text = '\n'.join(map_text) @@ -90,9 +90,9 @@ def main(): input = fi.read() result = generate_core_version_map( - input, - args.label, - otio.core.type_version_map() + input, + args.label, + otio.core.type_version_map() ) if args.dryrun: diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 715a7a227..3c1d0f86e 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -76,9 +76,9 @@ def serialize_json_to_string(root, schema_version_targets=None, indent=4): return _serialize_json_to_string( - _value_to_any(root), - schema_version_targets or {}, - indent + _value_to_any(root), + schema_version_targets or {}, + indent ) @@ -89,10 +89,10 @@ def serialize_json_to_file( indent=4 ): return _serialize_json_to_file( - _value_to_any(root), - filename, - schema_version_targets or {}, - indent + _value_to_any(root), + filename, + schema_version_targets or {}, + indent ) @@ -189,8 +189,11 @@ def wrapped_update(data): data.clear() data.update(modified) - register_downgrade_function(cls._serializable_label.split(".")[0], - version_to_upgrade_to, wrapped_update) + register_downgrade_function( + cls._serializable_label.split(".")[0], + version_to_upgrade_to, + wrapped_update + ) return func return decorator_func diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py index dddec453e..d4ad8ea9e 100644 --- a/src/py-opentimelineio/opentimelineio/versioning.py +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -35,9 +35,9 @@ def full_map(): """ result = copy.deepcopy(plugins.ActiveManifest().version_manifests) result.update( - { - "OTIO_CORE": core.release_to_schema_version_map(), - } + { + "OTIO_CORE": core.release_to_schema_version_map(), + } ) return result diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index b564cba15..e2068ac37 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -175,7 +175,7 @@ class DoubleReg(otio.core.SerializableObject): # not allowed to register a type twice with self.assertRaises(ValueError): @otio.core.register_type - class DoubleReg(otio.core.SerializableObject): + class DoubleReg(otio.core.SerializableObject): # noqa F811 _serializable_label = "Stuff.1" def test_upgrade_versions(self): diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index 143a9b451..ccf876eb2 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -6,8 +6,6 @@ import sys import subprocess -import opentimelineio as otio - from opentimelineio.console import ( autogen_serialized_datamodel as asd, autogen_plugin_documentation as apd, diff --git a/tests/test_version_manifest.py b/tests/test_version_manifest.py index 8b2ffedad..068c41cb4 100644 --- a/tests/test_version_manifest.py +++ b/tests/test_version_manifest.py @@ -28,22 +28,22 @@ def tearDown(self): def test_read_in_manifest(self): self.assertIn("TEST_FAMILY_NAME", self.man.version_manifests) self.assertIn( - "TEST_LABEL", - self.man.version_manifests["TEST_FAMILY_NAME"] + "TEST_LABEL", + self.man.version_manifests["TEST_FAMILY_NAME"] ) def test_full_map(self): d = otio.versioning.full_map() self.assertIn("TEST_FAMILY_NAME", d) self.assertIn( - "TEST_LABEL", - d["TEST_FAMILY_NAME"] + "TEST_LABEL", + d["TEST_FAMILY_NAME"] ) def test_fetch_map(self): self.assertEqual( - otio.versioning.fetch_map("TEST_FAMILY_NAME", "TEST_LABEL"), - {"ExampleSchema": 2, "EnvVarTestSchema":1, "Clip": 1} + otio.versioning.fetch_map("TEST_FAMILY_NAME", "TEST_LABEL"), + {"ExampleSchema": 2, "EnvVarTestSchema": 1, "Clip": 1} ) def test_env_variable_downgrade(self): @@ -64,7 +64,7 @@ def downgrade_2_to_1(_data_dict): # env variable should make a downgrade by default... os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( - "TEST_FAMILY_NAME:TEST_LABEL" + "TEST_FAMILY_NAME:TEST_LABEL" ) result = eval(otio.adapters.otio_json.write_to_string(evt)) self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.1") From dd26dedb743bfb69603a3581b86fc5dcc2c93afe Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:07:03 -0700 Subject: [PATCH 051/119] comment formatting for RTD Signed-off-by: ssteinbach --- src/py-opentimelineio/opentimelineio/versioning.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py index d4ad8ea9e..1efe27369 100644 --- a/src/py-opentimelineio/opentimelineio/versioning.py +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -10,6 +10,7 @@ def full_map(): """Return the full map of schema version sets, including core and plugins. Organized as follows: + ``` { "FAMILY_NAME": { "LABEL": { @@ -20,10 +21,12 @@ def full_map(): } } } + ``` The "OTIO_CORE" family is always provided and represents the built in schemas defined in the C++ core. IE: + ``` { "OTIO_CORE": { "0.15.0": { @@ -32,6 +35,7 @@ def full_map(): } } } + ``` """ result = copy.deepcopy(plugins.ActiveManifest().version_manifests) result.update( From 85f225005ca2bb61c2c2ce93b13dcbf49a88ba05 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:09:53 -0700 Subject: [PATCH 052/119] lint Signed-off-by: ssteinbach --- tests/test_serializable_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index e2068ac37..da3a77f1c 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -175,7 +175,7 @@ class DoubleReg(otio.core.SerializableObject): # not allowed to register a type twice with self.assertRaises(ValueError): @otio.core.register_type - class DoubleReg(otio.core.SerializableObject): # noqa F811 + class DoubleReg(otio.core.SerializableObject): # noqa: F811 _serializable_label = "Stuff.1" def test_upgrade_versions(self): From ac0c24f11cd8472c45003b9ddea7d6662a4d19bf Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:16:10 -0700 Subject: [PATCH 053/119] lint again Signed-off-by: ssteinbach --- tests/test_serializable_object.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index da3a77f1c..3e9c0d8fa 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -168,14 +168,15 @@ class FakeThing(otio.core.SerializableObject): @unittest.skip("@TODO: disabled pending discussion") def test_double_register_schema(self): @otio.core.register_type - class DoubleReg(otio.core.SerializableObject): + class DoubleReg(otio.core.SerializableObject): _serializable_label = "Stuff.1" foo_two = otio.core.serializable_field("foo_2", doc="test") + _ = DoubleReg() # quiet pyflakes # not allowed to register a type twice with self.assertRaises(ValueError): @otio.core.register_type - class DoubleReg(otio.core.SerializableObject): # noqa: F811 + class DoubleReg(otio.core.SerializableObject): _serializable_label = "Stuff.1" def test_upgrade_versions(self): From b1707bc4c55b4e86db19189f891f271b28d2ccb9 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:17:57 -0700 Subject: [PATCH 054/119] linty lint lint Signed-off-by: ssteinbach --- tests/test_serializable_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index 3e9c0d8fa..37f872d7b 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -168,7 +168,7 @@ class FakeThing(otio.core.SerializableObject): @unittest.skip("@TODO: disabled pending discussion") def test_double_register_schema(self): @otio.core.register_type - class DoubleReg(otio.core.SerializableObject): + class DoubleReg(otio.core.SerializableObject): _serializable_label = "Stuff.1" foo_two = otio.core.serializable_field("foo_2", doc="test") _ = DoubleReg() # quiet pyflakes From 75fb9c847cb258c0fda77d4338290e078f3f4901 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:25:14 -0700 Subject: [PATCH 055/119] update the doc model Signed-off-by: ssteinbach --- docs/tutorials/otio-serialized-schema-only-fields.md | 1 + docs/tutorials/otio-serialized-schema.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/tutorials/otio-serialized-schema-only-fields.md b/docs/tutorials/otio-serialized-schema-only-fields.md index 624f96aaf..e6ff566e9 100644 --- a/docs/tutorials/otio-serialized-schema-only-fields.md +++ b/docs/tutorials/otio-serialized-schema-only-fields.md @@ -121,6 +121,7 @@ parameters: - *hooks* - *media_linkers* - *schemadefs* +- *version_manifests* ### SerializableObject.1 diff --git a/docs/tutorials/otio-serialized-schema.md b/docs/tutorials/otio-serialized-schema.md index e59b68a90..3fbb4744a 100644 --- a/docs/tutorials/otio-serialized-schema.md +++ b/docs/tutorials/otio-serialized-schema.md @@ -254,6 +254,7 @@ parameters: - *hooks*: Hooks that hooks scripts can be attached to. - *media_linkers*: Media Linkers this manifest describes. - *schemadefs*: Schemadefs this manifest describes. +- *version_manifests*: Sets of versions to downgrade schemas to. ### SerializableObject.1 From 103679d8c494cb6ab3b8f8e1c77c5954ddb34452 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:45:20 -0700 Subject: [PATCH 056/119] update docs for better RTD readability Signed-off-by: ssteinbach --- .../opentimelineio/versioning.py | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py index 1efe27369..1f0278a78 100644 --- a/src/py-opentimelineio/opentimelineio/versioning.py +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -8,35 +8,38 @@ def full_map(): """Return the full map of schema version sets, including core and plugins. - Organized as follows: - ``` - { - "FAMILY_NAME": { - "LABEL": { - "SchemaName": schemaversion, - "Clip": 2, - "Timeline": 3, - ... + + .. code-block:: python + + { + "FAMILY_NAME": { + "LABEL": { + "SchemaName": schemaversion, + "Clip": 2, + "Timeline": 3, + ... + } } } - } - ``` + The "OTIO_CORE" family is always provided and represents the built in schemas defined in the C++ core. IE: - ``` - { + + .. code-block:: python + + { "OTIO_CORE": { "0.15.0": { "Clip": 2, ... } } - } - ``` + } """ + result = copy.deepcopy(plugins.ActiveManifest().version_manifests) result.update( { @@ -47,6 +50,25 @@ def full_map(): def fetch_map(family, label): + """Fetch the version map for the given family and label. OpenTimelineIO + includes a built in family called "OTIO_CORE", this is compiled into the + C++ core and represents the core interchange schemas of OpenTimelineIO. + + Users may define more family/label/schema:version mappings by way of the + version manifest plugins. + + Returns a dictionary mapping Schema name to schema version, like: + + .. code-block:: python + + { + "Clip": 2, + "Timeline": 1, + ... + } + + """ + if family == "OTIO_CORE": src = core.release_to_schema_version_map() else: From 0743f12c6286250fd69f3896ae4047a1bf82b671 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:50:25 -0700 Subject: [PATCH 057/119] bump plugin docs Signed-off-by: ssteinbach --- docs/tutorials/otio-plugins.md | 35 ++++------------------------------ 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index 0933098db..0f428cff3 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -136,41 +136,14 @@ De-serializes an OpenTimelineIO object from a json string OpenTimeline: An OpenTimeline object ``` - input_str -- write_to_file: -``` -Serializes an OpenTimelineIO object into a file - - Args: - - input_otio (OpenTimeline): An OpenTimeline object - filepath (str): The name of an otio file to write to - indent (int): number of spaces for each json indentation level. - Use -1 for no indentation or newlines. - - Returns: - bool: Write success - - Raises: - ValueError: on write error -``` +- write_to_file: - input_otio - filepath - - target_family_label_spec + - target_schema_versions - indent -- write_to_string: -``` -Serializes an OpenTimelineIO object into a string - - Args: - input_otio (OpenTimeline): An OpenTimeline object - indent (int): number of spaces for each json indentation level. Use - -1 for no indentation or newlines. - - Returns: - str: A json serialized string representation -``` +- write_to_string: - input_otio - - target_family_label_spec + - target_schema_versions - indent From 16cf426bd673561d0a461f08424b92522e846303 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 10:58:56 -0700 Subject: [PATCH 058/119] force sort of autogen version map Signed-off-by: ssteinbach --- .../opentimelineio/console/autogen_version_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py index 9820074a6..0b5f7c27e 100644 --- a/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py +++ b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py @@ -68,7 +68,7 @@ def generate_core_version_map(src_text, label, version_map): # iterate over the map and print the template out map_text = [] - for key, value in version_map.items(): + for key, value in sorted(version_map.items()): map_text.append( MAP_ITEM_TEMPLATE.format( indent=' ' * INDENT, From 2b93376547d859865d55d508c956f4656dbcb23f Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 2 Sep 2022 14:12:42 -0700 Subject: [PATCH 059/119] force unix line endings for the version map test Signed-off-by: ssteinbach --- .../opentimelineio/console/autogen_version_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py index 0b5f7c27e..a77d3616e 100644 --- a/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py +++ b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py @@ -107,7 +107,7 @@ def main(): delete=False ).name - with open(output, 'w') as fo: + with open(output, 'w', newline="\n") as fo: fo.write(result) print("Wrote CORE_VERSION_MAP to: '{}'.".format(output)) From b5584f11f1f80df5b814109e79e3b473bc5ea7dc Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sat, 3 Sep 2022 22:12:28 -0700 Subject: [PATCH 060/119] fix manifest layering for the version_manifests Signed-off-by: ssteinbach --- .../opentimelineio/plugins/manifest.py | 10 ++++ tests/test_version_manifest.py | 58 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/plugins/manifest.py b/src/py-opentimelineio/opentimelineio/plugins/manifest.py index 603618db2..3d612b943 100644 --- a/src/py-opentimelineio/opentimelineio/plugins/manifest.py +++ b/src/py-opentimelineio/opentimelineio/plugins/manifest.py @@ -139,7 +139,17 @@ def extend(self, another_manifest): self.media_linkers.extend(another_manifest.media_linkers) self.hook_scripts.extend(another_manifest.hook_scripts) + for family, label_map in another_manifest.version_manifests.items(): + # because self.version_manifests is an AnyDictionary instead of a + # vanilla python dictionary, it does not support the .set_default() + # method. + if family not in self.version_manifests: + self.version_manifests[family] = {} + self.version_manifests[family].update(label_map) + for trigger_name, hooks in another_manifest.hooks.items(): + # because self.hooks is an AnyDictionary instead of a vanilla + # python dictionary, it does not support the .set_default() method. if trigger_name not in self.hooks: self.hooks[trigger_name] = [] self.hooks[trigger_name].extend(hooks) diff --git a/tests/test_version_manifest.py b/tests/test_version_manifest.py index 068c41cb4..6d8bedb41 100644 --- a/tests/test_version_manifest.py +++ b/tests/test_version_manifest.py @@ -5,11 +5,48 @@ import unittest import os +import json import opentimelineio as otio from tests import utils +FIRST_MANIFEST = """{ + "OTIO_SCHEMA" : "PluginManifest.1", + "version_manifests": { + "UNIQUE_FAMILY": { + "TEST_LABEL": { + "second_thing": 3 + } + }, + "LAYERED_FAMILY": { + "June2022": { + "SimpleClass": 2 + }, + "May2022": { + "SimpleClass": 1 + } + } + } +} +""" + +SECOND_MANIFEST = """{ + "OTIO_SCHEMA" : "PluginManifest.1", + "version_manifests": { + "LAYERED_FAMILY": { + "May2022": { + "SimpleClass": 2 + }, + "April2022": { + "SimpleClass": 1 + } + } + } +} +""" + + class TestPlugin_VersionManifest(unittest.TestCase): def setUp(self): self.bak = otio.plugins.ActiveManifest() @@ -59,20 +96,35 @@ def downgrade_2_to_1(_data_dict): evt = EnvVarTestSchema() evt.foo_two = "asdf" - result = eval(otio.adapters.otio_json.write_to_string(evt)) + result = json.loads(otio.adapters.otio_json.write_to_string(evt)) self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.2") # env variable should make a downgrade by default... os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( "TEST_FAMILY_NAME:TEST_LABEL" ) - result = eval(otio.adapters.otio_json.write_to_string(evt)) + result = json.loads(otio.adapters.otio_json.write_to_string(evt)) self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.1") # ...but can still be overridden by passing in an argument - result = eval(otio.adapters.otio_json.write_to_string(evt, {})) + result = json.loads(otio.adapters.otio_json.write_to_string(evt, {})) self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.2") + def test_two_version_manifests(self): + """test that two manifests layer correctly""" + + fst = otio.plugins.manifest.manifest_from_string(FIRST_MANIFEST) + snd = otio.plugins.manifest.manifest_from_string(SECOND_MANIFEST) + fst.extend(snd) + + self.assertIn("UNIQUE_FAMILY", fst.version_manifests) + + lay_fam = fst.version_manifests["LAYERED_FAMILY"] + + self.assertIn("June2022", lay_fam) + self.assertIn("April2022", lay_fam) + self.assertEqual(lay_fam["May2022"]["SimpleClass"], 2) + if __name__ == '__main__': unittest.main() From 34d33a647ad4802c11933d7296ad275f873ec3a8 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sat, 3 Sep 2022 22:12:50 -0700 Subject: [PATCH 061/119] eval->json.loads Signed-off-by: ssteinbach --- tests/test_serializable_object.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index 37f872d7b..37a9d863d 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -7,6 +7,7 @@ import opentimelineio.test_utils as otio_test_utils import unittest +import json class OpenTimeTypeSerializerTest(unittest.TestCase): @@ -249,7 +250,7 @@ def downgrade_2_to_1_again(_data_dict): downgrade_target = {"FakeThingToDowngrade": 1} - result = eval( + result = json.loads( otio.adapters.otio_json.write_to_string(f, downgrade_target) ) From 5de438cfa6f2b9543c02e60ab52cda3df9412536 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sat, 3 Sep 2022 22:37:05 -0700 Subject: [PATCH 062/119] fix name of the downgrade_function_from decorator - also clean up argument names for the decorator Signed-off-by: ssteinbach --- .../opentimelineio/core/__init__.py | 35 ++++++++----------- tests/test_serializable_object.py | 4 +-- tests/test_version_manifest.py | 2 +- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 3c1d0f86e..9e0535634 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -63,7 +63,7 @@ 'set_type_record', 'add_method', 'upgrade_function_for', - 'downgrade_function_for', + 'downgrade_function_from', 'fetch_version_map', 'serializable_field', 'deprecated_field', @@ -155,35 +155,30 @@ def wrapped_update(data): return decorator_func -def downgrade_function_for(cls, version_to_upgrade_to): +def downgrade_function_from(cls, version_to_downgrade_from): """ - @TODO <- fix docs - Decorator for identifying schema class downgrade functions. - Example: .. code-block:: python - @upgrade_function_for(MyClass, 5) - def upgrade_to_version_five(data): - pass + @downgrade_function_from(MyClass, 5) + def downgrade_from_five_to_four(data): + return {"old_attr": data["new_attr"]} - This will get called to upgrade a schema of MyClass to version 5. MyClass - must be a class deriving from :class:`~SerializableObject`. + This will get called to downgrade a schema of MyClass from version 5 to + version 4. MyClass must be a class deriving from + :class:`~SerializableObject`. - The upgrade function should take a single argument - the dictionary to - upgrade, and return a dictionary with the fields upgraded. + The downgrade function should take a single argument - the dictionary to + downgrade, and return a dictionary with the fields downgraded. - Remember that you don't need to provide an upgrade function for upgrades - that add or remove fields, only for schema versions that change the field - names. - - :param type cls: class to upgrade - :param int version_to_upgrade_to: the version to upgrade to + :param type cls: class to downgrade + :param int version_to_downgrade_from: the function downgrading from this + version to (version - 1) """ def decorator_func(func): - """ Decorator for marking upgrade functions """ + """ Decorator for marking downgrade functions """ def wrapped_update(data): modified = func(data) data.clear() @@ -191,7 +186,7 @@ def wrapped_update(data): register_downgrade_function( cls._serializable_label.split(".")[0], - version_to_upgrade_to, + version_to_downgrade_from, wrapped_update ) return func diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index 37a9d863d..ebc9e3738 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -230,13 +230,13 @@ class FakeThing(otio.core.SerializableObject): _serializable_label = "FakeThingToDowngrade.2" foo_two = otio.core.serializable_field("foo_2") - @otio.core.downgrade_function_for(FakeThing, 2) + @otio.core.downgrade_function_from(FakeThing, 2) def downgrade_2_to_1(_data_dict): return {"foo": _data_dict["foo_2"]} # not allowed to overwrite registered functions with self.assertRaises(ValueError): - @otio.core.downgrade_function_for(FakeThing, 2) + @otio.core.downgrade_function_from(FakeThing, 2) def downgrade_2_to_1_again(_data_dict): raise RuntimeError("shouldn't see this ever") diff --git a/tests/test_version_manifest.py b/tests/test_version_manifest.py index 6d8bedb41..c0baca3d1 100644 --- a/tests/test_version_manifest.py +++ b/tests/test_version_manifest.py @@ -89,7 +89,7 @@ class EnvVarTestSchema(otio.core.SerializableObject): _serializable_label = "EnvVarTestSchema.2" foo_two = otio.core.serializable_field("foo_2") - @otio.core.downgrade_function_for(EnvVarTestSchema, 2) + @otio.core.downgrade_function_from(EnvVarTestSchema, 2) def downgrade_2_to_1(_data_dict): return {"foo": _data_dict["foo_2"]} From 457e9e97756e0da3f6360434a00c3dc31cd6e3ec Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:25:50 -0700 Subject: [PATCH 063/119] add docs about new downgrading schemas Signed-off-by: ssteinbach --- docs/tutorials/versioning-schemas.md | 248 ++++++++++++++++++++++++--- 1 file changed, 226 insertions(+), 22 deletions(-) diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 3adea62f4..13b3ab9f2 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -1,16 +1,209 @@ # Versioning Schemas +## Overview -During development, it is natural that the fields on objects in OTIO change. To accommodate this, OTIO has a system for handling version differences and upgrading older schemas to new ones. There are two components: +This document describes OpenTimelineIO's systems for dealing with different schema versions when reading files, writing files, and finally as part of the development process. It is intended for developers who are integrating OpenTimelineIO into their pipelines or applications. -1. `serializeable_label` on the class has a name and version field: `Foo.5` -- `Foo` is the schema name and `5` is the version. -2. `upgrade_function_for` decorator +TL;DR for users: OpenTimelineIO should be able to read files produced by older versions of the library and be able to write files that are compatible with older versions of the library from newer versions. +## Schema/Version Introduction -Changing a Field ---------------------- +Each SerializeableObject (the base class of OpenTimelineIO) has `schema_name` and `schema_version` fields. The `schema_name` is a string naming the schema, for example, `Clip`, and the `schema_version` is an integer of the current version number, for example, `3`. -For example, lets say you have class: +SerializeableObjects can be queried for these using the `.schema_name()` and `.schema_version()` methods. For a given release of the OpenTimelineIO library, in-memory objects the library creates will always be the same schema version. In other words, if `otio.schema.Clip()` instantiates an object with `schema_version` 2, there is no way to get an in-memory `Clip` object with version 1. + +OpenTimelineIO can still interoperate with older and newer versions of the library by way of the schema upgrading/downgrading system. As OpenTimelineIO deserializes json from a string or disk, it will upgrade the schemas to the version supported by the library before instantiating the concrete in-memory object. Similarly, when serializing OpenTimelineIO back to disk, the user can instruct OpenTimelineIO to downgrade the JSON to older versions of the schemas. In this way, a newer version of OpenTimelineIO can read files with older schemas, and a newer version of OpenTimelineIO can generate JSON with older schemas in it. + +## Schema Upgrading + +Once a type is registerd to OpenTimelineIO, deveopers may also register upgrade functions. In python, each upgrade function takes a dictionary and returns a dictionary. In C++, the AnyDictionary is manipulated in place. Each upgrade function is associated with a version number - this is the version number that it upgrades to. + +C++ Example (can be viewed/run in `examples/upgrade_example.cpp`): + +```cpp +class SimpleClass : public otio::SerializableObject +{ +public: + struct Schema + { + static auto constexpr name = "SimpleClass"; + static int constexpr version = 2; + }; + + void set_new_field(int64_t val) { _new_field = val; } + const int64_t new_field() const { return _new_field; } + +protected: + using Parent = SerializableObject; + + virtual ~SimpleClass() = default; + + virtual bool + read_from(Reader& reader) + { + auto result = ( + reader.read("new_field", &_new_field) + && Parent::read_from(reader) + ); + + return result; + } + + virtual void + write_to(Writer& writer) const + { + Parent::write_to(writer); + writer.write("new_field", _new_field); + } + +private: + int64_t _new_field; +}; + + // later, during execution: + + // register type and upgrade/downgrade functions + otio::TypeRegistry::instance().register_type(); + + // 1->2 + otio::TypeRegistry::instance().register_upgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["new_field"] = (*d)["my_field"]; + d->erase("my_field"); + } + ); +``` + +Python Example: + +```python +@otio.core.register_type +class SimpleClass(otio.core.SerializeableObject): + serializeable_label = "SimpleClass.2" + my_field = otio.core.serializeable_field("new_field", int) + +@otio.core.upgrade_function_for(SimpleClass, 2) +def upgrade_one_to_two(data): + return {"new_field" : data["my_field"] } +``` + +When upgrading schemas, OpenTimelineIO will call each upgrade function in order in an attempt to get to the current version. For example, if a schema is registered to have version 3, and a file with version 1 is read, OpenTimelineIO will attempt to call the 1->2 function, then the 2->3 function before instantiating the concrete class. + +## Schema Downgrading + +Similarly, once a type is registered, downgrade functions may be registered. Downgrade functions take a dictionary of the version specified and return a dictionary of the schema version one lower. For example, if a downgrade function is registered for version 5, that will downgrade from 5 to 4. + +C++ Example, building off the prior section SimpleClass example (can be viewed/run in `examples/upgrade_example.cpp`): + +```cpp +// 2->1 +otio::TypeRegistry::instance().register_downgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["my_field"] = (*d)["new_field"]; + d->erase("new_field"); + } +); +``` + +Python Example: + +```python +@otio.core.upgrade_function_for(SimpleClass, 2) +def downgrade_two_to_one(data): + return {"my_field" : data["new_field"] } +``` + +To specify what version of a schema to downgrade to, the serialization functions include an optional `schema_version_targets` argument which is a map of schema name to target schema version. During serialization, any schemas who are listed in the map and are of greater version than specified in the map will be converted to AnyDictionary and run through the necessary downgrade functions before being serialized. + +Example C++: + +```cpp +auto sc = otio::SerializableObject::Retainer(new SimpleClass()); +sc->set_new_field(12); + +// this will only downgrade the SimpleClass, to version 1 +otio::schema_version_map downgrade_manifest = { + {"SimpleClass", 1} +}; + +// write it out to disk, downgrading to version 1 +sc->to_json_file("/var/tmp/simpleclass.otio", &err, &downgrade_manifest); +``` + +Example python: + +```python +sc = SimpleClass() +otio.adapters.write_to_file( + sc, + "/path/to/output.otio", + target_schema_versions={"SimpleClass":1} +) +``` + +### Schema-Version Sets + +In addition to passing in dictionaries of desired target schema versions, OpenTimelineIO also provides some tools for having sets of schemas with an associated label. The core C++ library contains a compiled-in map of them, the `CORE_VERSION_MAP`. This is organized (as of v0.15.0) by library release versions label, ie "0.15.0", "0.14.0" and so on. + +In order to downgrade to version 0.15.0 for example: + +```cpp +auto downgrade_manifest = otio::CORE_VERSION_MAP["0.15.0"]; + +// write it out to disk, downgrading to version 1 +sc->to_json_file("/var/tmp/simpleclass.otio", &err, &downgrade_manifest); +``` + +In python, an additional level of indirection is provided, "FAMILY", which is intended to allow developers to define their own sets of target versions for their plugin schemas. For example, a studio might have a family named "MYFAMILY" under which they organize labels for their internal releases of their own plugins. + +These can be defined in a plugin manifest, which is a `.plugin_manifest.json` file found on the environment variable `${OTIO_PLUGIN_MANIFEST_PATH}`. + +For example: + +```python +{ + "OTIO_SCHEMA" : "PluginManifest.1", + "version_manifests": { + "MYFAMILY": { + "June2022": { + "SimpleClass": 2, + ... + }, + "May2022": { + "SimpleClass": 1, + ... + } + } + } +} +``` + +To fetch the version maps and work with this, the python API provides some additional functions: + +```python +downgrade_manifest = otio.versioning.fetch_map("MYFAMILY", "June2022") +otio.adapters.write_to_file( + sc, + "/path/to/file.otio", + downgrade_manifest +) +``` + +See the `versioning` module for more information on accessing these. + +## For Developers + +During the development of OpenTimelineIO schemas, whether they are in the core or in plugins, it is expected that schemas will change and evolve over time. Here are some processes for doing that. + +### Changing a Field + +Given `SimpleClass`: ```python import opentimelineio as otio @@ -21,33 +214,34 @@ class SimpleClass(otio.core.SerializeableObject): my_field = otio.core.serializeable_field("my_field", int) ``` - -And you want to change `my_field` to `new_field`. To do this: +And `my_field` needs to be renamed to `new_field`. To do this: - Make the change in the class - Bump the version number in the label -- add an upgrade function - -So after the changes, you'll have: +- add upgrade and downgrade functions ```python @otio.core.register_type class SimpleClass(otio.core.SerializeableObject): serializeable_label = "SimpleClass.2" - my_field = otio.core.serializeable_field("new_field", int) + new_field = otio.core.serializeable_field("new_field", int) @otio.core.upgrade_function_for(SimpleClass, 2) def upgrade_one_to_two(data): return {"new_field" : data["my_field"] } + +@otio.core.downgrade_function_from(SimpleClass, 2) +def downgrade_two_to_one(data): + return {"my_field": data["new_field"]} ``` -Lets change it again, so that `new_field` becomes `even_newer_field`. +Changing it again, now `new_field` becomes `even_newer_field`. ```python @otio.core.register_type class SimpleClass(otio.core.SerializeableObject): serializeable_label = "SimpleClass.2" - my_field = otio.core.serializeable_field("even_newer_field", int) + even_newer_field = otio.core.serializeable_field("even_newer_field", int) @otio.core.upgrade_function_for(SimpleClass, 2) def upgrade_one_to_two(data): @@ -57,12 +251,18 @@ def upgrade_one_to_two(data): @otio.core.upgrade_function_for(SimpleClass, 3) def upgrade_two_to_three(data): return {"even_newer_field" : data["new_field"] } -``` -Upgrade functions can be sparse - if version `3` to `4` doesn't require a function, for example, you don't need to write one. +@otio.core.downgrade_function_from(SimpleClass, 2) +def downgrade_two_to_one(data): + return {"my_field": data["new_field"]} -Adding or Removing a Field --------------------------------- +# ...and corresponding second downgrade function +@otio.core.downgrade_function_from(SimpleClass, 3) +def downgrade_two_to_one(data): + return {"new_field": data["even_newer_field"]} +``` + +### Adding or Removing a Field Starting from the same class: @@ -73,9 +273,11 @@ class SimpleClass(otio.core.SerializeableObject): my_field = otio.core.serializeable_field("my_field", int) ``` -Adding or Removing a field is simpler. In these cases, you don't need to write an upgrade function, since any new classes will be initialized through the constructor, and any removed fields will be ignored when reading from an older schema version. +If a change to a schema is to add a field, for which the default value is the correct value for an old schema, then no upgrade or downgrade function is needed. The parser ignores values that aren't in the schema. + +Additionally, upgrade functions will be called in order, but they need not cover every version number. So if there is an upgrade function for version 2 and 4, to get to version 4, OTIO will automatically apply function 2 and then function 4 in order, skipping the missing 3. -So lets add a new field: +Example of adding a field: ```python @otio.core.register_type @@ -85,11 +287,13 @@ class SimpleClass(otio.core.SerializeableObject): other_field = otio.core.serializeable_field("other_field", int) ``` -And then delete the original field: +Removing a field: ```python @otio.core.register_type class SimpleClass(otio.core.SerializeableObject): serializeable_label = "SimpleClass.3" other_field = otio.core.serializeable_field("other_field", int) -``` \ No newline at end of file +``` + +Similarly, when deleting a field, if the field is now ignored and does not contribute to computation, no upgrade is needed. From 8aeffc3de635fac8226b972d0c0f7854c18dcdcb Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:26:57 -0700 Subject: [PATCH 064/119] remove dead code Signed-off-by: ssteinbach --- src/opentimelineio/serialization.h | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index 5d9fa43d7..03f7ba006 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -14,23 +14,6 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -// -// { -// "OTIO_CORE": { -// ^ family -// "0.14.0": { -// ^ family_label -// "Clip": 1, -// ^ schema ^ schema_version -// ... -// }, -// "0.15.0": { -// ... -// }, -// ... -// }, -// "MY_COMPANY_PLUGIN_SETS": {} -// } std::string serialize_json_to_string( From 82f15c540dbf908be48c98b125c266bb5661c16f Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:27:07 -0700 Subject: [PATCH 065/119] unordered_map in serializableObject instead of map Signed-off-by: ssteinbach --- src/opentimelineio/serializableObject.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index a38e54090..f4fd4a2ce 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -536,18 +536,17 @@ class SerializableObject bool _any_equals(any const& lhs, any const& rhs); std::string _no_key; - // @TODO: should probably be unordered maps - std::map> + std::unordered_map> _write_dispatch_table; - std::map< + std::unordered_map< std::type_info const*, std::function> _equality_dispatch_table; - std::map> + std::unordered_map> _write_dispatch_table_by_name; - std::map _id_for_object; - std::map _next_id_for_type; + std::unordered_map _id_for_object; + std::unordered_map _next_id_for_type; class Encoder& _encoder; optional _downgrade_version_manifest; From 5712202ed307d3352a97b4c6175841495055e50d Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:27:23 -0700 Subject: [PATCH 066/119] all io_perf_test on by default Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index ea839adbc..3e8376bea 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -13,12 +13,12 @@ namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; using chrono_time_point = std::chrono::steady_clock::time_point; constexpr struct { - bool TO_JSON_STRING = true; + bool TO_JSON_STRING = true; bool TO_JSON_STRING_NO_DOWNGRADE = true; - bool TO_JSON_FILE = true; - bool TO_JSON_FILE_NO_DOWNGRADE = true; - bool CLONE_TEST = true; - bool SINGLE_CLIP_DOWNGRADE_TEST = true; + bool TO_JSON_FILE = true; + bool TO_JSON_FILE_NO_DOWNGRADE = true; + bool CLONE_TEST = true; + bool SINGLE_CLIP_DOWNGRADE_TEST = true; } RUN_STRUCT ; /// utility function for printing std::chrono elapsed time From 90fdac1b697cbb26ee8f9a2fa339034b01820df1 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:29:06 -0700 Subject: [PATCH 067/119] add upgrade_downgrade_example in C++ Signed-off-by: ssteinbach --- examples/CMakeLists.txt | 1 + examples/upgrade_downgrade_example.cpp | 104 +++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 examples/upgrade_downgrade_example.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index d42ac1375..960a99625 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -9,6 +9,7 @@ list(APPEND examples conform) list(APPEND examples flatten_video_tracks) list(APPEND examples summarize_timing) list(APPEND examples io_perf_test) +list(APPEND examples upgrade_downgrade_example) if(OTIO_PYTHON_INSTALL) list(APPEND examples python_adapters_child_process) list(APPEND examples python_adapters_embed) diff --git a/examples/upgrade_downgrade_example.cpp b/examples/upgrade_downgrade_example.cpp new file mode 100644 index 000000000..55ac40f1f --- /dev/null +++ b/examples/upgrade_downgrade_example.cpp @@ -0,0 +1,104 @@ +#include "opentimelineio/serializableObject.h" +#include "opentimelineio/typeRegistry.h" +#include + +// demonstrates a minimal custom SerializableObject written in C++ with an +// upgrade function + +namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; + +class SimpleClass : public otio::SerializableObject +{ +public: + struct Schema + { + static auto constexpr name = "SimpleClass"; + static int constexpr version = 2; + }; + + void set_new_field(int64_t val) { _new_field = val; } + const int64_t new_field() const { return _new_field; } + +protected: + using Parent = SerializableObject; + + virtual ~SimpleClass() = default; + + virtual bool + read_from(Reader& reader) + { + auto result = ( + reader.read("new_field", &_new_field) + && Parent::read_from(reader) + ); + + return result; + } + + virtual void + write_to(Writer& writer) const + { + Parent::write_to(writer); + writer.write("new_field", _new_field); + } + +private: + int64_t _new_field; +}; + +int +main( + int argc, + char *argv[] +) +{ + // register type and upgrade/downgrade functions + otio::TypeRegistry::instance().register_type(); + + // 1->2 + otio::TypeRegistry::instance().register_upgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["new_field"] = (*d)["my_field"]; + d->erase("my_field"); + } + ); + // 2->1 + otio::TypeRegistry::instance().register_downgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["my_field"] = (*d)["new_field"]; + d->erase("new_field"); + } + ); + + otio::ErrorStatus err; + + auto sc = otio::SerializableObject::Retainer(new SimpleClass()); + sc->set_new_field(12); + + // write it out to disk, without changing it + sc->to_json_file("/var/tmp/simpleclass.otio", &err); + + otio::schema_version_map downgrade_manifest = { + {"SimpleClass", 1} + }; + + // write it out to disk, downgrading to version 1 + sc->to_json_file("/var/tmp/simpleclass.otio", &err, &downgrade_manifest); + + // read it back, upgrading automatically back up to version 2 of the schema + otio::SerializableObject::Retainer sc2( + dynamic_cast( + SimpleClass::from_json_file("/var/tmp/simpleclass.otio", &err) + ) + ); + + assert(sc2->new_field() == sc->new_field()); + + return 0; +} From dcef19bb218ea4201dec32df9aa769d1fa533218 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:33:02 -0700 Subject: [PATCH 068/119] Add notes to environment variables markdown. Signed-off-by: ssteinbach --- docs/tutorials/otio-env-variables.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/otio-env-variables.md b/docs/tutorials/otio-env-variables.md index 3c8a113df..1bdacd3cf 100644 --- a/docs/tutorials/otio-env-variables.md +++ b/docs/tutorials/otio-env-variables.md @@ -5,11 +5,12 @@ various aspects of OTIO. ## Plugin Configuration -These variables must be set _before_ the OpenTimelineIO python library is imported. +These variables must be set _before_ the OpenTimelineIO python library is imported. They only impact the python library. The C++ has no environment variables. - `OTIO_PLUGIN_MANIFEST_PATH`: a ":" separated string with paths to .manifest.json files that contain OTIO plugin manifests. See: [Tutorial on how to write an adapter plugin](write-an-adapter). - `OTIO_DEFAULT_MEDIA_LINKER`: the name of the default media linker to use after reading a file, if "" then no media linker is automatically invoked. - `OTIO_DISABLE_PKG_RESOURCE_PLUGINS`: By default, OTIO will use the pkg_resource entry_points mechanism to discover plugins that have been installed into the current python environment. pkg_resources, however, can be slow in certain cases, so for users who wish to disable this behavior, this variable can be set to 1. +- `OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL`: if no downgrade arguments are passed to `write_to_file`/`write_to_string`, use the downgrade manifest specified by the family/label combination in the variable. Variable is of the form FAMILY:LABEL. Only one tuple of FAMILY:LABEL may be specified. ## Unit tests From d29ab26c5f5d2bd88993adf78925d775ac7e9211 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:44:40 -0700 Subject: [PATCH 069/119] typo Signed-off-by: ssteinbach --- docs/tutorials/otio-env-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/otio-env-variables.md b/docs/tutorials/otio-env-variables.md index 1bdacd3cf..7d46132ac 100644 --- a/docs/tutorials/otio-env-variables.md +++ b/docs/tutorials/otio-env-variables.md @@ -5,7 +5,7 @@ various aspects of OTIO. ## Plugin Configuration -These variables must be set _before_ the OpenTimelineIO python library is imported. They only impact the python library. The C++ has no environment variables. +These variables must be set _before_ the OpenTimelineIO python library is imported. They only impact the python library. The C++ library has no environment variables. - `OTIO_PLUGIN_MANIFEST_PATH`: a ":" separated string with paths to .manifest.json files that contain OTIO plugin manifests. See: [Tutorial on how to write an adapter plugin](write-an-adapter). - `OTIO_DEFAULT_MEDIA_LINKER`: the name of the default media linker to use after reading a file, if "" then no media linker is automatically invoked. From 5ab1e67e9287a0cde880974fc859994fad172a96 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Sun, 4 Sep 2022 22:47:06 -0700 Subject: [PATCH 070/119] typo Signed-off-by: ssteinbach --- docs/tutorials/versioning-schemas.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 13b3ab9f2..3d394c7d8 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -18,7 +18,7 @@ OpenTimelineIO can still interoperate with older and newer versions of the libra Once a type is registerd to OpenTimelineIO, deveopers may also register upgrade functions. In python, each upgrade function takes a dictionary and returns a dictionary. In C++, the AnyDictionary is manipulated in place. Each upgrade function is associated with a version number - this is the version number that it upgrades to. -C++ Example (can be viewed/run in `examples/upgrade_example.cpp`): +C++ Example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): ```cpp class SimpleClass : public otio::SerializableObject @@ -96,7 +96,7 @@ When upgrading schemas, OpenTimelineIO will call each upgrade function in order Similarly, once a type is registered, downgrade functions may be registered. Downgrade functions take a dictionary of the version specified and return a dictionary of the schema version one lower. For example, if a downgrade function is registered for version 5, that will downgrade from 5 to 4. -C++ Example, building off the prior section SimpleClass example (can be viewed/run in `examples/upgrade_example.cpp`): +C++ Example, building off the prior section SimpleClass example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): ```cpp // 2->1 From 0c680bb6b189c7baeaf5636362dd80710f17aef6 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Mon, 5 Sep 2022 20:17:56 -0700 Subject: [PATCH 071/119] Improve error handling and text for env var errors - the OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL has checking to make sure the format is correct and that the version/label requested are present. - Adds a custom exception that gets raised if there is a problem - Adds a unit test for testing this behavior Signed-off-by: ssteinbach --- .../opentimelineio/adapters/otio_json.py | 78 +++++++++++++------ .../opentimelineio/exceptions.py | 4 + tests/test_version_manifest.py | 18 +++++ 3 files changed, 78 insertions(+), 22 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 685886e29..5a37a2acb 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -5,12 +5,13 @@ from .. import ( core, - versioning + versioning, + exceptions ) import os -DEFAULT_VERSION_ENVVAR = "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" +_DEFAULT_VERSION_ENVVAR = "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" def read_from_file(filepath): @@ -39,6 +40,38 @@ def read_from_string(input_str): return core.deserialize_json_from_string(input_str) +def _fetch_downgrade_map_from_env(): + version_envvar = os.environ[_DEFAULT_VERSION_ENVVAR] + + try: + family, label = version_envvar.split(":") + except ValueError: + raise exceptions.InvalidEnvironmentVariableError( + "Environment variable '{}' is incorrectly formatted with '{}'." + "Variable must be formatted as 'FAMILY:LABEL'".format( + _DEFAULT_VERSION_ENVVAR, + version_envvar, + ) + ) + + try: + # technically fetch_map returns an AnyDictionary, but the pybind11 + # code wrapping the call to the serializer expects a python + # dictionary. This turns it back into a normal dictionary. + return dict(versioning.fetch_map(family, label)) + except KeyError: + raise exceptions.InvalidEnvironmentVariableError( + "Environment variable '{}' is requesting family '{}' and label" + " '{}', however this combination does not exist in the " + "currently loaded manifests. Full version map: {}".format( + _DEFAULT_VERSION_ENVVAR, + family, + label, + versioning.full_map() + ) + ) + + def write_to_string(input_otio, target_schema_versions=None, indent=4): """ Serializes an OpenTimelineIO object into a string @@ -49,25 +82,23 @@ def write_to_string(input_otio, target_schema_versions=None, indent=4): -1 for no indentation or newlines. If target_schema_versions is None and the environment variable - "{}" is set, will read a map out of + "{0}" is set, will read a map out of that for downgrade target. The variable should be of the form FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". Returns: str: A json serialized string representation - """.format(DEFAULT_VERSION_ENVVAR) - if target_schema_versions is None and DEFAULT_VERSION_ENVVAR in os.environ: - version_envvar = os.environ[DEFAULT_VERSION_ENVVAR] - family, label = version_envvar.split(":") - # @TODO: something isn't right, I shouldn't need to do this extra hop- - # if I don't, I end up with an AnyDictionary instead of a python - # {}, which pybind doesn't quite map into the std::map the way - # I'd hope when calling serialize_json_to_string() - target_schema_versions = versioning.fetch_map(family, label) - d = {} - d.update(target_schema_versions) - target_schema_versions = d + Raises: + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable "{0}". + """.format(_DEFAULT_VERSION_ENVVAR) + + if ( + target_schema_versions is None + and _DEFAULT_VERSION_ENVVAR in os.environ + ): + target_schema_versions = _fetch_downgrade_map_from_env() return core.serialize_json_to_string( input_otio, @@ -93,7 +124,7 @@ def write_to_file( Use -1 for no indentation or newlines. If target_schema_versions is None and the environment variable - "{}" is set, will read a map out of + "{0}" is set, will read a map out of that for downgrade target. The variable should be of the form FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". @@ -102,12 +133,15 @@ def write_to_file( Raises: ValueError: on write error - """.format(DEFAULT_VERSION_ENVVAR) - - if target_schema_versions is None and DEFAULT_VERSION_ENVVAR in os.environ: - version_envvar = os.environ[DEFAULT_VERSION_ENVVAR] - family, label = version_envvar.split(":") - target_schema_versions = versioning.fetch_map(family, label) + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable "{0}". + """.format(_DEFAULT_VERSION_ENVVAR) + + if ( + target_schema_versions is None + and _DEFAULT_VERSION_ENVVAR in os.environ + ): + target_schema_versions = _fetch_downgrade_map_from_env() return core.serialize_json_to_file( input_otio, diff --git a/src/py-opentimelineio/opentimelineio/exceptions.py b/src/py-opentimelineio/opentimelineio/exceptions.py index 32cb25cf5..964777756 100644 --- a/src/py-opentimelineio/opentimelineio/exceptions.py +++ b/src/py-opentimelineio/opentimelineio/exceptions.py @@ -75,3 +75,7 @@ class CannotTrimTransitionsError(OTIOError): class NoDefaultMediaLinkerError(OTIOError): pass + + +class InvalidEnvironmentVariableError(OTIOError): + pass diff --git a/tests/test_version_manifest.py b/tests/test_version_manifest.py index c0baca3d1..4fecd638e 100644 --- a/tests/test_version_manifest.py +++ b/tests/test_version_manifest.py @@ -110,6 +110,24 @@ def downgrade_2_to_1(_data_dict): result = json.loads(otio.adapters.otio_json.write_to_string(evt, {})) self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.2") + def test_garbage_env_variables(self): + cl = otio.schema.Clip() + invalid_env_error = otio.exceptions.InvalidEnvironmentVariableError + + # missing ":" + os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( + "invalid_formatting" + ) + with self.assertRaises(invalid_env_error): + otio.adapters.otio_json.write_to_string(cl) + + # asking for family/label that doesn't exist in the plugins + os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( + "nosuch:labelorfamily" + ) + with self.assertRaises(invalid_env_error): + otio.adapters.otio_json.write_to_string(cl) + def test_two_version_manifests(self): """test that two manifests layer correctly""" From 5f22bb52a7e391cffccdce346d0c86197e6b8562 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 08:03:16 -0700 Subject: [PATCH 072/119] various tightening, notes, and documentation Signed-off-by: ssteinbach --- docs/tutorials/versioning-schemas.md | 10 ++++++---- examples/io_perf_test.cpp | 11 ++++++++--- examples/upgrade_downgrade_example.cpp | 13 +++++++++---- src/opentimelineio/CORE_VERSION_MAP.cpp | 16 ++++++++-------- src/opentimelineio/CORE_VERSION_MAP.last.cpp | 16 ++++++++-------- src/opentimelineio/serializableObject.h | 2 -- src/opentimelineio/serialization.cpp | 4 ++-- src/opentimelineio/typeRegistry.h | 4 +++- .../opentimelineio-bindings/otio_bindings.cpp | 4 ++++ 9 files changed, 48 insertions(+), 32 deletions(-) diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 3d394c7d8..968247ac5 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -2,7 +2,7 @@ ## Overview -This document describes OpenTimelineIO's systems for dealing with different schema versions when reading files, writing files, and finally as part of the development process. It is intended for developers who are integrating OpenTimelineIO into their pipelines or applications. +This document describes OpenTimelineIO's systems for dealing with different schema versions when reading files, writing files, or during development of the library itself. It is intended for developers who are integrating OpenTimelineIO into their pipelines or applications, or working directly on OpenTimelineIO. TL;DR for users: OpenTimelineIO should be able to read files produced by older versions of the library and be able to write files that are compatible with older versions of the library from newer versions. @@ -277,7 +277,9 @@ If a change to a schema is to add a field, for which the default value is the co Additionally, upgrade functions will be called in order, but they need not cover every version number. So if there is an upgrade function for version 2 and 4, to get to version 4, OTIO will automatically apply function 2 and then function 4 in order, skipping the missing 3. -Example of adding a field: +Downgrade functions must be called in order with no gaps. + +Example of adding a field (`other_field`): ```python @otio.core.register_type @@ -287,7 +289,7 @@ class SimpleClass(otio.core.SerializeableObject): other_field = otio.core.serializeable_field("other_field", int) ``` -Removing a field: +Removing a field (`my_field`): ```python @otio.core.register_type @@ -296,4 +298,4 @@ class SimpleClass(otio.core.SerializeableObject): other_field = otio.core.serializeable_field("other_field", int) ``` -Similarly, when deleting a field, if the field is now ignored and does not contribute to computation, no upgrade is needed. +Similarly, when deleting a field, if the field is now ignored and does not contribute to computation, no upgrade or downgrade function is needed. diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 3e8376bea..aa6a87ee3 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -12,7 +12,8 @@ namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; using chrono_time_point = std::chrono::steady_clock::time_point; -constexpr struct { +const struct { + bool PRINT_CPP_VERSION_FAMILY = false; bool TO_JSON_STRING = true; bool TO_JSON_STRING_NO_DOWNGRADE = true; bool TO_JSON_FILE = true; @@ -59,9 +60,13 @@ main( char *argv[] ) { - print_version_map(); + if (RUN_STRUCT.PRINT_CPP_VERSION_FAMILY) + { + print_version_map(); + } - if (argc < 2) { + if (argc < 2) + { std::cerr << "usage: otio_io_perf_test path/to/timeline.otio"; std::cerr << std::endl; return 1; diff --git a/examples/upgrade_downgrade_example.cpp b/examples/upgrade_downgrade_example.cpp index 55ac40f1f..ff8f88268 100644 --- a/examples/upgrade_downgrade_example.cpp +++ b/examples/upgrade_downgrade_example.cpp @@ -2,11 +2,12 @@ #include "opentimelineio/typeRegistry.h" #include -// demonstrates a minimal custom SerializableObject written in C++ with an -// upgrade function +// demonstrates a minimal custom SerializableObject written in C++ with upgrade +// and downgrade functions namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; +// define the custom class class SimpleClass : public otio::SerializableObject { public: @@ -24,8 +25,9 @@ class SimpleClass : public otio::SerializableObject virtual ~SimpleClass() = default; + // methods for serialization virtual bool - read_from(Reader& reader) + read_from(Reader& reader) override { auto result = ( reader.read("new_field", &_new_field) @@ -35,8 +37,9 @@ class SimpleClass : public otio::SerializableObject return result; } + // ...and deserialization virtual void - write_to(Writer& writer) const + write_to(Writer& writer) const override { Parent::write_to(writer); writer.write("new_field", _new_field); @@ -100,5 +103,7 @@ main( assert(sc2->new_field() == sc->new_field()); + std::cout << "Upgrade/Downgrade demo complete." << std::endl; + return 0; } diff --git a/src/opentimelineio/CORE_VERSION_MAP.cpp b/src/opentimelineio/CORE_VERSION_MAP.cpp index 096cdd831..48e8c44f3 100644 --- a/src/opentimelineio/CORE_VERSION_MAP.cpp +++ b/src/opentimelineio/CORE_VERSION_MAP.cpp @@ -1,15 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Contributors to the OpenTimelineIO project // -// This document is automatically generated by running the -// `autogen_version_info` command, or by running `make version-info`. It is -// part of the unit tests suite and should be updated whenever schema versions -// change. If it needs to be updated, run: `make version-info-update` and this -// file should be regenerated. +// This document is automatically generated by running the `make version-map` +// make target. It is part of the unit tests suite and should be updated +// whenever schema versions change. If it needs to be updated, run: `make +// version-map-update` and this file should be regenerated. // -// This is a mapping of "family" to "label", where each label is then mapped to -// a "schema-version map", a mapping of schema name to schema version for that -// label. +// This maps a "Label" to a map of Schema name to Schema version. The intent is +// that these sets of schemas can be used for compatability with future +// versions of OTIO, so that a newer version of OTIO can target a compatability +// version of an older library. #include "opentimelineio/typeRegistry.h" namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { diff --git a/src/opentimelineio/CORE_VERSION_MAP.last.cpp b/src/opentimelineio/CORE_VERSION_MAP.last.cpp index 8276293a5..fdb321113 100644 --- a/src/opentimelineio/CORE_VERSION_MAP.last.cpp +++ b/src/opentimelineio/CORE_VERSION_MAP.last.cpp @@ -1,15 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Contributors to the OpenTimelineIO project // -// This document is automatically generated by running the -// `autogen_version_info` command, or by running `make version-info`. It is -// part of the unit tests suite and should be updated whenever schema versions -// change. If it needs to be updated, run: `make version-info-update` and this -// file should be regenerated. +// This document is automatically generated by running the `make version-map` +// make target. It is part of the unit tests suite and should be updated +// whenever schema versions change. If it needs to be updated, run: `make +// version-map-update` and this file should be regenerated. // -// This is a mapping of "family" to "label", where each label is then mapped to -// a "schema-version map", a mapping of schema name to schema version for that -// label. +// This maps a "Label" to a map of Schema name to Schema version. The intent is +// that these sets of schemas can be used for compatability with future +// versions of OTIO, so that a newer version of OTIO can target a compatability +// version of an older library. #include "opentimelineio/typeRegistry.h" namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index f4fd4a2ce..6101c0bc8 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -22,8 +22,6 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -using schema_version_map = std::unordered_map; - class SerializableObject { public: diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index cf52e5b81..3fa7beac1 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -978,8 +978,6 @@ SerializableObject::Writer::write( std::to_string(++_next_id_for_type[schema_type_name]); _id_for_object[value] = next_id; - std::string schema_str = ""; - // detect if downgrading needs to happen const std::string& schema_name = value->schema_name(); int schema_version = value->schema_version(); @@ -1023,6 +1021,8 @@ SerializableObject::Writer::write( } } + std::string schema_str = ""; + // if its an unknown schema, the schema name is computed from the // _original_schema_name and _original_schema_version attributes if (UnknownSchema const* us = dynamic_cast(value)) diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 1129df3e0..ce2fe54b9 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -21,7 +21,9 @@ class Encoder; class AnyDictionary; // typedefs for the schema downgrading system -using schema_version_map = std::unordered_map; +// @TODO: should we make version an int64_t? That would match what we can +// serialize natively, since we only serialize 64 bit signed ints. +using schema_version_map = std::unordered_map; using label_to_schema_version_map = std::unordered_map; extern const label_to_schema_version_map CORE_VERSION_MAP; diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 1dab6efe5..ebfe906bd 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -93,6 +93,10 @@ register_downgrade_function( int version_to_downgrade_from, py::object const& downgrade_function_obj) { + // @TODO: I'm actually surprised this works, since the python function + // seems to often make new dictionaries and relies on the return + // vallue... but here it doesn't seem like the return value matters. + // ...the C++ functions definitely work in place though. std::function downgrade_function = ( [downgrade_function_obj](AnyDictionary* d) { From 88de08af7f173f01bc15075be1c8419eca3686b2 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 08:08:40 -0700 Subject: [PATCH 073/119] Simplify docs Signed-off-by: ssteinbach --- .../opentimelineio/adapters/otio_json.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 5a37a2acb..d1c8c9655 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -82,7 +82,7 @@ def write_to_string(input_otio, target_schema_versions=None, indent=4): -1 for no indentation or newlines. If target_schema_versions is None and the environment variable - "{0}" is set, will read a map out of + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of that for downgrade target. The variable should be of the form FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". @@ -91,8 +91,9 @@ def write_to_string(input_otio, target_schema_versions=None, indent=4): Raises: otio.exceptions.InvalidEnvironmentVariableError: if there is a problem - with the default environment variable "{0}". - """.format(_DEFAULT_VERSION_ENVVAR) + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". + """ if ( target_schema_versions is None @@ -124,7 +125,7 @@ def write_to_file( Use -1 for no indentation or newlines. If target_schema_versions is None and the environment variable - "{0}" is set, will read a map out of + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of that for downgrade target. The variable should be of the form FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". @@ -134,7 +135,8 @@ def write_to_file( Raises: ValueError: on write error otio.exceptions.InvalidEnvironmentVariableError: if there is a problem - with the default environment variable "{0}". + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". """.format(_DEFAULT_VERSION_ENVVAR) if ( From 3038c7728315a9814a6889e91834586293c07f5e Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 08:14:37 -0700 Subject: [PATCH 074/119] update plugin autodocs Signed-off-by: ssteinbach --- docs/tutorials/otio-plugins.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index 0f428cff3..5e03120bd 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -141,7 +141,28 @@ De-serializes an OpenTimelineIO object from a json string - filepath - target_schema_versions - indent -- write_to_string: +- write_to_string: +``` +Serializes an OpenTimelineIO object into a string + + Args: + input_otio (OpenTimeline): An OpenTimeline object + indent (int): number of spaces for each json indentation level. Use + -1 for no indentation or newlines. + + If target_schema_versions is None and the environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". + + Returns: + str: A json serialized string representation + + Raises: + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". +``` - input_otio - target_schema_versions - indent From 110251c02b62ee3817663a9ceb60cc05d196fdf0 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 08:16:14 -0700 Subject: [PATCH 075/119] update docs more Signed-off-by: ssteinbach --- docs/tutorials/otio-plugins.md | 26 ++++++++++++++++++- .../opentimelineio/adapters/otio_json.py | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index 5e03120bd..ced77ed2c 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -136,7 +136,31 @@ De-serializes an OpenTimelineIO object from a json string OpenTimeline: An OpenTimeline object ``` - input_str -- write_to_file: +- write_to_file: +``` +Serializes an OpenTimelineIO object into a file + + Args: + + input_otio (OpenTimeline): An OpenTimeline object + filepath (str): The name of an otio file to write to + indent (int): number of spaces for each json indentation level. + Use -1 for no indentation or newlines. + + If target_schema_versions is None and the environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". + + Returns: + bool: Write success + + Raises: + ValueError: on write error + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". +``` - input_otio - filepath - target_schema_versions diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index d1c8c9655..775c73021 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -137,7 +137,7 @@ def write_to_file( otio.exceptions.InvalidEnvironmentVariableError: if there is a problem with the default environment variable "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". - """.format(_DEFAULT_VERSION_ENVVAR) + """ if ( target_schema_versions is None From 1f45885e03f2cad8ec83bf9417033382c601c26e Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 09:24:34 -0700 Subject: [PATCH 076/119] update docs to quiet warnings Signed-off-by: ssteinbach --- docs/tutorials/otio-plugins.md | 8 ++++---- .../opentimelineio/adapters/otio_json.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index ced77ed2c..4e737d240 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -158,8 +158,8 @@ Serializes an OpenTimelineIO object into a file Raises: ValueError: on write error otio.exceptions.InvalidEnvironmentVariableError: if there is a problem - with the default environment variable - "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". ``` - input_otio - filepath @@ -184,8 +184,8 @@ Serializes an OpenTimelineIO object into a string Raises: otio.exceptions.InvalidEnvironmentVariableError: if there is a problem - with the default environment variable - "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". ``` - input_otio - target_schema_versions diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 775c73021..9d761a451 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -91,8 +91,8 @@ def write_to_string(input_otio, target_schema_versions=None, indent=4): Raises: otio.exceptions.InvalidEnvironmentVariableError: if there is a problem - with the default environment variable - "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". """ if ( @@ -135,8 +135,8 @@ def write_to_file( Raises: ValueError: on write error otio.exceptions.InvalidEnvironmentVariableError: if there is a problem - with the default environment variable - "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". """ if ( From 2b9c779492b10fa66842b4d832380cba0fdc9529 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 09:44:20 -0700 Subject: [PATCH 077/119] remove dead code and use type alias Signed-off-by: ssteinbach --- src/opentimelineio/typeRegistry.cpp | 2 +- src/opentimelineio/typeRegistry.h | 9 +-------- .../opentimelineio-bindings/otio_bindings.cpp | 2 +- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 90254796d..4ea67e9f7 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -395,7 +395,7 @@ TypeRegistry::set_type_record( void TypeRegistry::type_version_map( - std::map& result) + schema_version_map& result) { std::lock_guard lock(_registry_mutex); diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index ce2fe54b9..295030ae8 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -142,15 +142,8 @@ class TypeRegistry std::string const& schema_name, ErrorStatus* error_status = nullptr); - int - version_number_of_schema( - const std::string& schema_name) - { - return _find_type_record(schema_name)->schema_version; - } - // for inspecting the type registry, build a map of schema name to version - void type_version_map(std::map& result); + void type_version_map(schema_version_map& result); private: TypeRegistry(); diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index ebfe906bd..e2f9f624c 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -249,7 +249,7 @@ Return an instance of the schema from data in the data_dict. )docstring"); m.def("type_version_map", []() { - std::map tmp; + schema_version_map tmp; TypeRegistry::instance().type_version_map(tmp); return tmp; }); From 3b197a23ddcae8444d4ec678fe77105665eac69a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 09:45:25 -0700 Subject: [PATCH 078/119] remove dead code Signed-off-by: ssteinbach --- src/opentimelineio/typeRegistry.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 295030ae8..691f0f2cd 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -207,15 +207,4 @@ class TypeRegistry friend class CloningEncoder; }; -// Functions for dealing with schema versioning - -/// add a new family:version:schema_version_map for downgrading -bool -add_family_label_version( - const std::string& family, - const std::string& label, - const schema_version_map& new_map, - ErrorStatus* err -); - }} // namespace opentimelineio::OPENTIMELINEIO_VERSION From cc1112a6bd17260f5a5ba39ebfb7cb93d2933b87 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 09:52:13 -0700 Subject: [PATCH 079/119] remove unhelpful comment Signed-off-by: ssteinbach --- src/opentimelineio/serialization.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 3fa7beac1..9bce632e7 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -116,14 +116,6 @@ class CloningEncoder : public Encoder virtual ~CloningEncoder() {} - /** - * 1. - * e = JSONEncoder(_downgrade) - * SerializableObject::Writer::write_root(e, _downgrade) - * [recurse until downgrade] - * ce = CloningEncoder(_downgrade) - * SerializableObject::Writer::write_root(e), - */ virtual bool encoding_to_anydict() override { From d8ebf235527f3d99f0b790d2fe2432d6c56931cd Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 10:24:38 -0700 Subject: [PATCH 080/119] don't erase the enabled flag per our policy Signed-off-by: ssteinbach --- src/opentimelineio/typeRegistry.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 4ea67e9f7..9e2ffede7 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -136,7 +136,6 @@ TypeRegistry::TypeRegistry() d->erase("media_references"); d->erase("active_media_reference_key"); - d->erase("enabled"); }); } From ed5aae7d4ec6a424896697d124459a7597aa9115 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 14:18:57 -0700 Subject: [PATCH 081/119] remove breadcrumbing from the downgrade function Signed-off-by: ssteinbach --- src/opentimelineio/typeRegistry.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 9e2ffede7..0c20dfae1 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -125,15 +125,6 @@ TypeRegistry::TypeRegistry() (*d)["media_reference"] = active_ref; - auto downgrade_md = AnyDictionary(); - downgrade_md["media_references"] = media_refs; - downgrade_md["active_media_reference_key"] = active_reference_key; - - d->set_default( - "metadata", - AnyDictionary() - )["downgrade_Clip.2_to_Clip.1"] = downgrade_md; - d->erase("media_references"); d->erase("active_media_reference_key"); }); From 1f7740490cb2f062532dfeeeee98a3f24ced57f3 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 15:36:38 -0700 Subject: [PATCH 082/119] optional -> pointer Signed-off-by: ssteinbach --- src/opentimelineio/serializableObject.cpp | 4 +-- src/opentimelineio/serializableObject.h | 11 +++---- src/opentimelineio/serialization.cpp | 29 ++++++++++--------- src/opentimelineio/serialization.h | 8 ++--- .../opentimelineio-bindings/otio_bindings.cpp | 22 ++++---------- 5 files changed, 32 insertions(+), 42 deletions(-) diff --git a/src/opentimelineio/serializableObject.cpp b/src/opentimelineio/serializableObject.cpp index 2e98bc8d6..bce6a89de 100644 --- a/src/opentimelineio/serializableObject.cpp +++ b/src/opentimelineio/serializableObject.cpp @@ -115,7 +115,7 @@ SerializableObject::is_unknown_schema() const std::string SerializableObject::to_json_string( ErrorStatus* error_status, - optional schema_version_targets, + const schema_version_map* schema_version_targets, int indent ) const { @@ -131,7 +131,7 @@ bool SerializableObject::to_json_file( std::string const& file_name, ErrorStatus* error_status, - optional schema_version_targets, + const schema_version_map* schema_version_targets, int indent) const { return serialize_json_to_file( diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index 6101c0bc8..acd190f65 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -16,6 +16,7 @@ #include "ImathBox.h" #include "serialization.h" +#include #include #include #include @@ -48,13 +49,13 @@ class SerializableObject to_json_file( std::string const& file_name, ErrorStatus* error_status = nullptr, - optional target_family_label_spec = {}, + const schema_version_map* target_family_label_spec = nullptr, int indent = 4) const; std::string to_json_string( ErrorStatus* error_status = nullptr, - optional target_family_label_spec = {}, + const schema_version_map* target_family_label_spec = nullptr, int indent = 4) const; static SerializableObject* from_json_file( @@ -402,7 +403,7 @@ class SerializableObject static bool write_root( any const& value, class Encoder& encoder, - optional downgrade_version_manifest= {}, + const schema_version_map* downgrade_version_manifest=nullptr, ErrorStatus* error_status = nullptr); void write(std::string const& key, bool value); @@ -513,7 +514,7 @@ class SerializableObject Writer( class Encoder& encoder, - optional downgrade_version_manifest + const schema_version_map* downgrade_version_manifest ) : _encoder(encoder), _downgrade_version_manifest(downgrade_version_manifest) @@ -547,7 +548,7 @@ class SerializableObject std::unordered_map _next_id_for_type; class Encoder& _encoder; - optional _downgrade_version_manifest; + const schema_version_map* _downgrade_version_manifest; friend class SerializableObject; }; diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 9bce632e7..7b6de98f8 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -8,6 +8,7 @@ #include "opentimelineio/anyDictionary.h" #include "opentimelineio/unknownSchema.h" #include "stringUtils.h" +#include #include #define RAPIDJSON_NAMESPACE OTIO_rapidjson @@ -105,7 +106,7 @@ class CloningEncoder : public Encoder CloningEncoder( CloningEncoder::ResultObjectPolicy result_object_policy, - optional schema_version_targets = {} + const schema_version_map* schema_version_targets = nullptr ) : _result_object_policy(result_object_policy), _downgrade_version_manifest(schema_version_targets) @@ -380,7 +381,7 @@ class CloningEncoder : public Encoder AnyDictionary m; m.swap(top.dict); - if (_downgrade_version_manifest.has_value()) + if (_downgrade_version_manifest != nullptr) { _downgrade_dictionary(m); } @@ -427,7 +428,7 @@ class CloningEncoder : public Encoder friend class SerializableObject; std::vector<_DictOrArray> _stack; ResultObjectPolicy _result_object_policy; - optional _downgrade_version_manifest = {}; + const schema_version_map* _downgrade_version_manifest = nullptr; void _downgrade_dictionary( @@ -447,10 +448,11 @@ class CloningEncoder : public Encoder const int sep = schema_string.rfind('.'); const std::string& schema_name = schema_string.substr(0, sep); - const auto& dg_man = *(*_downgrade_version_manifest); - const auto dg_version_it = dg_man.find(schema_name); + const auto dg_version_it = _downgrade_version_manifest->find( + schema_name + ); - if (dg_version_it == dg_man.end()) + if (dg_version_it == _downgrade_version_manifest->end()) { return; } @@ -829,7 +831,7 @@ bool SerializableObject::Writer::write_root( any const& value, Encoder& encoder, - optional schema_version_targets, + const schema_version_map* schema_version_targets, ErrorStatus* error_status ) { @@ -978,16 +980,17 @@ SerializableObject::Writer::write( // if there is a manifest & the encoder is not converting to AnyDictionary if ( - _downgrade_version_manifest.has_value() + _downgrade_version_manifest != nullptr && !_encoder.encoding_to_anydict() ) { - const auto& dg_man = *(*_downgrade_version_manifest); - const auto& target_version_it = dg_man.find(schema_name); + const auto& target_version_it = _downgrade_version_manifest->find( + schema_name + ); // ...and if that downgrade manifest specifies a target version for // this schema - if (target_version_it != dg_man.end()) + if (target_version_it != _downgrade_version_manifest->end()) { const int target_version = target_version_it->second; @@ -1237,7 +1240,7 @@ SerializableObject::clone(ErrorStatus* error_status) const std::string serialize_json_to_string( const any& value, - optional schema_version_targets, + const schema_version_map* schema_version_targets, ErrorStatus* error_status, int indent ) @@ -1279,7 +1282,7 @@ bool serialize_json_to_file( any const& value, std::string const& file_name, - optional schema_version_targets, + const schema_version_map* schema_version_targets, ErrorStatus* error_status, int indent) { diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index 03f7ba006..aa4772435 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -18,7 +18,7 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { std::string serialize_json_to_string( const any& value, - optional schema_version_targets = {}, + const schema_version_map* schema_version_targets = nullptr, ErrorStatus* error_status = nullptr, int indent = 4 ); @@ -27,11 +27,7 @@ bool serialize_json_to_file( const any& value, std::string const& file_name, - // @TODO: I think this wants to be an optional, - // but that isn't allowed, so maybe a const family_label_spec*? - // (to avoid the copy). - // these aren't inner loop functions, so isn't *that* crucial anyway. - optional schema_version_targets = {}, + const schema_version_map* schema_version_targets = nullptr, ErrorStatus* error_status = nullptr, int indent = 4 ); diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index e2f9f624c..166e1f691 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -156,17 +156,12 @@ PYBIND11_MODULE(_otio, m) { int indent ) { - optional pass_through = {}; - if (!schema_version_targets.empty()) - { - pass_through = {&schema_version_targets}; - } auto result = serialize_json_to_string( - pyAny->a, - pass_through, - ErrorStatusHandler(), - indent - ); + pyAny->a, + &schema_version_targets, + ErrorStatusHandler(), + indent + ); return result; }, @@ -181,15 +176,10 @@ PYBIND11_MODULE(_otio, m) { const schema_version_map& schema_version_targets, int indent ) { - optional pass_through = {}; - if (!schema_version_targets.empty()) - { - pass_through = {&schema_version_targets}; - } return serialize_json_to_file( pyAny->a, filename, - pass_through, + &schema_version_targets, ErrorStatusHandler(), indent ); From 93cae2aa459f79bb012abea567fe5016c70d51e3 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 16:10:11 -0700 Subject: [PATCH 083/119] AnyDictionary::get_default -> get_if_set - change the calling semantics to pass in containedType* and set if the value is present, otherwise return false Signed-off-by: ssteinbach --- src/opentimelineio/anyDictionary.h | 58 ++++++++++++++++++---------- src/opentimelineio/serialization.cpp | 9 ++--- src/opentimelineio/typeRegistry.cpp | 45 ++++++++++++++------- 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/src/opentimelineio/anyDictionary.h b/src/opentimelineio/anyDictionary.h index 0ead3b167..fe1927275 100644 --- a/src/opentimelineio/anyDictionary.h +++ b/src/opentimelineio/anyDictionary.h @@ -123,23 +123,35 @@ class AnyDictionary : private std::map map::swap(other); } - /// @TODO: need the right policy for accessing/returning membors + /// @TODO: remove all of these @{ + + // if key is in this, and the type of key matches the type of result, then + // set result to the value of any_cast(this[key]) and return true, + // otherwise return false template - containedType - get_default( + bool + get_if_set( const std::string& key, - const containedType& default_value + containedType* result ) { - const auto& it = this->find(key); - - return ( - it != this->end() - && it->second.type().hash_code() == typeid(containedType).hash_code() - ) ? - any_cast(it->second) - : default_value - ; + const auto it = this->find(key); + + if ( + (it != this->end()) + && ( + it->second.type().hash_code() + == typeid(containedType).hash_code() + ) + ) + { + *result = any_cast(it->second); + return true; + } + else + { + return false; + } } inline bool @@ -150,22 +162,28 @@ class AnyDictionary : private std::map return (this->find(key) != this->end()); } + // if key is in this, place the value in result and return true, otherwise + // store the value in result at key and return false template - containedType + bool set_default( const std::string& key, - const containedType& default_value + containedType* result ) { - const auto& d_it = this->find(key); - if (d_it != this->end()) + const auto d_it = this->find(key); + if ( + (d_it != this->end()) + && (d_it->second.type() == typeid(containedType)) + ) { - return any_cast(d_it->second); + *result = any_cast(d_it->second); + return true; } else { - this->insert({key, default_value}); - return default_value; + this->insert({key, *result}); + return false; } } diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 7b6de98f8..b23f465e1 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -435,17 +435,14 @@ class CloningEncoder : public Encoder AnyDictionary& m ) { - const std::string& schema_string = m.get_default( - "OTIO_SCHEMA", - std::string("") - ); + std::string schema_string = ""; - if (schema_string.empty()) + if (!m.get_if_set("OTIO_SCHEMA", &schema_string)) { return; } - const int sep = schema_string.rfind('.'); + const auto sep = schema_string.rfind('.'); const std::string& schema_name = schema_string.substr(0, sep); const auto dg_version_it = _downgrade_version_manifest->find( diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 0c20dfae1..96accaade 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -3,6 +3,7 @@ #include "opentimelineio/typeRegistry.h" +#include "anyDictionary.h" #include "opentimelineio/clip.h" #include "opentimelineio/composable.h" #include "opentimelineio/composition.h" @@ -113,21 +114,35 @@ TypeRegistry::TypeRegistry() }); // 2->1 - register_downgrade_function(Clip::Schema::name, 2, [](AnyDictionary* d) { - AnyDictionary media_refs = d->get_default("media_references", AnyDictionary()); - const std::string active_reference_key = ( - d->get_default("active_media_reference_key", std::string("")) - ); - AnyDictionary active_ref = AnyDictionary(); - if (active_reference_key != "") { - active_ref = media_refs.get_default(active_reference_key, AnyDictionary()); - } - - (*d)["media_reference"] = active_ref; - - d->erase("media_references"); - d->erase("active_media_reference_key"); - }); + register_downgrade_function( + Clip::Schema::name, + 2, + [](AnyDictionary* d) + { + AnyDictionary mrefs; + std::string active_rkey = ""; + + if (d->get_if_set("media_references", &mrefs)) + { + if ( + d->get_if_set( + "active_media_reference_key", + &active_rkey + ) + ) + { + AnyDictionary active_ref; + if (mrefs.get_if_set(active_rkey, &active_ref)) + { + (*d)["media_reference"] = active_ref; + } + } + } + + d->erase("media_references"); + d->erase("active_media_reference_key"); + } + ); } bool From 1dc926b11e27e74bb03f308d82cd0cf7965bbcf2 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 16:16:06 -0700 Subject: [PATCH 084/119] mark AnyDictionary methods const Signed-off-by: ssteinbach --- src/opentimelineio/anyDictionary.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/opentimelineio/anyDictionary.h b/src/opentimelineio/anyDictionary.h index fe1927275..c86333d3b 100644 --- a/src/opentimelineio/anyDictionary.h +++ b/src/opentimelineio/anyDictionary.h @@ -133,7 +133,7 @@ class AnyDictionary : private std::map get_if_set( const std::string& key, containedType* result - ) + ) const { const auto it = this->find(key); @@ -157,7 +157,7 @@ class AnyDictionary : private std::map inline bool has_key( const std::string& key - ) + ) const { return (this->find(key) != this->end()); } From 53399ef1228d9955eb5a8edf83cd94ac60497ec4 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Tue, 6 Sep 2022 20:00:46 -0700 Subject: [PATCH 085/119] remove dead code Signed-off-by: ssteinbach --- tests/test_serializable_object.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index ebc9e3738..cee6e9a2c 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -243,11 +243,6 @@ def downgrade_2_to_1_again(_data_dict): f = FakeThing() f.foo_two = "a thing here" - # downgrade_target = otio.core.fetch_version_map( - # "OTIO_CORE", - # otio.__version__ - # ) - downgrade_target = {"FakeThingToDowngrade": 1} result = json.loads( From 0ecc418a45d5a74a9222b5a980752a4a8b1549a0 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 15:38:48 -0700 Subject: [PATCH 086/119] create a temp directory for io_perf_test Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 57 +++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index aa6a87ee3..386b54053 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -1,4 +1,5 @@ #include +#include #include "opentimelineio/clip.h" #include "opentimelineio/typeRegistry.h" @@ -67,11 +68,23 @@ main( if (argc < 2) { - std::cerr << "usage: otio_io_perf_test path/to/timeline.otio"; + std::cerr << "usage: otio_io_perf_test path/to/timeline.otio [--keep-tmp]"; std::cerr << std::endl; return 1; } + bool keep_tmp = false; + if (argc > 2) + { + const std::string arg = argv[2]; + if (arg == "--keep-tmp") + { + keep_tmp = true; + } + } + + const std::string tmp_dir_path = examples::create_temp_dir(); + otio::ErrorStatus err; otio::schema_version_map downgrade_manifest = { @@ -97,7 +110,11 @@ main( otio::SerializableObject::Retainer cl = new otio::Clip("test"); cl->metadata()["example thing"] = "banana"; chrono_time_point begin = std::chrono::steady_clock::now(); - cl->to_json_file("/var/tmp/clip.otio", &err, &downgrade_manifest); + cl->to_json_file( + examples::normalize_path(tmp_dir_path + "/clip.otio"), + &err, + &downgrade_manifest + ); chrono_time_point end = std::chrono::steady_clock::now(); print_elapsed_time("downgrade clip", begin, end); } @@ -105,10 +122,14 @@ main( otio::any tl; std::string fname = std::string(argv[1]); + // read file chrono_time_point begin = std::chrono::steady_clock::now(); otio::SerializableObject::Retainer timeline( dynamic_cast( - otio::Timeline::from_json_file(argv[1], &err) + otio::Timeline::from_json_file( + examples::normalize_path(argv[1]), + &err + ) ) ); chrono_time_point end = std::chrono::steady_clock::now(); @@ -156,7 +177,7 @@ main( { begin = std::chrono::steady_clock::now(); timeline.value->to_json_file( - "/var/tmp/io_perf_test.otio", + examples::normalize_path(tmp_dir_path + "/io_perf_test.otio"), &err, &downgrade_manifest ); @@ -167,10 +188,36 @@ main( if (RUN_STRUCT.TO_JSON_FILE_NO_DOWNGRADE) { begin = std::chrono::steady_clock::now(); - timeline.value->to_json_file("/var/tmp/io_perf_test.nodowngrade.otio", &err, {}); + timeline.value->to_json_file( + examples::normalize_path( + tmp_dir_path + + "/io_perf_test.nodowngrade.otio" + ), + &err, + {} + ); end = std::chrono::steady_clock::now(); print_elapsed_time("serialize_json_to_file [no downgrade]", begin, end); } + std::cout << "All files written to: " << tmp_dir_path << std::endl; + if (keep_tmp) + { + std::cout << "Temp directory preserved. All files written to: "; + std::cout << tmp_dir_path << std::endl; + } + else + { + // clean up + const auto tmp_files = examples::glob(tmp_dir_path, "*"); + for (auto fp : tmp_files) + { + remove(fp.c_str()); + } + remove(tmp_dir_path.c_str()); + std::cout << "cleaned up tmp dir, pass --keep-tmp to preserve"; + std::cout << " output." << std::endl; + } + return 0; } From 040eecc4ed1f29150a628147a39bfa2b06250910 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 15:39:06 -0700 Subject: [PATCH 087/119] review note Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 386b54053..5b0e30617 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -24,7 +24,7 @@ const struct { } RUN_STRUCT ; /// utility function for printing std::chrono elapsed time -const void +void print_elapsed_time( const std::string& message, const chrono_time_point& begin, From 887b02ccb209dea5feaec8008c4d9f2dc98cb4ae Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 15:40:35 -0700 Subject: [PATCH 088/119] remove printout of tmp dir path if it is removed Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 5b0e30617..37d6e4704 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -200,7 +200,6 @@ main( print_elapsed_time("serialize_json_to_file [no downgrade]", begin, end); } - std::cout << "All files written to: " << tmp_dir_path << std::endl; if (keep_tmp) { std::cout << "Temp directory preserved. All files written to: "; From 81d8c6516cfd410685ca6a88fdff1daaebce0756 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 15:40:56 -0700 Subject: [PATCH 089/119] clean up imports Signed-off-by: ssteinbach --- src/opentimelineio/typeRegistry.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 691f0f2cd..47ec0e14e 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -6,12 +6,12 @@ #include "opentimelineio/any.h" #include "opentimelineio/errorStatus.h" #include "opentimelineio/version.h" + #include #include -#include -#include #include #include +#include #include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { From e3aa7a3d6a8e7f4edaa21ebe9404a6b7f56629d3 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 15:41:02 -0700 Subject: [PATCH 090/119] stray const Signed-off-by: ssteinbach --- examples/upgrade_downgrade_example.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/upgrade_downgrade_example.cpp b/examples/upgrade_downgrade_example.cpp index ff8f88268..9861a9eb4 100644 --- a/examples/upgrade_downgrade_example.cpp +++ b/examples/upgrade_downgrade_example.cpp @@ -18,7 +18,7 @@ class SimpleClass : public otio::SerializableObject }; void set_new_field(int64_t val) { _new_field = val; } - const int64_t new_field() const { return _new_field; } + int64_t new_field() const { return _new_field; } protected: using Parent = SerializableObject; From b9b5d11fd8e4bb6afe93def71c7df6847d77621a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 16:18:18 -0700 Subject: [PATCH 091/119] sanitize line endings in serialized versions test Signed-off-by: ssteinbach --- tests/test_serialized_schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index ccf876eb2..9cadfca92 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -87,7 +87,7 @@ def test_core_version_map_generator(self): target_fp = os.path.join(root, "CORE_VERSION_MAP.cpp") with open(target_fp) as fi: - baseline_text = fi.read() + baseline_text = "\n".join(ln for ln in fi.read().splitlines()) proc = subprocess.Popen( [ @@ -104,6 +104,9 @@ def test_core_version_map_generator(self): test_text = stdout.decode("utf-8")[:-1] + # sanitize the line endings + test_text = "\n".join(ln for ln in test_text.splitlines()) + self.maxDiff = None self.longMessage = True self.assertMultiLineEqual( From 13b11b62d3825ed06a1614fcc723f8c8d5741a0e Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 16:34:06 -0700 Subject: [PATCH 092/119] remove empty lines from comparison in version test Signed-off-by: ssteinbach --- tests/test_serialized_schema.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index 9cadfca92..7cc7b78ed 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -87,7 +87,13 @@ def test_core_version_map_generator(self): target_fp = os.path.join(root, "CORE_VERSION_MAP.cpp") with open(target_fp) as fi: - baseline_text = "\n".join(ln for ln in fi.read().splitlines()) + # sanitize line endings and remove empty lines for cross-windows + # /*nix consistent behavior + baseline_text = "\n".join( + ln + for ln in fi.read().splitlines() + if ln + ) proc = subprocess.Popen( [ @@ -105,7 +111,11 @@ def test_core_version_map_generator(self): test_text = stdout.decode("utf-8")[:-1] # sanitize the line endings - test_text = "\n".join(ln for ln in test_text.splitlines()) + test_text = "\n".join( + ln + for ln in fi.read().splitlines() + if ln + ) self.maxDiff = None self.longMessage = True From 96ccfa093b8b725a1adc0a29cfe4ceb3e9d188f3 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 19:53:12 -0700 Subject: [PATCH 093/119] lint Signed-off-by: ssteinbach --- tests/test_serialized_schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index 7cc7b78ed..85e542df6 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -112,9 +112,9 @@ def test_core_version_map_generator(self): # sanitize the line endings test_text = "\n".join( - ln - for ln in fi.read().splitlines() - if ln + ln + for ln in fi.read().splitlines() + if ln ) self.maxDiff = None From f268dccf2c121c36037f07d17121d33400fffe1e Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 20:02:04 -0700 Subject: [PATCH 094/119] typo Signed-off-by: ssteinbach --- tests/test_serialized_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index 85e542df6..7a486f29c 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -113,7 +113,7 @@ def test_core_version_map_generator(self): # sanitize the line endings test_text = "\n".join( ln - for ln in fi.read().splitlines() + for ln in test_text.splitlines() if ln ) From d6bb4b566408cb961e7470d368561e0ba3bd6e44 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 20:09:13 -0700 Subject: [PATCH 095/119] comment out debug code Signed-off-by: ssteinbach --- tests/test_builtin_adapters.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_builtin_adapters.py b/tests/test_builtin_adapters.py index a3f3771a6..8c9bb2cd0 100755 --- a/tests/test_builtin_adapters.py +++ b/tests/test_builtin_adapters.py @@ -70,11 +70,12 @@ def test_disk_vs_string(self): self.maxDiff = None - with open("/var/tmp/in_memory.otio", "w") as fo: - fo.write(in_memory) - - with open("/var/tmp/on_disk.otio", "w") as fo: - fo.write(on_disk) + # for debugging + # with open("/var/tmp/in_memory.otio", "w") as fo: + # fo.write(in_memory) + # + # with open("/var/tmp/on_disk.otio", "w") as fo: + # fo.write(on_disk) self.assertEqual(in_memory, on_disk) From 0633af1bcedc095a15fe87a32df27471d247fdfb Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 20:11:31 -0700 Subject: [PATCH 096/119] add license headers Signed-off-by: ssteinbach --- examples/io_perf_test.cpp | 3 +++ examples/upgrade_downgrade_example.cpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 37d6e4704..16744b56e 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + #include #include diff --git a/examples/upgrade_downgrade_example.cpp b/examples/upgrade_downgrade_example.cpp index 9861a9eb4..532fc9abc 100644 --- a/examples/upgrade_downgrade_example.cpp +++ b/examples/upgrade_downgrade_example.cpp @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + #include "opentimelineio/serializableObject.h" #include "opentimelineio/typeRegistry.h" #include From 0ff70368420724cef58d31092413062cef72099f Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 20:19:16 -0700 Subject: [PATCH 097/119] link to file for examples Signed-off-by: ssteinbach --- docs/tutorials/versioning-schemas.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 968247ac5..8149b9ba7 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -18,7 +18,7 @@ OpenTimelineIO can still interoperate with older and newer versions of the libra Once a type is registerd to OpenTimelineIO, deveopers may also register upgrade functions. In python, each upgrade function takes a dictionary and returns a dictionary. In C++, the AnyDictionary is manipulated in place. Each upgrade function is associated with a version number - this is the version number that it upgrades to. -C++ Example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): +C++ Example (can be viewed/run in [examples/upgrade_downgrade_example.cpp](https://github.com/AcademySoftwareFoundation/OpenTimelineIO/blob/main/examples/upgrade_downgrade_example.cpp)): ```cpp class SimpleClass : public otio::SerializableObject From f9696b4e4038bb1c04dc47a6459b6a1b6f216215 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 20:19:24 -0700 Subject: [PATCH 098/119] spell pass Signed-off-by: ssteinbach --- docs/tutorials/versioning-schemas.md | 50 ++++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 8149b9ba7..3b1954ffc 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -8,15 +8,15 @@ TL;DR for users: OpenTimelineIO should be able to read files produced by older v ## Schema/Version Introduction -Each SerializeableObject (the base class of OpenTimelineIO) has `schema_name` and `schema_version` fields. The `schema_name` is a string naming the schema, for example, `Clip`, and the `schema_version` is an integer of the current version number, for example, `3`. +Each SerializableObject (the base class of OpenTimelineIO) has `schema_name` and `schema_version` fields. The `schema_name` is a string naming the schema, for example, `Clip`, and the `schema_version` is an integer of the current version number, for example, `3`. -SerializeableObjects can be queried for these using the `.schema_name()` and `.schema_version()` methods. For a given release of the OpenTimelineIO library, in-memory objects the library creates will always be the same schema version. In other words, if `otio.schema.Clip()` instantiates an object with `schema_version` 2, there is no way to get an in-memory `Clip` object with version 1. +SerializableObjects can be queried for these using the `.schema_name()` and `.schema_version()` methods. For a given release of the OpenTimelineIO library, in-memory objects the library creates will always be the same schema version. In other words, if `otio.schema.Clip()` instantiates an object with `schema_version` 2, there is no way to get an in-memory `Clip` object with version 1. OpenTimelineIO can still interoperate with older and newer versions of the library by way of the schema upgrading/downgrading system. As OpenTimelineIO deserializes json from a string or disk, it will upgrade the schemas to the version supported by the library before instantiating the concrete in-memory object. Similarly, when serializing OpenTimelineIO back to disk, the user can instruct OpenTimelineIO to downgrade the JSON to older versions of the schemas. In this way, a newer version of OpenTimelineIO can read files with older schemas, and a newer version of OpenTimelineIO can generate JSON with older schemas in it. ## Schema Upgrading -Once a type is registerd to OpenTimelineIO, deveopers may also register upgrade functions. In python, each upgrade function takes a dictionary and returns a dictionary. In C++, the AnyDictionary is manipulated in place. Each upgrade function is associated with a version number - this is the version number that it upgrades to. +Once a type is registered to OpenTimelineIO, developers may also register upgrade functions. In python, each upgrade function takes a dictionary and returns a dictionary. In C++, the AnyDictionary is manipulated in place. Each upgrade function is associated with a version number - this is the version number that it upgrades to. C++ Example (can be viewed/run in [examples/upgrade_downgrade_example.cpp](https://github.com/AcademySoftwareFoundation/OpenTimelineIO/blob/main/examples/upgrade_downgrade_example.cpp)): @@ -81,9 +81,9 @@ Python Example: ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.2" - my_field = otio.core.serializeable_field("new_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + my_field = otio.core.serializable_field("new_field", int) @otio.core.upgrade_function_for(SimpleClass, 2) def upgrade_one_to_two(data): @@ -209,9 +209,9 @@ Given `SimpleClass`: import opentimelineio as otio @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.1" - my_field = otio.core.serializeable_field("my_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.1" + my_field = otio.core.serializable_field("my_field", int) ``` And `my_field` needs to be renamed to `new_field`. To do this: @@ -222,9 +222,9 @@ And `my_field` needs to be renamed to `new_field`. To do this: ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.2" - new_field = otio.core.serializeable_field("new_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + new_field = otio.core.serializable_field("new_field", int) @otio.core.upgrade_function_for(SimpleClass, 2) def upgrade_one_to_two(data): @@ -239,9 +239,9 @@ Changing it again, now `new_field` becomes `even_newer_field`. ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.2" - even_newer_field = otio.core.serializeable_field("even_newer_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + even_newer_field = otio.core.serializable_field("even_newer_field", int) @otio.core.upgrade_function_for(SimpleClass, 2) def upgrade_one_to_two(data): @@ -268,9 +268,9 @@ Starting from the same class: ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.1" - my_field = otio.core.serializeable_field("my_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.1" + my_field = otio.core.serializable_field("my_field", int) ``` If a change to a schema is to add a field, for which the default value is the correct value for an old schema, then no upgrade or downgrade function is needed. The parser ignores values that aren't in the schema. @@ -283,19 +283,19 @@ Example of adding a field (`other_field`): ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.2" - my_field = otio.core.serializeable_field("my_field", int) - other_field = otio.core.serializeable_field("other_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + my_field = otio.core.serializable_field("my_field", int) + other_field = otio.core.serializable_field("other_field", int) ``` Removing a field (`my_field`): ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.3" - other_field = otio.core.serializeable_field("other_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.3" + other_field = otio.core.serializable_field("other_field", int) ``` Similarly, when deleting a field, if the field is now ignored and does not contribute to computation, no upgrade or downgrade function is needed. From 68310cabc9b5e187a1e72f9a361978e20ef2a2db Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 20:22:53 -0700 Subject: [PATCH 099/119] remove dead code from core Signed-off-by: ssteinbach --- src/py-opentimelineio/opentimelineio/core/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 9e0535634..1a3833234 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -26,8 +26,6 @@ set_type_record, _serialize_json_to_string, _serialize_json_to_file, - type_version_map, - release_to_schema_version_map, ) from . _core_utils import ( # noqa @@ -70,7 +68,6 @@ 'serialize_json_to_string', 'serialize_json_to_file', 'register_type', - 'type_version_map', ] From 68a0263abe4f9417b202961452aaec2ef29b5ceb Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 20:23:16 -0700 Subject: [PATCH 100/119] spell fix Signed-off-by: ssteinbach --- src/py-opentimelineio/opentimelineio/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 1a3833234..546f33e5c 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -193,7 +193,7 @@ def wrapped_update(data): def serializable_field(name, required_type=None, doc=None): """ - Convienence function for adding attributes to child classes of + Convenience function for adding attributes to child classes of :class:`~SerializableObject` in such a way that they will be serialized/deserialized automatically. From 5d1e78a68d8daad5b561a3eb8a6bb92f441ceaff Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 21:00:17 -0700 Subject: [PATCH 101/119] add license header Signed-off-by: ssteinbach --- src/py-opentimelineio/opentimelineio/versioning.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py index 1f0278a78..3b7c254a5 100644 --- a/src/py-opentimelineio/opentimelineio/versioning.py +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -1,3 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the OpenTimelineIO project import copy from . import ( From 1349dd70b32fdc1c6dcc04169814c57371c8d2a7 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 21:00:30 -0700 Subject: [PATCH 102/119] doc tuning Signed-off-by: ssteinbach --- docs/tutorials/versioning-schemas.md | 6 +++--- .../opentimelineio/core/__init__.py | 14 ++++++++------ src/py-opentimelineio/opentimelineio/versioning.py | 3 +++ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 3b1954ffc..8b598b2f2 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -31,7 +31,7 @@ public: }; void set_new_field(int64_t val) { _new_field = val; } - const int64_t new_field() const { return _new_field; } + int64_t new_field() const { return _new_field; } protected: using Parent = SerializableObject; @@ -96,7 +96,7 @@ When upgrading schemas, OpenTimelineIO will call each upgrade function in order Similarly, once a type is registered, downgrade functions may be registered. Downgrade functions take a dictionary of the version specified and return a dictionary of the schema version one lower. For example, if a downgrade function is registered for version 5, that will downgrade from 5 to 4. -C++ Example, building off the prior section SimpleClass example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): +C++ Example, building off the prior section SimpleClass example (can be viewed/run in [examples/upgrade_downgrade_example.cpp](https://github.com/AcademySoftwareFoundation/OpenTimelineIO/blob/main/examples/upgrade_downgrade_example.cpp)): ```cpp // 2->1 @@ -195,7 +195,7 @@ otio.adapters.write_to_file( ) ``` -See the `versioning` module for more information on accessing these. +See the [versioning module](../api/python/opentimelineio.versioning.rst) for more information on accessing these. ## For Developers diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 546f33e5c..f99c7b006 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project +"""Core implementation details and wrappers around the C++ library""" + from .. _otio import ( # noqa # errors CannotComputeAvailableRangeError, @@ -26,6 +28,8 @@ set_type_record, _serialize_json_to_string, _serialize_json_to_file, + type_version_map, + release_to_schema_version_map, ) from . _core_utils import ( # noqa @@ -55,19 +59,17 @@ 'flatten_stack', 'install_external_keepalive_monitor', 'instance_from_schema', - 'register_serializable_object_type', - 'register_upgrade_function', - 'register_downgrade_function', 'set_type_record', 'add_method', 'upgrade_function_for', 'downgrade_function_from', - 'fetch_version_map', 'serializable_field', 'deprecated_field', 'serialize_json_to_string', 'serialize_json_to_file', 'register_type', + 'type_version_map', + 'release_to_schema_version_map', ] @@ -134,7 +136,7 @@ def upgrade_to_version_five(data): that add or remove fields, only for schema versions that change the field names. - :param type cls: class to upgrade + :param typing.Type[SerializableObject] cls: class to upgrade :param int version_to_upgrade_to: the version to upgrade to """ @@ -169,7 +171,7 @@ def downgrade_from_five_to_four(data): The downgrade function should take a single argument - the dictionary to downgrade, and return a dictionary with the fields downgraded. - :param type cls: class to downgrade + :param typing.Type[SerializableObject] cls: class to downgrade :param int version_to_downgrade_from: the function downgrading from this version to (version - 1) """ diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py index 3b7c254a5..e5a553764 100644 --- a/src/py-opentimelineio/opentimelineio/versioning.py +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project + +"""Tools for fetching the family->label->schema:version maps""" + import copy from . import ( From ea353dd353838a74a7fe4cb24c9beaca7fb32b3c Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 22:11:51 -0700 Subject: [PATCH 103/119] hiding links that won't exist until PR is landed Signed-off-by: ssteinbach --- docs/tutorials/versioning-schemas.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 8b598b2f2..950acd97d 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -18,7 +18,8 @@ OpenTimelineIO can still interoperate with older and newer versions of the libra Once a type is registered to OpenTimelineIO, developers may also register upgrade functions. In python, each upgrade function takes a dictionary and returns a dictionary. In C++, the AnyDictionary is manipulated in place. Each upgrade function is associated with a version number - this is the version number that it upgrades to. -C++ Example (can be viewed/run in [examples/upgrade_downgrade_example.cpp](https://github.com/AcademySoftwareFoundation/OpenTimelineIO/blob/main/examples/upgrade_downgrade_example.cpp)): +C++ Example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): + ```cpp class SimpleClass : public otio::SerializableObject @@ -96,7 +97,8 @@ When upgrading schemas, OpenTimelineIO will call each upgrade function in order Similarly, once a type is registered, downgrade functions may be registered. Downgrade functions take a dictionary of the version specified and return a dictionary of the schema version one lower. For example, if a downgrade function is registered for version 5, that will downgrade from 5 to 4. -C++ Example, building off the prior section SimpleClass example (can be viewed/run in [examples/upgrade_downgrade_example.cpp](https://github.com/AcademySoftwareFoundation/OpenTimelineIO/blob/main/examples/upgrade_downgrade_example.cpp)): +C++ Example, building off the prior section SimpleClass example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): + ```cpp // 2->1 From 6685d415d58247589280070f215fb3db729d4530 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Wed, 7 Sep 2022 22:12:13 -0700 Subject: [PATCH 104/119] updating docs Signed-off-by: ssteinbach --- docs/tutorials/otio-plugins.md | 2 +- src/py-opentimelineio/opentimelineio/adapters/otio_json.py | 2 +- src/py-opentimelineio/opentimelineio/versioning.py | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index 4e737d240..120cffd6f 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -106,7 +106,7 @@ OpenTimelineIO Final Cut Pro 7 XML Adapter. ### otio_json ``` -This adapter lets you read and write native .otio files +Adapter for reading and writing native .otio json files. ``` *source*: `opentimelineio/adapters/otio_json.py` diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 9d761a451..4bcffdb08 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project -"""This adapter lets you read and write native .otio files""" +"""Adapter for reading and writing native .otio json files.""" from .. import ( core, diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py index e5a553764..4fd0e9126 100644 --- a/src/py-opentimelineio/opentimelineio/versioning.py +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -43,6 +43,9 @@ def full_map(): } } } + + :returns: full map of schema version sets, including core and plugins + :rtype: dict[str, dict[str, dict[str, int]]] """ result = copy.deepcopy(plugins.ActiveManifest().version_manifests) @@ -72,6 +75,10 @@ def fetch_map(family, label): ... } + :param str family: family of labels (ie: "OTIO_CORE") + :param str label: label of schema-version map (ie: "0.15.0") + :returns: a dictionary mapping Schema name to schema version + :rtype: dict[str, int] """ if family == "OTIO_CORE": From 4b7f93f5fd2837f4ff72208f7ae7f2cadf215ef7 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 10:23:11 -0700 Subject: [PATCH 105/119] Add docs to core/__init__.py --- .../opentimelineio/core/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index f99c7b006..865d14729 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -96,6 +96,20 @@ def serialize_json_to_file( def register_type(classobj, schemaname=None): + """Decorator for registering a SerializableObject type + + Example: + + .. code-block:: python + + @otio.core.register_type + class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + ... + + :param typing.Type[SerializableObject] cls: class to register + :param str schemaname: Schema name (default: parse from serializable_label) + """ label = classobj._serializable_label if schemaname is None: schema_name, schema_version = label.split(".", 2) @@ -156,6 +170,8 @@ def wrapped_update(data): def downgrade_function_from(cls, version_to_downgrade_from): """ + Decorator for identifying schema class downgrade functions. + Example: .. code-block:: python From 91cfd7a720a412afacfee00b86d11a9cbc2a7a25 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 12:19:16 -0700 Subject: [PATCH 106/119] add docstrings to core module --- .../opentimelineio-bindings/otio_bindings.cpp | 36 ++++++++++++++++--- .../opentimelineio/core/__init__.py | 20 +++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index 166e1f691..ad0e7d72a 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -193,13 +193,30 @@ PYBIND11_MODULE(_otio, m) { any result; deserialize_json_from_string(input, &result, ErrorStatusHandler()); return any_to_py(result, true /*top_level*/); - }, "input"_a) + }, "input"_a, + R"docstring(Deserialize json string to in-memory objects. + +:param str input: json string to deserialize + +:returns: root object in the string (usually a Timeline or SerializableCollection) +:rtype: SerializableObject + +)docstring") .def("deserialize_json_from_file", [](std::string filename) { any result; deserialize_json_from_file(filename, &result, ErrorStatusHandler()); return any_to_py(result, true /*top_level*/); - }, "filename"_a); + }, + "filename"_a, + R"docstring(Deserialize json file to in-memory objects. + +:param str filename: path to json file to read + +:returns: root object in the file (usually a Timeline or SerializableCollection) +:rtype: SerializableObject + +)docstring"); py::class_(m, "PyAny") // explicitly map python bool, int and double classes so that they @@ -242,7 +259,12 @@ Return an instance of the schema from data in the data_dict. schema_version_map tmp; TypeRegistry::instance().type_version_map(tmp); return tmp; - }); + }, + R"docstring(Fetch the currently registered schemas and their versions. + +:returns: Map of all registered schema names to their current versions. +:rtype: dict[str, int])docstring" + ); m.def("register_upgrade_function", ®ister_upgrade_function, "schema_name"_a, "version_to_upgrade_to"_a, @@ -253,7 +275,13 @@ Return an instance of the schema from data in the data_dict. "downgrade_function"_a); m.def( "release_to_schema_version_map", - [](){ return label_to_schema_version_map(CORE_VERSION_MAP);} + [](){ return label_to_schema_version_map(CORE_VERSION_MAP);}, + R"docstring(Fetch the compiled in CORE_VERSION_MAP. + +The CORE_VERSION_MAP maps OTIO release versions to maps of schema name to schema version and is autogenerated by the OpenTimelineIO build and release system. For example: `{"0.15.0": {"Clip": 2, ...}}` + +:returns: dictionary mapping core version label to schema_version_map +:rtype: dict[str, dict[str, int]])docstring" ); m.def("flatten_stack", [](Stack* s) { return flatten_stack(s, ErrorStatusHandler()); diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 865d14729..ca223c445 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -74,6 +74,16 @@ def serialize_json_to_string(root, schema_version_targets=None, indent=4): + """Serialize root to a json string. Optionally downgrade resulting schemas + to schema_version_targets. + + :param SerializableObject root: root object to serialize + :param dict[str, int] schema_version_targets: optional dictionary mapping schema name to desired schema version, for downgrading the result to be compatible with older versions of OpenTimelineIO. + :param int indent: number of spaces for each json indentation level. Use -1 for no indentation or newlines. + + :returns: resulting json string + :rtype: str + """ return _serialize_json_to_string( _value_to_any(root), schema_version_targets or {}, @@ -87,6 +97,16 @@ def serialize_json_to_file( schema_version_targets=None, indent=4 ): + """Serialize root to a json file. Optionally downgrade resulting schemas + to schema_version_targets. + + :param SerializableObject root: root object to serialize + :param dict[str, int] schema_version_targets: optional dictionary mapping schema name to desired schema version, for downgrading the result to be compatible with older versions of OpenTimelineIO. + :param int indent: number of spaces for each json indentation level. Use -1 for no indentation or newlines. + + :returns: true for success, false for failure + :rtype: bool + """ return _serialize_json_to_file( _value_to_any(root), filename, From 02d4b953e58f954056c6d1008f4ea4e625515468 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 14:35:24 -0700 Subject: [PATCH 107/119] lint --- .../opentimelineio/core/__init__.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index ca223c445..922eec480 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -78,8 +78,14 @@ def serialize_json_to_string(root, schema_version_targets=None, indent=4): to schema_version_targets. :param SerializableObject root: root object to serialize - :param dict[str, int] schema_version_targets: optional dictionary mapping schema name to desired schema version, for downgrading the result to be compatible with older versions of OpenTimelineIO. - :param int indent: number of spaces for each json indentation level. Use -1 for no indentation or newlines. + :param dict[str, int] schema_version_targets: optional dictionary mapping + schema name to desired schema + version, for downgrading the + result to be compatible with + older versions of + OpenTimelineIO. + :param int indent: number of spaces for each json indentation level. Use -1 + for no indentation or newlines. :returns: resulting json string :rtype: str @@ -101,8 +107,14 @@ def serialize_json_to_file( to schema_version_targets. :param SerializableObject root: root object to serialize - :param dict[str, int] schema_version_targets: optional dictionary mapping schema name to desired schema version, for downgrading the result to be compatible with older versions of OpenTimelineIO. - :param int indent: number of spaces for each json indentation level. Use -1 for no indentation or newlines. + :param dict[str, int] schema_version_targets: optional dictionary mapping + schema name to desired schema + version, for downgrading the + result to be compatible with + older versions of + OpenTimelineIO. + :param int indent: number of spaces for each json indentation level. Use -1 + for no indentation or newlines. :returns: true for success, false for failure :rtype: bool From 05cadb2e307fb879c02f891d1455c71b6b809bce Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 15:35:00 -0700 Subject: [PATCH 108/119] Removed need for root() method and use any/swap --- src/opentimelineio/serialization.cpp | 30 ++++++++-------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index b23f465e1..911086cdf 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -91,7 +91,7 @@ class Encoder }; /** - * This encoder builds up a dictionary as its method of "encoding". + * This encoder builds up a AnyDictionary as its method of "encoding". * The dictionary is than handed off to a CloningDecoder, to complete * copying of a SerializableObject instance. */ @@ -390,20 +390,6 @@ class CloningEncoder : public Encoder _store(any(std::move(m))); } - // @TODO: what kind of ownership policy here? - AnyDictionary - root() - { - if (_root.type() == typeid(AnyDictionary)) - { - return any_cast(_root); - } - else - { - return AnyDictionary(); - } - } - private: any _root; SerializableObject::Reader::_Resolver _resolver; @@ -973,12 +959,13 @@ SerializableObject::Writer::write( const std::string& schema_name = value->schema_name(); int schema_version = value->schema_version(); - optional downgraded = {}; + any downgraded = {}; // if there is a manifest & the encoder is not converting to AnyDictionary if ( - _downgrade_version_manifest != nullptr - && !_encoder.encoding_to_anydict() + (_downgrade_version_manifest != nullptr) + && (!_downgrade_version_manifest->empty()) + && (!_encoder.encoding_to_anydict()) ) { const auto& target_version_it = _downgrade_version_manifest->find( @@ -1007,7 +994,7 @@ SerializableObject::Writer::write( return; } - downgraded = { e.root() }; + downgraded.swap(e._root); schema_version = target_version; } } @@ -1040,10 +1027,9 @@ SerializableObject::Writer::write( // write the contents of the object to the encoder, either the downgraded // anydictionary or the SerializableObject - if (downgraded.has_value()) + if (!(downgraded.empty())) { - // the inner loop of write( - for (const auto& kv : (*downgraded)) + for (const auto& kv : any_cast(downgraded)) { this->write(kv.first, kv.second); } From 58f6418b2497d01c3abf39deade3e029ad78bec2 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 16:33:16 -0700 Subject: [PATCH 109/119] checkpoint perf 1.66s --- src/opentimelineio/serialization.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 911086cdf..b79986081 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -377,11 +377,13 @@ class CloningEncoder : public Encoder return; } - // otherwise, build out as an anydictionary & downgrade AnyDictionary m; m.swap(top.dict); - if (_downgrade_version_manifest != nullptr) + if ( + (_downgrade_version_manifest != nullptr) + && (!_downgrade_version_manifest->empty()) + ) { _downgrade_dictionary(m); } @@ -448,6 +450,7 @@ class CloningEncoder : public Encoder current_version = std::stoi(schema_vers); } + // @TODO: is 0 a legitimate schema version? if (current_version < 0) { _internal_error( @@ -455,8 +458,8 @@ class CloningEncoder : public Encoder "Could not parse version number from Schema" " string: %s", schema_string.c_str() - ) - ); + ) + ); return; } @@ -464,14 +467,12 @@ class CloningEncoder : public Encoder const auto& type_rec = ( TypeRegistry::instance()._find_type_record(schema_name) - ); + ); - while (current_version > target_version ) + while (current_version > target_version) { const auto& next_dg_fn = ( - type_rec->downgrade_functions.find( - current_version - ) + type_rec->downgrade_functions.find(current_version) ); if (next_dg_fn == type_rec->downgrade_functions.end()) @@ -482,8 +483,8 @@ class CloningEncoder : public Encoder "going from version %d to version %d.", current_version, target_version - ) - ); + ) + ); return; } From e30d36f82485aa2a13886e0145a6164fe7072d81 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 16:44:10 -0700 Subject: [PATCH 110/119] add replace_back function (further work needed) --- src/opentimelineio/serialization.cpp | 38 ++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index b79986081..72d5a9662 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -140,6 +140,41 @@ class CloningEncoder : public Encoder _stack.back().cur_key = key; } + void _replace_back(AnyDictionary&& a) + { + if (has_errored()) + { + return; + } + + if (_stack.size() == 1) + { + any newstack(std::move(a)); + _root.swap(newstack); + } + else + { + // if (_stack[_stack.size() - 1].is_dict) + // { + // _stack[_stack.size() - 1].dict.swap(a); + // } + + _stack.pop_back(); + auto& top = _stack.back(); + if (top.is_dict) + { + // top.dict.swap(a); + top.dict.emplace(_stack.back().cur_key, a); + } + else + { + any newstack(std::move(a)); + top.array.emplace_back(newstack); + } + } + + } + void _store(any&& a) { if (has_errored()) @@ -388,8 +423,7 @@ class CloningEncoder : public Encoder _downgrade_dictionary(m); } - _stack.pop_back(); - _store(any(std::move(m))); + _replace_back(std::move(m)); } private: From 37ae0c3b7a6b9b502338d69d942c0b244d62c11a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 20:43:33 -0700 Subject: [PATCH 111/119] io_perf_test asserts --- examples/io_perf_test.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 16744b56e..bb5d870f5 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Contributors to the OpenTimelineIO project +#include #include #include @@ -17,6 +18,7 @@ namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; using chrono_time_point = std::chrono::steady_clock::time_point; const struct { + bool FIXED_TMP = true; bool PRINT_CPP_VERSION_FAMILY = false; bool TO_JSON_STRING = true; bool TO_JSON_STRING_NO_DOWNGRADE = true; @@ -46,7 +48,7 @@ void print_version_map() { std::cerr << "current version map: " << std::endl; - for (auto kv_lbl: otio::CORE_VERSION_MAP) + for (const auto& kv_lbl: otio::CORE_VERSION_MAP) { std::cerr << " " << kv_lbl.first << std::endl; for (auto kv_schema_version : kv_lbl.second) @@ -71,8 +73,8 @@ main( if (argc < 2) { - std::cerr << "usage: otio_io_perf_test path/to/timeline.otio [--keep-tmp]"; - std::cerr << std::endl; + std::cerr << "usage: otio_io_perf_test path/to/timeline.otio "; + std::cerr << "[--keep-tmp]" << std::endl; return 1; } @@ -86,9 +88,14 @@ main( } } - const std::string tmp_dir_path = examples::create_temp_dir(); + const std::string tmp_dir_path = ( + RUN_STRUCT.FIXED_TMP + ? "/var/tmp/ioperftest" + : examples::create_temp_dir() + ); otio::ErrorStatus err; + assert(!otio::is_error(err)); otio::schema_version_map downgrade_manifest = { {"FakeSchema", 3}, @@ -119,6 +126,7 @@ main( &downgrade_manifest ); chrono_time_point end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); print_elapsed_time("downgrade clip", begin, end); } @@ -136,6 +144,7 @@ main( ) ); chrono_time_point end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); if (!timeline) { examples::print_error(err); @@ -153,6 +162,7 @@ main( &downgrade_manifest ); end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); if (otio::is_error(err)) { @@ -167,6 +177,7 @@ main( begin = std::chrono::steady_clock::now(); const std::string result = timeline.value->to_json_string(&err, {}); end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); if (otio::is_error(err)) { @@ -185,6 +196,7 @@ main( &downgrade_manifest ); end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); print_elapsed_time("serialize_json_to_file", begin, end); } @@ -200,10 +212,11 @@ main( {} ); end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); print_elapsed_time("serialize_json_to_file [no downgrade]", begin, end); } - if (keep_tmp) + if (keep_tmp || RUN_STRUCT.FIXED_TMP) { std::cout << "Temp directory preserved. All files written to: "; std::cout << tmp_dir_path << std::endl; From efee6487b7a77244b9004fe3ab2085edbb08bd3d Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Thu, 8 Sep 2022 21:21:54 -0700 Subject: [PATCH 112/119] use insert directly for up/downgrade functions --- src/opentimelineio/typeRegistry.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 96accaade..f75229c3e 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -236,12 +236,13 @@ TypeRegistry::register_upgrade_function( std::lock_guard lock(_registry_mutex); if (auto r = _find_type_record(schema_name)) { - if (r->upgrade_functions.find(version_to_upgrade_to) == - r->upgrade_functions.end()) - { - r->upgrade_functions[version_to_upgrade_to] = upgrade_function; - return true; - } + auto result = r->upgrade_functions.insert( + { + version_to_upgrade_to, + upgrade_function + } + ); + return result.second; } return false; @@ -256,14 +257,13 @@ TypeRegistry::register_downgrade_function( std::lock_guard lock(_registry_mutex); if (auto r = _find_type_record(schema_name)) { - if (r->downgrade_functions.find(version_to_downgrade_from) == - r->downgrade_functions.end()) - { - r->downgrade_functions[version_to_downgrade_from] = ( - downgrade_function - ); - return true; - } + auto result = r->downgrade_functions.insert( + { + version_to_downgrade_from, + downgrade_function + } + ); + return result.second; } return false; From b49033c1c03a7b9fcce4b8b30f65d7a0827718f3 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 9 Sep 2022 14:26:59 -0700 Subject: [PATCH 113/119] Performance tuning - cache the child cloning encoder across the entire serialization - const & sprinkled in - remove dead code --- src/opentimelineio/serializableObject.h | 13 +++-- src/opentimelineio/serialization.cpp | 71 ++++++++++++++----------- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index acd190f65..d25b7c2d5 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -23,6 +23,8 @@ namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { +class CloningEncoder; + class SerializableObject { public: @@ -450,7 +452,7 @@ class SerializableObject AnyVector av; av.reserve(value.size()); - for (auto e: value) + for (const auto& e: value) { av.emplace_back(_to_any(e)); } @@ -462,7 +464,7 @@ class SerializableObject static any _to_any(std::map const& value) { AnyDictionary am; - for (auto e: value) + for (const auto& e: value) { am.emplace(e.first, _to_any(e.second)); } @@ -476,7 +478,7 @@ class SerializableObject AnyVector av; av.reserve(value.size()); - for (auto e: value) + for (const auto& e: value) { av.emplace_back(_to_any(e)); } @@ -523,6 +525,8 @@ class SerializableObject _build_dispatch_tables(); } + ~Writer(); + Writer(Writer const&) = delete; Writer operator=(Writer const&) = delete; @@ -547,6 +551,9 @@ class SerializableObject std::unordered_map _id_for_object; std::unordered_map _next_id_for_type; + Writer* _child_writer = nullptr; + CloningEncoder* _child_cloning_encoder = nullptr; + class Encoder& _encoder; const schema_version_map* _downgrade_version_manifest; friend class SerializableObject; diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 72d5a9662..0f54e1079 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -154,17 +154,11 @@ class CloningEncoder : public Encoder } else { - // if (_stack[_stack.size() - 1].is_dict) - // { - // _stack[_stack.size() - 1].dict.swap(a); - // } - _stack.pop_back(); auto& top = _stack.back(); if (top.is_dict) { - // top.dict.swap(a); - top.dict.emplace(_stack.back().cur_key, a); + top.dict.emplace(top.cur_key, a); } else { @@ -1016,20 +1010,26 @@ SerializableObject::Writer::write( // and the current_version is greater than the target version if (schema_version > target_version) { - CloningEncoder e( - CloningEncoder::ResultObjectPolicy::OnlyAnyDictionary, - _downgrade_version_manifest - ); + if (_child_writer == nullptr) + { + _child_cloning_encoder = new CloningEncoder( + CloningEncoder::ResultObjectPolicy::OnlyAnyDictionary, + _downgrade_version_manifest + ); + _child_writer = new Writer(*_child_cloning_encoder, {}); + } + else { + _child_cloning_encoder->_stack.clear(); + } - Writer w(e, {}); - w.write(w._no_key, value); + _child_writer->write(_child_writer->_no_key, value); - if (e.has_errored(&_encoder._error_status)) + if (_child_cloning_encoder->has_errored(&_encoder._error_status)) { return; } - downgraded.swap(e._root); + downgraded.swap(_child_cloning_encoder->_root); schema_version = target_version; } } @@ -1138,12 +1138,6 @@ SerializableObject::Writer::write( std::string const& key, any const& value) { - // if (value.empty()) - // { - // _encoder.write_key(key); - // _encoder.write_null_value(); - // return; - // } std::type_info const& type = value.type(); _encoder_write_key(key); @@ -1152,19 +1146,24 @@ SerializableObject::Writer::write( if (e == _write_dispatch_table.end()) { /* - * Using the address of a type_info suffers from aliasing across compilation units. - * If we fail on a lookup, we fallback on the by_name table, but that's slow because - * we have to keep making a string each time. + * Using the address of a type_info suffers from aliasing across + * compilation units. If we fail on a lookup, we fallback on the + * by_name table, but that's slow because we have to keep making a + * string each time. * - * So when we fail, we insert the address of the type_info that failed to be found, - * so that we'll catch it the next time. This ensures we fail exactly once per alias - * per type while using this writer. + * So when we fail, we insert the address of the type_info that failed + * to be found, so that we'll catch it the next time. This ensures we + * fail exactly once per alias per type while using this writer. */ - auto backup_e = _write_dispatch_table_by_name.find(type.name()); + const auto& backup_e = _write_dispatch_table_by_name.find(type.name()); if (backup_e != _write_dispatch_table_by_name.end()) { - _write_dispatch_table[&type] = backup_e->second; - e = _write_dispatch_table.find(&type); + e = _write_dispatch_table.insert( + { + &type, + backup_e->second + } + ).first; } } @@ -1353,4 +1352,16 @@ serialize_json_to_file( } +SerializableObject::Writer::~Writer() +{ + if (_child_writer) + { + delete _child_writer; + } + if (_child_cloning_encoder) + { + delete _child_cloning_encoder; + } +} + }} // namespace opentimelineio::OPENTIMELINEIO_VERSION From 3e00dad5fc962253e4478ad4cd53e9bb66a6801f Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 9 Sep 2022 14:27:39 -0700 Subject: [PATCH 114/119] add nullptr check and compare type hash code --- src/opentimelineio/anyDictionary.h | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/opentimelineio/anyDictionary.h b/src/opentimelineio/anyDictionary.h index c86333d3b..a017b3185 100644 --- a/src/opentimelineio/anyDictionary.h +++ b/src/opentimelineio/anyDictionary.h @@ -135,6 +135,11 @@ class AnyDictionary : private std::map containedType* result ) const { + if (result == nullptr) + { + return false; + } + const auto it = this->find(key); if ( @@ -171,10 +176,19 @@ class AnyDictionary : private std::map containedType* result ) { + if (result == nullptr) + { + return false; + } + const auto d_it = this->find(key); + if ( (d_it != this->end()) - && (d_it->second.type() == typeid(containedType)) + && ( + d_it->second.type().hash_code() + == typeid(containedType).hash_code() + ) ) { *result = any_cast(d_it->second); From cf5a5ef7ef3145ef669c9ace933cd1ab5938db7e Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 9 Sep 2022 14:30:35 -0700 Subject: [PATCH 115/119] print ratio of duration in the io_perf_test --- examples/io_perf_test.cpp | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index bb5d870f5..2464eedf7 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -29,7 +29,7 @@ const struct { } RUN_STRUCT ; /// utility function for printing std::chrono elapsed time -void +double print_elapsed_time( const std::string& message, const chrono_time_point& begin, @@ -41,7 +41,11 @@ print_elapsed_time( end - begin ).count() ); - std::cout << message << ": " << dur/1000000.0 << " [s]" << std::endl; + + double result = dur/1000000.0; + std::cout << message << ": " << result << " [s]" << std::endl; + + return result; } void @@ -154,6 +158,7 @@ main( print_elapsed_time("deserialize_json_from_file", begin, end); + double str_dg, str_nodg; if (RUN_STRUCT.TO_JSON_STRING) { begin = std::chrono::steady_clock::now(); @@ -169,7 +174,7 @@ main( examples::print_error(err); return 1; } - print_elapsed_time("serialize_json_to_string", begin, end); + str_dg = print_elapsed_time("serialize_json_to_string", begin, end); } if (RUN_STRUCT.TO_JSON_STRING_NO_DOWNGRADE) @@ -184,9 +189,20 @@ main( examples::print_error(err); return 1; } - print_elapsed_time("serialize_json_to_string [no downgrade]", begin, end); + str_nodg = print_elapsed_time( + "serialize_json_to_string [no downgrade]", + begin, + end + ); } + if (RUN_STRUCT.TO_JSON_STRING && RUN_STRUCT.TO_JSON_STRING_NO_DOWNGRADE) + { + std::cout << " JSON to string no_dg/dg: " << str_dg / str_nodg; + std::cout << std::endl; + } + + double file_dg, file_nodg; if (RUN_STRUCT.TO_JSON_FILE) { begin = std::chrono::steady_clock::now(); @@ -197,7 +213,7 @@ main( ); end = std::chrono::steady_clock::now(); assert(!otio::is_error(err)); - print_elapsed_time("serialize_json_to_file", begin, end); + file_dg = print_elapsed_time("serialize_json_to_file", begin, end); } if (RUN_STRUCT.TO_JSON_FILE_NO_DOWNGRADE) @@ -213,7 +229,17 @@ main( ); end = std::chrono::steady_clock::now(); assert(!otio::is_error(err)); - print_elapsed_time("serialize_json_to_file [no downgrade]", begin, end); + file_nodg = print_elapsed_time( + "serialize_json_to_file [no downgrade]", + begin, + end + ); + } + + if (RUN_STRUCT.TO_JSON_FILE && RUN_STRUCT.TO_JSON_FILE_NO_DOWNGRADE) + { + std::cout << " JSON to file no_dg/dg: " << file_dg / file_nodg; + std::cout << std::endl; } if (keep_tmp || RUN_STRUCT.FIXED_TMP) From d97241b46d8b326cd6209e841069ccbeb6132a8a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 9 Sep 2022 14:35:37 -0700 Subject: [PATCH 116/119] use std::chrono... to get a float seconds count --- examples/io_perf_test.cpp | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 2464eedf7..5a1d03400 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -1,17 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Contributors to the OpenTimelineIO project -#include #include -#include #include "opentimelineio/clip.h" #include "opentimelineio/typeRegistry.h" +#include "opentimelineio/any.h" +#include "opentimelineio/serialization.h" +#include "opentimelineio/deserialization.h" +#include "opentimelineio/timeline.h" + #include "util.h" -#include -#include -#include -#include namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; @@ -28,6 +27,11 @@ const struct { bool SINGLE_CLIP_DOWNGRADE_TEST = true; } RUN_STRUCT ; +// typedef std::chrono::duration fsec; + // auto t0 = Time::now(); + // auto t1 = Time::now(); + // fsec fs = t1 - t0; + /// utility function for printing std::chrono elapsed time double print_elapsed_time( @@ -36,16 +40,11 @@ print_elapsed_time( const chrono_time_point& end ) { - const auto dur = ( - std::chrono::duration_cast( - end - begin - ).count() - ); + const std::chrono::duration dur = end - begin; - double result = dur/1000000.0; - std::cout << message << ": " << result << " [s]" << std::endl; + std::cout << message << ": " << dur.count() << " [s]" << std::endl; - return result; + return dur.count(); } void From 381a76288ae4c4d87976ca4601a24c117e99e520 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 9 Sep 2022 15:08:29 -0700 Subject: [PATCH 117/119] one more string copy prevented --- examples/io_perf_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp index 5a1d03400..920adfbeb 100644 --- a/examples/io_perf_test.cpp +++ b/examples/io_perf_test.cpp @@ -250,7 +250,7 @@ main( { // clean up const auto tmp_files = examples::glob(tmp_dir_path, "*"); - for (auto fp : tmp_files) + for (const auto& fp : tmp_files) { remove(fp.c_str()); } From 83c99799b6f4dcdc602b4d2afc129b53d8037e30 Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 9 Sep 2022 15:10:33 -0700 Subject: [PATCH 118/119] remove double exception checks --- .../opentimelineio-bindings/otio_bindings.cpp | 36 ++++++++++++++++--- tests/test_serializable_object.py | 20 ++++++----- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index ad0e7d72a..706b79d34 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -39,6 +39,9 @@ static void register_python_type(py::object class_object, return r.take_value(); }; + + // @TODO: further discussion required about preventing double registering +#if 0 if ( !TypeRegistry::instance().register_type( schema_name, @@ -55,6 +58,15 @@ static void register_python_type(py::object class_object, "Schema: " + schema_name + " has already been registered." ); } +#else + TypeRegistry::instance().register_type( + schema_name, + schema_version, + nullptr, + create, + schema_name + ); +#endif } static bool register_upgrade_function(std::string const& schema_name, @@ -68,6 +80,8 @@ static bool register_upgrade_function(std::string const& schema_name, upgrade_function_obj(dobj); }; + // further discussion required about preventing double registering +#if 0 if ( !TypeRegistry::instance().register_upgrade_function( schema_name, @@ -85,6 +99,13 @@ static bool register_upgrade_function(std::string const& schema_name, } return true; +#else + return TypeRegistry::instance().register_upgrade_function( + schema_name, + version_to_upgrade_to, + upgrade_function + ); +#endif } static bool @@ -93,10 +114,6 @@ register_downgrade_function( int version_to_downgrade_from, py::object const& downgrade_function_obj) { - // @TODO: I'm actually surprised this works, since the python function - // seems to often make new dictionaries and relies on the return - // vallue... but here it doesn't seem like the return value matters. - // ...the C++ functions definitely work in place though. std::function downgrade_function = ( [downgrade_function_obj](AnyDictionary* d) { @@ -108,6 +125,8 @@ register_downgrade_function( } ); + // further discussion required about preventing double registering +#if 0 if ( !TypeRegistry::instance().register_downgrade_function( schema_name, @@ -123,8 +142,15 @@ register_downgrade_function( ); return false; } - return true; +#else + return TypeRegistry::instance().register_downgrade_function( + schema_name, + version_to_downgrade_from, + downgrade_function + ) ; +#endif + } static void set_type_record(SerializableObject* so, std::string schema_name) { diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index cee6e9a2c..6c8bf6d09 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -196,11 +196,12 @@ def upgrade_one_to_two(_data_dict): def upgrade_one_to_two_three(_data_dict): return {"foo_3": _data_dict["foo_2"]} + # @TODO: further discussion required # not allowed to overwrite registered functions - with self.assertRaises(ValueError): - @otio.core.upgrade_function_for(FakeThing, 3) - def upgrade_one_to_two_three_again(_data_dict): - raise RuntimeError("shouldn't see this ever") + # with self.assertRaises(ValueError): + # @otio.core.upgrade_function_for(FakeThing, 3) + # def upgrade_one_to_two_three_again(_data_dict): + # raise RuntimeError("shouldn't see this ever") ft = otio.core.instance_from_schema("NewStuff", 1, {"foo": "bar"}) self.assertEqual(ft._dynamic_fields['foo_3'], "bar") @@ -234,11 +235,12 @@ class FakeThing(otio.core.SerializableObject): def downgrade_2_to_1(_data_dict): return {"foo": _data_dict["foo_2"]} - # not allowed to overwrite registered functions - with self.assertRaises(ValueError): - @otio.core.downgrade_function_from(FakeThing, 2) - def downgrade_2_to_1_again(_data_dict): - raise RuntimeError("shouldn't see this ever") + # @TODO: further discussion required + # # not allowed to overwrite registered functions + # with self.assertRaises(ValueError): + # @otio.core.downgrade_function_from(FakeThing, 2) + # def downgrade_2_to_1_again(_data_dict): + # raise RuntimeError("shouldn't see this ever") f = FakeThing() f.foo_two = "a thing here" From 026c8f04d4eea4dc9f294a1ffd0f3f11b693cd9a Mon Sep 17 00:00:00 2001 From: ssteinbach Date: Fri, 9 Sep 2022 15:24:02 -0700 Subject: [PATCH 119/119] Remove some unneeded imports. --- src/opentimelineio/anyDictionary.h | 1 - src/opentimelineio/serializableObject.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/opentimelineio/anyDictionary.h b/src/opentimelineio/anyDictionary.h index a017b3185..3512897e7 100644 --- a/src/opentimelineio/anyDictionary.h +++ b/src/opentimelineio/anyDictionary.h @@ -8,7 +8,6 @@ #include #include -#include #include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index d25b7c2d5..46f34dad2 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -16,9 +16,7 @@ #include "ImathBox.h" #include "serialization.h" -#include #include -#include #include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION {