diff --git a/.gitignore b/.gitignore index 053fa5a82..d1dfcb6a7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ htmlcov .vscode/ xcuserdata/ .venv/ +.cache # Pycharm metadata .idea/ diff --git a/Makefile b/Makefile index c89d0f4db..b64fd167c 100644 --- a/Makefile +++ b/Makefile @@ -182,6 +182,13 @@ doc-plugins: doc-plugins-update: @python src/py-opentimelineio/opentimelineio/console/autogen_plugin_documentation.py -o docs/tutorials/otio-plugins.md --public-only --sanitized-paths +# build the CORE_VERSION_MAP cpp file +version-map: + @python src/py-opentimelineio/opentimelineio/console/autogen_version_map.py -i src/opentimelineio/CORE_VERSION_MAP.last.cpp --dryrun + +version-map-update: + @python src/py-opentimelineio/opentimelineio/console/autogen_version_map.py -i src/opentimelineio/CORE_VERSION_MAP.last.cpp -o src/opentimelineio/CORE_VERSION_MAP.cpp + # generate documentation in html doc-html: @# if you just want to build the docs yourself outside of RTD diff --git a/docs/tutorials/otio-env-variables.md b/docs/tutorials/otio-env-variables.md index 3c8a113df..7d46132ac 100644 --- a/docs/tutorials/otio-env-variables.md +++ b/docs/tutorials/otio-env-variables.md @@ -5,11 +5,12 @@ various aspects of OTIO. ## Plugin Configuration -These variables must be set _before_ the OpenTimelineIO python library is imported. +These variables must be set _before_ the OpenTimelineIO python library is imported. They only impact the python library. The C++ library has no environment variables. - `OTIO_PLUGIN_MANIFEST_PATH`: a ":" separated string with paths to .manifest.json files that contain OTIO plugin manifests. See: [Tutorial on how to write an adapter plugin](write-an-adapter). - `OTIO_DEFAULT_MEDIA_LINKER`: the name of the default media linker to use after reading a file, if "" then no media linker is automatically invoked. - `OTIO_DISABLE_PKG_RESOURCE_PLUGINS`: By default, OTIO will use the pkg_resource entry_points mechanism to discover plugins that have been installed into the current python environment. pkg_resources, however, can be slow in certain cases, so for users who wish to disable this behavior, this variable can be set to 1. +- `OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL`: if no downgrade arguments are passed to `write_to_file`/`write_to_string`, use the downgrade manifest specified by the family/label combination in the variable. Variable is of the form FAMILY:LABEL. Only one tuple of FAMILY:LABEL may be specified. ## Unit tests diff --git a/docs/tutorials/otio-plugins.md b/docs/tutorials/otio-plugins.md index a3c538e9a..120cffd6f 100644 --- a/docs/tutorials/otio-plugins.md +++ b/docs/tutorials/otio-plugins.md @@ -106,7 +106,7 @@ OpenTimelineIO Final Cut Pro 7 XML Adapter. ### otio_json ``` -This adapter lets you read and write native .otio files +Adapter for reading and writing native .otio json files. ``` *source*: `opentimelineio/adapters/otio_json.py` @@ -147,14 +147,23 @@ Serializes an OpenTimelineIO object into a file indent (int): number of spaces for each json indentation level. Use -1 for no indentation or newlines. + If target_schema_versions is None and the environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". + Returns: bool: Write success Raises: ValueError: on write error + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". ``` - input_otio - filepath + - target_schema_versions - indent - write_to_string: ``` @@ -165,10 +174,21 @@ Serializes an OpenTimelineIO object into a string indent (int): number of spaces for each json indentation level. Use -1 for no indentation or newlines. + If target_schema_versions is None and the environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". + Returns: str: A json serialized string representation + + Raises: + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". ``` - input_otio + - target_schema_versions - indent diff --git a/docs/tutorials/otio-serialized-schema-only-fields.md b/docs/tutorials/otio-serialized-schema-only-fields.md index 624f96aaf..e6ff566e9 100644 --- a/docs/tutorials/otio-serialized-schema-only-fields.md +++ b/docs/tutorials/otio-serialized-schema-only-fields.md @@ -121,6 +121,7 @@ parameters: - *hooks* - *media_linkers* - *schemadefs* +- *version_manifests* ### SerializableObject.1 diff --git a/docs/tutorials/otio-serialized-schema.md b/docs/tutorials/otio-serialized-schema.md index e59b68a90..3fbb4744a 100644 --- a/docs/tutorials/otio-serialized-schema.md +++ b/docs/tutorials/otio-serialized-schema.md @@ -254,6 +254,7 @@ parameters: - *hooks*: Hooks that hooks scripts can be attached to. - *media_linkers*: Media Linkers this manifest describes. - *schemadefs*: Schemadefs this manifest describes. +- *version_manifests*: Sets of versions to downgrade schemas to. ### SerializableObject.1 diff --git a/docs/tutorials/versioning-schemas.md b/docs/tutorials/versioning-schemas.md index 3adea62f4..950acd97d 100644 --- a/docs/tutorials/versioning-schemas.md +++ b/docs/tutorials/versioning-schemas.md @@ -1,53 +1,249 @@ # Versioning Schemas +## Overview -During development, it is natural that the fields on objects in OTIO change. To accommodate this, OTIO has a system for handling version differences and upgrading older schemas to new ones. There are two components: +This document describes OpenTimelineIO's systems for dealing with different schema versions when reading files, writing files, or during development of the library itself. It is intended for developers who are integrating OpenTimelineIO into their pipelines or applications, or working directly on OpenTimelineIO. -1. `serializeable_label` on the class has a name and version field: `Foo.5` -- `Foo` is the schema name and `5` is the version. -2. `upgrade_function_for` decorator +TL;DR for users: OpenTimelineIO should be able to read files produced by older versions of the library and be able to write files that are compatible with older versions of the library from newer versions. +## Schema/Version Introduction -Changing a Field ---------------------- +Each SerializableObject (the base class of OpenTimelineIO) has `schema_name` and `schema_version` fields. The `schema_name` is a string naming the schema, for example, `Clip`, and the `schema_version` is an integer of the current version number, for example, `3`. -For example, lets say you have class: +SerializableObjects can be queried for these using the `.schema_name()` and `.schema_version()` methods. For a given release of the OpenTimelineIO library, in-memory objects the library creates will always be the same schema version. In other words, if `otio.schema.Clip()` instantiates an object with `schema_version` 2, there is no way to get an in-memory `Clip` object with version 1. + +OpenTimelineIO can still interoperate with older and newer versions of the library by way of the schema upgrading/downgrading system. As OpenTimelineIO deserializes json from a string or disk, it will upgrade the schemas to the version supported by the library before instantiating the concrete in-memory object. Similarly, when serializing OpenTimelineIO back to disk, the user can instruct OpenTimelineIO to downgrade the JSON to older versions of the schemas. In this way, a newer version of OpenTimelineIO can read files with older schemas, and a newer version of OpenTimelineIO can generate JSON with older schemas in it. + +## Schema Upgrading + +Once a type is registered to OpenTimelineIO, developers may also register upgrade functions. In python, each upgrade function takes a dictionary and returns a dictionary. In C++, the AnyDictionary is manipulated in place. Each upgrade function is associated with a version number - this is the version number that it upgrades to. + +C++ Example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): + + +```cpp +class SimpleClass : public otio::SerializableObject +{ +public: + struct Schema + { + static auto constexpr name = "SimpleClass"; + static int constexpr version = 2; + }; + + void set_new_field(int64_t val) { _new_field = val; } + int64_t new_field() const { return _new_field; } + +protected: + using Parent = SerializableObject; + + virtual ~SimpleClass() = default; + + virtual bool + read_from(Reader& reader) + { + auto result = ( + reader.read("new_field", &_new_field) + && Parent::read_from(reader) + ); + + return result; + } + + virtual void + write_to(Writer& writer) const + { + Parent::write_to(writer); + writer.write("new_field", _new_field); + } + +private: + int64_t _new_field; +}; + + // later, during execution: + + // register type and upgrade/downgrade functions + otio::TypeRegistry::instance().register_type(); + + // 1->2 + otio::TypeRegistry::instance().register_upgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["new_field"] = (*d)["my_field"]; + d->erase("my_field"); + } + ); +``` + +Python Example: + +```python +@otio.core.register_type +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + my_field = otio.core.serializable_field("new_field", int) + +@otio.core.upgrade_function_for(SimpleClass, 2) +def upgrade_one_to_two(data): + return {"new_field" : data["my_field"] } +``` + +When upgrading schemas, OpenTimelineIO will call each upgrade function in order in an attempt to get to the current version. For example, if a schema is registered to have version 3, and a file with version 1 is read, OpenTimelineIO will attempt to call the 1->2 function, then the 2->3 function before instantiating the concrete class. + +## Schema Downgrading + +Similarly, once a type is registered, downgrade functions may be registered. Downgrade functions take a dictionary of the version specified and return a dictionary of the schema version one lower. For example, if a downgrade function is registered for version 5, that will downgrade from 5 to 4. + +C++ Example, building off the prior section SimpleClass example (can be viewed/run in `examples/upgrade_downgrade_example.cpp`): + + +```cpp +// 2->1 +otio::TypeRegistry::instance().register_downgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["my_field"] = (*d)["new_field"]; + d->erase("new_field"); + } +); +``` + +Python Example: + +```python +@otio.core.upgrade_function_for(SimpleClass, 2) +def downgrade_two_to_one(data): + return {"my_field" : data["new_field"] } +``` + +To specify what version of a schema to downgrade to, the serialization functions include an optional `schema_version_targets` argument which is a map of schema name to target schema version. During serialization, any schemas who are listed in the map and are of greater version than specified in the map will be converted to AnyDictionary and run through the necessary downgrade functions before being serialized. + +Example C++: + +```cpp +auto sc = otio::SerializableObject::Retainer(new SimpleClass()); +sc->set_new_field(12); + +// this will only downgrade the SimpleClass, to version 1 +otio::schema_version_map downgrade_manifest = { + {"SimpleClass", 1} +}; + +// write it out to disk, downgrading to version 1 +sc->to_json_file("/var/tmp/simpleclass.otio", &err, &downgrade_manifest); +``` + +Example python: + +```python +sc = SimpleClass() +otio.adapters.write_to_file( + sc, + "/path/to/output.otio", + target_schema_versions={"SimpleClass":1} +) +``` + +### Schema-Version Sets + +In addition to passing in dictionaries of desired target schema versions, OpenTimelineIO also provides some tools for having sets of schemas with an associated label. The core C++ library contains a compiled-in map of them, the `CORE_VERSION_MAP`. This is organized (as of v0.15.0) by library release versions label, ie "0.15.0", "0.14.0" and so on. + +In order to downgrade to version 0.15.0 for example: + +```cpp +auto downgrade_manifest = otio::CORE_VERSION_MAP["0.15.0"]; + +// write it out to disk, downgrading to version 1 +sc->to_json_file("/var/tmp/simpleclass.otio", &err, &downgrade_manifest); +``` + +In python, an additional level of indirection is provided, "FAMILY", which is intended to allow developers to define their own sets of target versions for their plugin schemas. For example, a studio might have a family named "MYFAMILY" under which they organize labels for their internal releases of their own plugins. + +These can be defined in a plugin manifest, which is a `.plugin_manifest.json` file found on the environment variable `${OTIO_PLUGIN_MANIFEST_PATH}`. + +For example: + +```python +{ + "OTIO_SCHEMA" : "PluginManifest.1", + "version_manifests": { + "MYFAMILY": { + "June2022": { + "SimpleClass": 2, + ... + }, + "May2022": { + "SimpleClass": 1, + ... + } + } + } +} +``` + +To fetch the version maps and work with this, the python API provides some additional functions: + +```python +downgrade_manifest = otio.versioning.fetch_map("MYFAMILY", "June2022") +otio.adapters.write_to_file( + sc, + "/path/to/file.otio", + downgrade_manifest +) +``` + +See the [versioning module](../api/python/opentimelineio.versioning.rst) for more information on accessing these. + +## For Developers + +During the development of OpenTimelineIO schemas, whether they are in the core or in plugins, it is expected that schemas will change and evolve over time. Here are some processes for doing that. + +### Changing a Field + +Given `SimpleClass`: ```python import opentimelineio as otio @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.1" - my_field = otio.core.serializeable_field("my_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.1" + my_field = otio.core.serializable_field("my_field", int) ``` - -And you want to change `my_field` to `new_field`. To do this: +And `my_field` needs to be renamed to `new_field`. To do this: - Make the change in the class - Bump the version number in the label -- add an upgrade function - -So after the changes, you'll have: +- add upgrade and downgrade functions ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.2" - my_field = otio.core.serializeable_field("new_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + new_field = otio.core.serializable_field("new_field", int) @otio.core.upgrade_function_for(SimpleClass, 2) def upgrade_one_to_two(data): return {"new_field" : data["my_field"] } + +@otio.core.downgrade_function_from(SimpleClass, 2) +def downgrade_two_to_one(data): + return {"my_field": data["new_field"]} ``` -Lets change it again, so that `new_field` becomes `even_newer_field`. +Changing it again, now `new_field` becomes `even_newer_field`. ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.2" - my_field = otio.core.serializeable_field("even_newer_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + even_newer_field = otio.core.serializable_field("even_newer_field", int) @otio.core.upgrade_function_for(SimpleClass, 2) def upgrade_one_to_two(data): @@ -57,39 +253,51 @@ def upgrade_one_to_two(data): @otio.core.upgrade_function_for(SimpleClass, 3) def upgrade_two_to_three(data): return {"even_newer_field" : data["new_field"] } -``` -Upgrade functions can be sparse - if version `3` to `4` doesn't require a function, for example, you don't need to write one. +@otio.core.downgrade_function_from(SimpleClass, 2) +def downgrade_two_to_one(data): + return {"my_field": data["new_field"]} + +# ...and corresponding second downgrade function +@otio.core.downgrade_function_from(SimpleClass, 3) +def downgrade_two_to_one(data): + return {"new_field": data["even_newer_field"]} +``` -Adding or Removing a Field --------------------------------- +### Adding or Removing a Field Starting from the same class: ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.1" - my_field = otio.core.serializeable_field("my_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.1" + my_field = otio.core.serializable_field("my_field", int) ``` -Adding or Removing a field is simpler. In these cases, you don't need to write an upgrade function, since any new classes will be initialized through the constructor, and any removed fields will be ignored when reading from an older schema version. +If a change to a schema is to add a field, for which the default value is the correct value for an old schema, then no upgrade or downgrade function is needed. The parser ignores values that aren't in the schema. -So lets add a new field: +Additionally, upgrade functions will be called in order, but they need not cover every version number. So if there is an upgrade function for version 2 and 4, to get to version 4, OTIO will automatically apply function 2 and then function 4 in order, skipping the missing 3. + +Downgrade functions must be called in order with no gaps. + +Example of adding a field (`other_field`): ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.2" - my_field = otio.core.serializeable_field("my_field", int) - other_field = otio.core.serializeable_field("other_field", int) +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + my_field = otio.core.serializable_field("my_field", int) + other_field = otio.core.serializable_field("other_field", int) ``` -And then delete the original field: +Removing a field (`my_field`): ```python @otio.core.register_type -class SimpleClass(otio.core.SerializeableObject): - serializeable_label = "SimpleClass.3" - other_field = otio.core.serializeable_field("other_field", int) -``` \ No newline at end of file +class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.3" + other_field = otio.core.serializable_field("other_field", int) +``` + +Similarly, when deleting a field, if the field is now ignored and does not contribute to computation, no upgrade or downgrade function is needed. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index de02cade0..960a99625 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -8,6 +8,8 @@ include_directories(${PROJECT_SOURCE_DIR}/src list(APPEND examples conform) list(APPEND examples flatten_video_tracks) list(APPEND examples summarize_timing) +list(APPEND examples io_perf_test) +list(APPEND examples upgrade_downgrade_example) if(OTIO_PYTHON_INSTALL) list(APPEND examples python_adapters_child_process) list(APPEND examples python_adapters_embed) diff --git a/examples/io_perf_test.cpp b/examples/io_perf_test.cpp new file mode 100644 index 000000000..920adfbeb --- /dev/null +++ b/examples/io_perf_test.cpp @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + +#include + +#include "opentimelineio/clip.h" +#include "opentimelineio/typeRegistry.h" +#include "opentimelineio/any.h" +#include "opentimelineio/serialization.h" +#include "opentimelineio/deserialization.h" +#include "opentimelineio/timeline.h" + +#include "util.h" + +namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; + +using chrono_time_point = std::chrono::steady_clock::time_point; + +const struct { + bool FIXED_TMP = true; + bool PRINT_CPP_VERSION_FAMILY = false; + bool TO_JSON_STRING = true; + bool TO_JSON_STRING_NO_DOWNGRADE = true; + bool TO_JSON_FILE = true; + bool TO_JSON_FILE_NO_DOWNGRADE = true; + bool CLONE_TEST = true; + bool SINGLE_CLIP_DOWNGRADE_TEST = true; +} RUN_STRUCT ; + +// typedef std::chrono::duration fsec; + // auto t0 = Time::now(); + // auto t1 = Time::now(); + // fsec fs = t1 - t0; + +/// utility function for printing std::chrono elapsed time +double +print_elapsed_time( + const std::string& message, + const chrono_time_point& begin, + const chrono_time_point& end +) +{ + const std::chrono::duration dur = end - begin; + + std::cout << message << ": " << dur.count() << " [s]" << std::endl; + + return dur.count(); +} + +void +print_version_map() +{ + std::cerr << "current version map: " << std::endl; + for (const auto& kv_lbl: otio::CORE_VERSION_MAP) + { + std::cerr << " " << kv_lbl.first << std::endl; + for (auto kv_schema_version : kv_lbl.second) + { + std::cerr << " \"" << kv_schema_version.first << "\": "; + std::cerr << kv_schema_version.second << std::endl; + } + } + +} + +int +main( + int argc, + char *argv[] +) +{ + if (RUN_STRUCT.PRINT_CPP_VERSION_FAMILY) + { + print_version_map(); + } + + if (argc < 2) + { + std::cerr << "usage: otio_io_perf_test path/to/timeline.otio "; + std::cerr << "[--keep-tmp]" << std::endl; + return 1; + } + + bool keep_tmp = false; + if (argc > 2) + { + const std::string arg = argv[2]; + if (arg == "--keep-tmp") + { + keep_tmp = true; + } + } + + const std::string tmp_dir_path = ( + RUN_STRUCT.FIXED_TMP + ? "/var/tmp/ioperftest" + : examples::create_temp_dir() + ); + + otio::ErrorStatus err; + assert(!otio::is_error(err)); + + otio::schema_version_map downgrade_manifest = { + {"FakeSchema", 3}, + {"Clip", 1}, + {"OtherThing", 12000} + }; + + if (RUN_STRUCT.CLONE_TEST) + { + otio::SerializableObject::Retainer cl = new otio::Clip("test"); + cl->metadata()["example thing"] = "banana"; + const auto intermediate = cl->clone(&err); + assert(intermediate != nullptr); + const auto cl_clone = dynamic_cast(intermediate); + assert(cl_clone != nullptr); + assert(!otio::is_error(err)); + assert(cl->name() == cl_clone->name()); + } + + if (RUN_STRUCT.SINGLE_CLIP_DOWNGRADE_TEST) + { + otio::SerializableObject::Retainer cl = new otio::Clip("test"); + cl->metadata()["example thing"] = "banana"; + chrono_time_point begin = std::chrono::steady_clock::now(); + cl->to_json_file( + examples::normalize_path(tmp_dir_path + "/clip.otio"), + &err, + &downgrade_manifest + ); + chrono_time_point end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); + print_elapsed_time("downgrade clip", begin, end); + } + + otio::any tl; + std::string fname = std::string(argv[1]); + + // read file + chrono_time_point begin = std::chrono::steady_clock::now(); + otio::SerializableObject::Retainer timeline( + dynamic_cast( + otio::Timeline::from_json_file( + examples::normalize_path(argv[1]), + &err + ) + ) + ); + chrono_time_point end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); + if (!timeline) + { + examples::print_error(err); + return 1; + } + + print_elapsed_time("deserialize_json_from_file", begin, end); + + + double str_dg, str_nodg; + if (RUN_STRUCT.TO_JSON_STRING) + { + begin = std::chrono::steady_clock::now(); + const std::string result = timeline.value->to_json_string( + &err, + &downgrade_manifest + ); + end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); + + if (otio::is_error(err)) + { + examples::print_error(err); + return 1; + } + str_dg = print_elapsed_time("serialize_json_to_string", begin, end); + } + + if (RUN_STRUCT.TO_JSON_STRING_NO_DOWNGRADE) + { + begin = std::chrono::steady_clock::now(); + const std::string result = timeline.value->to_json_string(&err, {}); + end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); + + if (otio::is_error(err)) + { + examples::print_error(err); + return 1; + } + str_nodg = print_elapsed_time( + "serialize_json_to_string [no downgrade]", + begin, + end + ); + } + + if (RUN_STRUCT.TO_JSON_STRING && RUN_STRUCT.TO_JSON_STRING_NO_DOWNGRADE) + { + std::cout << " JSON to string no_dg/dg: " << str_dg / str_nodg; + std::cout << std::endl; + } + + double file_dg, file_nodg; + if (RUN_STRUCT.TO_JSON_FILE) + { + begin = std::chrono::steady_clock::now(); + timeline.value->to_json_file( + examples::normalize_path(tmp_dir_path + "/io_perf_test.otio"), + &err, + &downgrade_manifest + ); + end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); + file_dg = print_elapsed_time("serialize_json_to_file", begin, end); + } + + if (RUN_STRUCT.TO_JSON_FILE_NO_DOWNGRADE) + { + begin = std::chrono::steady_clock::now(); + timeline.value->to_json_file( + examples::normalize_path( + tmp_dir_path + + "/io_perf_test.nodowngrade.otio" + ), + &err, + {} + ); + end = std::chrono::steady_clock::now(); + assert(!otio::is_error(err)); + file_nodg = print_elapsed_time( + "serialize_json_to_file [no downgrade]", + begin, + end + ); + } + + if (RUN_STRUCT.TO_JSON_FILE && RUN_STRUCT.TO_JSON_FILE_NO_DOWNGRADE) + { + std::cout << " JSON to file no_dg/dg: " << file_dg / file_nodg; + std::cout << std::endl; + } + + if (keep_tmp || RUN_STRUCT.FIXED_TMP) + { + std::cout << "Temp directory preserved. All files written to: "; + std::cout << tmp_dir_path << std::endl; + } + else + { + // clean up + const auto tmp_files = examples::glob(tmp_dir_path, "*"); + for (const auto& fp : tmp_files) + { + remove(fp.c_str()); + } + remove(tmp_dir_path.c_str()); + std::cout << "cleaned up tmp dir, pass --keep-tmp to preserve"; + std::cout << " output." << std::endl; + } + + return 0; +} diff --git a/examples/upgrade_downgrade_example.cpp b/examples/upgrade_downgrade_example.cpp new file mode 100644 index 000000000..532fc9abc --- /dev/null +++ b/examples/upgrade_downgrade_example.cpp @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project + +#include "opentimelineio/serializableObject.h" +#include "opentimelineio/typeRegistry.h" +#include + +// demonstrates a minimal custom SerializableObject written in C++ with upgrade +// and downgrade functions + +namespace otio = opentimelineio::OPENTIMELINEIO_VERSION; + +// define the custom class +class SimpleClass : public otio::SerializableObject +{ +public: + struct Schema + { + static auto constexpr name = "SimpleClass"; + static int constexpr version = 2; + }; + + void set_new_field(int64_t val) { _new_field = val; } + int64_t new_field() const { return _new_field; } + +protected: + using Parent = SerializableObject; + + virtual ~SimpleClass() = default; + + // methods for serialization + virtual bool + read_from(Reader& reader) override + { + auto result = ( + reader.read("new_field", &_new_field) + && Parent::read_from(reader) + ); + + return result; + } + + // ...and deserialization + virtual void + write_to(Writer& writer) const override + { + Parent::write_to(writer); + writer.write("new_field", _new_field); + } + +private: + int64_t _new_field; +}; + +int +main( + int argc, + char *argv[] +) +{ + // register type and upgrade/downgrade functions + otio::TypeRegistry::instance().register_type(); + + // 1->2 + otio::TypeRegistry::instance().register_upgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["new_field"] = (*d)["my_field"]; + d->erase("my_field"); + } + ); + // 2->1 + otio::TypeRegistry::instance().register_downgrade_function( + SimpleClass::Schema::name, + 2, + [](otio::AnyDictionary* d) + { + (*d)["my_field"] = (*d)["new_field"]; + d->erase("new_field"); + } + ); + + otio::ErrorStatus err; + + auto sc = otio::SerializableObject::Retainer(new SimpleClass()); + sc->set_new_field(12); + + // write it out to disk, without changing it + sc->to_json_file("/var/tmp/simpleclass.otio", &err); + + otio::schema_version_map downgrade_manifest = { + {"SimpleClass", 1} + }; + + // write it out to disk, downgrading to version 1 + sc->to_json_file("/var/tmp/simpleclass.otio", &err, &downgrade_manifest); + + // read it back, upgrading automatically back up to version 2 of the schema + otio::SerializableObject::Retainer sc2( + dynamic_cast( + SimpleClass::from_json_file("/var/tmp/simpleclass.otio", &err) + ) + ); + + assert(sc2->new_field() == sc->new_field()); + + std::cout << "Upgrade/Downgrade demo complete." << std::endl; + + return 0; +} diff --git a/src/opentimelineio/CMakeLists.txt b/src/opentimelineio/CMakeLists.txt index a5b484b47..29cc3a8a8 100644 --- a/src/opentimelineio/CMakeLists.txt +++ b/src/opentimelineio/CMakeLists.txt @@ -72,6 +72,7 @@ add_library(opentimelineio ${OTIO_SHARED_OR_STATIC_LIB} transition.cpp typeRegistry.cpp unknownSchema.cpp + CORE_VERSION_MAP.cpp ${OPENTIMELINEIO_HEADER_FILES}) add_library(OTIO::opentimelineio ALIAS opentimelineio) diff --git a/src/opentimelineio/CORE_VERSION_MAP.cpp b/src/opentimelineio/CORE_VERSION_MAP.cpp new file mode 100644 index 000000000..48e8c44f3 --- /dev/null +++ b/src/opentimelineio/CORE_VERSION_MAP.cpp @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project +// +// This document is automatically generated by running the `make version-map` +// make target. It is part of the unit tests suite and should be updated +// whenever schema versions change. If it needs to be updated, run: `make +// version-map-update` and this file should be regenerated. +// +// This maps a "Label" to a map of Schema name to Schema version. The intent is +// that these sets of schemas can be used for compatability with future +// versions of OTIO, so that a newer version of OTIO can target a compatability +// version of an older library. +#include "opentimelineio/typeRegistry.h" + +namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { + +const label_to_schema_version_map CORE_VERSION_MAP { + { "0.15.0.dev1", + { + { "Adapter", 1 }, + { "Clip", 2 }, + { "Composable", 1 }, + { "Composition", 1 }, + { "Effect", 1 }, + { "ExternalReference", 1 }, + { "FreezeFrame", 1 }, + { "Gap", 1 }, + { "GeneratorReference", 1 }, + { "HookScript", 1 }, + { "ImageSequenceReference", 1 }, + { "Item", 1 }, + { "LinearTimeWarp", 1 }, + { "Marker", 2 }, + { "MediaLinker", 1 }, + { "MediaReference", 1 }, + { "MissingReference", 1 }, + { "PluginManifest", 1 }, + { "SchemaDef", 1 }, + { "SerializableCollection", 1 }, + { "SerializableObject", 1 }, + { "SerializableObjectWithMetadata", 1 }, + { "Stack", 1 }, + { "Test", 1 }, + { "TimeEffect", 1 }, + { "Timeline", 1 }, + { "Track", 1 }, + { "Transition", 1 }, + { "UnknownSchema", 1 }, + } + }, + // {next} +}; + +} } // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/CORE_VERSION_MAP.last.cpp b/src/opentimelineio/CORE_VERSION_MAP.last.cpp new file mode 100644 index 000000000..fdb321113 --- /dev/null +++ b/src/opentimelineio/CORE_VERSION_MAP.last.cpp @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Contributors to the OpenTimelineIO project +// +// This document is automatically generated by running the `make version-map` +// make target. It is part of the unit tests suite and should be updated +// whenever schema versions change. If it needs to be updated, run: `make +// version-map-update` and this file should be regenerated. +// +// This maps a "Label" to a map of Schema name to Schema version. The intent is +// that these sets of schemas can be used for compatability with future +// versions of OTIO, so that a newer version of OTIO can target a compatability +// version of an older library. +#include "opentimelineio/typeRegistry.h" + +namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { + +const label_to_schema_version_map CORE_VERSION_MAP { + // {next} +}; + +} } // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/anyDictionary.h b/src/opentimelineio/anyDictionary.h index 3d9be59a4..3512897e7 100644 --- a/src/opentimelineio/anyDictionary.h +++ b/src/opentimelineio/anyDictionary.h @@ -122,6 +122,85 @@ class AnyDictionary : private std::map map::swap(other); } + /// @TODO: remove all of these @{ + + // if key is in this, and the type of key matches the type of result, then + // set result to the value of any_cast(this[key]) and return true, + // otherwise return false + template + bool + get_if_set( + const std::string& key, + containedType* result + ) const + { + if (result == nullptr) + { + return false; + } + + const auto it = this->find(key); + + if ( + (it != this->end()) + && ( + it->second.type().hash_code() + == typeid(containedType).hash_code() + ) + ) + { + *result = any_cast(it->second); + return true; + } + else + { + return false; + } + } + + inline bool + has_key( + const std::string& key + ) const + { + return (this->find(key) != this->end()); + } + + // if key is in this, place the value in result and return true, otherwise + // store the value in result at key and return false + template + bool + set_default( + const std::string& key, + containedType* result + ) + { + if (result == nullptr) + { + return false; + } + + const auto d_it = this->find(key); + + if ( + (d_it != this->end()) + && ( + d_it->second.type().hash_code() + == typeid(containedType).hash_code() + ) + ) + { + *result = any_cast(d_it->second); + return true; + } + else + { + this->insert({key, *result}); + return false; + } + } + + using map::empty; using map::max_size; using map::size; diff --git a/src/opentimelineio/serializableObject.cpp b/src/opentimelineio/serializableObject.cpp index fb749bd1e..bce6a89de 100644 --- a/src/opentimelineio/serializableObject.cpp +++ b/src/opentimelineio/serializableObject.cpp @@ -5,6 +5,7 @@ #include "opentimelineio/deserialization.h" #include "opentimelineio/serialization.h" #include "stringUtils.h" +#include "typeRegistry.h" namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { @@ -112,18 +113,34 @@ SerializableObject::is_unknown_schema() const } std::string -SerializableObject::to_json_string(ErrorStatus* error_status, int indent) const +SerializableObject::to_json_string( + ErrorStatus* error_status, + const schema_version_map* schema_version_targets, + int indent +) const { return serialize_json_to_string( - any(Retainer<>(this)), error_status, indent); + any(Retainer<>(this)), + schema_version_targets, + error_status, + indent + ); } bool SerializableObject::to_json_file( - std::string const& file_name, ErrorStatus* error_status, int indent) const + std::string const& file_name, + ErrorStatus* error_status, + const schema_version_map* schema_version_targets, + int indent) const { return serialize_json_to_file( - any(Retainer<>(this)), file_name, error_status, indent); + any(Retainer<>(this)), + file_name, + schema_version_targets, + error_status, + indent + ); } SerializableObject* diff --git a/src/opentimelineio/serializableObject.h b/src/opentimelineio/serializableObject.h index 2df315859..46f34dad2 100644 --- a/src/opentimelineio/serializableObject.h +++ b/src/opentimelineio/serializableObject.h @@ -14,12 +14,15 @@ #include "opentimelineio/version.h" #include "ImathBox.h" +#include "serialization.h" #include -#include +#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { +class CloningEncoder; + class SerializableObject { public: @@ -42,12 +45,18 @@ class SerializableObject */ bool possibly_delete(); - bool to_json_file( + bool + to_json_file( std::string const& file_name, ErrorStatus* error_status = nullptr, + const schema_version_map* target_family_label_spec = nullptr, int indent = 4) const; + std::string - to_json_string(ErrorStatus* error_status = nullptr, int indent = 4) const; + to_json_string( + ErrorStatus* error_status = nullptr, + const schema_version_map* target_family_label_spec = nullptr, + int indent = 4) const; static SerializableObject* from_json_file( std::string const& file_name, ErrorStatus* error_status = nullptr); @@ -394,6 +403,7 @@ class SerializableObject static bool write_root( any const& value, class Encoder& encoder, + const schema_version_map* downgrade_version_manifest=nullptr, ErrorStatus* error_status = nullptr); void write(std::string const& key, bool value); @@ -440,7 +450,7 @@ class SerializableObject AnyVector av; av.reserve(value.size()); - for (auto e: value) + for (const auto& e: value) { av.emplace_back(_to_any(e)); } @@ -452,7 +462,7 @@ class SerializableObject static any _to_any(std::map const& value) { AnyDictionary am; - for (auto e: value) + for (const auto& e: value) { am.emplace(e.first, _to_any(e.second)); } @@ -466,7 +476,7 @@ class SerializableObject AnyVector av; av.reserve(value.size()); - for (auto e: value) + for (const auto& e: value) { av.emplace_back(_to_any(e)); } @@ -502,12 +512,19 @@ class SerializableObject } ///@} - Writer(class Encoder& encoder) - : _encoder(encoder) + Writer( + class Encoder& encoder, + const schema_version_map* downgrade_version_manifest + ) + : _encoder(encoder), + _downgrade_version_manifest(downgrade_version_manifest) + { _build_dispatch_tables(); } + ~Writer(); + Writer(Writer const&) = delete; Writer operator=(Writer const&) = delete; @@ -520,19 +537,23 @@ class SerializableObject bool _any_equals(any const& lhs, any const& rhs); std::string _no_key; - std::map> + std::unordered_map> _write_dispatch_table; - std::map< + std::unordered_map< std::type_info const*, std::function> _equality_dispatch_table; - std::map> + std::unordered_map> _write_dispatch_table_by_name; - std::map _id_for_object; - std::map _next_id_for_type; + std::unordered_map _id_for_object; + std::unordered_map _next_id_for_type; + + Writer* _child_writer = nullptr; + CloningEncoder* _child_cloning_encoder = nullptr; class Encoder& _encoder; + const schema_version_map* _downgrade_version_manifest; friend class SerializableObject; }; diff --git a/src/opentimelineio/serialization.cpp b/src/opentimelineio/serialization.cpp index 0c9ad8c68..0f54e1079 100644 --- a/src/opentimelineio/serialization.cpp +++ b/src/opentimelineio/serialization.cpp @@ -1,9 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Contributors to the OpenTimelineIO project +#include "errorStatus.h" +#include "nonstd/optional.hpp" #include "opentimelineio/serializableObject.h" +#include "opentimelineio/serialization.h" +#include "opentimelineio/anyDictionary.h" #include "opentimelineio/unknownSchema.h" #include "stringUtils.h" +#include +#include #define RAPIDJSON_NAMESPACE OTIO_rapidjson #include @@ -51,6 +57,8 @@ class Encoder bool has_errored() { return is_error(_error_status); } + virtual bool encoding_to_anydict() { return false; } + virtual void start_object() = 0; virtual void end_object() = 0; @@ -83,23 +91,39 @@ class Encoder }; /** - * This encoder builds up a dictionary as its method of "encoding". + * This encoder builds up a AnyDictionary as its method of "encoding". * The dictionary is than handed off to a CloningDecoder, to complete * copying of a SerializableObject instance. */ class CloningEncoder : public Encoder { public: - CloningEncoder(bool actually_clone) + enum class ResultObjectPolicy { + CloneBackToSerializableObject = 0, + MathTypesConcreteAnyDictionaryResult, + OnlyAnyDictionary, + }; + + CloningEncoder( + CloningEncoder::ResultObjectPolicy result_object_policy, + const schema_version_map* schema_version_targets = nullptr + ) : + _result_object_policy(result_object_policy), + _downgrade_version_manifest(schema_version_targets) { using namespace std::placeholders; _error_function = std::bind(&CloningEncoder::_error, this, _1); - _actually_clone = actually_clone; } virtual ~CloningEncoder() {} - void write_key(std::string const& key) + virtual bool + encoding_to_anydict() override + { + return (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary); + } + + void write_key(std::string const& key) override { if (has_errored()) { @@ -116,6 +140,35 @@ class CloningEncoder : public Encoder _stack.back().cur_key = key; } + void _replace_back(AnyDictionary&& a) + { + if (has_errored()) + { + return; + } + + if (_stack.size() == 1) + { + any newstack(std::move(a)); + _root.swap(newstack); + } + else + { + _stack.pop_back(); + auto& top = _stack.back(); + if (top.is_dict) + { + top.dict.emplace(top.cur_key, a); + } + else + { + any newstack(std::move(a)); + top.array.emplace_back(newstack); + } + } + + } + void _store(any&& a) { if (has_errored()) @@ -141,36 +194,124 @@ class CloningEncoder : public Encoder } } - void write_null_value() { _store(any()); } - - void write_value(bool value) { _store(any(value)); } - - void write_value(int value) { _store(any(value)); } - - void write_value(int64_t value) { _store(any(value)); } - - void write_value(uint64_t value) { _store(any(value)); } - - void write_value(std::string const& value) { _store(any(value)); } - - void write_value(double value) { _store(any(value)); } - - void write_value(RationalTime const& value) { _store(any(value)); } - - void write_value(TimeRange const& value) { _store(any(value)); } - - void write_value(TimeTransform const& value) { _store(any(value)); } + void write_null_value() override { _store(any()); } + void write_value(bool value) override { _store(any(value)); } + void write_value(int value) override { _store(any(value)); } + void write_value(int64_t value) override { _store(any(value)); } + void write_value(uint64_t value) override { _store(any(value)); } + void write_value(std::string const& value) override { _store(any(value)); } + void write_value(double value) override { _store(any(value)); } + + void + write_value(RationalTime const& value) override + { + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result = { + {"OTIO_SCHEMA", "RationalTime.1"}, + {"value", value.value()}, + {"rate", value.rate()}, + }; + _store(any(std::move(result))); + } + else + { + _store(any(value)); + } + } + void + write_value(TimeRange const& value) override + { + + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result = { + {"OTIO_SCHEMA", "TimeRange.1"}, + {"duration", value.duration()}, + {"start_time", value.start_time()}, + }; + _store(any(std::move(result))); + } + else + { + _store(any(value)); + } - void write_value(SerializableObject::ReferenceId value) + } + void + write_value(TimeTransform const& value) override { - _store(any(value)); + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "TimeTransform.1"}, + {"offset", value.offset()}, + {"rate", value.rate()}, + {"scale", value.scale()}, + }; + _store(any(std::move(result))); + } + else + { + _store(any(value)); + } } + void + write_value(SerializableObject::ReferenceId value) override + { + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "SerializableObjectRef.1"}, + {"id", value.id.c_str()}, + }; + _store(any(std::move(result))); + } + else + { + _store(any(value)); + } + _store(any(value)); + } + + void + write_value(Imath::V2d const& value) + { + + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "V2d.1"}, + {"x", value.x}, + {"y", value.y}, + }; + _store(any(std::move(result))); + } + else + { + _store(any(value)); + } - void write_value(Imath::V2d const& value) { _store(any(value)); } + } - void write_value(Imath::Box2d const& value) { _store(any(value)); } + void + write_value(Imath::Box2d const& value) override + { + if (_result_object_policy == ResultObjectPolicy::OnlyAnyDictionary) + { + AnyDictionary result { + {"OTIO_SCHEMA", "Box2d.1"}, + {"min", value.min}, + {"max", value.max}, + }; + _store(any(std::move(result))); + } else { + _store(any(value)); + } + } + // @} - void start_array(size_t /* n */) + void start_array(size_t /* n */) override { if (has_errored()) { @@ -180,7 +321,7 @@ class CloningEncoder : public Encoder _stack.emplace_back(_DictOrArray{ false /* is_dict*/ }); } - void start_object() + void start_object() override { if (has_errored()) { @@ -190,7 +331,7 @@ class CloningEncoder : public Encoder _stack.emplace_back(_DictOrArray{ true /* is_dict*/ }); } - void end_array() + void end_array() override { if (has_errored()) { @@ -221,7 +362,7 @@ class CloningEncoder : public Encoder } } - void end_object() + void end_object() override { if (has_errored()) { @@ -229,40 +370,54 @@ class CloningEncoder : public Encoder } if (_stack.empty()) + { + _internal_error( + "Encoder::end_object() called without matching start_object()" + ); + return; + } + + auto& top = _stack.back(); + if (!top.is_dict) { _internal_error( "Encoder::end_object() called without matching start_object()"); + _stack.pop_back(); + + return; } - else + + /* + * Convert back to SerializableObject* right here. + */ + if ( + _result_object_policy + == ResultObjectPolicy::CloneBackToSerializableObject + ) { - auto& top = _stack.back(); - if (!top.is_dict) - { - _internal_error( - "Encoder::end_object() called without matching start_object()"); - _stack.pop_back(); - } - else - { - /* - * Convert back to SerializableObject* right here. - */ - if (_actually_clone) - { - SerializableObject::Reader reader( - top.dict, _error_function, nullptr); - _stack.pop_back(); - _store(reader._decode(_resolver)); - } - else - { - AnyDictionary m; - m.swap(top.dict); - _stack.pop_back(); - _store(any(std::move(m))); - } - } + SerializableObject::Reader reader( + top.dict, + _error_function, + nullptr + ); + _stack.pop_back(); + _store(reader._decode(_resolver)); + + return; } + + AnyDictionary m; + m.swap(top.dict); + + if ( + (_downgrade_version_manifest != nullptr) + && (!_downgrade_version_manifest->empty()) + ) + { + _downgrade_dictionary(m); + } + + _replace_back(std::move(m)); } private: @@ -280,6 +435,7 @@ class CloningEncoder : public Encoder std::string cur_key; }; + void _internal_error(std::string const& err_msg) { _error(ErrorStatus(ErrorStatus::INTERNAL_ERROR, err_msg)); @@ -287,7 +443,87 @@ class CloningEncoder : public Encoder friend class SerializableObject; std::vector<_DictOrArray> _stack; - bool _actually_clone; + ResultObjectPolicy _result_object_policy; + const schema_version_map* _downgrade_version_manifest = nullptr; + + void + _downgrade_dictionary( + AnyDictionary& m + ) + { + std::string schema_string = ""; + + if (!m.get_if_set("OTIO_SCHEMA", &schema_string)) + { + return; + } + + const auto sep = schema_string.rfind('.'); + const std::string& schema_name = schema_string.substr(0, sep); + + const auto dg_version_it = _downgrade_version_manifest->find( + schema_name + ); + + if (dg_version_it == _downgrade_version_manifest->end()) + { + return; + } + + const std::string& schema_vers = schema_string.substr(sep+1); + int current_version = -1; + + if (!schema_vers.empty()) + { + current_version = std::stoi(schema_vers); + } + + // @TODO: is 0 a legitimate schema version? + if (current_version < 0) + { + _internal_error( + string_printf( + "Could not parse version number from Schema" + " string: %s", + schema_string.c_str() + ) + ); + return; + } + + const int target_version = (dg_version_it->second); + + const auto& type_rec = ( + TypeRegistry::instance()._find_type_record(schema_name) + ); + + while (current_version > target_version) + { + const auto& next_dg_fn = ( + type_rec->downgrade_functions.find(current_version) + ); + + if (next_dg_fn == type_rec->downgrade_functions.end()) + { + _internal_error( + string_printf( + "No downgrader function available for " + "going from version %d to version %d.", + current_version, + target_version + ) + ); + return; + } + + // apply it + next_dg_fn->second(&m); + + current_version --; + } + + m["OTIO_SCHEMA"] = schema_name + "." + std::to_string(current_version); + } }; template @@ -454,6 +690,7 @@ SerializableObject::Writer::_build_dispatch_tables() /* * These are basically atomic writes to the encoder: */ + auto& wt = _write_dispatch_table; wt[&typeid(void)] = [this](any const&) { _encoder.write_null_value(); }; wt[&typeid(bool)] = [this](any const& value) { @@ -506,7 +743,7 @@ SerializableObject::Writer::_build_dispatch_tables() * Install a backup table, using the actual type name as a key. * This is to deal with type aliasing across compilation units. */ - for (auto e: wt) + for (const auto& e: wt) { _write_dispatch_table_by_name[e.first->name()] = e.second; } @@ -551,7 +788,7 @@ SerializableObject::Writer::_any_dict_equals(any const& lhs, any const& rhs) auto r_it = rd.begin(); - for (auto l_it: ld) + for (const auto& l_it: ld) { if (r_it == rd.end()) { @@ -604,9 +841,13 @@ SerializableObject::Writer::_any_equals(any const& lhs, any const& rhs) bool SerializableObject::Writer::write_root( - any const& value, Encoder& encoder, ErrorStatus* error_status) + any const& value, + Encoder& encoder, + const schema_version_map* schema_version_targets, + ErrorStatus* error_status +) { - Writer w(encoder); + Writer w(encoder, schema_version_targets); w.write(w._no_key, value); return !encoder.has_errored(error_status); } @@ -688,7 +929,10 @@ SerializableObject::Writer::write( } void -SerializableObject::Writer::write(std::string const& key, TimeTransform value) +SerializableObject::Writer::write( + std::string const& key, + TimeTransform value +) { _encoder_write_key(key); _encoder.write_value(value); @@ -696,8 +940,10 @@ SerializableObject::Writer::write(std::string const& key, TimeTransform value) void SerializableObject::Writer::write( - std::string const& key, SerializableObject const* value) -{ + std::string const& key, + SerializableObject const* value +) +{ _encoder_write_key(key); if (!value) { @@ -738,28 +984,97 @@ SerializableObject::Writer::write( std::to_string(++_next_id_for_type[schema_type_name]); _id_for_object[value] = next_id; - _encoder.start_object(); + // detect if downgrading needs to happen + const std::string& schema_name = value->schema_name(); + int schema_version = value->schema_version(); + + any downgraded = {}; + + // if there is a manifest & the encoder is not converting to AnyDictionary + if ( + (_downgrade_version_manifest != nullptr) + && (!_downgrade_version_manifest->empty()) + && (!_encoder.encoding_to_anydict()) + ) + { + const auto& target_version_it = _downgrade_version_manifest->find( + schema_name + ); + + // ...and if that downgrade manifest specifies a target version for + // this schema + if (target_version_it != _downgrade_version_manifest->end()) + { + const int target_version = target_version_it->second; - _encoder.write_key("OTIO_SCHEMA"); + // and the current_version is greater than the target version + if (schema_version > target_version) + { + if (_child_writer == nullptr) + { + _child_cloning_encoder = new CloningEncoder( + CloningEncoder::ResultObjectPolicy::OnlyAnyDictionary, + _downgrade_version_manifest + ); + _child_writer = new Writer(*_child_cloning_encoder, {}); + } + else { + _child_cloning_encoder->_stack.clear(); + } + + _child_writer->write(_child_writer->_no_key, value); + if (_child_cloning_encoder->has_errored(&_encoder._error_status)) + { + return; + } + + downgraded.swap(_child_cloning_encoder->_root); + schema_version = target_version; + } + } + } + + std::string schema_str = ""; + + // if its an unknown schema, the schema name is computed from the + // _original_schema_name and _original_schema_version attributes if (UnknownSchema const* us = dynamic_cast(value)) { - _encoder.write_value(string_printf( - "%s.%d", - us->_original_schema_name.c_str(), - us->_original_schema_version)); + schema_str = ( + us->_original_schema_name + + "." + + std::to_string(us->_original_schema_version) + ); } else { - _encoder.write_value(string_printf( - "%s.%d", value->schema_name().c_str(), value->schema_version())); + // otherwise, use the schema_name and schema_version attributes + schema_str = schema_name + "." + std::to_string(schema_version); } + _encoder.start_object(); + #ifdef OTIO_INSTANCING_SUPPORT _encoder.write_key("OTIO_REF_ID"); _encoder.write_value(next_id); #endif - value->write_to(*this); + + // write the contents of the object to the encoder, either the downgraded + // anydictionary or the SerializableObject + if (!(downgraded.empty())) + { + for (const auto& kv : any_cast(downgraded)) + { + this->write(kv.first, kv.second); + } + } + else + { + _encoder.write_key("OTIO_SCHEMA"); + _encoder.write_value(schema_str); + value->write_to(*this); + } _encoder.end_object(); @@ -794,7 +1109,7 @@ SerializableObject::Writer::write( _encoder.start_object(); - for (auto e: value) + for (const auto& e: value) { write(e.first, e.second); } @@ -810,7 +1125,7 @@ SerializableObject::Writer::write( _encoder.start_array(value.size()); - for (auto e: value) + for (const auto& e: value) { write(_no_key, e); } @@ -819,7 +1134,9 @@ SerializableObject::Writer::write( } void -SerializableObject::Writer::write(std::string const& key, any const& value) +SerializableObject::Writer::write( + std::string const& key, + any const& value) { std::type_info const& type = value.type(); @@ -829,19 +1146,24 @@ SerializableObject::Writer::write(std::string const& key, any const& value) if (e == _write_dispatch_table.end()) { /* - * Using the address of a type_info suffers from aliasing across compilation units. - * If we fail on a lookup, we fallback on the by_name table, but that's slow because - * we have to keep making a string each time. + * Using the address of a type_info suffers from aliasing across + * compilation units. If we fail on a lookup, we fallback on the + * by_name table, but that's slow because we have to keep making a + * string each time. * - * So when we fail, we insert the address of the type_info that failed to be found, - * so that we'll catch it the next time. This ensures we fail exactly once per alias - * per type while using this writer. + * So when we fail, we insert the address of the type_info that failed + * to be found, so that we'll catch it the next time. This ensures we + * fail exactly once per alias per type while using this writer. */ - auto backup_e = _write_dispatch_table_by_name.find(type.name()); + const auto& backup_e = _write_dispatch_table_by_name.find(type.name()); if (backup_e != _write_dispatch_table_by_name.end()) { - _write_dispatch_table[&type] = backup_e->second; - e = _write_dispatch_table.find(&type); + e = _write_dispatch_table.insert( + { + &type, + backup_e->second + } + ).first; } } @@ -885,9 +1207,13 @@ SerializableObject::is_equivalent_to(SerializableObject const& other) const return false; } - CloningEncoder e1(false), e2(false); - SerializableObject::Writer w1(e1); - SerializableObject::Writer w2(e2); + const auto policy = ( + CloningEncoder::ResultObjectPolicy::MathTypesConcreteAnyDictionaryResult + ); + + CloningEncoder e1(policy), e2(policy); + SerializableObject::Writer w1(e1, {}); + SerializableObject::Writer w2(e2, {}); w1.write(w1._no_key, any(Retainer<>(this))); w2.write(w2._no_key, any(Retainer<>(&other))); @@ -900,8 +1226,10 @@ SerializableObject::is_equivalent_to(SerializableObject const& other) const SerializableObject* SerializableObject::clone(ErrorStatus* error_status) const { - CloningEncoder e(true /* actually_clone*/); - SerializableObject::Writer w(e); + CloningEncoder e( + CloningEncoder::ResultObjectPolicy::CloneBackToSerializableObject + ); + SerializableObject::Writer w(e, {}); w.write(w._no_key, any(Retainer<>(this))); if (e.has_errored(error_status)) @@ -924,59 +1252,58 @@ SerializableObject::clone(ErrorStatus* error_status) const : nullptr; } + +// to json_string std::string serialize_json_to_string( - any const& value, ErrorStatus* error_status, int indent) + const any& value, + const schema_version_map* schema_version_targets, + ErrorStatus* error_status, + int indent +) { - OTIO_rapidjson::StringBuffer s; + OTIO_rapidjson::StringBuffer output_string_buffer; + + OTIO_rapidjson::PrettyWriter< + decltype(output_string_buffer), + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::CrtAllocator, + OTIO_rapidjson::kWriteNanAndInfFlag> + json_writer(output_string_buffer); - if (indent < 0) + if (indent >= 0) { - OTIO_rapidjson::Writer< - decltype(s), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(s); - JSONEncoder json_encoder(json_writer); - - if (!SerializableObject::Writer::write_root( - value, json_encoder, error_status)) - { - return std::string(); - } + json_writer.SetIndent(' ', indent); } - else - { - OTIO_rapidjson::PrettyWriter< - decltype(s), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(s); - JSONEncoder json_encoder(json_writer); + JSONEncoder json_encoder(json_writer); - json_writer.SetIndent(' ', indent); - if (!SerializableObject::Writer::write_root( - value, json_encoder, error_status)) - { - return std::string(); - } + if ( + !SerializableObject::Writer::write_root( + value, + json_encoder, + schema_version_targets, + error_status + ) + ) + { + return std::string(); } - return std::string(s.GetString()); + return std::string(output_string_buffer.GetString()); + } bool serialize_json_to_file( any const& value, std::string const& file_name, + const schema_version_map* schema_version_targets, ErrorStatus* error_status, int indent) { + #if defined(_WINDOWS) const int wlen = MultiByteToWideChar(CP_UTF8, 0, file_name.c_str(), -1, NULL, 0); @@ -986,6 +1313,7 @@ serialize_json_to_file( #else // _WINDOWS std::ofstream os(file_name); #endif // _WINDOWS + if (!os.is_open()) { if (error_status) @@ -999,36 +1327,41 @@ serialize_json_to_file( OTIO_rapidjson::OStreamWrapper osw(os); bool status; - if (indent < 0) - { - OTIO_rapidjson::Writer< - decltype(osw), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(osw); - JSONEncoder json_encoder(json_writer); - status = SerializableObject::Writer::write_root( - value, json_encoder, error_status); - } - else - { - OTIO_rapidjson::PrettyWriter< - decltype(osw), - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::UTF8<>, - OTIO_rapidjson::CrtAllocator, - OTIO_rapidjson::kWriteNanAndInfFlag> - json_writer(osw); - JSONEncoder json_encoder(json_writer); + OTIO_rapidjson::PrettyWriter< + decltype(osw), + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::UTF8<>, + OTIO_rapidjson::CrtAllocator, + OTIO_rapidjson::kWriteNanAndInfFlag> + json_writer(osw); + JSONEncoder json_encoder(json_writer); + if (indent >= 0) + { json_writer.SetIndent(' ', indent); - status = SerializableObject::Writer::write_root( - value, json_encoder, error_status); } + status = SerializableObject::Writer::write_root( + value, + json_encoder, + schema_version_targets, + error_status + ); + return status; } + +SerializableObject::Writer::~Writer() +{ + if (_child_writer) + { + delete _child_writer; + } + if (_child_cloning_encoder) + { + delete _child_cloning_encoder; + } +} + }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/serialization.h b/src/opentimelineio/serialization.h index e716740a3..aa4772435 100644 --- a/src/opentimelineio/serialization.h +++ b/src/opentimelineio/serialization.h @@ -6,18 +6,30 @@ #include "opentimelineio/any.h" #include "opentimelineio/errorStatus.h" #include "opentimelineio/version.h" +#include "opentimelineio/typeRegistry.h" +#include #include +#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { -std::string serialize_json_to_string( - const any& value, ErrorStatus* error_status = nullptr, int indent = 4); -bool serialize_json_to_file( +std::string +serialize_json_to_string( + const any& value, + const schema_version_map* schema_version_targets = nullptr, + ErrorStatus* error_status = nullptr, + int indent = 4 +); + +bool +serialize_json_to_file( const any& value, std::string const& file_name, + const schema_version_map* schema_version_targets = nullptr, ErrorStatus* error_status = nullptr, - int indent = 4); + int indent = 4 +); }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/track.cpp b/src/opentimelineio/track.cpp index 9118340a5..44e027666 100644 --- a/src/opentimelineio/track.cpp +++ b/src/opentimelineio/track.cpp @@ -135,7 +135,7 @@ TimeRange Track::available_range(ErrorStatus* error_status) const { RationalTime duration; - for (auto child: children()) + for (const auto& child: children()) { if (auto item = dynamic_retainer_cast(child)) { @@ -261,7 +261,7 @@ Track::range_of_all_children(ErrorStatus* error_status) const } RationalTime last_end_time(0, rate); - for (auto child: children()) + for (const auto& child: children()) { if (auto transition = dynamic_retainer_cast(child)) { @@ -300,7 +300,7 @@ Track::available_image_bounds(ErrorStatus* error_status) const { optional box; bool found_first_clip = false; - for (auto child: children()) + for (const auto& child: children()) { if (auto clip = dynamic_cast(child.value)) { diff --git a/src/opentimelineio/typeRegistry.cpp b/src/opentimelineio/typeRegistry.cpp index 4aca5a9fa..f75229c3e 100644 --- a/src/opentimelineio/typeRegistry.cpp +++ b/src/opentimelineio/typeRegistry.cpp @@ -3,6 +3,7 @@ #include "opentimelineio/typeRegistry.h" +#include "anyDictionary.h" #include "opentimelineio/clip.h" #include "opentimelineio/composable.h" #include "opentimelineio/composition.h" @@ -30,8 +31,6 @@ #include #include -//#include -//#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { @@ -113,6 +112,37 @@ TypeRegistry::TypeRegistry() d->erase("media_reference"); }); + + // 2->1 + register_downgrade_function( + Clip::Schema::name, + 2, + [](AnyDictionary* d) + { + AnyDictionary mrefs; + std::string active_rkey = ""; + + if (d->get_if_set("media_references", &mrefs)) + { + if ( + d->get_if_set( + "active_media_reference_key", + &active_rkey + ) + ) + { + AnyDictionary active_ref; + if (mrefs.get_if_set(active_rkey, &active_ref)) + { + (*d)["media_reference"] = active_ref; + } + } + } + + d->erase("media_references"); + d->erase("active_media_reference_key"); + } + ); } bool @@ -125,6 +155,25 @@ TypeRegistry::register_type( { std::lock_guard lock(_registry_mutex); + // auto existing_tr = _find_type_record(schema_name); + // + // // if the exact type record has already been added (happens in unit tests + // // and re-setting manifest stuff) + // if (existing_tr) + // { + // if ( + // existing_tr->schema_name == schema_name + // && existing_tr->schema_version == schema_version + // && existing_tr->class_name == class_name + // && ( + // existing_tr->create.target() + // == create.target() + // ) + // ) { + // return true; + // } + // } + if (!_find_type_record(schema_name)) { _TypeRecord* r = @@ -187,12 +236,34 @@ TypeRegistry::register_upgrade_function( std::lock_guard lock(_registry_mutex); if (auto r = _find_type_record(schema_name)) { - if (r->upgrade_functions.find(version_to_upgrade_to) == - r->upgrade_functions.end()) - { - r->upgrade_functions[version_to_upgrade_to] = upgrade_function; - return true; - } + auto result = r->upgrade_functions.insert( + { + version_to_upgrade_to, + upgrade_function + } + ); + return result.second; + } + + return false; +} + +bool +TypeRegistry::register_downgrade_function( + std::string const& schema_name, + int version_to_downgrade_from, + std::function downgrade_function) +{ + std::lock_guard lock(_registry_mutex); + if (auto r = _find_type_record(schema_name)) + { + auto result = r->downgrade_functions.insert( + { + version_to_downgrade_from, + downgrade_function + } + ); + return result.second; } return false; @@ -250,7 +321,7 @@ TypeRegistry::_instance_from_schema( } else if (schema_version < type_record->schema_version) { - for (auto e: type_record->upgrade_functions) + for (const auto& e: type_record->upgrade_functions) { if (schema_version <= e.first && e.first <= type_record->schema_version) @@ -327,4 +398,16 @@ TypeRegistry::set_type_record( return false; } +void +TypeRegistry::type_version_map( + schema_version_map& result) +{ + std::lock_guard lock(_registry_mutex); + + for (const auto& pair: _type_records) { + const auto record_ptr = pair.second; + result[record_ptr->schema_name] = record_ptr->schema_version; + } +} + }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/opentimelineio/typeRegistry.h b/src/opentimelineio/typeRegistry.h index 20cf90e8f..47ec0e14e 100644 --- a/src/opentimelineio/typeRegistry.h +++ b/src/opentimelineio/typeRegistry.h @@ -6,17 +6,28 @@ #include "opentimelineio/any.h" #include "opentimelineio/errorStatus.h" #include "opentimelineio/version.h" + #include #include -#include #include #include +#include +#include namespace opentimelineio { namespace OPENTIMELINEIO_VERSION { class SerializableObject; +class Encoder; class AnyDictionary; +// typedefs for the schema downgrading system +// @TODO: should we make version an int64_t? That would match what we can +// serialize natively, since we only serialize 64 bit signed ints. +using schema_version_map = std::unordered_map; +using label_to_schema_version_map = std::unordered_map; + +extern const label_to_schema_version_map CORE_VERSION_MAP; + class TypeRegistry { public: @@ -93,6 +104,24 @@ class TypeRegistry CLASS::schema_name, version_to_upgrade_to, upgrade_function); } + /// Downgrade function from version_to_downgrade_from to + /// version_to_downgrade_from - 1 + bool register_downgrade_function( + std::string const& schema_name, + int version_to_downgrade_from, + std::function downgrade_function); + + /// Convenience API for C++ developers. See the documentation of the + /// non-templated register_downgrade_function() for details. + template + bool register_downgrade_function( + int version_to_upgrade_to, + std::function upgrade_function) + { + return register_downgrade_function( + CLASS::schema_name, version_to_upgrade_to, upgrade_function); + } + SerializableObject* instance_from_schema( std::string const& schema_name, int schema_version, @@ -113,6 +142,9 @@ class TypeRegistry std::string const& schema_name, ErrorStatus* error_status = nullptr); + // for inspecting the type registry, build a map of schema name to version + void type_version_map(schema_version_map& result); + private: TypeRegistry(); @@ -127,6 +159,7 @@ class TypeRegistry std::function create; std::map> upgrade_functions; + std::map> downgrade_functions; _TypeRecord( std::string _schema_name, @@ -144,6 +177,7 @@ class TypeRegistry friend class TypeRegistry; friend class SerializableObject; + friend class CloningEncoder; }; // helper functions for lookup @@ -170,6 +204,7 @@ class TypeRegistry std::map _type_records_by_type_name; friend class SerializableObject; + friend class CloningEncoder; }; }} // namespace opentimelineio::OPENTIMELINEIO_VERSION diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp index ae8472c7f..706b79d34 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_bindings.cpp @@ -16,6 +16,9 @@ namespace py = pybind11; using namespace pybind11::literals; +// temporarily disabling this feature while I chew on it +const static bool EXCEPTION_ON_DOUBLE_REGISTER = false; + static void register_python_type(py::object class_object, std::string schema_name, int schema_version) { @@ -36,8 +39,34 @@ static void register_python_type(py::object class_object, return r.take_value(); }; - TypeRegistry::instance().register_type(schema_name, schema_version, - nullptr, create, schema_name); + + // @TODO: further discussion required about preventing double registering +#if 0 + if ( + !TypeRegistry::instance().register_type( + schema_name, + schema_version, + nullptr, + create, + schema_name + ) + && EXCEPTION_ON_DOUBLE_REGISTER + ) { + auto err = ErrorStatusHandler(); + err.error_status = ErrorStatus( + ErrorStatus::INTERNAL_ERROR, + "Schema: " + schema_name + " has already been registered." + ); + } +#else + TypeRegistry::instance().register_type( + schema_name, + schema_version, + nullptr, + create, + schema_name + ); +#endif } static bool register_upgrade_function(std::string const& schema_name, @@ -51,8 +80,77 @@ static bool register_upgrade_function(std::string const& schema_name, upgrade_function_obj(dobj); }; - return TypeRegistry::instance().register_upgrade_function(schema_name, version_to_upgrade_to, - upgrade_function); + // further discussion required about preventing double registering +#if 0 + if ( + !TypeRegistry::instance().register_upgrade_function( + schema_name, + version_to_upgrade_to, + upgrade_function + ) //&& EXCEPTION_ON_DOUBLE_REGISTER + ) + { + auto err = ErrorStatusHandler(); + err.error_status = ErrorStatus( + ErrorStatus::INTERNAL_ERROR, + "Upgrade function already exists for " + schema_name + ); + return false; + } + + return true; +#else + return TypeRegistry::instance().register_upgrade_function( + schema_name, + version_to_upgrade_to, + upgrade_function + ); +#endif +} + +static bool +register_downgrade_function( + std::string const& schema_name, + int version_to_downgrade_from, + py::object const& downgrade_function_obj) +{ + std::function downgrade_function = ( + [downgrade_function_obj](AnyDictionary* d) + { + py::gil_scoped_acquire acquire; + + auto ptr = d->get_or_create_mutation_stamp(); + py::object dobj = py::cast((AnyDictionaryProxy*)ptr); + downgrade_function_obj(dobj); + } + ); + + // further discussion required about preventing double registering +#if 0 + if ( + !TypeRegistry::instance().register_downgrade_function( + schema_name, + version_to_downgrade_from, + downgrade_function + ) //&& EXCEPTION_ON_DOUBLE_REGISTER + ) + { + auto err = ErrorStatusHandler(); + err.error_status = ErrorStatus( + ErrorStatus::INTERNAL_ERROR, + "Downgrade function already exists for " + schema_name + ); + return false; + } + return true; +#else + return TypeRegistry::instance().register_downgrade_function( + schema_name, + version_to_downgrade_from, + downgrade_function + ) ; +#endif + } static void set_type_record(SerializableObject* so, std::string schema_name) { @@ -76,26 +174,75 @@ PYBIND11_MODULE(_otio, m) { otio_serializable_object_bindings(m); otio_tests_bindings(m); - m.def("_serialize_json_to_string", - [](PyAny* pyAny, int indent) { - return serialize_json_to_string(pyAny->a, ErrorStatusHandler(), indent); - }, "value"_a, "indent"_a) + m.def( + "_serialize_json_to_string", + []( + PyAny* pyAny, + const schema_version_map& schema_version_targets, + int indent + ) + { + auto result = serialize_json_to_string( + pyAny->a, + &schema_version_targets, + ErrorStatusHandler(), + indent + ); + + return result; + }, + "value"_a, + "schema_version_targets"_a, + "indent"_a + ) .def("_serialize_json_to_file", - [](PyAny* pyAny, std::string filename, int indent) { - return serialize_json_to_file(pyAny->a, filename, ErrorStatusHandler(), indent); - }, "value"_a, "filename"_a, "indent"_a) + []( + PyAny* pyAny, + std::string filename, + const schema_version_map& schema_version_targets, + int indent + ) { + return serialize_json_to_file( + pyAny->a, + filename, + &schema_version_targets, + ErrorStatusHandler(), + indent + ); + }, + "value"_a, + "filename"_a, + "schema_version_targets"_a, + "indent"_a) .def("deserialize_json_from_string", [](std::string input) { any result; deserialize_json_from_string(input, &result, ErrorStatusHandler()); return any_to_py(result, true /*top_level*/); - }, "input"_a) + }, "input"_a, + R"docstring(Deserialize json string to in-memory objects. + +:param str input: json string to deserialize + +:returns: root object in the string (usually a Timeline or SerializableCollection) +:rtype: SerializableObject + +)docstring") .def("deserialize_json_from_file", [](std::string filename) { any result; deserialize_json_from_file(filename, &result, ErrorStatusHandler()); return any_to_py(result, true /*top_level*/); - }, "filename"_a); + }, + "filename"_a, + R"docstring(Deserialize json file to in-memory objects. + +:param str filename: path to json file to read + +:returns: root object in the file (usually a Timeline or SerializableCollection) +:rtype: SerializableObject + +)docstring"); py::class_(m, "PyAny") // explicitly map python bool, int and double classes so that they @@ -133,10 +280,35 @@ Return an instance of the schema from data in the data_dict. :raises UnsupportedSchemaError: when the requested schema version is greater than the registered schema version. )docstring"); + m.def("type_version_map", + []() { + schema_version_map tmp; + TypeRegistry::instance().type_version_map(tmp); + return tmp; + }, + R"docstring(Fetch the currently registered schemas and their versions. + +:returns: Map of all registered schema names to their current versions. +:rtype: dict[str, int])docstring" + ); m.def("register_upgrade_function", ®ister_upgrade_function, "schema_name"_a, "version_to_upgrade_to"_a, "upgrade_function"_a); + m.def("register_downgrade_function", ®ister_downgrade_function, + "schema_name"_a, + "version_to_downgrade_from"_a, + "downgrade_function"_a); + m.def( + "release_to_schema_version_map", + [](){ return label_to_schema_version_map(CORE_VERSION_MAP);}, + R"docstring(Fetch the compiled in CORE_VERSION_MAP. + +The CORE_VERSION_MAP maps OTIO release versions to maps of schema name to schema version and is autogenerated by the OpenTimelineIO build and release system. For example: `{"0.15.0": {"Clip": 2, ...}}` + +:returns: dictionary mapping core version label to schema_version_map +:rtype: dict[str, dict[str, int]])docstring" + ); m.def("flatten_stack", [](Stack* s) { return flatten_stack(s, ErrorStatusHandler()); }, "in_stack"_a); diff --git a/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp b/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp index ffa8616d5..76b496e5a 100644 --- a/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp +++ b/src/py-opentimelineio/opentimelineio-bindings/otio_serializableObjects.cpp @@ -163,10 +163,10 @@ static void define_bases1(py::module m) { .def("clone", [](SerializableObject* so) { return so->clone(ErrorStatusHandler()); }) .def("to_json_string", [](SerializableObject* so, int indent) { - return so->to_json_string(ErrorStatusHandler(), indent); }, + return so->to_json_string(ErrorStatusHandler(), {}, indent); }, "indent"_a = 4) .def("to_json_file", [](SerializableObject* so, std::string file_name, int indent) { - return so->to_json_file(file_name, ErrorStatusHandler(), indent); }, + return so->to_json_file(file_name, ErrorStatusHandler(), {}, indent); }, "file_name"_a, "indent"_a = 4) .def_static("from_json_file", [](std::string file_name) { diff --git a/src/py-opentimelineio/opentimelineio/__init__.py b/src/py-opentimelineio/opentimelineio/__init__.py index 3d3bab5fb..036907f4f 100644 --- a/src/py-opentimelineio/opentimelineio/__init__.py +++ b/src/py-opentimelineio/opentimelineio/__init__.py @@ -22,5 +22,6 @@ adapters, hooks, algorithms, - url_utils + url_utils, + versioning, ) diff --git a/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json b/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json index 0662e7ba2..4525f5392 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json +++ b/src/py-opentimelineio/opentimelineio/adapters/builtin_adapters.plugin_manifest.json @@ -49,5 +49,7 @@ "post_media_linker" : [], "pre_adapter_write" : [], "post_adapter_write" : [] + }, + "version_manifests": { } } diff --git a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py index 3db34c108..4bcffdb08 100644 --- a/src/py-opentimelineio/opentimelineio/adapters/otio_json.py +++ b/src/py-opentimelineio/opentimelineio/adapters/otio_json.py @@ -1,14 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project -"""This adapter lets you read and write native .otio files""" +"""Adapter for reading and writing native .otio json files.""" from .. import ( - core + core, + versioning, + exceptions ) +import os -# @TODO: Implement out of process plugins that hand around JSON +_DEFAULT_VERSION_ENVVAR = "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" def read_from_file(filepath): @@ -37,7 +40,39 @@ def read_from_string(input_str): return core.deserialize_json_from_string(input_str) -def write_to_string(input_otio, indent=4): +def _fetch_downgrade_map_from_env(): + version_envvar = os.environ[_DEFAULT_VERSION_ENVVAR] + + try: + family, label = version_envvar.split(":") + except ValueError: + raise exceptions.InvalidEnvironmentVariableError( + "Environment variable '{}' is incorrectly formatted with '{}'." + "Variable must be formatted as 'FAMILY:LABEL'".format( + _DEFAULT_VERSION_ENVVAR, + version_envvar, + ) + ) + + try: + # technically fetch_map returns an AnyDictionary, but the pybind11 + # code wrapping the call to the serializer expects a python + # dictionary. This turns it back into a normal dictionary. + return dict(versioning.fetch_map(family, label)) + except KeyError: + raise exceptions.InvalidEnvironmentVariableError( + "Environment variable '{}' is requesting family '{}' and label" + " '{}', however this combination does not exist in the " + "currently loaded manifests. Full version map: {}".format( + _DEFAULT_VERSION_ENVVAR, + family, + label, + versioning.full_map() + ) + ) + + +def write_to_string(input_otio, target_schema_versions=None, indent=4): """ Serializes an OpenTimelineIO object into a string @@ -46,13 +81,39 @@ def write_to_string(input_otio, indent=4): indent (int): number of spaces for each json indentation level. Use\ -1 for no indentation or newlines. + If target_schema_versions is None and the environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". + Returns: str: A json serialized string representation - """ - return core.serialize_json_to_string(input_otio, indent) + Raises: + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". + """ -def write_to_file(input_otio, filepath, indent=4): + if ( + target_schema_versions is None + and _DEFAULT_VERSION_ENVVAR in os.environ + ): + target_schema_versions = _fetch_downgrade_map_from_env() + + return core.serialize_json_to_string( + input_otio, + target_schema_versions, + indent + ) + + +def write_to_file( + input_otio, + filepath, + target_schema_versions=None, + indent=4 +): """ Serializes an OpenTimelineIO object into a file @@ -63,10 +124,30 @@ def write_to_file(input_otio, filepath, indent=4): indent (int): number of spaces for each json indentation level.\ Use -1 for no indentation or newlines. + If target_schema_versions is None and the environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" is set, will read a map out of + that for downgrade target. The variable should be of the form + FAMILY:LABEL, for example "MYSTUDIO:JUNE2022". + Returns: bool: Write success Raises: ValueError: on write error + otio.exceptions.InvalidEnvironmentVariableError: if there is a problem + with the default environment variable + "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL". """ - return core.serialize_json_to_file(input_otio, filepath, indent) + + if ( + target_schema_versions is None + and _DEFAULT_VERSION_ENVVAR in os.environ + ): + target_schema_versions = _fetch_downgrade_map_from_env() + + return core.serialize_json_to_file( + input_otio, + filepath, + target_schema_versions, + indent + ) diff --git a/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py new file mode 100644 index 000000000..a77d3616e --- /dev/null +++ b/src/py-opentimelineio/opentimelineio/console/autogen_version_map.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the OpenTimelineIO project + +"""Generate the CORE_VERSION_MAP for this version of OTIO""" + +import argparse +import tempfile + +import opentimelineio as otio + + +LABEL_MAP_TEMPLATE = """{{ "{label}", + {{ +{sv_map} + }} + }}, + // {{next}}""" +MAP_ITEM_TEMPLATE = '{indent}{{ "{key}", {value} }},' +INDENT = 12 + + +def _parsed_args(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "-d", + "--dryrun", + default=False, + action="store_true", + help="write to stdout instead of printing to file." + ) + parser.add_argument( + "-l", + "--label", + default=otio.__version__, + # @TODO - should we strip the .dev1 label? that would probably be + # more consistent since we don't do sub-beta releases + help="Version label to assign this schema map to." + ) + parser.add_argument( + "-i", + "--input", + type=str, + default=None, + required=True, + help="Path to CORE_VERSION_MAP.last.cpp" + ) + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help="Path to where CORE_VERSION_MAP.cpp should be written to." + ) + + return parser.parse_args() + + +def generate_core_version_map(src_text, label, version_map): + # turn the braces in the .cpp file into python-format template compatible + # form ({{ }} where needed) + src_text = src_text.replace("{", "{{").replace("}", "}}") + src_text = src_text.replace("// {{next}}", "{next}") + + # iterate over the map and print the template out + map_text = [] + for key, value in sorted(version_map.items()): + map_text.append( + MAP_ITEM_TEMPLATE.format( + indent=' ' * INDENT, + key=key, + value=value + ) + ) + map_text = '\n'.join(map_text) + + # assemble the result + next_text = LABEL_MAP_TEMPLATE.format(label=label, sv_map=map_text) + return src_text.format(label=label, next=next_text) + + +def main(): + args = _parsed_args() + + with open(args.input, 'r') as fi: + input = fi.read() + + result = generate_core_version_map( + input, + args.label, + otio.core.type_version_map() + ) + + if args.dryrun: + print(result) + return + + output = args.output + if not output: + output = tempfile.NamedTemporaryFile( + 'w', + suffix="CORE_VERSION_MAP.cpp", + delete=False + ).name + + with open(output, 'w', newline="\n") as fo: + fo.write(result) + + print("Wrote CORE_VERSION_MAP to: '{}'.".format(output)) + + +if __name__ == '__main__': + main() diff --git a/src/py-opentimelineio/opentimelineio/core/__init__.py b/src/py-opentimelineio/opentimelineio/core/__init__.py index 4aa455bba..922eec480 100644 --- a/src/py-opentimelineio/opentimelineio/core/__init__.py +++ b/src/py-opentimelineio/opentimelineio/core/__init__.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the OpenTimelineIO project +"""Core implementation details and wrappers around the C++ library""" + from .. _otio import ( # noqa # errors CannotComputeAvailableRangeError, @@ -22,9 +24,12 @@ instance_from_schema, register_serializable_object_type, register_upgrade_function, + register_downgrade_function, set_type_record, _serialize_json_to_string, _serialize_json_to_file, + type_version_map, + release_to_schema_version_map, ) from . _core_utils import ( # noqa @@ -54,28 +59,89 @@ 'flatten_stack', 'install_external_keepalive_monitor', 'instance_from_schema', - 'register_serializable_object_type', - 'register_upgrade_function', 'set_type_record', 'add_method', 'upgrade_function_for', + 'downgrade_function_from', 'serializable_field', 'deprecated_field', 'serialize_json_to_string', 'serialize_json_to_file', - 'register_type' + 'register_type', + 'type_version_map', + 'release_to_schema_version_map', ] -def serialize_json_to_string(root, indent=4): - return _serialize_json_to_string(_value_to_any(root), indent) +def serialize_json_to_string(root, schema_version_targets=None, indent=4): + """Serialize root to a json string. Optionally downgrade resulting schemas + to schema_version_targets. + :param SerializableObject root: root object to serialize + :param dict[str, int] schema_version_targets: optional dictionary mapping + schema name to desired schema + version, for downgrading the + result to be compatible with + older versions of + OpenTimelineIO. + :param int indent: number of spaces for each json indentation level. Use -1 + for no indentation or newlines. -def serialize_json_to_file(root, filename, indent=4): - return _serialize_json_to_file(_value_to_any(root), filename, indent) + :returns: resulting json string + :rtype: str + """ + return _serialize_json_to_string( + _value_to_any(root), + schema_version_targets or {}, + indent + ) + + +def serialize_json_to_file( + root, + filename, + schema_version_targets=None, + indent=4 +): + """Serialize root to a json file. Optionally downgrade resulting schemas + to schema_version_targets. + + :param SerializableObject root: root object to serialize + :param dict[str, int] schema_version_targets: optional dictionary mapping + schema name to desired schema + version, for downgrading the + result to be compatible with + older versions of + OpenTimelineIO. + :param int indent: number of spaces for each json indentation level. Use -1 + for no indentation or newlines. + + :returns: true for success, false for failure + :rtype: bool + """ + return _serialize_json_to_file( + _value_to_any(root), + filename, + schema_version_targets or {}, + indent + ) def register_type(classobj, schemaname=None): + """Decorator for registering a SerializableObject type + + Example: + + .. code-block:: python + + @otio.core.register_type + class SimpleClass(otio.core.SerializableObject): + serializable_label = "SimpleClass.2" + ... + + :param typing.Type[SerializableObject] cls: class to register + :param str schemaname: Schema name (default: parse from serializable_label) + """ label = classobj._serializable_label if schemaname is None: schema_name, schema_version = label.split(".", 2) @@ -116,7 +182,7 @@ def upgrade_to_version_five(data): that add or remove fields, only for schema versions that change the field names. - :param type cls: class to upgrade + :param typing.Type[SerializableObject] cls: class to upgrade :param int version_to_upgrade_to: the version to upgrade to """ @@ -134,9 +200,50 @@ def wrapped_update(data): return decorator_func +def downgrade_function_from(cls, version_to_downgrade_from): + """ + Decorator for identifying schema class downgrade functions. + + Example: + + .. code-block:: python + + @downgrade_function_from(MyClass, 5) + def downgrade_from_five_to_four(data): + return {"old_attr": data["new_attr"]} + + This will get called to downgrade a schema of MyClass from version 5 to + version 4. MyClass must be a class deriving from + :class:`~SerializableObject`. + + The downgrade function should take a single argument - the dictionary to + downgrade, and return a dictionary with the fields downgraded. + + :param typing.Type[SerializableObject] cls: class to downgrade + :param int version_to_downgrade_from: the function downgrading from this + version to (version - 1) + """ + + def decorator_func(func): + """ Decorator for marking downgrade functions """ + def wrapped_update(data): + modified = func(data) + data.clear() + data.update(modified) + + register_downgrade_function( + cls._serializable_label.split(".")[0], + version_to_downgrade_from, + wrapped_update + ) + return func + + return decorator_func + + def serializable_field(name, required_type=None, doc=None): """ - Convienence function for adding attributes to child classes of + Convenience function for adding attributes to child classes of :class:`~SerializableObject` in such a way that they will be serialized/deserialized automatically. diff --git a/src/py-opentimelineio/opentimelineio/exceptions.py b/src/py-opentimelineio/opentimelineio/exceptions.py index 32cb25cf5..964777756 100644 --- a/src/py-opentimelineio/opentimelineio/exceptions.py +++ b/src/py-opentimelineio/opentimelineio/exceptions.py @@ -75,3 +75,7 @@ class CannotTrimTransitionsError(OTIOError): class NoDefaultMediaLinkerError(OTIOError): pass + + +class InvalidEnvironmentVariableError(OTIOError): + pass diff --git a/src/py-opentimelineio/opentimelineio/plugins/manifest.py b/src/py-opentimelineio/opentimelineio/plugins/manifest.py index 6ef6e9bd1..3d612b943 100644 --- a/src/py-opentimelineio/opentimelineio/plugins/manifest.py +++ b/src/py-opentimelineio/opentimelineio/plugins/manifest.py @@ -32,6 +32,7 @@ 'schemadefs', 'hook_scripts', 'hooks', + 'version_manifests', ] @@ -90,6 +91,8 @@ def __init__(self): self.hooks = {} self.hook_scripts = [] + self.version_manifests = {} + adapters = core.serializable_field( "adapters", type([]), @@ -115,6 +118,11 @@ def __init__(self): type([]), "Scripts that can be attached to hooks." ) + version_manifests = core.serializable_field( + "version_manifests", + type({}), + "Sets of versions to downgrade schemas to." + ) def extend(self, another_manifest): """ @@ -131,7 +139,17 @@ def extend(self, another_manifest): self.media_linkers.extend(another_manifest.media_linkers) self.hook_scripts.extend(another_manifest.hook_scripts) + for family, label_map in another_manifest.version_manifests.items(): + # because self.version_manifests is an AnyDictionary instead of a + # vanilla python dictionary, it does not support the .set_default() + # method. + if family not in self.version_manifests: + self.version_manifests[family] = {} + self.version_manifests[family].update(label_map) + for trigger_name, hooks in another_manifest.hooks.items(): + # because self.hooks is an AnyDictionary instead of a vanilla + # python dictionary, it does not support the .set_default() method. if trigger_name not in self.hooks: self.hooks[trigger_name] = [] self.hooks[trigger_name].extend(hooks) diff --git a/src/py-opentimelineio/opentimelineio/versioning.py b/src/py-opentimelineio/opentimelineio/versioning.py new file mode 100644 index 000000000..4fd0e9126 --- /dev/null +++ b/src/py-opentimelineio/opentimelineio/versioning.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the OpenTimelineIO project + +"""Tools for fetching the family->label->schema:version maps""" + +import copy + +from . import ( + core, + plugins +) + + +def full_map(): + """Return the full map of schema version sets, including core and plugins. + Organized as follows: + + .. code-block:: python + + { + "FAMILY_NAME": { + "LABEL": { + "SchemaName": schemaversion, + "Clip": 2, + "Timeline": 3, + ... + } + } + } + + + The "OTIO_CORE" family is always provided and represents the built in + schemas defined in the C++ core. + IE: + + .. code-block:: python + + { + "OTIO_CORE": { + "0.15.0": { + "Clip": 2, + ... + } + } + } + + :returns: full map of schema version sets, including core and plugins + :rtype: dict[str, dict[str, dict[str, int]]] + """ + + result = copy.deepcopy(plugins.ActiveManifest().version_manifests) + result.update( + { + "OTIO_CORE": core.release_to_schema_version_map(), + } + ) + return result + + +def fetch_map(family, label): + """Fetch the version map for the given family and label. OpenTimelineIO + includes a built in family called "OTIO_CORE", this is compiled into the + C++ core and represents the core interchange schemas of OpenTimelineIO. + + Users may define more family/label/schema:version mappings by way of the + version manifest plugins. + + Returns a dictionary mapping Schema name to schema version, like: + + .. code-block:: python + + { + "Clip": 2, + "Timeline": 1, + ... + } + + :param str family: family of labels (ie: "OTIO_CORE") + :param str label: label of schema-version map (ie: "0.15.0") + :returns: a dictionary mapping Schema name to schema version + :rtype: dict[str, int] + """ + + if family == "OTIO_CORE": + src = core.release_to_schema_version_map() + else: + src = plugins.ActiveManifest().version_manifests[family] + + return copy.deepcopy(src[label]) diff --git a/tests/baselines/adapter_plugin_manifest.plugin_manifest.json b/tests/baselines/adapter_plugin_manifest.plugin_manifest.json index f3daafe0d..839dcadcc 100644 --- a/tests/baselines/adapter_plugin_manifest.plugin_manifest.json +++ b/tests/baselines/adapter_plugin_manifest.plugin_manifest.json @@ -23,5 +23,14 @@ "post_adapter_read" : [], "post_adapter_write" : ["post write example hook"], "post_media_linker" : ["example hook"] + }, + "version_manifests" : { + "TEST_FAMILY_NAME": { + "TEST_LABEL": { + "ExampleSchema":2, + "EnvVarTestSchema":1, + "Clip": 1 + } + } } } diff --git a/tests/test_adapter_plugin.py b/tests/test_adapter_plugin.py index 6e92b9583..5477fe2a1 100755 --- a/tests/test_adapter_plugin.py +++ b/tests/test_adapter_plugin.py @@ -222,7 +222,8 @@ def test_deduplicate_env_variable_paths(self): if bak_env is not None: os.environ['OTIO_PLUGIN_MANIFEST_PATH'] = bak_env else: - del os.environ['OTIO_PLUGIN_MANIFEST_PATH'] + if "OTIO_PLUGIN_MANIFEST_PATH" in os.environ: + del os.environ['OTIO_PLUGIN_MANIFEST_PATH'] def test_find_manifest_by_environment_variable(self): basename = "unittest.plugin_manifest.json" diff --git a/tests/test_builtin_adapters.py b/tests/test_builtin_adapters.py index 4fa707a27..8c9bb2cd0 100755 --- a/tests/test_builtin_adapters.py +++ b/tests/test_builtin_adapters.py @@ -68,6 +68,15 @@ def test_disk_vs_string(self): with open(temp_file, 'r') as f: on_disk = f.read() + self.maxDiff = None + + # for debugging + # with open("/var/tmp/in_memory.otio", "w") as fo: + # fo.write(in_memory) + # + # with open("/var/tmp/on_disk.otio", "w") as fo: + # fo.write(on_disk) + self.assertEqual(in_memory, on_disk) def test_adapters_fetch(self): diff --git a/tests/test_serializable_object.py b/tests/test_serializable_object.py index d9079bcd2..6c8bf6d09 100755 --- a/tests/test_serializable_object.py +++ b/tests/test_serializable_object.py @@ -7,6 +7,7 @@ import opentimelineio.test_utils as otio_test_utils import unittest +import json class OpenTimeTypeSerializerTest(unittest.TestCase): @@ -56,6 +57,7 @@ def test_copy_lib(self): # deep copy so_cp = copy.deepcopy(so) + self.assertIsNotNone(so_cp) self.assertIsOTIOEquivalentTo(so, so_cp) so_cp.metadata["foo"] = "bar" @@ -78,48 +80,6 @@ class Foo(otio.core.SerializableObjectWithMetadata): self.assertEqual(Foo, type(foo_copy)) - def test_schema_versioning(self): - @otio.core.register_type - class FakeThing(otio.core.SerializableObject): - _serializable_label = "Stuff.1" - foo_two = otio.core.serializable_field("foo_2", doc="test") - ft = FakeThing() - - self.assertEqual(ft.schema_name(), "Stuff") - self.assertEqual(ft.schema_version(), 1) - - with self.assertRaises(otio.exceptions.UnsupportedSchemaError): - otio.core.instance_from_schema( - "Stuff", - 2, - {"foo": "bar"} - ) - - ft = otio.core.instance_from_schema("Stuff", 1, {"foo": "bar"}) - self.assertEqual(ft._dynamic_fields['foo'], "bar") - - @otio.core.register_type - class FakeThing(otio.core.SerializableObject): - _serializable_label = "NewStuff.4" - foo_two = otio.core.serializable_field("foo_2") - - @otio.core.upgrade_function_for(FakeThing, 2) - def upgrade_one_to_two(_data_dict): - return {"foo_2": _data_dict["foo"]} - - @otio.core.upgrade_function_for(FakeThing, 3) - def upgrade_one_to_two_three(_data_dict): - return {"foo_3": _data_dict["foo_2"]} - - ft = otio.core.instance_from_schema("NewStuff", 1, {"foo": "bar"}) - self.assertEqual(ft._dynamic_fields['foo_3'], "bar") - - ft = otio.core.instance_from_schema("NewStuff", 3, {"foo_2": "bar"}) - self.assertEqual(ft._dynamic_fields['foo_3'], "bar") - - ft = otio.core.instance_from_schema("NewStuff", 4, {"foo_3": "bar"}) - self.assertEqual(ft._dynamic_fields['foo_3'], "bar") - def test_equality(self): o1 = otio.core.SerializableObject() o2 = otio.core.SerializableObject() @@ -177,5 +137,128 @@ def test_cycle_detection(self): o.clone() +class VersioningTests(unittest.TestCase, otio_test_utils.OTIOAssertions): + def test_schema_definition(self): + """define a schema and instantiate it from python""" + + # ensure that the type hasn't already been registered + self.assertNotIn("Stuff", otio.core.type_version_map()) + + @otio.core.register_type + class FakeThing(otio.core.SerializableObject): + _serializable_label = "Stuff.1" + foo_two = otio.core.serializable_field("foo_2", doc="test") + ft = FakeThing() + + self.assertEqual(ft.schema_name(), "Stuff") + self.assertEqual(ft.schema_version(), 1) + + with self.assertRaises(otio.exceptions.UnsupportedSchemaError): + otio.core.instance_from_schema( + "Stuff", + 2, + {"foo": "bar"} + ) + + version_map = otio.core.type_version_map() + self.assertEqual(version_map["Stuff"], 1) + + ft = otio.core.instance_from_schema("Stuff", 1, {"foo": "bar"}) + self.assertEqual(ft._dynamic_fields['foo'], "bar") + + @unittest.skip("@TODO: disabled pending discussion") + def test_double_register_schema(self): + @otio.core.register_type + class DoubleReg(otio.core.SerializableObject): + _serializable_label = "Stuff.1" + foo_two = otio.core.serializable_field("foo_2", doc="test") + _ = DoubleReg() # quiet pyflakes + + # not allowed to register a type twice + with self.assertRaises(ValueError): + @otio.core.register_type + class DoubleReg(otio.core.SerializableObject): + _serializable_label = "Stuff.1" + + def test_upgrade_versions(self): + """Test adding an upgrade functions for a type""" + + @otio.core.register_type + class FakeThing(otio.core.SerializableObject): + _serializable_label = "NewStuff.4" + foo_two = otio.core.serializable_field("foo_2") + + @otio.core.upgrade_function_for(FakeThing, 2) + def upgrade_one_to_two(_data_dict): + return {"foo_2": _data_dict["foo"]} + + @otio.core.upgrade_function_for(FakeThing, 3) + def upgrade_one_to_two_three(_data_dict): + return {"foo_3": _data_dict["foo_2"]} + + # @TODO: further discussion required + # not allowed to overwrite registered functions + # with self.assertRaises(ValueError): + # @otio.core.upgrade_function_for(FakeThing, 3) + # def upgrade_one_to_two_three_again(_data_dict): + # raise RuntimeError("shouldn't see this ever") + + ft = otio.core.instance_from_schema("NewStuff", 1, {"foo": "bar"}) + self.assertEqual(ft._dynamic_fields['foo_3'], "bar") + + ft = otio.core.instance_from_schema("NewStuff", 3, {"foo_2": "bar"}) + self.assertEqual(ft._dynamic_fields['foo_3'], "bar") + + ft = otio.core.instance_from_schema("NewStuff", 4, {"foo_3": "bar"}) + self.assertEqual(ft._dynamic_fields['foo_3'], "bar") + + def test_upgrade_rename(self): + """test that upgrading system handles schema renames correctly""" + + @otio.core.register_type + class FakeThingToRename(otio.core.SerializableObject): + _serializable_label = "ThingToRename.2" + my_field = otio.core.serializable_field("my_field", doc="example") + + thing = otio.core.type_version_map() + self.assertTrue(thing) + + def test_downgrade_version(self): + """ test a python defined downgrade function""" + + @otio.core.register_type + class FakeThing(otio.core.SerializableObject): + _serializable_label = "FakeThingToDowngrade.2" + foo_two = otio.core.serializable_field("foo_2") + + @otio.core.downgrade_function_from(FakeThing, 2) + def downgrade_2_to_1(_data_dict): + return {"foo": _data_dict["foo_2"]} + + # @TODO: further discussion required + # # not allowed to overwrite registered functions + # with self.assertRaises(ValueError): + # @otio.core.downgrade_function_from(FakeThing, 2) + # def downgrade_2_to_1_again(_data_dict): + # raise RuntimeError("shouldn't see this ever") + + f = FakeThing() + f.foo_two = "a thing here" + + downgrade_target = {"FakeThingToDowngrade": 1} + + result = json.loads( + otio.adapters.otio_json.write_to_string(f, downgrade_target) + ) + + self.assertDictEqual( + result, + { + "OTIO_SCHEMA": "FakeThingToDowngrade.1", + "foo": "a thing here", + } + ) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_serialized_schema.py b/tests/test_serialized_schema.py index 3dc76a917..7a486f29c 100644 --- a/tests/test_serialized_schema.py +++ b/tests/test_serialized_schema.py @@ -3,10 +3,13 @@ import unittest import os +import sys +import subprocess from opentimelineio.console import ( autogen_serialized_datamodel as asd, autogen_plugin_documentation as apd, + autogen_version_map as avm ) @@ -40,7 +43,7 @@ def test_serialized_schema(self): @unittest.skipIf( os.environ.get("OTIO_DISABLE_SERIALIZED_SCHEMA_TEST"), - "Serialized schema test disabled because " + "Plugin documentation test disabled because " "$OTIO_DISABLE_SERIALIZED_SCHEMA_TEST is set to something other than ''" ) class PluginDocumentationTester(unittest.TestCase): @@ -69,5 +72,62 @@ def test_plugin_documentation(self): ) +@unittest.skipIf( + os.environ.get("OTIO_DISABLE_SERIALIZED_SCHEMA_TEST"), + "CORE_VERSION_MAP generation test disabled because " + "$OTIO_DISABLE_SERIALIZED_SCHEMA_TEST is set to something other than ''" +) +class CoreVersionMapGenerationTester(unittest.TestCase): + def test_core_version_map_generator(self): + """Verify the current CORE_VERSION_MAP matches the checked in one.""" + + pt = os.path.dirname(os.path.dirname(__file__)) + root = os.path.join(pt, "src", "opentimelineio") + template_fp = os.path.join(root, "CORE_VERSION_MAP.last.cpp") + target_fp = os.path.join(root, "CORE_VERSION_MAP.cpp") + + with open(target_fp) as fi: + # sanitize line endings and remove empty lines for cross-windows + # /*nix consistent behavior + baseline_text = "\n".join( + ln + for ln in fi.read().splitlines() + if ln + ) + + proc = subprocess.Popen( + [ + sys.executable, + avm.__file__, + "-i", + template_fp, + "-d", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, _ = proc.communicate() + + test_text = stdout.decode("utf-8")[:-1] + + # sanitize the line endings + test_text = "\n".join( + ln + for ln in test_text.splitlines() + if ln + ) + + self.maxDiff = None + self.longMessage = True + self.assertMultiLineEqual( + baseline_text, + test_text, + "\n The CORE_VERSION_MAP has changed and the autogenerated one in" + " {} needs to be updated. run: `make version-map-update`".format( + target_fp + ) + ) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_version_manifest.py b/tests/test_version_manifest.py new file mode 100644 index 000000000..4fecd638e --- /dev/null +++ b/tests/test_version_manifest.py @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the OpenTimelineIO project + +"""unit tests for the version manifest plugin system""" + +import unittest +import os +import json + +import opentimelineio as otio +from tests import utils + + +FIRST_MANIFEST = """{ + "OTIO_SCHEMA" : "PluginManifest.1", + "version_manifests": { + "UNIQUE_FAMILY": { + "TEST_LABEL": { + "second_thing": 3 + } + }, + "LAYERED_FAMILY": { + "June2022": { + "SimpleClass": 2 + }, + "May2022": { + "SimpleClass": 1 + } + } + } +} +""" + +SECOND_MANIFEST = """{ + "OTIO_SCHEMA" : "PluginManifest.1", + "version_manifests": { + "LAYERED_FAMILY": { + "May2022": { + "SimpleClass": 2 + }, + "April2022": { + "SimpleClass": 1 + } + } + } +} +""" + + +class TestPlugin_VersionManifest(unittest.TestCase): + def setUp(self): + self.bak = otio.plugins.ActiveManifest() + self.man = utils.create_manifest() + otio.plugins.manifest._MANIFEST = self.man + + if "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" in os.environ: + del os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] + + def tearDown(self): + otio.plugins.manifest._MANIFEST = self.bak + utils.remove_manifest(self.man) + if "OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL" in os.environ: + del os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] + + def test_read_in_manifest(self): + self.assertIn("TEST_FAMILY_NAME", self.man.version_manifests) + self.assertIn( + "TEST_LABEL", + self.man.version_manifests["TEST_FAMILY_NAME"] + ) + + def test_full_map(self): + d = otio.versioning.full_map() + self.assertIn("TEST_FAMILY_NAME", d) + self.assertIn( + "TEST_LABEL", + d["TEST_FAMILY_NAME"] + ) + + def test_fetch_map(self): + self.assertEqual( + otio.versioning.fetch_map("TEST_FAMILY_NAME", "TEST_LABEL"), + {"ExampleSchema": 2, "EnvVarTestSchema": 1, "Clip": 1} + ) + + def test_env_variable_downgrade(self): + @otio.core.register_type + class EnvVarTestSchema(otio.core.SerializableObject): + _serializable_label = "EnvVarTestSchema.2" + foo_two = otio.core.serializable_field("foo_2") + + @otio.core.downgrade_function_from(EnvVarTestSchema, 2) + def downgrade_2_to_1(_data_dict): + return {"foo": _data_dict["foo_2"]} + + evt = EnvVarTestSchema() + evt.foo_two = "asdf" + + result = json.loads(otio.adapters.otio_json.write_to_string(evt)) + self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.2") + + # env variable should make a downgrade by default... + os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( + "TEST_FAMILY_NAME:TEST_LABEL" + ) + result = json.loads(otio.adapters.otio_json.write_to_string(evt)) + self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.1") + + # ...but can still be overridden by passing in an argument + result = json.loads(otio.adapters.otio_json.write_to_string(evt, {})) + self.assertEqual(result["OTIO_SCHEMA"], "EnvVarTestSchema.2") + + def test_garbage_env_variables(self): + cl = otio.schema.Clip() + invalid_env_error = otio.exceptions.InvalidEnvironmentVariableError + + # missing ":" + os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( + "invalid_formatting" + ) + with self.assertRaises(invalid_env_error): + otio.adapters.otio_json.write_to_string(cl) + + # asking for family/label that doesn't exist in the plugins + os.environ["OTIO_DEFAULT_TARGET_VERSION_FAMILY_LABEL"] = ( + "nosuch:labelorfamily" + ) + with self.assertRaises(invalid_env_error): + otio.adapters.otio_json.write_to_string(cl) + + def test_two_version_manifests(self): + """test that two manifests layer correctly""" + + fst = otio.plugins.manifest.manifest_from_string(FIRST_MANIFEST) + snd = otio.plugins.manifest.manifest_from_string(SECOND_MANIFEST) + fst.extend(snd) + + self.assertIn("UNIQUE_FAMILY", fst.version_manifests) + + lay_fam = fst.version_manifests["LAYERED_FAMILY"] + + self.assertIn("June2022", lay_fam) + self.assertIn("April2022", lay_fam) + self.assertEqual(lay_fam["May2022"]["SimpleClass"], 2) + + +if __name__ == '__main__': + unittest.main()