From 26aa7c07c24e5712b85191a75e4a724b65e4ed9d Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 26 Jun 2024 16:15:49 +0000 Subject: [PATCH 001/261] tests sync --- .../docker-test/ya.make | 26 +++++++++++++++++++ .../postgres_intergrations/library/docker.py | 0 .../postgres_intergrations/library/ya.make | 15 +++++++++++ ydb/tests/postgres_intergrations/ya.make | 4 +++ ydb/tests/ya.make | 1 + 5 files changed, 46 insertions(+) create mode 100644 ydb/tests/postgres_intergrations/docker-test/ya.make create mode 100644 ydb/tests/postgres_intergrations/library/docker.py create mode 100644 ydb/tests/postgres_intergrations/library/ya.make create mode 100644 ydb/tests/postgres_intergrations/ya.make diff --git a/ydb/tests/postgres_intergrations/docker-test/ya.make b/ydb/tests/postgres_intergrations/docker-test/ya.make new file mode 100644 index 000000000000..d8625049ad54 --- /dev/null +++ b/ydb/tests/postgres_intergrations/docker-test/ya.make @@ -0,0 +1,26 @@ +PY3TEST() + +FORK_TEST_FILES() +TIMEOUT(600) +SIZE(LARGE) + +ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") +DEPENDS( + ydb/apps/ydbd +) + +ALL_PYTEST_SRCS( + +) + +PEERDIR( + contrib/python/requests + contrib/python/tornado/tornado-4 + ydb/tests/library + ydb/tests/oss/ydb_sdk_import + ydb/public/sdk/python +) + +REQUIREMENTS(ram:10) + +END() diff --git a/ydb/tests/postgres_intergrations/library/docker.py b/ydb/tests/postgres_intergrations/library/docker.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/ydb/tests/postgres_intergrations/library/ya.make b/ydb/tests/postgres_intergrations/library/ya.make new file mode 100644 index 000000000000..4b35ca54046e --- /dev/null +++ b/ydb/tests/postgres_intergrations/library/ya.make @@ -0,0 +1,15 @@ +PY3TEST() + +ALL_PY_SRCS() + +PEERDIR( + contrib/python/requests + contrib/python/tornado/tornado-4 + ydb/tests/library + ydb/tests/oss/ydb_sdk_import + ydb/public/sdk/python +) + +REQUIREMENTS(ram:10) + +END() diff --git a/ydb/tests/postgres_intergrations/ya.make b/ydb/tests/postgres_intergrations/ya.make new file mode 100644 index 000000000000..32f99dfa427c --- /dev/null +++ b/ydb/tests/postgres_intergrations/ya.make @@ -0,0 +1,4 @@ +RECURSE( + docker-test + library +) diff --git a/ydb/tests/ya.make b/ydb/tests/ya.make index 2f828ced77aa..13c3aa5e7afc 100644 --- a/ydb/tests/ya.make +++ b/ydb/tests/ya.make @@ -6,6 +6,7 @@ RECURSE( olap oss perf + postgres_integrations supp tools ) From 8cb18b9db7ed059cd9b826db534c38c53cc7094e Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 9 Jul 2024 19:22:55 +0000 Subject: [PATCH 002/261] initialize test selector --- .../docker-test/ya.make | 26 ------------------- .../postgres_intergrations/library/docker.py | 0 .../postgres_intergrations/library/ya.make | 15 ----------- ydb/tests/postgres_intergrations/ya.make | 4 --- 4 files changed, 45 deletions(-) delete mode 100644 ydb/tests/postgres_intergrations/docker-test/ya.make delete mode 100644 ydb/tests/postgres_intergrations/library/docker.py delete mode 100644 ydb/tests/postgres_intergrations/library/ya.make delete mode 100644 ydb/tests/postgres_intergrations/ya.make diff --git a/ydb/tests/postgres_intergrations/docker-test/ya.make b/ydb/tests/postgres_intergrations/docker-test/ya.make deleted file mode 100644 index d8625049ad54..000000000000 --- a/ydb/tests/postgres_intergrations/docker-test/ya.make +++ /dev/null @@ -1,26 +0,0 @@ -PY3TEST() - -FORK_TEST_FILES() -TIMEOUT(600) -SIZE(LARGE) - -ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") -DEPENDS( - ydb/apps/ydbd -) - -ALL_PYTEST_SRCS( - -) - -PEERDIR( - contrib/python/requests - contrib/python/tornado/tornado-4 - ydb/tests/library - ydb/tests/oss/ydb_sdk_import - ydb/public/sdk/python -) - -REQUIREMENTS(ram:10) - -END() diff --git a/ydb/tests/postgres_intergrations/library/docker.py b/ydb/tests/postgres_intergrations/library/docker.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/ydb/tests/postgres_intergrations/library/ya.make b/ydb/tests/postgres_intergrations/library/ya.make deleted file mode 100644 index 4b35ca54046e..000000000000 --- a/ydb/tests/postgres_intergrations/library/ya.make +++ /dev/null @@ -1,15 +0,0 @@ -PY3TEST() - -ALL_PY_SRCS() - -PEERDIR( - contrib/python/requests - contrib/python/tornado/tornado-4 - ydb/tests/library - ydb/tests/oss/ydb_sdk_import - ydb/public/sdk/python -) - -REQUIREMENTS(ram:10) - -END() diff --git a/ydb/tests/postgres_intergrations/ya.make b/ydb/tests/postgres_intergrations/ya.make deleted file mode 100644 index 32f99dfa427c..000000000000 --- a/ydb/tests/postgres_intergrations/ya.make +++ /dev/null @@ -1,4 +0,0 @@ -RECURSE( - docker-test - library -) From 1ea36d7bce32e8b5dec4b7b1b67a439f8d7d9582 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 9 Jul 2024 19:25:08 +0000 Subject: [PATCH 003/261] initialize test selector --- .../postgres_integrations/library/__init__.py | 6 + .../postgres_integrations/library/docker.py | 2 + .../library/pytest_integration.py | 42 + .../postgres_integrations/library/ya.make | 16 + .../psycopg2/conftest.py | 19 + .../psycopg2/data/full-test-list.txt | 742 ++++++++++++++++++ .../psycopg2/data/unit-tests.txt | 114 +++ .../psycopg2/docker_wrapper_test.py | 4 + .../postgres_integrations/psycopg2/ya.make | 37 + 9 files changed, 982 insertions(+) create mode 100644 ydb/tests/postgres_integrations/library/__init__.py create mode 100644 ydb/tests/postgres_integrations/library/docker.py create mode 100644 ydb/tests/postgres_integrations/library/pytest_integration.py create mode 100644 ydb/tests/postgres_integrations/library/ya.make create mode 100644 ydb/tests/postgres_integrations/psycopg2/conftest.py create mode 100644 ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt create mode 100644 ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt create mode 100644 ydb/tests/postgres_integrations/psycopg2/docker_wrapper_test.py create mode 100644 ydb/tests/postgres_integrations/psycopg2/ya.make diff --git a/ydb/tests/postgres_integrations/library/__init__.py b/ydb/tests/postgres_integrations/library/__init__.py new file mode 100644 index 000000000000..56167a3b5eeb --- /dev/null +++ b/ydb/tests/postgres_integrations/library/__init__.py @@ -0,0 +1,6 @@ +__all__ = [ + 'IntegrationTests' +] + +from .pytest_integration import * + diff --git a/ydb/tests/postgres_integrations/library/docker.py b/ydb/tests/postgres_integrations/library/docker.py new file mode 100644 index 000000000000..b367352b91b0 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/docker.py @@ -0,0 +1,2 @@ +from typing import Optional + diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py new file mode 100644 index 000000000000..787c91a3882f --- /dev/null +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -0,0 +1,42 @@ +from typing import List, Set, Optional +from os import path + +import pytest + +class IntegrationTests: + _folder: str + _all_tests: Set[str] + _selected_test: Set[str] + + def __init__(self, folder): + self._folder = folder + + self._all_tests = _read_tests(folder) + self._selected_test = set(self._all_tests) + + def pytest_generate_tests(self, metafunc: pytest.Metafunc): + """ + Return tests for run through pytest. + """ + all_tests = list(self._all_tests) + all_tests.sort() + metafunc.parametrize('testname', all_tests, ids=all_tests) + + def pytest_deselected(self, items: List[pytest.Item]): + for item in items: + test_name = item.callspec.id + print("rekby-test: removed", test_name) + self._selected_test.remove(test_name) + + def execute_test(self, testname: str): + pass + +def _read_tests(folder: str) -> Set[str]: + with open(path.join(folder, "full-test-list.txt"), "rt") as f: + all = set(line.strip() for line in f.readlines()) + + with open(path.join(folder, "unit-tests.txt"), "rt") as f: + unit = set(f.readlines()) + + test_list_for_run = all - unit + return test_list_for_run diff --git a/ydb/tests/postgres_integrations/library/ya.make b/ydb/tests/postgres_integrations/library/ya.make new file mode 100644 index 000000000000..525effab05ef --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ya.make @@ -0,0 +1,16 @@ +PY3_LIBRARY() + + +ALL_PY_SRCS() + +PEERDIR( + contrib/python/requests + contrib/python/tornado/tornado-4 + ydb/tests/library + ydb/tests/oss/ydb_sdk_import + ydb/public/sdk/python +) + +REQUIREMENTS(ram:10) + +END() diff --git a/ydb/tests/postgres_integrations/psycopg2/conftest.py b/ydb/tests/postgres_integrations/psycopg2/conftest.py new file mode 100644 index 000000000000..7fecb16136d1 --- /dev/null +++ b/ydb/tests/postgres_integrations/psycopg2/conftest.py @@ -0,0 +1,19 @@ + +from os import path +from typing import Sequence + +import yatest + +import pytest + + +from ydb.tests.postgres_integrations.library import IntegrationTests + +integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/psycopg2/data")) + +def pytest_generate_tests(metafunc: pytest.Metafunc): + integrations.pytest_generate_tests(metafunc) + + +def pytest_deselected(items: Sequence[pytest.Item]): + integrations.pytest_deselected(items) diff --git a/ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt b/ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt new file mode 100644 index 000000000000..a778bb393f35 --- /dev/null +++ b/ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt @@ -0,0 +1,742 @@ +python-psycopg2/tests.test_async.AsyncTests.test_async_after_async +python-psycopg2/tests.test_async.AsyncTests.test_async_callproc +python-psycopg2/tests.test_async.AsyncTests.test_async_connection_error_message +python-psycopg2/tests.test_async.AsyncTests.test_async_cursor_gone +python-psycopg2/tests.test_async.AsyncTests.test_async_dont_read_all +python-psycopg2/tests.test_async.AsyncTests.test_async_executemany +python-psycopg2/tests.test_async.AsyncTests.test_async_fetch_wrong_cursor +python-psycopg2/tests.test_async.AsyncTests.test_async_iter +python-psycopg2/tests.test_async.AsyncTests.test_async_named_cursor +python-psycopg2/tests.test_async.AsyncTests.test_async_scroll +python-psycopg2/tests.test_async.AsyncTests.test_async_select +python-psycopg2/tests.test_async.AsyncTests.test_async_subclass +python-psycopg2/tests.test_async.AsyncTests.test_close +python-psycopg2/tests.test_async.AsyncTests.test_commit_while_async +python-psycopg2/tests.test_async.AsyncTests.test_connection_setup +python-psycopg2/tests.test_async.AsyncTests.test_copy_no_hang +python-psycopg2/tests.test_async.AsyncTests.test_copy_while_async +python-psycopg2/tests.test_async.AsyncTests.test_error +python-psycopg2/tests.test_async.AsyncTests.test_error_two_cursors +python-psycopg2/tests.test_async.AsyncTests.test_fetch_after_async +python-psycopg2/tests.test_async.AsyncTests.test_flush_on_write +python-psycopg2/tests.test_async.AsyncTests.test_lobject_while_async +python-psycopg2/tests.test_async.AsyncTests.test_non_block_after_notification +python-psycopg2/tests.test_async.AsyncTests.test_notices +python-psycopg2/tests.test_async.AsyncTests.test_notify +python-psycopg2/tests.test_async.AsyncTests.test_poll_conn_for_notification +python-psycopg2/tests.test_async.AsyncTests.test_poll_noop +python-psycopg2/tests.test_async.AsyncTests.test_reset_while_async +python-psycopg2/tests.test_async.AsyncTests.test_rollback_while_async +python-psycopg2/tests.test_async.AsyncTests.test_scroll +python-psycopg2/tests.test_async.AsyncTests.test_set_parameters_while_async +python-psycopg2/tests.test_async.AsyncTests.test_stop_on_first_error +python-psycopg2/tests.test_async.AsyncTests.test_sync_poll +python-psycopg2/tests.test_bugX000.DateTimeAllocationBugTestCase.test_date_time_allocation_bug +python-psycopg2/tests.test_bug_gc.StolenReferenceTestCase.test_stolen_reference_bug +python-psycopg2/tests.test_cancel.CancelTests.test_async_cancel +python-psycopg2/tests.test_cancel.CancelTests.test_cancel +python-psycopg2/tests.test_cancel.CancelTests.test_empty_cancel +python-psycopg2/tests.test_connection.AutocommitTests.test_closed +python-psycopg2/tests.test_connection.AutocommitTests.test_default_no_autocommit +python-psycopg2/tests.test_connection.AutocommitTests.test_set_autocommit +python-psycopg2/tests.test_connection.AutocommitTests.test_set_intrans_error +python-psycopg2/tests.test_connection.AutocommitTests.test_set_session_autocommit +python-psycopg2/tests.test_connection.ConnectionTests.test_cleanup_on_badconn_close +python-psycopg2/tests.test_connection.ConnectionTests.test_close_idempotent +python-psycopg2/tests.test_connection.ConnectionTests.test_closed_attribute +python-psycopg2/tests.test_connection.ConnectionTests.test_commit_concurrency +python-psycopg2/tests.test_connection.ConnectionTests.test_concurrent_execution +python-psycopg2/tests.test_connection.ConnectionTests.test_connect_cursor_factory +python-psycopg2/tests.test_connection.ConnectionTests.test_connect_no_string +python-psycopg2/tests.test_connection.ConnectionTests.test_connect_nonnormal_envvar +python-psycopg2/tests.test_connection.ConnectionTests.test_cursor_closed_attribute +python-psycopg2/tests.test_connection.ConnectionTests.test_cursor_factory +python-psycopg2/tests.test_connection.ConnectionTests.test_cursor_factory_none +python-psycopg2/tests.test_connection.ConnectionTests.test_encoding_name +python-psycopg2/tests.test_connection.ConnectionTests.test_failed_init_status +python-psycopg2/tests.test_connection.ConnectionTests.test_get_native_connection +python-psycopg2/tests.test_connection.ConnectionTests.test_multiprocess_close +python-psycopg2/tests.test_connection.ConnectionTests.test_notices +python-psycopg2/tests.test_connection.ConnectionTests.test_notices_consistent_order +python-psycopg2/tests.test_connection.ConnectionTests.test_notices_deque +python-psycopg2/tests.test_connection.ConnectionTests.test_notices_limited +python-psycopg2/tests.test_connection.ConnectionTests.test_notices_noappend +python-psycopg2/tests.test_connection.ConnectionTests.test_pgconn_ptr +python-psycopg2/tests.test_connection.ConnectionTests.test_protocol_version +python-psycopg2/tests.test_connection.ConnectionTests.test_reset +python-psycopg2/tests.test_connection.ConnectionTests.test_server_version +python-psycopg2/tests.test_connection.ConnectionTests.test_tpc_unsupported +python-psycopg2/tests.test_connection.ConnectionTests.test_weakref +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_cancel_fails_prepared +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_recovered_xids +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_status_after_recover +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_commit +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_commit_one_phase +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_commit_recovered +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_recover_non_dbapi_connection +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_rollback +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_rollback_one_phase +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_rollback_recovered +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_unparsed_roundtrip +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_construction +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_encoding +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_from_string +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_roundtrip +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_to_string +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_unicode +python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_unicode_unparsed +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_attribs_segfault +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_encoding +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_autocommit +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_closed +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_read_committed +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_serializable +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level_abort +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level_autocommit +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level_default +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_setattr_isolation_level_int +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_setattr_isolation_level_invalid +python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_setattr_isolation_level_str +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_database_is_a_keyword +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_arguments +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_param +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_string +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_escape +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_get_dsn_parameters +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_no_dsn_munging +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_null_args +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_merging +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_validation +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_url_is_cool +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_bad_param +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn_uri +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_str_subclass +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_key +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_value +python-psycopg2/tests.test_connection.PasswordLeakTestCase.test_leak +python-psycopg2/tests.test_connection.PasswordLeakTestCase.test_url_leak +python-psycopg2/tests.test_connection.SignalTestCase.test_bug_551_no_returning +python-psycopg2/tests.test_connection.SignalTestCase.test_bug_551_returning +python-psycopg2/tests.test_connection.TestConnectionInfo.test_backend_pid +python-psycopg2/tests.test_connection.TestConnectionInfo.test_dbname +python-psycopg2/tests.test_connection.TestConnectionInfo.test_dsn_parameters +python-psycopg2/tests.test_connection.TestConnectionInfo.test_error_message +python-psycopg2/tests.test_connection.TestConnectionInfo.test_host +python-psycopg2/tests.test_connection.TestConnectionInfo.test_host_readonly +python-psycopg2/tests.test_connection.TestConnectionInfo.test_needs_password +python-psycopg2/tests.test_connection.TestConnectionInfo.test_options +python-psycopg2/tests.test_connection.TestConnectionInfo.test_parameter_status +python-psycopg2/tests.test_connection.TestConnectionInfo.test_password +python-psycopg2/tests.test_connection.TestConnectionInfo.test_port +python-psycopg2/tests.test_connection.TestConnectionInfo.test_protocol_version +python-psycopg2/tests.test_connection.TestConnectionInfo.test_server_version +python-psycopg2/tests.test_connection.TestConnectionInfo.test_socket +python-psycopg2/tests.test_connection.TestConnectionInfo.test_ssl_attribute +python-psycopg2/tests.test_connection.TestConnectionInfo.test_ssl_in_use +python-psycopg2/tests.test_connection.TestConnectionInfo.test_ssl_not_supported +python-psycopg2/tests.test_connection.TestConnectionInfo.test_status +python-psycopg2/tests.test_connection.TestConnectionInfo.test_transaction_status +python-psycopg2/tests.test_connection.TestConnectionInfo.test_used_password +python-psycopg2/tests.test_connection.TestConnectionInfo.test_user +python-psycopg2/tests.test_connection.TestEncryptPassword.test_bad_types +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_bad_before_libpq_10 +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_bad_libpq_10 +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_md5 +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_password_post_9_6 +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_scram +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_scram_pre_10 +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_server +python-psycopg2/tests.test_connection.TransactionControlTests.test_bad_isolation_level +python-psycopg2/tests.test_connection.TransactionControlTests.test_closed +python-psycopg2/tests.test_connection.TransactionControlTests.test_idempotence_check +python-psycopg2/tests.test_connection.TransactionControlTests.test_mixing_session_attribs +python-psycopg2/tests.test_connection.TransactionControlTests.test_not_in_transaction +python-psycopg2/tests.test_connection.TransactionControlTests.test_set_default +python-psycopg2/tests.test_connection.TransactionControlTests.test_set_deferrable +python-psycopg2/tests.test_connection.TransactionControlTests.test_set_deferrable_error +python-psycopg2/tests.test_connection.TransactionControlTests.test_set_isolation_level +python-psycopg2/tests.test_connection.TransactionControlTests.test_set_isolation_level_str +python-psycopg2/tests.test_connection.TransactionControlTests.test_set_read_only +python-psycopg2/tests.test_connection.TransactionControlTests.test_setattr_deferrable +python-psycopg2/tests.test_connection.TransactionControlTests.test_setattr_read_only +python-psycopg2/tests.test_copy.CopyTests.test_copy_bytes +python-psycopg2/tests.test_copy.CopyTests.test_copy_expert_file_refcount +python-psycopg2/tests.test_copy.CopyTests.test_copy_expert_textiobase +python-psycopg2/tests.test_copy.CopyTests.test_copy_from +python-psycopg2/tests.test_copy.CopyTests.test_copy_from_cols +python-psycopg2/tests.test_copy.CopyTests.test_copy_from_cols_err +python-psycopg2/tests.test_copy.CopyTests.test_copy_from_insane_size +python-psycopg2/tests.test_copy.CopyTests.test_copy_from_propagate_error +python-psycopg2/tests.test_copy.CopyTests.test_copy_from_segfault +python-psycopg2/tests.test_copy.CopyTests.test_copy_funny_names +python-psycopg2/tests.test_copy.CopyTests.test_copy_no_column_limit +python-psycopg2/tests.test_copy.CopyTests.test_copy_query +python-psycopg2/tests.test_copy.CopyTests.test_copy_rowcount +python-psycopg2/tests.test_copy.CopyTests.test_copy_rowcount_error +python-psycopg2/tests.test_copy.CopyTests.test_copy_text +python-psycopg2/tests.test_copy.CopyTests.test_copy_to +python-psycopg2/tests.test_copy.CopyTests.test_copy_to_propagate_error +python-psycopg2/tests.test_copy.CopyTests.test_copy_to_segfault +python-psycopg2/tests.test_cursor.CursorTests.test_bad_placeholder +python-psycopg2/tests.test_cursor.CursorTests.test_bad_subclass +python-psycopg2/tests.test_cursor.CursorTests.test_callproc_badparam +python-psycopg2/tests.test_cursor.CursorTests.test_callproc_dict +python-psycopg2/tests.test_cursor.CursorTests.test_cast +python-psycopg2/tests.test_cursor.CursorTests.test_cast_specificity +python-psycopg2/tests.test_cursor.CursorTests.test_close_idempotent +python-psycopg2/tests.test_cursor.CursorTests.test_column_refcount +python-psycopg2/tests.test_cursor.CursorTests.test_description_attribs +python-psycopg2/tests.test_cursor.CursorTests.test_description_extra_attribs +python-psycopg2/tests.test_cursor.CursorTests.test_description_slice +python-psycopg2/tests.test_cursor.CursorTests.test_empty_query +python-psycopg2/tests.test_cursor.CursorTests.test_executemany_propagate_exceptions +python-psycopg2/tests.test_cursor.CursorTests.test_external_close_async +python-psycopg2/tests.test_cursor.CursorTests.test_external_close_sync +python-psycopg2/tests.test_cursor.CursorTests.test_modify_closed +python-psycopg2/tests.test_cursor.CursorTests.test_mogrify_decimal_explodes +python-psycopg2/tests.test_cursor.CursorTests.test_mogrify_leak_on_multiple_reference +python-psycopg2/tests.test_cursor.CursorTests.test_mogrify_unicode +python-psycopg2/tests.test_cursor.CursorTests.test_null_name +python-psycopg2/tests.test_cursor.CursorTests.test_pgresult_ptr +python-psycopg2/tests.test_cursor.CursorTests.test_pickle_description +python-psycopg2/tests.test_cursor.CursorTests.test_rowcount_on_executemany_returning +python-psycopg2/tests.test_cursor.CursorTests.test_weakref +python-psycopg2/tests.test_cursor.NamedCursorTests.test_invalid_name +python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_default_itersize +python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_efficient +python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_itersize +python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_rownumber +python-psycopg2/tests.test_cursor.NamedCursorTests.test_named_cursor_stealing +python-psycopg2/tests.test_cursor.NamedCursorTests.test_named_noop_close +python-psycopg2/tests.test_cursor.NamedCursorTests.test_not_scrollable +python-psycopg2/tests.test_cursor.NamedCursorTests.test_scroll +python-psycopg2/tests.test_cursor.NamedCursorTests.test_scroll_named +python-psycopg2/tests.test_cursor.NamedCursorTests.test_scrollable +python-psycopg2/tests.test_cursor.NamedCursorTests.test_stolen_named_cursor_close +python-psycopg2/tests.test_cursor.NamedCursorTests.test_withhold +python-psycopg2/tests.test_cursor.NamedCursorTests.test_withhold_autocommit +python-psycopg2/tests.test_cursor.NamedCursorTests.test_withhold_no_begin +python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_date +python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_datetime +python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_infinity_tz +python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_negative_timedelta +python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_time +python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_timedelta +python-psycopg2/tests.test_dates.DatetimeTests.test_default_tzinfo +python-psycopg2/tests.test_dates.DatetimeTests.test_fotz_tzinfo +python-psycopg2/tests.test_dates.DatetimeTests.test_interval_iso_8601_not_supported +python-psycopg2/tests.test_dates.DatetimeTests.test_interval_overflow +python-psycopg2/tests.test_dates.DatetimeTests.test_large_interval +python-psycopg2/tests.test_dates.DatetimeTests.test_micros_rounding +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_10k_date +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_10k_datetime +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_bc_date +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_bc_datetime +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_date +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime_microseconds +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime_no_timezone +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime_timezone +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_incomplete_date +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_incomplete_datetime +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_incomplete_time +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_infinity +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_interval +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_negative_interval +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_date +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_datetime +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_interval +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_time +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time_microseconds +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time_no_timezone +python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time_timezone +python-psycopg2/tests.test_dates.DatetimeTests.test_redshift_day +python-psycopg2/tests.test_dates.DatetimeTests.test_time_24 +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_date +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_date_array +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetime +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetime_array +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetimetz +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetimetz_array +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_interval +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_interval_array +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_time +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_time_array +python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_timetz +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_no_args +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_timedelta +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_instance_caching +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_pickle +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_name +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_negative_offset +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_positive_offset +python-psycopg2/tests.test_dates.FromTicksTestCase.test_date_value_error_sec_59_99 +python-psycopg2/tests.test_dates.FromTicksTestCase.test_time_value_error_sec_59_99 +python-psycopg2/tests.test_dates.FromTicksTestCase.test_timestamp_value_error_sec_59_99 +python-psycopg2/tests.test_errcodes.ErrocodeTests.test_ambiguous_names +python-psycopg2/tests.test_errcodes.ErrocodeTests.test_lookup_threadsafe +python-psycopg2/tests.test_errors.ErrorsTests.test_connection_exceptions_backwards_compatibility +python-psycopg2/tests.test_errors.ErrorsTests.test_exception_class +python-psycopg2/tests.test_errors.ErrorsTests.test_exception_class_fallback +python-psycopg2/tests.test_errors.ErrorsTests.test_has_base_exceptions +python-psycopg2/tests.test_errors.ErrorsTests.test_lookup +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchAll +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchMany +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchManyNoarg +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchOne +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorIter +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorIterRowNumber +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorNotGreedy +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchAll +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchMany +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchManyNoarg +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchOne +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealIter +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealIterRowNumber +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testPickleRealDictRow +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testRealMeansReal +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_copy +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_iter_methods +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_mod +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_order +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_pop +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictConnCursorArgs +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchAll +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchMany +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchManyNoarg +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchOne +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorIter +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorIterRowNumber +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorNotGreedy +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchAll +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchMany +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchManyNoarg +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchOne +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorIter +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorIterRowNumber +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testPickleDictRow +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testUpdateRow +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.test_copy +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.test_iter_methods +python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.test_order +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_bad_col_names +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_cache +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_cursor_args +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_executemany +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchall +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchmany +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchmany_noarg +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchone +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_iter +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_max_cache +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_minimal_generation +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_fetchall +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_fetchmany +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_fetchone +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_rownumber +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_no_result_no_surprise +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_nonascii_name +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_not_greedy +python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_record_updated +python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_composed +python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_empty +python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_many +python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_one +python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_pages +python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_tuples +python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_unicode +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_composed +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_dicts +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_empty +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_invalid_sql +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_many +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_one +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_pages +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_percent_escape +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_returning +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_tuples +python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_unicode +python-psycopg2/tests.test_fast_executemany.TestPaginate.test_paginate +python-psycopg2/tests.test_green.CallbackErrorTestCase.test_errors_named_cursor +python-psycopg2/tests.test_green.CallbackErrorTestCase.test_errors_on_connection +python-psycopg2/tests.test_green.CallbackErrorTestCase.test_errors_on_query +python-psycopg2/tests.test_green.GreenTestCase.test_copy_no_hang +python-psycopg2/tests.test_green.GreenTestCase.test_dont_freak_out +python-psycopg2/tests.test_green.GreenTestCase.test_error_in_callback +python-psycopg2/tests.test_green.GreenTestCase.test_flush_on_write +python-psycopg2/tests.test_green.GreenTestCase.test_non_block_after_notice +python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_cidr_adapt +python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_cidr_array_cast +python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_cidr_cast +python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_inet_adapt +python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_inet_array_cast +python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_inet_cast +python-psycopg2/tests.test_lobject.LargeObject64Tests.test_seek_tell_truncate_greater_than_2gb +python-psycopg2/tests.test_lobject.LargeObjectNot64Tests.test_seek_larger_than_2gb +python-psycopg2/tests.test_lobject.LargeObjectNot64Tests.test_truncate_larger_than_2gb +python-psycopg2/tests.test_lobject.LargeObjectTests.test_close +python-psycopg2/tests.test_lobject.LargeObjectTests.test_close_after_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_close_connection_gone +python-psycopg2/tests.test_lobject.LargeObjectTests.test_close_twice +python-psycopg2/tests.test_lobject.LargeObjectTests.test_connection_needed +python-psycopg2/tests.test_lobject.LargeObjectTests.test_create +python-psycopg2/tests.test_lobject.LargeObjectTests.test_create_with_existing_oid +python-psycopg2/tests.test_lobject.LargeObjectTests.test_create_with_oid +python-psycopg2/tests.test_lobject.LargeObjectTests.test_export +python-psycopg2/tests.test_lobject.LargeObjectTests.test_export_after_close +python-psycopg2/tests.test_lobject.LargeObjectTests.test_export_after_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_factory +python-psycopg2/tests.test_lobject.LargeObjectTests.test_import +python-psycopg2/tests.test_lobject.LargeObjectTests.test_large_oid +python-psycopg2/tests.test_lobject.LargeObjectTests.test_mode_defaults +python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_existing +python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_for_write +python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_mode_n +python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_non_existent +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_close +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_tpc_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_tpc_prepare +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_binary +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_large +python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_text +python-psycopg2/tests.test_lobject.LargeObjectTests.test_seek_after_close +python-psycopg2/tests.test_lobject.LargeObjectTests.test_seek_after_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_seek_tell +python-psycopg2/tests.test_lobject.LargeObjectTests.test_tell_after_close +python-psycopg2/tests.test_lobject.LargeObjectTests.test_tell_after_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_unlink +python-psycopg2/tests.test_lobject.LargeObjectTests.test_unlink_after_close +python-psycopg2/tests.test_lobject.LargeObjectTests.test_unlink_after_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_write +python-psycopg2/tests.test_lobject.LargeObjectTests.test_write_after_close +python-psycopg2/tests.test_lobject.LargeObjectTests.test_write_after_commit +python-psycopg2/tests.test_lobject.LargeObjectTests.test_write_large +python-psycopg2/tests.test_lobject.LargeObjectTruncateTests.test_truncate +python-psycopg2/tests.test_lobject.LargeObjectTruncateTests.test_truncate_after_close +python-psycopg2/tests.test_lobject.LargeObjectTruncateTests.test_truncate_after_commit +python-psycopg2/tests.test_module.ConnectTestCase.test_async +python-psycopg2/tests.test_module.ConnectTestCase.test_dsn +python-psycopg2/tests.test_module.ConnectTestCase.test_empty_param +python-psycopg2/tests.test_module.ConnectTestCase.test_escape +python-psycopg2/tests.test_module.ConnectTestCase.test_factory +python-psycopg2/tests.test_module.ConnectTestCase.test_generic_keywords +python-psycopg2/tests.test_module.ConnectTestCase.test_int_port_param +python-psycopg2/tests.test_module.ConnectTestCase.test_no_keywords +python-psycopg2/tests.test_module.ConnectTestCase.test_params_merging +python-psycopg2/tests.test_module.ConnectTestCase.test_supported_keywords +python-psycopg2/tests.test_module.ConnectTestCase.test_there_might_be_nothing +python-psycopg2/tests.test_module.ExceptionsTestCase.test_9_3_diagnostics +python-psycopg2/tests.test_module.ExceptionsTestCase.test_9_6_diagnostics +python-psycopg2/tests.test_module.ExceptionsTestCase.test_attributes +python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_attributes +python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_copy +python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_from_commit +python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_independent +python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_life +python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_values +python-psycopg2/tests.test_module.ExceptionsTestCase.test_pickle +python-psycopg2/tests.test_module.ExceptionsTestCase.test_pickle_connection_error +python-psycopg2/tests.test_module.TestExtensionModule.test_import_internal +python-psycopg2/tests.test_module.TestVersionDiscovery.test_libpq_version +python-psycopg2/tests.test_notify.NotifiesTests.test_compare +python-psycopg2/tests.test_notify.NotifiesTests.test_compare_tuple +python-psycopg2/tests.test_notify.NotifiesTests.test_hash +python-psycopg2/tests.test_notify.NotifiesTests.test_many_notifies +python-psycopg2/tests.test_notify.NotifiesTests.test_notifies_received_on_execute +python-psycopg2/tests.test_notify.NotifiesTests.test_notifies_received_on_poll +python-psycopg2/tests.test_notify.NotifiesTests.test_notify_attributes +python-psycopg2/tests.test_notify.NotifiesTests.test_notify_deque +python-psycopg2/tests.test_notify.NotifiesTests.test_notify_init +python-psycopg2/tests.test_notify.NotifiesTests.test_notify_noappend +python-psycopg2/tests.test_notify.NotifiesTests.test_notify_object +python-psycopg2/tests.test_notify.NotifiesTests.test_notify_payload +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_commit_in_tpc_fails +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_rollback_in_tpc_fails +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_begin +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_begin_in_tpc_transaction_fails +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_begin_in_transaction_fails +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_commit_with_prepare +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_commit_without_prepare +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_rollback_with_prepare +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_rollback_without_prepare +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_xid +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_BINARY +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Binary +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_DATETIME +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Date +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Exceptions +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_ExceptionsAsConnectionAttributes +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_NUMBER +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_None +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_ROWID +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_STRING +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Time +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Timestamp +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_apilevel +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_arraysize +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_callproc +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_close +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_commit +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_connect +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_cursor +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_cursor_isolation +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_description +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_execute +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_executemany +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_fetchall +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_fetchmany +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_fetchone +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_mixedfetch +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_nextset +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_paramstyle +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_rollback +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_rowcount +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_setinputsizes +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_setoutputsize +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_setoutputsize_basic +python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_threadsafety +python-psycopg2/tests.test_quote.QuotingTestCase.test_binary +python-psycopg2/tests.test_quote.QuotingTestCase.test_bytes +python-psycopg2/tests.test_quote.QuotingTestCase.test_koi8 +python-psycopg2/tests.test_quote.QuotingTestCase.test_latin1 +python-psycopg2/tests.test_quote.QuotingTestCase.test_string +python-psycopg2/tests.test_quote.QuotingTestCase.test_string_null_terminator +python-psycopg2/tests.test_quote.QuotingTestCase.test_unicode +python-psycopg2/tests.test_quote.TestQuotedIdentifier.test_identifier +python-psycopg2/tests.test_quote.TestQuotedIdentifier.test_unicode_ident +python-psycopg2/tests.test_quote.TestQuotedString.test_encoding_from_conn +python-psycopg2/tests.test_quote.TestStringAdapter.test_adapt_bytes +python-psycopg2/tests.test_quote.TestStringAdapter.test_connection_wins_anyway +python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_default +python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_error +python-psycopg2/tests.test_quote.TestStringAdapter.test_set_encoding +python-psycopg2/tests.test_replication.AsyncReplicationTest.test_async_replication +python-psycopg2/tests.test_replication.ReplicationTest.test_create_replication_slot +python-psycopg2/tests.test_replication.ReplicationTest.test_datestyle +python-psycopg2/tests.test_replication.ReplicationTest.test_keepalive +python-psycopg2/tests.test_replication.ReplicationTest.test_logical_replication_connection +python-psycopg2/tests.test_replication.ReplicationTest.test_physical_replication_connection +python-psycopg2/tests.test_replication.ReplicationTest.test_start_and_recover_from_error +python-psycopg2/tests.test_replication.ReplicationTest.test_start_on_missing_replication_slot +python-psycopg2/tests.test_replication.ReplicationTest.test_start_replication_expert_sql +python-psycopg2/tests.test_replication.ReplicationTest.test_stop_replication +python-psycopg2/tests.test_sql.ComposedTest.test_class +python-psycopg2/tests.test_sql.ComposedTest.test_eq +python-psycopg2/tests.test_sql.ComposedTest.test_iter +python-psycopg2/tests.test_sql.ComposedTest.test_join +python-psycopg2/tests.test_sql.ComposedTest.test_repr +python-psycopg2/tests.test_sql.ComposedTest.test_seq +python-psycopg2/tests.test_sql.ComposedTest.test_sum +python-psycopg2/tests.test_sql.ComposedTest.test_sum_inplace +python-psycopg2/tests.test_sql.IdentifierTests.test_as_str +python-psycopg2/tests.test_sql.IdentifierTests.test_class +python-psycopg2/tests.test_sql.IdentifierTests.test_eq +python-psycopg2/tests.test_sql.IdentifierTests.test_init +python-psycopg2/tests.test_sql.IdentifierTests.test_join +python-psycopg2/tests.test_sql.IdentifierTests.test_repr +python-psycopg2/tests.test_sql.IdentifierTests.test_strings +python-psycopg2/tests.test_sql.LiteralTests.test_class +python-psycopg2/tests.test_sql.LiteralTests.test_eq +python-psycopg2/tests.test_sql.LiteralTests.test_init +python-psycopg2/tests.test_sql.LiteralTests.test_must_be_adaptable +python-psycopg2/tests.test_sql.LiteralTests.test_repr +python-psycopg2/tests.test_sql.LiteralTests.test_wrapped +python-psycopg2/tests.test_sql.PlaceholderTest.test_bad_name +python-psycopg2/tests.test_sql.PlaceholderTest.test_class +python-psycopg2/tests.test_sql.PlaceholderTest.test_eq +python-psycopg2/tests.test_sql.PlaceholderTest.test_name +python-psycopg2/tests.test_sql.PlaceholderTest.test_repr +python-psycopg2/tests.test_sql.PlaceholderTest.test_repr_name +python-psycopg2/tests.test_sql.SQLTests.test_class +python-psycopg2/tests.test_sql.SQLTests.test_eq +python-psycopg2/tests.test_sql.SQLTests.test_init +python-psycopg2/tests.test_sql.SQLTests.test_join +python-psycopg2/tests.test_sql.SQLTests.test_multiply +python-psycopg2/tests.test_sql.SQLTests.test_repr +python-psycopg2/tests.test_sql.SQLTests.test_string +python-psycopg2/tests.test_sql.SQLTests.test_sum +python-psycopg2/tests.test_sql.SQLTests.test_sum_inplace +python-psycopg2/tests.test_sql.SqlFormatTests.test_braces_escape +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_bad_args_type +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs_auto +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_empty +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_literal +python-psycopg2/tests.test_sql.SqlFormatTests.test_copy +python-psycopg2/tests.test_sql.SqlFormatTests.test_dict +python-psycopg2/tests.test_sql.SqlFormatTests.test_execute +python-psycopg2/tests.test_sql.SqlFormatTests.test_executemany +python-psycopg2/tests.test_sql.SqlFormatTests.test_must_be_adaptable +python-psycopg2/tests.test_sql.SqlFormatTests.test_must_be_composable +python-psycopg2/tests.test_sql.SqlFormatTests.test_no_modifiers +python-psycopg2/tests.test_sql.SqlFormatTests.test_percent_escape +python-psycopg2/tests.test_sql.SqlFormatTests.test_pos +python-psycopg2/tests.test_sql.SqlFormatTests.test_pos_spec +python-psycopg2/tests.test_sql.ValuesTest.test_default +python-psycopg2/tests.test_sql.ValuesTest.test_null +python-psycopg2/tests.test_transaction.DeadlockSerializationTests.test_deadlock +python-psycopg2/tests.test_transaction.DeadlockSerializationTests.test_serialisation_failure +python-psycopg2/tests.test_transaction.QueryCancellationTests.test_statement_timeout +python-psycopg2/tests.test_transaction.TransactionTests.test_commit +python-psycopg2/tests.test_transaction.TransactionTests.test_failed_commit +python-psycopg2/tests.test_transaction.TransactionTests.test_rollback +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_most_specific +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype_3 +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_conform_subclass_precedence +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank_hex +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_escaped_mixed +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_escaped_octal +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex_upper +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_null +python-psycopg2/tests.test_types_basic.TypesBasicTests.testAdaptBytearray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testAdaptMemoryview +python-psycopg2/tests.test_types_basic.TypesBasicTests.testArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testArrayEscape +python-psycopg2/tests.test_types_basic.TypesBasicTests.testArrayMalformed +python-psycopg2/tests.test_types_basic.TypesBasicTests.testArrayOfNulls +python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinary +python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryEmptyString +python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryNone +python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryRoundTrip +python-psycopg2/tests.test_types_basic.TypesBasicTests.testBoolean +python-psycopg2/tests.test_types_basic.TypesBasicTests.testByteaHexCheckFalsePositive +python-psycopg2/tests.test_types_basic.TypesBasicTests.testBytesArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testDecimal +python-psycopg2/tests.test_types_basic.TypesBasicTests.testEmptyArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testEmptyArrayNoCast +python-psycopg2/tests.test_types_basic.TypesBasicTests.testEmptyArrayRegression +python-psycopg2/tests.test_types_basic.TypesBasicTests.testFloatInf +python-psycopg2/tests.test_types_basic.TypesBasicTests.testFloatNan +python-psycopg2/tests.test_types_basic.TypesBasicTests.testGenericArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testGenericArrayNull +python-psycopg2/tests.test_types_basic.TypesBasicTests.testIntEnum +python-psycopg2/tests.test_types_basic.TypesBasicTests.testNegNumber +python-psycopg2/tests.test_types_basic.TypesBasicTests.testNestedArrays +python-psycopg2/tests.test_types_basic.TypesBasicTests.testNestedEmptyArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testNetworkArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testNumber +python-psycopg2/tests.test_types_basic.TypesBasicTests.testQuoting +python-psycopg2/tests.test_types_basic.TypesBasicTests.testTextArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testTypeRoundtripBytes +python-psycopg2/tests.test_types_basic.TypesBasicTests.testTypeRoundtripBytesArray +python-psycopg2/tests.test_types_basic.TypesBasicTests.testUnicode +python-psycopg2/tests.test_types_basic.TypesBasicTests.testUnicodeArray +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_cast_composite +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_cast_nested +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_array +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_namespace +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_namespace_path +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_not_found +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_weird_name +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_empty_string +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_from_tables +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_non_dbapi_connection +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_none_fast_path +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_none_in_record +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_register_globally +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_register_on_connection +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_register_on_cursor +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_subclass +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_tokenization +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_wrong_schema +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_adapt_8 +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_adapt_9 +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_array_cast +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_array_cast_oid +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_non_dbapi_connection +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_oid +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_parse +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_register_conn +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_register_curs +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_register_globally +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_roundtrip +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_roundtrip_array +python-psycopg2/tests.test_types_extras.JsonTestCase.test_adapt +python-psycopg2/tests.test_types_extras.JsonTestCase.test_adapt_dumps +python-psycopg2/tests.test_types_extras.JsonTestCase.test_adapt_subclass +python-psycopg2/tests.test_types_extras.JsonTestCase.test_default_cast +python-psycopg2/tests.test_types_extras.JsonTestCase.test_loads +python-psycopg2/tests.test_types_extras.JsonTestCase.test_no_array_oid +python-psycopg2/tests.test_types_extras.JsonTestCase.test_no_conn_curs +python-psycopg2/tests.test_types_extras.JsonTestCase.test_null +python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_default +python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_globally +python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_on_connection +python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_on_cursor +python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_on_dict +python-psycopg2/tests.test_types_extras.JsonTestCase.test_scs +python-psycopg2/tests.test_types_extras.JsonTestCase.test_str +python-psycopg2/tests.test_types_extras.JsonTestCase.test_type_not_available +python-psycopg2/tests.test_types_extras.JsonbTestCase.test_default_cast +python-psycopg2/tests.test_types_extras.JsonbTestCase.test_loads +python-psycopg2/tests.test_types_extras.JsonbTestCase.test_null +python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_default +python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_globally +python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_on_connection +python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_on_cursor +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_adapt_date_range +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_adapt_number_range +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_adapt_numeric_range +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_date +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_empty +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_inf +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_null +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_numbers +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_timestamp +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_timestamptz +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_rang_weird_name +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_range_escaping +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_range_not_found +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_register_range_adapter +python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_schema_range +python-psycopg2/tests.test_types_extras.RangeTestCase.test_bad_bounds +python-psycopg2/tests.test_types_extras.RangeTestCase.test_bounds +python-psycopg2/tests.test_types_extras.RangeTestCase.test_empty +python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_hash +python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_subclass +python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_wrong_type +python-psycopg2/tests.test_types_extras.RangeTestCase.test_ge_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_gt_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_in +python-psycopg2/tests.test_types_extras.RangeTestCase.test_keywords +python-psycopg2/tests.test_types_extras.RangeTestCase.test_le_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_lt_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_nobounds +python-psycopg2/tests.test_types_extras.RangeTestCase.test_nonzero +python-psycopg2/tests.test_types_extras.RangeTestCase.test_noparam +python-psycopg2/tests.test_types_extras.RangeTestCase.test_pickling +python-psycopg2/tests.test_types_extras.RangeTestCase.test_str +python-psycopg2/tests.test_types_extras.RangeTestCase.test_str_datetime +python-psycopg2/tests.test_types_extras.TypesExtrasTests.testINET +python-psycopg2/tests.test_types_extras.TypesExtrasTests.testINETARRAY +python-psycopg2/tests.test_types_extras.TypesExtrasTests.testUUID +python-psycopg2/tests.test_types_extras.TypesExtrasTests.testUUIDARRAY +python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_adapt_fail +python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_inet_conform +python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_point_array +python-psycopg2/tests.test_with.WithConnectionTestCase.test_cant_reenter +python-psycopg2/tests.test_with.WithConnectionTestCase.test_subclass_commit +python-psycopg2/tests.test_with.WithConnectionTestCase.test_subclass_rollback +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_autocommit +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_autocommit_pgerror +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_autocommit_pyerror +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_closed +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_connect_idiom +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_error_db +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_error_python +python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_ok +python-psycopg2/tests.test_with.WithCursorTestCase.test_exception_swallow +python-psycopg2/tests.test_with.WithCursorTestCase.test_named_with_noop +python-psycopg2/tests.test_with.WithCursorTestCase.test_subclass +python-psycopg2/tests.test_with.WithCursorTestCase.test_with_error +python-psycopg2/tests.test_with.WithCursorTestCase.test_with_ok diff --git a/ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt b/ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt new file mode 100644 index 000000000000..82c9e871ef30 --- /dev/null +++ b/ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt @@ -0,0 +1,114 @@ +python-psycopg2/tests.test_bugX000.DateTimeAllocationBugTestCase.test_date_time_allocation_bug +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_database_is_a_keyword +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_arguments +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_param +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_string +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_escape +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_no_dsn_munging +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_null_args +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_merging +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_validation +python-psycopg2/tests.test_connection.MakeDsnTestCase.test_url_is_cool +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_bad_param +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn_uri +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_str_subclass +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_key +python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_value +python-psycopg2/tests.test_connection.PasswordLeakTestCase.test_url_leak +python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_md5 +python-psycopg2/tests.test_cursor.CursorTests.test_bad_subclass +python-psycopg2/tests.test_cursor.CursorTests.test_column_refcount +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_no_args +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_timedelta +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_instance_caching +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_pickle +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_name +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_negative_offset +python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_positive_offset +python-psycopg2/tests.test_dates.FromTicksTestCase.test_date_value_error_sec_59_99 +python-psycopg2/tests.test_dates.FromTicksTestCase.test_timestamp_value_error_sec_59_99 +python-psycopg2/tests.test_dates.FromTicksTestCase.test_time_value_error_sec_59_99 +python-psycopg2/tests.test_errcodes.ErrocodeTests.test_ambiguous_names +python-psycopg2/tests.test_errcodes.ErrocodeTests.test_lookup_threadsafe +python-psycopg2/tests.test_errors.ErrorsTests.test_connection_exceptions_backwards_compatibility +python-psycopg2/tests.test_errors.ErrorsTests.test_has_base_exceptions +python-psycopg2/tests.test_errors.ErrorsTests.test_lookup +python-psycopg2/tests.test_fast_executemany.TestPaginate.test_paginate +python-psycopg2/tests.test_module.ConnectTestCase.test_async +python-psycopg2/tests.test_module.ConnectTestCase.test_dsn +python-psycopg2/tests.test_module.ConnectTestCase.test_empty_param +python-psycopg2/tests.test_module.ConnectTestCase.test_escape +python-psycopg2/tests.test_module.ConnectTestCase.test_factory +python-psycopg2/tests.test_module.ConnectTestCase.test_generic_keywords +python-psycopg2/tests.test_module.ConnectTestCase.test_int_port_param +python-psycopg2/tests.test_module.ConnectTestCase.test_no_keywords +python-psycopg2/tests.test_module.ConnectTestCase.test_params_merging +python-psycopg2/tests.test_module.ConnectTestCase.test_supported_keywords +python-psycopg2/tests.test_module.ConnectTestCase.test_there_might_be_nothing +python-psycopg2/tests.test_module.TestExtensionModule.test_import_internal +python-psycopg2/tests.test_module.TestVersionDiscovery.test_libpq_version +python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_default +python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_error +python-psycopg2/tests.test_quote.TestStringAdapter.test_set_encoding +python-psycopg2/tests.test_sql.ComposedTest.test_class +python-psycopg2/tests.test_sql.ComposedTest.test_eq +python-psycopg2/tests.test_sql.ComposedTest.test_iter +python-psycopg2/tests.test_sql.ComposedTest.test_repr +python-psycopg2/tests.test_sql.ComposedTest.test_seq +python-psycopg2/tests.test_sql.IdentifierTests.test_class +python-psycopg2/tests.test_sql.IdentifierTests.test_eq +python-psycopg2/tests.test_sql.IdentifierTests.test_init +python-psycopg2/tests.test_sql.IdentifierTests.test_join +python-psycopg2/tests.test_sql.IdentifierTests.test_repr +python-psycopg2/tests.test_sql.IdentifierTests.test_strings +python-psycopg2/tests.test_sql.LiteralTests.test_class +python-psycopg2/tests.test_sql.LiteralTests.test_eq +python-psycopg2/tests.test_sql.LiteralTests.test_init +python-psycopg2/tests.test_sql.LiteralTests.test_wrapped +python-psycopg2/tests.test_sql.PlaceholderTest.test_bad_name +python-psycopg2/tests.test_sql.PlaceholderTest.test_class +python-psycopg2/tests.test_sql.PlaceholderTest.test_eq +python-psycopg2/tests.test_sql.PlaceholderTest.test_name +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_bad_args_type +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs +python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs_auto +python-psycopg2/tests.test_sql.SqlFormatTests.test_must_be_composable +python-psycopg2/tests.test_sql.SqlFormatTests.test_no_modifiers +python-psycopg2/tests.test_sql.SQLTests.test_class +python-psycopg2/tests.test_sql.SQLTests.test_eq +python-psycopg2/tests.test_sql.SQLTests.test_init +python-psycopg2/tests.test_sql.SQLTests.test_string +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_most_specific +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype_3 +python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_conform_subclass_precedence +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank_hex +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_escaped_mixed +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_escaped_octal +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex_upper +python-psycopg2/tests.test_types_basic.ByteaParserTest.test_null +python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryEmptyString +python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_tokenization +python-psycopg2/tests.test_types_extras.HstoreTestCase.test_parse +python-psycopg2/tests.test_types_extras.RangeTestCase.test_bad_bounds +python-psycopg2/tests.test_types_extras.RangeTestCase.test_bounds +python-psycopg2/tests.test_types_extras.RangeTestCase.test_empty +python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_hash +python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_subclass +python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_wrong_type +python-psycopg2/tests.test_types_extras.RangeTestCase.test_ge_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_gt_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_in +python-psycopg2/tests.test_types_extras.RangeTestCase.test_keywords +python-psycopg2/tests.test_types_extras.RangeTestCase.test_le_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_lt_ordering +python-psycopg2/tests.test_types_extras.RangeTestCase.test_nobounds +python-psycopg2/tests.test_types_extras.RangeTestCase.test_nonzero +python-psycopg2/tests.test_types_extras.RangeTestCase.test_noparam +python-psycopg2/tests.test_types_extras.RangeTestCase.test_pickling +python-psycopg2/tests.test_types_extras.RangeTestCase.test_str +python-psycopg2/tests.test_types_extras.RangeTestCase.test_str_datetime +python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_adapt_fail diff --git a/ydb/tests/postgres_integrations/psycopg2/docker_wrapper_test.py b/ydb/tests/postgres_integrations/psycopg2/docker_wrapper_test.py new file mode 100644 index 000000000000..5779f8b5e91b --- /dev/null +++ b/ydb/tests/postgres_integrations/psycopg2/docker_wrapper_test.py @@ -0,0 +1,4 @@ +from .conftest import integrations + +def test(testname): + integrations.execute_test(testname) diff --git a/ydb/tests/postgres_integrations/psycopg2/ya.make b/ydb/tests/postgres_integrations/psycopg2/ya.make new file mode 100644 index 000000000000..6e7e4552a820 --- /dev/null +++ b/ydb/tests/postgres_integrations/psycopg2/ya.make @@ -0,0 +1,37 @@ +PY3TEST() + +FORK_TEST_FILES() +TIMEOUT(600) +SIZE(LARGE) + +TAG( + ya:fat +) + +ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") +DEPENDS( + ydb/apps/ydbd +) + +TEST_SRCS( + conftest.py + docker_wrapper_test.py +) + + +DATA( + arcadia/ydb/tests/postgres_integrations/psycopg2/data +) + +PEERDIR( + contrib/python/requests + contrib/python/tornado/tornado-4 + ydb/tests/library + ydb/tests/postgres_integrations/library + ydb/tests/oss/ydb_sdk_import + ydb/public/sdk/python +) + +REQUIREMENTS(ram:10) + +END() From a5ab2be568db766407dfe4cefb8565108d6e4638 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 9 Jul 2024 19:25:33 +0000 Subject: [PATCH 004/261] ya.make --- ydb/tests/postgres_integrations/ya.make | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 ydb/tests/postgres_integrations/ya.make diff --git a/ydb/tests/postgres_integrations/ya.make b/ydb/tests/postgres_integrations/ya.make new file mode 100644 index 000000000000..94d799013f0c --- /dev/null +++ b/ydb/tests/postgres_integrations/ya.make @@ -0,0 +1,4 @@ +RECURSE( + psycopg2 + library +) From b23e2c33704db22220c070252d518b21257c6b9a Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 9 Jul 2024 20:10:36 +0000 Subject: [PATCH 005/261] initial docker code --- .../{psycopg2 => go-libpq}/conftest.py | 2 +- .../go-libpq/data/.gitignore | 2 + .../go-run-separate-tests.bash | 36 + .../go-libpq/data/docker-compose-host.yaml | 22 + .../go-libpq/data/docker-compose.yaml | 40 + .../go-libpq/data/docker-init.bash | 24 + .../go-libpq/data/docker-start.bash | 45 ++ .../go-libpq/data/full-test-list.txt | 236 ++++++ .../go-libpq/data/patch.diff | 227 ++++++ .../go-libpq/data/run-test.bash | 10 + .../go-libpq/data/unit-tests.txt | 106 +++ .../docker_wrapper_test.py | 0 .../{psycopg2 => go-libpq}/ya.make | 2 +- .../library/pytest_integration.py | 56 +- .../postgres_integrations/library/ya.make | 1 + .../psycopg2/data/full-test-list.txt | 742 ------------------ .../psycopg2/data/unit-tests.txt | 114 --- 17 files changed, 805 insertions(+), 860 deletions(-) rename ydb/tests/postgres_integrations/{psycopg2 => go-libpq}/conftest.py (90%) create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/.gitignore create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml create mode 100755 ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash create mode 100755 ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/patch.diff create mode 100755 ydb/tests/postgres_integrations/go-libpq/data/run-test.bash create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt rename ydb/tests/postgres_integrations/{psycopg2 => go-libpq}/docker_wrapper_test.py (100%) rename ydb/tests/postgres_integrations/{psycopg2 => go-libpq}/ya.make (88%) delete mode 100644 ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt delete mode 100644 ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt diff --git a/ydb/tests/postgres_integrations/psycopg2/conftest.py b/ydb/tests/postgres_integrations/go-libpq/conftest.py similarity index 90% rename from ydb/tests/postgres_integrations/psycopg2/conftest.py rename to ydb/tests/postgres_integrations/go-libpq/conftest.py index 7fecb16136d1..bdf601c4be56 100644 --- a/ydb/tests/postgres_integrations/psycopg2/conftest.py +++ b/ydb/tests/postgres_integrations/go-libpq/conftest.py @@ -9,7 +9,7 @@ from ydb.tests.postgres_integrations.library import IntegrationTests -integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/psycopg2/data")) +integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) def pytest_generate_tests(metafunc: pytest.Metafunc): integrations.pytest_generate_tests(metafunc) diff --git a/ydb/tests/postgres_integrations/go-libpq/data/.gitignore b/ydb/tests/postgres_integrations/go-libpq/data/.gitignore new file mode 100644 index 000000000000..f6a04b18adcd --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/.gitignore @@ -0,0 +1,2 @@ +/sources/ +/test-result/ \ No newline at end of file diff --git a/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash b/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash new file mode 100644 index 000000000000..a114b7b9419c --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash @@ -0,0 +1,36 @@ +#!/bin/bash + +set -eu + +ONE_TEST_TIMEOUT=5s +TEST_BINARY=./test.binary +[ -z "${SKIP_TESTS:-}" ] && SKIP_TESTS='^$' + +GO_LIST_FILTER="^Test" + +if [ -n "${YDB_PG_TESTNAME:-}" ]; then + GO_LIST_FILTER="^${YDB_PG_TESTNAME%%/*}\$" # Cut subtest name: TestAbc/sub -> ^TestAbc$ +fi + +echo "Get test list" +TESTS=$($TEST_BINARY --test.list "$GO_LIST_FILTER" | sort) + + +echo "Skip tests: '$SKIP_TESTS'" +echo "Shell $SHELL" + +rm -f /test-result/raw/result.txt +for TEST_NAME in $TESTS; do + echo -n "Test: $TEST_NAME " + if echo "$TEST_NAME" | grep -Eq "$SKIP_TESTS"; then + echo skip + continue + else + echo start + fi + CMD="$TEST_BINARY --test.run '^$TEST_NAME\$' --test.v --test.timeout='$ONE_TEST_TIMEOUT'" + echo "$CMD" + bash -c "$CMD" >> /test-result/raw/result.txt 2>&1 || true +done + +go-junit-report < /test-result/raw/result.txt > /test-result/raw/result.xml diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml new file mode 100644 index 000000000000..cfbb3d4d485c --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose-host.yaml @@ -0,0 +1,22 @@ +version: "3" +services: + project: + network_mode: host + + image: ydb-test/go-pqlib + build: + context: ../../.. + dockerfile: languages/go/libpq/Dockerfile + network: host + environment: + - PGUSER=${YDB_PG_USER:-root} + - PGPASSWORD=${YDB_PG_PASSWORD:-1234} + - PGHOST=${YDB_PG_HOST:-ydb} + - PGPORT=${YDB_PG_PORT:-5432} + - PGDATABASE=${YDB_PG_DATABASE:-local} + - PQGOSSLTESTS=0 + - PQSSLCERTTEST_PATH=certs + - YDB_PG_TESTNAME=${YDB_PG_TESTNAME:-} + volumes: + - ./exchange:/exchange + - ./test-result/:/test-result diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml new file mode 100644 index 000000000000..7c5fb48a1548 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-compose.yaml @@ -0,0 +1,40 @@ +version: "3" +services: + ydb: + image: ghcr.io/ydb-platform/local-ydb:nightly + environment: + - "YDB_DEFAULT_LOG_LEVEL=DEBUG" + - "GRPC_TLS_PORT=2135" + - "GRPC_PORT=2136" + - "MON_PORT=8765" + - "YDB_USE_IN_MEMORY_PDISKS=true" + - "POSTGRES_USER=${YDB_PG_USER:-root}" + - "POSTGRES_PASSWORD=${YDB_PG_PASSWORD:-1234}" + - "YDB_FEATURE_FLAGS=enable_temp_tables" + - "YDB_TABLE_ENABLE_PREPARED_DDL=true" + healthcheck: + test: "/bin/sh /health_check" + interval: 1s + start_period: 1m + project: + depends_on: + ydb: + condition: service_healthy + + image: ydb-test/go-pqlib + build: + context: ../../.. + dockerfile: languages/go/libpq/Dockerfile + network: host + environment: + - PGUSER=${YDB_PG_USER:-root} + - PGPASSWORD=${YDB_PG_PASSWORD:-1234} + - PGHOST=${YDB_PG_HOST:-ydb} + - PGPORT=${YDB_PG_PORT:-5432} + - PGDATABASE=${YDB_PG_DATABASE:-/local} + - PQGOSSLTESTS=0 + - PQSSLCERTTEST_PATH=certs + - YDB_PG_TESTNAME=${YDB_PG_TESTNAME:-} + volumes: + - ./exchange:/exchange + - ./test-result/:/test-result diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash b/ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash new file mode 100755 index 000000000000..9a8c5eff9af4 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-init.bash @@ -0,0 +1,24 @@ +#!/bin/bash + +set -eu + +apt-get update && apt-get install -y patch + +go install github.com/jstemmer/go-junit-report/v2@v2.0.0 + +mkdir -p /original-sources +cd /original-sources + +wget https://github.com/lib/pq/archive/refs/tags/v1.10.9.tar.gz -O libpq.tar.gz +tar --strip-components=1 -zxvf libpq.tar.gz +rm -f libpq.tar.gz + +mkdir -p /project/sources/ +cp -R /original-sources/. /project/sources/ + +cd /project/sources/ +[ -e /patch.diff ] && patch -s -p0 < /patch.diff + +# cache binary +echo "Build test binary" +go test -c -o ./test.binary diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash new file mode 100755 index 000000000000..6a12a7a3a470 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash @@ -0,0 +1,45 @@ +#!/bin/bash + +set -eu + +echo "Start script" + +rm -rf /test-result 2> /dev/null || true + +mkdir -p /exchange +mkdir -p /test-result/raw + +if [ -e /exchange/sources ]; then + echo "Skip prepare sources, because it is exist" +else + echo "Copy sources" + mkdir -p /exchange/sources + cp -R /project/sources/. /exchange/sources + chmod -R a+rw /exchange/sources +fi + +cd /project/sources/ + +export YDB_PG_TESTNAME="${YDB_PG_TESTNAME:-}" # set YDB_PG_TESTNAME to empty string if it not set + +if [ -n "${YDB_PG_TESTNAME:-}" ]; then + SKIP_TESTS="^\$" +fi + +echo "Run test: '$YDB_PG_TESTNAME'" + +echo "Start test" + +# PQTEST_BINARY_PARAMETERS=no go test -test.skip="$SKIP_TESTS" + +export SKIP_TESTS + +PQTEST_BINARY_PARAMETERS=no /go-run-separate-tests.bash + +sed -e 's|classname=""|classname="golang-lib-pq"|' -i /test-result/raw/result.xml + +if [ -n "${YDB_PG_TESTNAME:-}" ]; then + cat /test-result/raw/result.txt +fi + +chmod -R a+rw /test-result \ No newline at end of file diff --git a/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt b/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt new file mode 100644 index 000000000000..7cc782f5115e --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt @@ -0,0 +1,236 @@ +golang-lib-pq/Test64BitErrorChecking +golang-lib-pq/TestAppendEncodedText +golang-lib-pq/TestAppendEscapedText +golang-lib-pq/TestAppendEscapedTextExistingBuffer +golang-lib-pq/TestArrayScanBackend +golang-lib-pq/TestArrayScanner +golang-lib-pq/TestArrayValueBackend +golang-lib-pq/TestArrayValuer +golang-lib-pq/TestBadConn +golang-lib-pq/TestBinaryByteSliceToInt +golang-lib-pq/TestBinaryByteSlicetoUUID +golang-lib-pq/TestBindError +golang-lib-pq/TestBoolArrayScanBytes +golang-lib-pq/TestBoolArrayScanEmpty +golang-lib-pq/TestBoolArrayScanError +golang-lib-pq/TestBoolArrayScanNil +golang-lib-pq/TestBoolArrayScanString +golang-lib-pq/TestBoolArrayScanUnsupported +golang-lib-pq/TestBoolArrayValue +golang-lib-pq/TestByteSliceToText +golang-lib-pq/TestByteaArrayScanBytes +golang-lib-pq/TestByteaArrayScanEmpty +golang-lib-pq/TestByteaArrayScanError +golang-lib-pq/TestByteaArrayScanNil +golang-lib-pq/TestByteaArrayScanString +golang-lib-pq/TestByteaArrayScanUnsupported +golang-lib-pq/TestByteaArrayValue +golang-lib-pq/TestByteaOutputFormatEncoding +golang-lib-pq/TestByteaOutputFormats +golang-lib-pq/TestCloseBadConn +golang-lib-pq/TestCommit +golang-lib-pq/TestCommitInFailedTransaction +golang-lib-pq/TestCommitInFailedTransactionWithCancelContext +golang-lib-pq/TestConnClose +golang-lib-pq/TestConnExecDeadlock +golang-lib-pq/TestConnListen +golang-lib-pq/TestConnPing +golang-lib-pq/TestConnPrepareContext +golang-lib-pq/TestConnPrepareContext/context.Background +golang-lib-pq/TestConnPrepareContext/context.WithTimeout +golang-lib-pq/TestConnPrepareContext/context.WithTimeout_exceeded +golang-lib-pq/TestConnUnlisten +golang-lib-pq/TestConnUnlistenAll +golang-lib-pq/TestConnectorWithNoticeHandler_Simple +golang-lib-pq/TestConnectorWithNotificationHandler_Simple +golang-lib-pq/TestContextCancelBegin +golang-lib-pq/TestContextCancelExec +golang-lib-pq/TestContextCancelQuery +golang-lib-pq/TestCopyFromError +golang-lib-pq/TestCopyInBinaryError +golang-lib-pq/TestCopyInMultipleValues +golang-lib-pq/TestCopyInRaiseStmtTrigger +golang-lib-pq/TestCopyInSchemaStmt +golang-lib-pq/TestCopyInStmt +golang-lib-pq/TestCopyInStmtAffectedRows +golang-lib-pq/TestCopyInTypes +golang-lib-pq/TestCopyInWrongType +golang-lib-pq/TestCopyOutsideOfTxnError +golang-lib-pq/TestCopyRespLoopConnectionError +golang-lib-pq/TestCopySyntaxError +golang-lib-pq/TestDataType +golang-lib-pq/TestDataTypeLength +golang-lib-pq/TestDataTypeName +golang-lib-pq/TestDataTypePrecisionScale +golang-lib-pq/TestDecodeBool +golang-lib-pq/TestDecodeUUIDBackend +golang-lib-pq/TestDecodeUUIDBinaryError +golang-lib-pq/TestEmptyQuery +golang-lib-pq/TestEmptyResultSetColumns +golang-lib-pq/TestEncodeAndParseTs +golang-lib-pq/TestEncodeDecode +golang-lib-pq/TestErrorClass +golang-lib-pq/TestErrorDuringStartup +golang-lib-pq/TestErrorDuringStartupClosesConn +golang-lib-pq/TestErrorOnExec +golang-lib-pq/TestErrorOnQuery +golang-lib-pq/TestErrorOnQueryRowSimpleQuery +golang-lib-pq/TestErrorSQLState +golang-lib-pq/TestExec +golang-lib-pq/TestFloat32ArrayScanBytes +golang-lib-pq/TestFloat32ArrayScanEmpty +golang-lib-pq/TestFloat32ArrayScanError +golang-lib-pq/TestFloat32ArrayScanNil +golang-lib-pq/TestFloat32ArrayScanString +golang-lib-pq/TestFloat32ArrayScanUnsupported +golang-lib-pq/TestFloat32ArrayValue +golang-lib-pq/TestFloat64ArrayScanBytes +golang-lib-pq/TestFloat64ArrayScanEmpty +golang-lib-pq/TestFloat64ArrayScanError +golang-lib-pq/TestFloat64ArrayScanNil +golang-lib-pq/TestFloat64ArrayScanString +golang-lib-pq/TestFloat64ArrayScanUnsupported +golang-lib-pq/TestFloat64ArrayValue +golang-lib-pq/TestFormatAndParseTimestamp +golang-lib-pq/TestFormatTs +golang-lib-pq/TestFormatTsBackend +golang-lib-pq/TestFullParseURL +golang-lib-pq/TestGenericArrayScanDelimiter +golang-lib-pq/TestGenericArrayScanErrors +golang-lib-pq/TestGenericArrayScanScannerArrayBytes +golang-lib-pq/TestGenericArrayScanScannerArrayString +golang-lib-pq/TestGenericArrayScanScannerSliceBytes +golang-lib-pq/TestGenericArrayScanScannerSliceEmpty +golang-lib-pq/TestGenericArrayScanScannerSliceNil +golang-lib-pq/TestGenericArrayScanScannerSliceString +golang-lib-pq/TestGenericArrayScanUnsupported +golang-lib-pq/TestGenericArrayValue +golang-lib-pq/TestGenericArrayValueErrors +golang-lib-pq/TestGenericArrayValueUnsupported +golang-lib-pq/TestHasCorrectRootGroupPermissions +golang-lib-pq/TestIPv6LoopbackParseURL +golang-lib-pq/TestInfinityTimestamp +golang-lib-pq/TestInt32ArrayScanBytes +golang-lib-pq/TestInt32ArrayScanEmpty +golang-lib-pq/TestInt32ArrayScanError +golang-lib-pq/TestInt32ArrayScanNil +golang-lib-pq/TestInt32ArrayScanString +golang-lib-pq/TestInt32ArrayScanUnsupported +golang-lib-pq/TestInt32ArrayValue +golang-lib-pq/TestInt64ArrayScanBytes +golang-lib-pq/TestInt64ArrayScanEmpty +golang-lib-pq/TestInt64ArrayScanError +golang-lib-pq/TestInt64ArrayScanNil +golang-lib-pq/TestInt64ArrayScanString +golang-lib-pq/TestInt64ArrayScanUnsupported +golang-lib-pq/TestInt64ArrayValue +golang-lib-pq/TestInvalidProtocolParseURL +golang-lib-pq/TestIsUTF8 +golang-lib-pq/TestIssue1046 +golang-lib-pq/TestIssue1062 +golang-lib-pq/TestIssue186 +golang-lib-pq/TestIssue196 +golang-lib-pq/TestIssue282 +golang-lib-pq/TestIssue494 +golang-lib-pq/TestIssue617 +golang-lib-pq/TestListenerClose +golang-lib-pq/TestListenerConnCloseWhileQueryIsExecuting +golang-lib-pq/TestListenerFailedQuery +golang-lib-pq/TestListenerListen +golang-lib-pq/TestListenerPing +golang-lib-pq/TestListenerReconnect +golang-lib-pq/TestListenerUnlisten +golang-lib-pq/TestListenerUnlistenAll +golang-lib-pq/TestMinimalURL +golang-lib-pq/TestMultipleEmptyResult +golang-lib-pq/TestMultipleResult +golang-lib-pq/TestMultipleSimpleQuery +golang-lib-pq/TestNewConnector_Connect +golang-lib-pq/TestNewConnector_Driver +golang-lib-pq/TestNewConnector_WorksWithOpenDB +golang-lib-pq/TestNewListenerConn +golang-lib-pq/TestNoData +golang-lib-pq/TestNotifyExtra +golang-lib-pq/TestNullAfterNonNull +golang-lib-pq/TestOpenURL +golang-lib-pq/TestParameterCountMismatch +golang-lib-pq/TestParseArray +golang-lib-pq/TestParseArrayError +golang-lib-pq/TestParseComplete +golang-lib-pq/TestParseEnviron +golang-lib-pq/TestParseErrorInExtendedQuery +golang-lib-pq/TestParseOpts +golang-lib-pq/TestParseTs +golang-lib-pq/TestParseTsErrors +golang-lib-pq/TestPgpass +golang-lib-pq/TestPing +golang-lib-pq/TestQueryCancelRace +golang-lib-pq/TestQueryCancelledReused +golang-lib-pq/TestQueryRowBugWorkaround +golang-lib-pq/TestQuickClose +golang-lib-pq/TestQuoteIdentifier +golang-lib-pq/TestQuoteLiteral +golang-lib-pq/TestReadFloatPrecision +golang-lib-pq/TestReconnect +golang-lib-pq/TestReturning +golang-lib-pq/TestRowsCloseBeforeDone +golang-lib-pq/TestRowsColumnTypes +golang-lib-pq/TestRowsResultTag +golang-lib-pq/TestRuntimeParameters +golang-lib-pq/TestSNISupport +golang-lib-pq/TestSNISupport/SNI_is_not_passed_when_disabled +golang-lib-pq/TestSNISupport/SNI_is_not_set_for_IPv4 +golang-lib-pq/TestSNISupport/SNI_is_passed_when_asked_for +golang-lib-pq/TestSNISupport/SNI_is_set_by_default +golang-lib-pq/TestSSLClientCertificates +golang-lib-pq/TestSSLConnection +golang-lib-pq/TestSSLRequireWithRootCert +golang-lib-pq/TestSSLVerifyCA +golang-lib-pq/TestSSLVerifyFull +golang-lib-pq/TestScanNilTimestamp +golang-lib-pq/TestScanTimestamp +golang-lib-pq/TestSimpleParseURL +golang-lib-pq/TestSimpleQuery +golang-lib-pq/TestStatment +golang-lib-pq/TestStmtExecContext +golang-lib-pq/TestStmtExecContext/context.Background +golang-lib-pq/TestStmtExecContext/context.WithTimeout +golang-lib-pq/TestStmtExecContext/context.WithTimeout_exceeded +golang-lib-pq/TestStmtQueryContext +golang-lib-pq/TestStmtQueryContext/context.Background +golang-lib-pq/TestStmtQueryContext/context.WithTimeout +golang-lib-pq/TestStmtQueryContext/context.WithTimeout_exceeded +golang-lib-pq/TestStringArrayScanBytes +golang-lib-pq/TestStringArrayScanEmpty +golang-lib-pq/TestStringArrayScanError +golang-lib-pq/TestStringArrayScanNil +golang-lib-pq/TestStringArrayScanString +golang-lib-pq/TestStringArrayScanUnsupported +golang-lib-pq/TestStringArrayValue +golang-lib-pq/TestStringToBytea +golang-lib-pq/TestStringToUUID +golang-lib-pq/TestStringWithNul +golang-lib-pq/TestTextByteSliceToInt +golang-lib-pq/TestTextByteSliceToUUID +golang-lib-pq/TestTextDecodeIntoString +golang-lib-pq/TestTimeWithTimezone +golang-lib-pq/TestTimeWithTimezone/11:59:59+00:00_=>_0000-01-01T11:59:59Z +golang-lib-pq/TestTimeWithTimezone/11:59:59+04:00_=>_0000-01-01T11:59:59+04:00 +golang-lib-pq/TestTimeWithTimezone/11:59:59+04:01:02_=>_0000-01-01T11:59:59+04:01 +golang-lib-pq/TestTimeWithTimezone/11:59:59-04:01:02_=>_0000-01-01T11:59:59-04:01 +golang-lib-pq/TestTimeWithTimezone/24:00+00_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithTimezone/24:00-04:00_=>_0000-01-02T00:00:00-04:00 +golang-lib-pq/TestTimeWithTimezone/24:00:00+00_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithTimezone/24:00:00.0+00_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithTimezone/24:00:00.000000+00_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithTimezone/24:00Z_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithoutTimezone +golang-lib-pq/TestTimeWithoutTimezone/11:59:59_=>_0000-01-01T11:59:59Z +golang-lib-pq/TestTimeWithoutTimezone/24:00:00.000000_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithoutTimezone/24:00:00.0_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithoutTimezone/24:00:00_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimeWithoutTimezone/24:00_=>_0000-01-02T00:00:00Z +golang-lib-pq/TestTimestampWithOutTimezone +golang-lib-pq/TestTimestampWithTimeZone +golang-lib-pq/TestTxOptions +golang-lib-pq/TestXactMultiStmt diff --git a/ydb/tests/postgres_integrations/go-libpq/data/patch.diff b/ydb/tests/postgres_integrations/go-libpq/data/patch.diff new file mode 100644 index 000000000000..32c1d9828074 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/patch.diff @@ -0,0 +1,227 @@ +diff -ruN /original-sources/conn_test.go ./conn_test.go +--- /original-sources/conn_test.go 2023-04-26 04:34:24.000000000 +0000 ++++ ./conn_test.go 2023-09-15 09:16:17.844086739 +0000 +@@ -230,7 +230,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err := db.Exec("CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -318,7 +318,7 @@ + + if !r1.Next() { + if r.Err() != nil { +- t.Fatal(r1.Err()) ++ t.Fatal(r.Err()) + } + t.Fatal("expected row") + } +@@ -862,7 +862,7 @@ + defer db.Close() + + // stmt.exec() +- _, err := db.Exec("CREATE TEMP TABLE notnulltemp (a varchar(10) not null)") ++ _, err := db.Exec("CREATE TEMP TABLE notnulltemp (a varchar(10) not null, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -973,7 +973,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("create temp table test (i integer)") ++ _, err := db.Exec("create temp table test (i integer, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -1014,7 +1014,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE distributors (did integer default 0, dname text)") ++ _, err := db.Exec("CREATE TEMP TABLE distributors (did integer default 0, dname text, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -1067,7 +1067,7 @@ + } + defer txn.Rollback() + +- rows, err := txn.Query("CREATE TEMP TABLE foo(f1 int)") ++ rows, err := txn.Query("CREATE TEMP TABLE foo(f1 int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -1290,7 +1290,7 @@ + + if !r.Next() { + if r.Err() != nil { +- t.Fatal(err) ++ t.Fatal(r.Err()) + } + t.Fatal("expected row") + } +@@ -1305,7 +1305,7 @@ + + if !r.Next() { + if r.Err() != nil { +- t.Fatal(err) ++ t.Fatal(r.Err()) + } + t.Fatal("expected row") + } +@@ -1351,11 +1351,11 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err := db.Exec("CREATE TEMP TABLE temp (a int primary key)") + if err != nil { + t.Fatal(err) + } +- sqlInsert := "INSERT INTO temp VALUES (1)" ++ sqlInsert := "INSERT INTO temp (a) VALUES (1)" + sqlSelect := "SELECT * FROM temp" + tx, err := db.Begin() + if err != nil { +@@ -1501,7 +1501,7 @@ + } + + value, success := tryGetParameterValue() +- if success != test.success && !test.success { ++ if success != test.success && !success { + t.Fatalf("%v: unexpected error: %v", test.conninfo, err) + } + if success != test.success { +@@ -1603,7 +1603,7 @@ + ra int64 + }{ + { +- query: "CREATE TEMP TABLE temp (a int)", ++ query: "CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)", + tag: "CREATE TABLE", + }, + { +@@ -1623,19 +1623,19 @@ + }, + // Multiple statements that don't return rows should return the last tag. + { +- query: "CREATE TEMP TABLE t (a int); DROP TABLE t", ++ query: "CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); DROP TABLE t", + tag: "DROP TABLE", + }, + // Ensure a rows-returning query in any position among various tags-returing + // statements will prefer the rows. + { +- query: "SELECT 1; CREATE TEMP TABLE t (a int); DROP TABLE t", ++ query: "SELECT 1; CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); DROP TABLE t", + }, + { +- query: "CREATE TEMP TABLE t (a int); SELECT 1; DROP TABLE t", ++ query: "CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); SELECT 1; DROP TABLE t", + }, + { +- query: "CREATE TEMP TABLE t (a int); DROP TABLE t; SELECT 1", ++ query: "CREATE TEMP TABLE t (a int, _stub_id Serial PRIMARY KEY); DROP TABLE t; SELECT 1", + }, + } + +@@ -1775,7 +1775,7 @@ + db := openTestConn(t) + defer db.Close() + +- _, err := db.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err := db.Exec("CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +diff -ruN /original-sources/copy_test.go ./copy_test.go +--- /original-sources/copy_test.go 2023-04-26 04:34:24.000000000 +0000 ++++ ./copy_test.go 2023-09-15 09:14:56.207034622 +0000 +@@ -56,7 +56,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -125,7 +125,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -195,7 +195,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, text VARCHAR, blob BYTEA, nothing VARCHAR)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, text VARCHAR, blob BYTEA, nothing VARCHAR, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -254,7 +254,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -302,7 +302,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -327,7 +327,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (num INTEGER, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -383,7 +383,7 @@ + t.Fatal(err) + } + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, _stub_id Serial PRIMARY KEY)") + if err != nil { + t.Fatal(err) + } +@@ -463,7 +463,7 @@ + } + defer txn.Rollback() + +- _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar)") ++ _, err = txn.Exec("CREATE TEMP TABLE temp (a int, b varchar, _stub_id Serial PRIMARY KEY)") + if err != nil { + b.Fatal(err) + } +diff -ruN /original-sources/issues_test.go ./issues_test.go +--- /original-sources/issues_test.go 2023-04-26 04:34:24.000000000 +0000 ++++ ./issues_test.go 2023-08-22 09:35:23.189760257 +0000 +@@ -113,7 +113,7 @@ + time.Sleep(10 * time.Millisecond) + cancel() + }() +- row := db.QueryRowContext(ctx, "select pg_sleep(0.5)") ++ row := db.QueryRowContext(ctx, "select pg_sleep(4::float8)") + var pgSleepVoid string + err := row.Scan(&pgSleepVoid) + if pgErr := (*Error)(nil); !(errors.As(err, &pgErr) && pgErr.Code == cancelErrorCode) { diff --git a/ydb/tests/postgres_integrations/go-libpq/data/run-test.bash b/ydb/tests/postgres_integrations/go-libpq/data/run-test.bash new file mode 100755 index 000000000000..0a9e4945a0c0 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/run-test.bash @@ -0,0 +1,10 @@ +#!/bin/bash +# https://github.com/lib/pq + + +set -eu + +LOCAL_DIR=$(dirname "$0") +LOCAL_DIR=$(realpath "$LOCAL_DIR") + +scripts/run-test.bash "$LOCAL_DIR" diff --git a/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt b/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt new file mode 100644 index 000000000000..9f62657f9464 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt @@ -0,0 +1,106 @@ +golang-lib-pq/TestAppendEncodedText +golang-lib-pq/TestAppendEscapedText +golang-lib-pq/TestAppendEscapedTextExistingBuffer +golang-lib-pq/TestArrayScanner +golang-lib-pq/TestArrayValuer +golang-lib-pq/TestBadConn +golang-lib-pq/TestBoolArrayScanBytes +golang-lib-pq/TestBoolArrayScanEmpty +golang-lib-pq/TestBoolArrayScanError +golang-lib-pq/TestBoolArrayScanNil +golang-lib-pq/TestBoolArrayScanString +golang-lib-pq/TestBoolArrayScanUnsupported +golang-lib-pq/TestBoolArrayValue +golang-lib-pq/TestByteaArrayScanBytes +golang-lib-pq/TestByteaArrayScanEmpty +golang-lib-pq/TestByteaArrayScanError +golang-lib-pq/TestByteaArrayScanNil +golang-lib-pq/TestByteaArrayScanString +golang-lib-pq/TestByteaArrayScanUnsupported +golang-lib-pq/TestByteaArrayValue +golang-lib-pq/TestByteaOutputFormatEncoding +golang-lib-pq/TestCloseBadConn +golang-lib-pq/TestConnPrepareContext/context.WithTimeout_exceeded +golang-lib-pq/TestCopyInSchemaStmt +golang-lib-pq/TestCopyInStmt +golang-lib-pq/TestDataType +golang-lib-pq/TestDataTypeLength +golang-lib-pq/TestDataTypeName +golang-lib-pq/TestDataTypePrecisionScale +golang-lib-pq/TestDecodeUUIDBinaryError +golang-lib-pq/TestErrorDuringStartup +golang-lib-pq/TestErrorDuringStartupClosesConn +golang-lib-pq/TestErrorSQLState +golang-lib-pq/TestFloat32ArrayScanBytes +golang-lib-pq/TestFloat32ArrayScanEmpty +golang-lib-pq/TestFloat32ArrayScanError +golang-lib-pq/TestFloat32ArrayScanNil +golang-lib-pq/TestFloat32ArrayScanString +golang-lib-pq/TestFloat32ArrayScanUnsupported +golang-lib-pq/TestFloat32ArrayValue +golang-lib-pq/TestFloat64ArrayScanBytes +golang-lib-pq/TestFloat64ArrayScanEmpty +golang-lib-pq/TestFloat64ArrayScanError +golang-lib-pq/TestFloat64ArrayScanNil +golang-lib-pq/TestFloat64ArrayScanString +golang-lib-pq/TestFloat64ArrayScanUnsupported +golang-lib-pq/TestFloat64ArrayValue +golang-lib-pq/TestFormatAndParseTimestamp +golang-lib-pq/TestFormatTs +golang-lib-pq/TestFullParseURL +golang-lib-pq/TestGenericArrayScanDelimiter +golang-lib-pq/TestGenericArrayScanErrors +golang-lib-pq/TestGenericArrayScanScannerArrayBytes +golang-lib-pq/TestGenericArrayScanScannerArrayString +golang-lib-pq/TestGenericArrayScanScannerSliceBytes +golang-lib-pq/TestGenericArrayScanScannerSliceEmpty +golang-lib-pq/TestGenericArrayScanScannerSliceNil +golang-lib-pq/TestGenericArrayScanScannerSliceString +golang-lib-pq/TestGenericArrayScanUnsupported +golang-lib-pq/TestGenericArrayValue +golang-lib-pq/TestGenericArrayValueErrors +golang-lib-pq/TestGenericArrayValueUnsupported +golang-lib-pq/TestIPv6LoopbackParseURL +golang-lib-pq/TestInt32ArrayScanBytes +golang-lib-pq/TestInt32ArrayScanEmpty +golang-lib-pq/TestInt32ArrayScanError +golang-lib-pq/TestInt32ArrayScanNil +golang-lib-pq/TestInt32ArrayScanString +golang-lib-pq/TestInt32ArrayScanUnsupported +golang-lib-pq/TestInt32ArrayValue +golang-lib-pq/TestInt64ArrayScanBytes +golang-lib-pq/TestInt64ArrayScanEmpty +golang-lib-pq/TestInt64ArrayScanError +golang-lib-pq/TestInt64ArrayScanNil +golang-lib-pq/TestInt64ArrayScanString +golang-lib-pq/TestInt64ArrayScanUnsupported +golang-lib-pq/TestInt64ArrayValue +golang-lib-pq/TestInvalidProtocolParseURL +golang-lib-pq/TestIsUTF8 +golang-lib-pq/TestMinimalURL +golang-lib-pq/TestParseArray +golang-lib-pq/TestParseArrayError +golang-lib-pq/TestParseComplete +golang-lib-pq/TestParseEnviron +golang-lib-pq/TestParseOpts +golang-lib-pq/TestParseTs +golang-lib-pq/TestParseTsErrors +golang-lib-pq/TestQuoteIdentifier +golang-lib-pq/TestQuoteLiteral +golang-lib-pq/TestSNISupport +golang-lib-pq/TestSNISupport/SNI_is_not_passed_when_disabled +golang-lib-pq/TestSNISupport/SNI_is_not_set_for_IPv4 +golang-lib-pq/TestSNISupport/SNI_is_passed_when_asked_for +golang-lib-pq/TestSNISupport/SNI_is_set_by_default +golang-lib-pq/TestScanNilTimestamp +golang-lib-pq/TestScanTimestamp +golang-lib-pq/TestSimpleParseURL +golang-lib-pq/TestStringArrayScanBytes +golang-lib-pq/TestStringArrayScanEmpty +golang-lib-pq/TestStringArrayScanError +golang-lib-pq/TestStringArrayScanNil +golang-lib-pq/TestStringArrayScanString +golang-lib-pq/TestStringArrayScanUnsupported +golang-lib-pq/TestStringArrayValue +golang-lib-pq/TestStringWithNul +golang-lib-pq/TestTextDecodeIntoString diff --git a/ydb/tests/postgres_integrations/psycopg2/docker_wrapper_test.py b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py similarity index 100% rename from ydb/tests/postgres_integrations/psycopg2/docker_wrapper_test.py rename to ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py diff --git a/ydb/tests/postgres_integrations/psycopg2/ya.make b/ydb/tests/postgres_integrations/go-libpq/ya.make similarity index 88% rename from ydb/tests/postgres_integrations/psycopg2/ya.make rename to ydb/tests/postgres_integrations/go-libpq/ya.make index 6e7e4552a820..cd9534b5a32a 100644 --- a/ydb/tests/postgres_integrations/psycopg2/ya.make +++ b/ydb/tests/postgres_integrations/go-libpq/ya.make @@ -20,7 +20,7 @@ TEST_SRCS( DATA( - arcadia/ydb/tests/postgres_integrations/psycopg2/data + arcadia/ydb/tests/postgres_integrations/go-libpq/data ) PEERDIR( diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 787c91a3882f..2bb18b3a7047 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -1,19 +1,27 @@ from typing import List, Set, Optional from os import path +import docker + import pytest + class IntegrationTests: _folder: str + _image_name: str _all_tests: Set[str] _selected_test: Set[str] + _docker_executed: bool - def __init__(self, folder): + def __init__(self, folder: str, image_name: str = 'ydb-pg-test-image'): self._folder = folder + self._image_name = image_name self._all_tests = _read_tests(folder) self._selected_test = set(self._all_tests) + self._docker_executed = False + def pytest_generate_tests(self, metafunc: pytest.Metafunc): """ Return tests for run through pytest. @@ -29,7 +37,51 @@ def pytest_deselected(self, items: List[pytest.Item]): self._selected_test.remove(test_name) def execute_test(self, testname: str): - pass + self._run_tests_in_docker(testname) + + def _run_tests_in_docker(self, test_name: Optional[str]): + if self._docker_executed: + return + self._docker_executed = True + + if test_name is None: + test_name="" + + client: docker.Client = docker.from_env() + + client.images.build( + path = self._folder, + tag=self._image_name, + network_mode='host', + ) + logs = client.containers.run( + self._image_name, + auto_remove=True, + environment = [ + "PGUSER=root", + "PGPASSWORD=1234", + "PGHOST=ydb", + "PGPORT=5432", + "PGDATABASE=local", + "PQGOSSLTESTS=0", + "PQSSLCERTTEST_PATH=certs", + f"YDB_PG_TESTNAME={test_name}", + ], + mounts = [ + docker.types.Mount( + target="/exchange", + source=path.join(self._folder, "exchange"), + type="bind", + ), + docker.types.Mount( + target="/test-result", + source=path.join(self._folder, "test-result"), + type="bind", + ), + ], + network_mode='host', + ) + print(logs) def _read_tests(folder: str) -> Set[str]: with open(path.join(folder, "full-test-list.txt"), "rt") as f: diff --git a/ydb/tests/postgres_integrations/library/ya.make b/ydb/tests/postgres_integrations/library/ya.make index 525effab05ef..f18b45d46ce8 100644 --- a/ydb/tests/postgres_integrations/library/ya.make +++ b/ydb/tests/postgres_integrations/library/ya.make @@ -4,6 +4,7 @@ PY3_LIBRARY() ALL_PY_SRCS() PEERDIR( + contrib/docker contrib/python/requests contrib/python/tornado/tornado-4 ydb/tests/library diff --git a/ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt b/ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt deleted file mode 100644 index a778bb393f35..000000000000 --- a/ydb/tests/postgres_integrations/psycopg2/data/full-test-list.txt +++ /dev/null @@ -1,742 +0,0 @@ -python-psycopg2/tests.test_async.AsyncTests.test_async_after_async -python-psycopg2/tests.test_async.AsyncTests.test_async_callproc -python-psycopg2/tests.test_async.AsyncTests.test_async_connection_error_message -python-psycopg2/tests.test_async.AsyncTests.test_async_cursor_gone -python-psycopg2/tests.test_async.AsyncTests.test_async_dont_read_all -python-psycopg2/tests.test_async.AsyncTests.test_async_executemany -python-psycopg2/tests.test_async.AsyncTests.test_async_fetch_wrong_cursor -python-psycopg2/tests.test_async.AsyncTests.test_async_iter -python-psycopg2/tests.test_async.AsyncTests.test_async_named_cursor -python-psycopg2/tests.test_async.AsyncTests.test_async_scroll -python-psycopg2/tests.test_async.AsyncTests.test_async_select -python-psycopg2/tests.test_async.AsyncTests.test_async_subclass -python-psycopg2/tests.test_async.AsyncTests.test_close -python-psycopg2/tests.test_async.AsyncTests.test_commit_while_async -python-psycopg2/tests.test_async.AsyncTests.test_connection_setup -python-psycopg2/tests.test_async.AsyncTests.test_copy_no_hang -python-psycopg2/tests.test_async.AsyncTests.test_copy_while_async -python-psycopg2/tests.test_async.AsyncTests.test_error -python-psycopg2/tests.test_async.AsyncTests.test_error_two_cursors -python-psycopg2/tests.test_async.AsyncTests.test_fetch_after_async -python-psycopg2/tests.test_async.AsyncTests.test_flush_on_write -python-psycopg2/tests.test_async.AsyncTests.test_lobject_while_async -python-psycopg2/tests.test_async.AsyncTests.test_non_block_after_notification -python-psycopg2/tests.test_async.AsyncTests.test_notices -python-psycopg2/tests.test_async.AsyncTests.test_notify -python-psycopg2/tests.test_async.AsyncTests.test_poll_conn_for_notification -python-psycopg2/tests.test_async.AsyncTests.test_poll_noop -python-psycopg2/tests.test_async.AsyncTests.test_reset_while_async -python-psycopg2/tests.test_async.AsyncTests.test_rollback_while_async -python-psycopg2/tests.test_async.AsyncTests.test_scroll -python-psycopg2/tests.test_async.AsyncTests.test_set_parameters_while_async -python-psycopg2/tests.test_async.AsyncTests.test_stop_on_first_error -python-psycopg2/tests.test_async.AsyncTests.test_sync_poll -python-psycopg2/tests.test_bugX000.DateTimeAllocationBugTestCase.test_date_time_allocation_bug -python-psycopg2/tests.test_bug_gc.StolenReferenceTestCase.test_stolen_reference_bug -python-psycopg2/tests.test_cancel.CancelTests.test_async_cancel -python-psycopg2/tests.test_cancel.CancelTests.test_cancel -python-psycopg2/tests.test_cancel.CancelTests.test_empty_cancel -python-psycopg2/tests.test_connection.AutocommitTests.test_closed -python-psycopg2/tests.test_connection.AutocommitTests.test_default_no_autocommit -python-psycopg2/tests.test_connection.AutocommitTests.test_set_autocommit -python-psycopg2/tests.test_connection.AutocommitTests.test_set_intrans_error -python-psycopg2/tests.test_connection.AutocommitTests.test_set_session_autocommit -python-psycopg2/tests.test_connection.ConnectionTests.test_cleanup_on_badconn_close -python-psycopg2/tests.test_connection.ConnectionTests.test_close_idempotent -python-psycopg2/tests.test_connection.ConnectionTests.test_closed_attribute -python-psycopg2/tests.test_connection.ConnectionTests.test_commit_concurrency -python-psycopg2/tests.test_connection.ConnectionTests.test_concurrent_execution -python-psycopg2/tests.test_connection.ConnectionTests.test_connect_cursor_factory -python-psycopg2/tests.test_connection.ConnectionTests.test_connect_no_string -python-psycopg2/tests.test_connection.ConnectionTests.test_connect_nonnormal_envvar -python-psycopg2/tests.test_connection.ConnectionTests.test_cursor_closed_attribute -python-psycopg2/tests.test_connection.ConnectionTests.test_cursor_factory -python-psycopg2/tests.test_connection.ConnectionTests.test_cursor_factory_none -python-psycopg2/tests.test_connection.ConnectionTests.test_encoding_name -python-psycopg2/tests.test_connection.ConnectionTests.test_failed_init_status -python-psycopg2/tests.test_connection.ConnectionTests.test_get_native_connection -python-psycopg2/tests.test_connection.ConnectionTests.test_multiprocess_close -python-psycopg2/tests.test_connection.ConnectionTests.test_notices -python-psycopg2/tests.test_connection.ConnectionTests.test_notices_consistent_order -python-psycopg2/tests.test_connection.ConnectionTests.test_notices_deque -python-psycopg2/tests.test_connection.ConnectionTests.test_notices_limited -python-psycopg2/tests.test_connection.ConnectionTests.test_notices_noappend -python-psycopg2/tests.test_connection.ConnectionTests.test_pgconn_ptr -python-psycopg2/tests.test_connection.ConnectionTests.test_protocol_version -python-psycopg2/tests.test_connection.ConnectionTests.test_reset -python-psycopg2/tests.test_connection.ConnectionTests.test_server_version -python-psycopg2/tests.test_connection.ConnectionTests.test_tpc_unsupported -python-psycopg2/tests.test_connection.ConnectionTests.test_weakref -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_cancel_fails_prepared -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_recovered_xids -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_status_after_recover -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_commit -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_commit_one_phase -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_commit_recovered -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_recover_non_dbapi_connection -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_rollback -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_rollback_one_phase -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_tpc_rollback_recovered -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_unparsed_roundtrip -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_construction -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_encoding -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_from_string -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_roundtrip -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_to_string -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_unicode -python-psycopg2/tests.test_connection.ConnectionTwoPhaseTests.test_xid_unicode_unparsed -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_attribs_segfault -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_encoding -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_autocommit -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_closed -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_read_committed -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_isolation_level_serializable -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level_abort -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level_autocommit -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_set_isolation_level_default -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_setattr_isolation_level_int -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_setattr_isolation_level_invalid -python-psycopg2/tests.test_connection.IsolationLevelsTestCase.test_setattr_isolation_level_str -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_database_is_a_keyword -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_arguments -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_param -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_string -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_escape -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_get_dsn_parameters -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_no_dsn_munging -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_null_args -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_merging -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_validation -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_url_is_cool -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_bad_param -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn_uri -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_str_subclass -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_key -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_value -python-psycopg2/tests.test_connection.PasswordLeakTestCase.test_leak -python-psycopg2/tests.test_connection.PasswordLeakTestCase.test_url_leak -python-psycopg2/tests.test_connection.SignalTestCase.test_bug_551_no_returning -python-psycopg2/tests.test_connection.SignalTestCase.test_bug_551_returning -python-psycopg2/tests.test_connection.TestConnectionInfo.test_backend_pid -python-psycopg2/tests.test_connection.TestConnectionInfo.test_dbname -python-psycopg2/tests.test_connection.TestConnectionInfo.test_dsn_parameters -python-psycopg2/tests.test_connection.TestConnectionInfo.test_error_message -python-psycopg2/tests.test_connection.TestConnectionInfo.test_host -python-psycopg2/tests.test_connection.TestConnectionInfo.test_host_readonly -python-psycopg2/tests.test_connection.TestConnectionInfo.test_needs_password -python-psycopg2/tests.test_connection.TestConnectionInfo.test_options -python-psycopg2/tests.test_connection.TestConnectionInfo.test_parameter_status -python-psycopg2/tests.test_connection.TestConnectionInfo.test_password -python-psycopg2/tests.test_connection.TestConnectionInfo.test_port -python-psycopg2/tests.test_connection.TestConnectionInfo.test_protocol_version -python-psycopg2/tests.test_connection.TestConnectionInfo.test_server_version -python-psycopg2/tests.test_connection.TestConnectionInfo.test_socket -python-psycopg2/tests.test_connection.TestConnectionInfo.test_ssl_attribute -python-psycopg2/tests.test_connection.TestConnectionInfo.test_ssl_in_use -python-psycopg2/tests.test_connection.TestConnectionInfo.test_ssl_not_supported -python-psycopg2/tests.test_connection.TestConnectionInfo.test_status -python-psycopg2/tests.test_connection.TestConnectionInfo.test_transaction_status -python-psycopg2/tests.test_connection.TestConnectionInfo.test_used_password -python-psycopg2/tests.test_connection.TestConnectionInfo.test_user -python-psycopg2/tests.test_connection.TestEncryptPassword.test_bad_types -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_bad_before_libpq_10 -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_bad_libpq_10 -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_md5 -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_password_post_9_6 -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_scram -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_scram_pre_10 -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_server -python-psycopg2/tests.test_connection.TransactionControlTests.test_bad_isolation_level -python-psycopg2/tests.test_connection.TransactionControlTests.test_closed -python-psycopg2/tests.test_connection.TransactionControlTests.test_idempotence_check -python-psycopg2/tests.test_connection.TransactionControlTests.test_mixing_session_attribs -python-psycopg2/tests.test_connection.TransactionControlTests.test_not_in_transaction -python-psycopg2/tests.test_connection.TransactionControlTests.test_set_default -python-psycopg2/tests.test_connection.TransactionControlTests.test_set_deferrable -python-psycopg2/tests.test_connection.TransactionControlTests.test_set_deferrable_error -python-psycopg2/tests.test_connection.TransactionControlTests.test_set_isolation_level -python-psycopg2/tests.test_connection.TransactionControlTests.test_set_isolation_level_str -python-psycopg2/tests.test_connection.TransactionControlTests.test_set_read_only -python-psycopg2/tests.test_connection.TransactionControlTests.test_setattr_deferrable -python-psycopg2/tests.test_connection.TransactionControlTests.test_setattr_read_only -python-psycopg2/tests.test_copy.CopyTests.test_copy_bytes -python-psycopg2/tests.test_copy.CopyTests.test_copy_expert_file_refcount -python-psycopg2/tests.test_copy.CopyTests.test_copy_expert_textiobase -python-psycopg2/tests.test_copy.CopyTests.test_copy_from -python-psycopg2/tests.test_copy.CopyTests.test_copy_from_cols -python-psycopg2/tests.test_copy.CopyTests.test_copy_from_cols_err -python-psycopg2/tests.test_copy.CopyTests.test_copy_from_insane_size -python-psycopg2/tests.test_copy.CopyTests.test_copy_from_propagate_error -python-psycopg2/tests.test_copy.CopyTests.test_copy_from_segfault -python-psycopg2/tests.test_copy.CopyTests.test_copy_funny_names -python-psycopg2/tests.test_copy.CopyTests.test_copy_no_column_limit -python-psycopg2/tests.test_copy.CopyTests.test_copy_query -python-psycopg2/tests.test_copy.CopyTests.test_copy_rowcount -python-psycopg2/tests.test_copy.CopyTests.test_copy_rowcount_error -python-psycopg2/tests.test_copy.CopyTests.test_copy_text -python-psycopg2/tests.test_copy.CopyTests.test_copy_to -python-psycopg2/tests.test_copy.CopyTests.test_copy_to_propagate_error -python-psycopg2/tests.test_copy.CopyTests.test_copy_to_segfault -python-psycopg2/tests.test_cursor.CursorTests.test_bad_placeholder -python-psycopg2/tests.test_cursor.CursorTests.test_bad_subclass -python-psycopg2/tests.test_cursor.CursorTests.test_callproc_badparam -python-psycopg2/tests.test_cursor.CursorTests.test_callproc_dict -python-psycopg2/tests.test_cursor.CursorTests.test_cast -python-psycopg2/tests.test_cursor.CursorTests.test_cast_specificity -python-psycopg2/tests.test_cursor.CursorTests.test_close_idempotent -python-psycopg2/tests.test_cursor.CursorTests.test_column_refcount -python-psycopg2/tests.test_cursor.CursorTests.test_description_attribs -python-psycopg2/tests.test_cursor.CursorTests.test_description_extra_attribs -python-psycopg2/tests.test_cursor.CursorTests.test_description_slice -python-psycopg2/tests.test_cursor.CursorTests.test_empty_query -python-psycopg2/tests.test_cursor.CursorTests.test_executemany_propagate_exceptions -python-psycopg2/tests.test_cursor.CursorTests.test_external_close_async -python-psycopg2/tests.test_cursor.CursorTests.test_external_close_sync -python-psycopg2/tests.test_cursor.CursorTests.test_modify_closed -python-psycopg2/tests.test_cursor.CursorTests.test_mogrify_decimal_explodes -python-psycopg2/tests.test_cursor.CursorTests.test_mogrify_leak_on_multiple_reference -python-psycopg2/tests.test_cursor.CursorTests.test_mogrify_unicode -python-psycopg2/tests.test_cursor.CursorTests.test_null_name -python-psycopg2/tests.test_cursor.CursorTests.test_pgresult_ptr -python-psycopg2/tests.test_cursor.CursorTests.test_pickle_description -python-psycopg2/tests.test_cursor.CursorTests.test_rowcount_on_executemany_returning -python-psycopg2/tests.test_cursor.CursorTests.test_weakref -python-psycopg2/tests.test_cursor.NamedCursorTests.test_invalid_name -python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_default_itersize -python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_efficient -python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_itersize -python-psycopg2/tests.test_cursor.NamedCursorTests.test_iter_named_cursor_rownumber -python-psycopg2/tests.test_cursor.NamedCursorTests.test_named_cursor_stealing -python-psycopg2/tests.test_cursor.NamedCursorTests.test_named_noop_close -python-psycopg2/tests.test_cursor.NamedCursorTests.test_not_scrollable -python-psycopg2/tests.test_cursor.NamedCursorTests.test_scroll -python-psycopg2/tests.test_cursor.NamedCursorTests.test_scroll_named -python-psycopg2/tests.test_cursor.NamedCursorTests.test_scrollable -python-psycopg2/tests.test_cursor.NamedCursorTests.test_stolen_named_cursor_close -python-psycopg2/tests.test_cursor.NamedCursorTests.test_withhold -python-psycopg2/tests.test_cursor.NamedCursorTests.test_withhold_autocommit -python-psycopg2/tests.test_cursor.NamedCursorTests.test_withhold_no_begin -python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_date -python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_datetime -python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_infinity_tz -python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_negative_timedelta -python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_time -python-psycopg2/tests.test_dates.DatetimeTests.test_adapt_timedelta -python-psycopg2/tests.test_dates.DatetimeTests.test_default_tzinfo -python-psycopg2/tests.test_dates.DatetimeTests.test_fotz_tzinfo -python-psycopg2/tests.test_dates.DatetimeTests.test_interval_iso_8601_not_supported -python-psycopg2/tests.test_dates.DatetimeTests.test_interval_overflow -python-psycopg2/tests.test_dates.DatetimeTests.test_large_interval -python-psycopg2/tests.test_dates.DatetimeTests.test_micros_rounding -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_10k_date -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_10k_datetime -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_bc_date -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_bc_datetime -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_date -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime_microseconds -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime_no_timezone -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_datetime_timezone -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_incomplete_date -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_incomplete_datetime -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_incomplete_time -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_infinity -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_interval -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_negative_interval -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_date -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_datetime -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_interval -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_null_time -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time_microseconds -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time_no_timezone -python-psycopg2/tests.test_dates.DatetimeTests.test_parse_time_timezone -python-psycopg2/tests.test_dates.DatetimeTests.test_redshift_day -python-psycopg2/tests.test_dates.DatetimeTests.test_time_24 -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_date -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_date_array -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetime -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetime_array -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetimetz -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_datetimetz_array -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_interval -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_interval_array -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_time -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_time_array -python-psycopg2/tests.test_dates.DatetimeTests.test_type_roundtrip_timetz -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_no_args -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_timedelta -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_instance_caching -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_pickle -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_name -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_negative_offset -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_positive_offset -python-psycopg2/tests.test_dates.FromTicksTestCase.test_date_value_error_sec_59_99 -python-psycopg2/tests.test_dates.FromTicksTestCase.test_time_value_error_sec_59_99 -python-psycopg2/tests.test_dates.FromTicksTestCase.test_timestamp_value_error_sec_59_99 -python-psycopg2/tests.test_errcodes.ErrocodeTests.test_ambiguous_names -python-psycopg2/tests.test_errcodes.ErrocodeTests.test_lookup_threadsafe -python-psycopg2/tests.test_errors.ErrorsTests.test_connection_exceptions_backwards_compatibility -python-psycopg2/tests.test_errors.ErrorsTests.test_exception_class -python-psycopg2/tests.test_errors.ErrorsTests.test_exception_class_fallback -python-psycopg2/tests.test_errors.ErrorsTests.test_has_base_exceptions -python-psycopg2/tests.test_errors.ErrorsTests.test_lookup -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchAll -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchMany -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchManyNoarg -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorFetchOne -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorIter -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorIterRowNumber -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorRealWithNamedCursorNotGreedy -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchAll -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchMany -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchManyNoarg -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealFetchOne -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealIter -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testDictCursorWithPlainCursorRealIterRowNumber -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testPickleRealDictRow -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.testRealMeansReal -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_copy -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_iter_methods -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_mod -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_order -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorRealTests.test_pop -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictConnCursorArgs -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchAll -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchMany -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchManyNoarg -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorFetchOne -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorIter -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorIterRowNumber -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithNamedCursorNotGreedy -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchAll -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchMany -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchManyNoarg -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorFetchOne -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorIter -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testDictCursorWithPlainCursorIterRowNumber -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testPickleDictRow -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.testUpdateRow -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.test_copy -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.test_iter_methods -python-psycopg2/tests.test_extras_dictcursor.ExtrasDictCursorTests.test_order -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_bad_col_names -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_cache -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_cursor_args -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_executemany -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchall -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchmany -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchmany_noarg -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_fetchone -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_iter -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_max_cache -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_minimal_generation -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_fetchall -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_fetchmany -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_fetchone -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_named_rownumber -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_no_result_no_surprise -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_nonascii_name -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_not_greedy -python-psycopg2/tests.test_extras_dictcursor.NamedTupleCursorTest.test_record_updated -python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_composed -python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_empty -python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_many -python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_one -python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_pages -python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_tuples -python-psycopg2/tests.test_fast_executemany.TestExecuteBatch.test_unicode -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_composed -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_dicts -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_empty -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_invalid_sql -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_many -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_one -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_pages -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_percent_escape -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_returning -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_tuples -python-psycopg2/tests.test_fast_executemany.TestExecuteValues.test_unicode -python-psycopg2/tests.test_fast_executemany.TestPaginate.test_paginate -python-psycopg2/tests.test_green.CallbackErrorTestCase.test_errors_named_cursor -python-psycopg2/tests.test_green.CallbackErrorTestCase.test_errors_on_connection -python-psycopg2/tests.test_green.CallbackErrorTestCase.test_errors_on_query -python-psycopg2/tests.test_green.GreenTestCase.test_copy_no_hang -python-psycopg2/tests.test_green.GreenTestCase.test_dont_freak_out -python-psycopg2/tests.test_green.GreenTestCase.test_error_in_callback -python-psycopg2/tests.test_green.GreenTestCase.test_flush_on_write -python-psycopg2/tests.test_green.GreenTestCase.test_non_block_after_notice -python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_cidr_adapt -python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_cidr_array_cast -python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_cidr_cast -python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_inet_adapt -python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_inet_array_cast -python-psycopg2/tests.test_ipaddress.NetworkingTestCase.test_inet_cast -python-psycopg2/tests.test_lobject.LargeObject64Tests.test_seek_tell_truncate_greater_than_2gb -python-psycopg2/tests.test_lobject.LargeObjectNot64Tests.test_seek_larger_than_2gb -python-psycopg2/tests.test_lobject.LargeObjectNot64Tests.test_truncate_larger_than_2gb -python-psycopg2/tests.test_lobject.LargeObjectTests.test_close -python-psycopg2/tests.test_lobject.LargeObjectTests.test_close_after_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_close_connection_gone -python-psycopg2/tests.test_lobject.LargeObjectTests.test_close_twice -python-psycopg2/tests.test_lobject.LargeObjectTests.test_connection_needed -python-psycopg2/tests.test_lobject.LargeObjectTests.test_create -python-psycopg2/tests.test_lobject.LargeObjectTests.test_create_with_existing_oid -python-psycopg2/tests.test_lobject.LargeObjectTests.test_create_with_oid -python-psycopg2/tests.test_lobject.LargeObjectTests.test_export -python-psycopg2/tests.test_lobject.LargeObjectTests.test_export_after_close -python-psycopg2/tests.test_lobject.LargeObjectTests.test_export_after_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_factory -python-psycopg2/tests.test_lobject.LargeObjectTests.test_import -python-psycopg2/tests.test_lobject.LargeObjectTests.test_large_oid -python-psycopg2/tests.test_lobject.LargeObjectTests.test_mode_defaults -python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_existing -python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_for_write -python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_mode_n -python-psycopg2/tests.test_lobject.LargeObjectTests.test_open_non_existent -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_close -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_tpc_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_after_tpc_prepare -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_binary -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_large -python-psycopg2/tests.test_lobject.LargeObjectTests.test_read_text -python-psycopg2/tests.test_lobject.LargeObjectTests.test_seek_after_close -python-psycopg2/tests.test_lobject.LargeObjectTests.test_seek_after_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_seek_tell -python-psycopg2/tests.test_lobject.LargeObjectTests.test_tell_after_close -python-psycopg2/tests.test_lobject.LargeObjectTests.test_tell_after_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_unlink -python-psycopg2/tests.test_lobject.LargeObjectTests.test_unlink_after_close -python-psycopg2/tests.test_lobject.LargeObjectTests.test_unlink_after_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_write -python-psycopg2/tests.test_lobject.LargeObjectTests.test_write_after_close -python-psycopg2/tests.test_lobject.LargeObjectTests.test_write_after_commit -python-psycopg2/tests.test_lobject.LargeObjectTests.test_write_large -python-psycopg2/tests.test_lobject.LargeObjectTruncateTests.test_truncate -python-psycopg2/tests.test_lobject.LargeObjectTruncateTests.test_truncate_after_close -python-psycopg2/tests.test_lobject.LargeObjectTruncateTests.test_truncate_after_commit -python-psycopg2/tests.test_module.ConnectTestCase.test_async -python-psycopg2/tests.test_module.ConnectTestCase.test_dsn -python-psycopg2/tests.test_module.ConnectTestCase.test_empty_param -python-psycopg2/tests.test_module.ConnectTestCase.test_escape -python-psycopg2/tests.test_module.ConnectTestCase.test_factory -python-psycopg2/tests.test_module.ConnectTestCase.test_generic_keywords -python-psycopg2/tests.test_module.ConnectTestCase.test_int_port_param -python-psycopg2/tests.test_module.ConnectTestCase.test_no_keywords -python-psycopg2/tests.test_module.ConnectTestCase.test_params_merging -python-psycopg2/tests.test_module.ConnectTestCase.test_supported_keywords -python-psycopg2/tests.test_module.ConnectTestCase.test_there_might_be_nothing -python-psycopg2/tests.test_module.ExceptionsTestCase.test_9_3_diagnostics -python-psycopg2/tests.test_module.ExceptionsTestCase.test_9_6_diagnostics -python-psycopg2/tests.test_module.ExceptionsTestCase.test_attributes -python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_attributes -python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_copy -python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_from_commit -python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_independent -python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_life -python-psycopg2/tests.test_module.ExceptionsTestCase.test_diagnostics_values -python-psycopg2/tests.test_module.ExceptionsTestCase.test_pickle -python-psycopg2/tests.test_module.ExceptionsTestCase.test_pickle_connection_error -python-psycopg2/tests.test_module.TestExtensionModule.test_import_internal -python-psycopg2/tests.test_module.TestVersionDiscovery.test_libpq_version -python-psycopg2/tests.test_notify.NotifiesTests.test_compare -python-psycopg2/tests.test_notify.NotifiesTests.test_compare_tuple -python-psycopg2/tests.test_notify.NotifiesTests.test_hash -python-psycopg2/tests.test_notify.NotifiesTests.test_many_notifies -python-psycopg2/tests.test_notify.NotifiesTests.test_notifies_received_on_execute -python-psycopg2/tests.test_notify.NotifiesTests.test_notifies_received_on_poll -python-psycopg2/tests.test_notify.NotifiesTests.test_notify_attributes -python-psycopg2/tests.test_notify.NotifiesTests.test_notify_deque -python-psycopg2/tests.test_notify.NotifiesTests.test_notify_init -python-psycopg2/tests.test_notify.NotifiesTests.test_notify_noappend -python-psycopg2/tests.test_notify.NotifiesTests.test_notify_object -python-psycopg2/tests.test_notify.NotifiesTests.test_notify_payload -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_commit_in_tpc_fails -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_rollback_in_tpc_fails -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_begin -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_begin_in_tpc_transaction_fails -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_begin_in_transaction_fails -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_commit_with_prepare -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_commit_without_prepare -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_rollback_with_prepare -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_tpc_rollback_without_prepare -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2TPCTests.test_xid -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_BINARY -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Binary -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_DATETIME -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Date -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Exceptions -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_ExceptionsAsConnectionAttributes -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_NUMBER -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_None -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_ROWID -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_STRING -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Time -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_Timestamp -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_apilevel -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_arraysize -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_callproc -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_close -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_commit -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_connect -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_cursor -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_cursor_isolation -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_description -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_execute -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_executemany -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_fetchall -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_fetchmany -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_fetchone -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_mixedfetch -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_nextset -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_paramstyle -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_rollback -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_rowcount -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_setinputsizes -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_setoutputsize -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_setoutputsize_basic -python-psycopg2/tests.test_psycopg2_dbapi20.Psycopg2Tests.test_threadsafety -python-psycopg2/tests.test_quote.QuotingTestCase.test_binary -python-psycopg2/tests.test_quote.QuotingTestCase.test_bytes -python-psycopg2/tests.test_quote.QuotingTestCase.test_koi8 -python-psycopg2/tests.test_quote.QuotingTestCase.test_latin1 -python-psycopg2/tests.test_quote.QuotingTestCase.test_string -python-psycopg2/tests.test_quote.QuotingTestCase.test_string_null_terminator -python-psycopg2/tests.test_quote.QuotingTestCase.test_unicode -python-psycopg2/tests.test_quote.TestQuotedIdentifier.test_identifier -python-psycopg2/tests.test_quote.TestQuotedIdentifier.test_unicode_ident -python-psycopg2/tests.test_quote.TestQuotedString.test_encoding_from_conn -python-psycopg2/tests.test_quote.TestStringAdapter.test_adapt_bytes -python-psycopg2/tests.test_quote.TestStringAdapter.test_connection_wins_anyway -python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_default -python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_error -python-psycopg2/tests.test_quote.TestStringAdapter.test_set_encoding -python-psycopg2/tests.test_replication.AsyncReplicationTest.test_async_replication -python-psycopg2/tests.test_replication.ReplicationTest.test_create_replication_slot -python-psycopg2/tests.test_replication.ReplicationTest.test_datestyle -python-psycopg2/tests.test_replication.ReplicationTest.test_keepalive -python-psycopg2/tests.test_replication.ReplicationTest.test_logical_replication_connection -python-psycopg2/tests.test_replication.ReplicationTest.test_physical_replication_connection -python-psycopg2/tests.test_replication.ReplicationTest.test_start_and_recover_from_error -python-psycopg2/tests.test_replication.ReplicationTest.test_start_on_missing_replication_slot -python-psycopg2/tests.test_replication.ReplicationTest.test_start_replication_expert_sql -python-psycopg2/tests.test_replication.ReplicationTest.test_stop_replication -python-psycopg2/tests.test_sql.ComposedTest.test_class -python-psycopg2/tests.test_sql.ComposedTest.test_eq -python-psycopg2/tests.test_sql.ComposedTest.test_iter -python-psycopg2/tests.test_sql.ComposedTest.test_join -python-psycopg2/tests.test_sql.ComposedTest.test_repr -python-psycopg2/tests.test_sql.ComposedTest.test_seq -python-psycopg2/tests.test_sql.ComposedTest.test_sum -python-psycopg2/tests.test_sql.ComposedTest.test_sum_inplace -python-psycopg2/tests.test_sql.IdentifierTests.test_as_str -python-psycopg2/tests.test_sql.IdentifierTests.test_class -python-psycopg2/tests.test_sql.IdentifierTests.test_eq -python-psycopg2/tests.test_sql.IdentifierTests.test_init -python-psycopg2/tests.test_sql.IdentifierTests.test_join -python-psycopg2/tests.test_sql.IdentifierTests.test_repr -python-psycopg2/tests.test_sql.IdentifierTests.test_strings -python-psycopg2/tests.test_sql.LiteralTests.test_class -python-psycopg2/tests.test_sql.LiteralTests.test_eq -python-psycopg2/tests.test_sql.LiteralTests.test_init -python-psycopg2/tests.test_sql.LiteralTests.test_must_be_adaptable -python-psycopg2/tests.test_sql.LiteralTests.test_repr -python-psycopg2/tests.test_sql.LiteralTests.test_wrapped -python-psycopg2/tests.test_sql.PlaceholderTest.test_bad_name -python-psycopg2/tests.test_sql.PlaceholderTest.test_class -python-psycopg2/tests.test_sql.PlaceholderTest.test_eq -python-psycopg2/tests.test_sql.PlaceholderTest.test_name -python-psycopg2/tests.test_sql.PlaceholderTest.test_repr -python-psycopg2/tests.test_sql.PlaceholderTest.test_repr_name -python-psycopg2/tests.test_sql.SQLTests.test_class -python-psycopg2/tests.test_sql.SQLTests.test_eq -python-psycopg2/tests.test_sql.SQLTests.test_init -python-psycopg2/tests.test_sql.SQLTests.test_join -python-psycopg2/tests.test_sql.SQLTests.test_multiply -python-psycopg2/tests.test_sql.SQLTests.test_repr -python-psycopg2/tests.test_sql.SQLTests.test_string -python-psycopg2/tests.test_sql.SQLTests.test_sum -python-psycopg2/tests.test_sql.SQLTests.test_sum_inplace -python-psycopg2/tests.test_sql.SqlFormatTests.test_braces_escape -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_bad_args_type -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs_auto -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_empty -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_literal -python-psycopg2/tests.test_sql.SqlFormatTests.test_copy -python-psycopg2/tests.test_sql.SqlFormatTests.test_dict -python-psycopg2/tests.test_sql.SqlFormatTests.test_execute -python-psycopg2/tests.test_sql.SqlFormatTests.test_executemany -python-psycopg2/tests.test_sql.SqlFormatTests.test_must_be_adaptable -python-psycopg2/tests.test_sql.SqlFormatTests.test_must_be_composable -python-psycopg2/tests.test_sql.SqlFormatTests.test_no_modifiers -python-psycopg2/tests.test_sql.SqlFormatTests.test_percent_escape -python-psycopg2/tests.test_sql.SqlFormatTests.test_pos -python-psycopg2/tests.test_sql.SqlFormatTests.test_pos_spec -python-psycopg2/tests.test_sql.ValuesTest.test_default -python-psycopg2/tests.test_sql.ValuesTest.test_null -python-psycopg2/tests.test_transaction.DeadlockSerializationTests.test_deadlock -python-psycopg2/tests.test_transaction.DeadlockSerializationTests.test_serialisation_failure -python-psycopg2/tests.test_transaction.QueryCancellationTests.test_statement_timeout -python-psycopg2/tests.test_transaction.TransactionTests.test_commit -python-psycopg2/tests.test_transaction.TransactionTests.test_failed_commit -python-psycopg2/tests.test_transaction.TransactionTests.test_rollback -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_most_specific -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype_3 -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_conform_subclass_precedence -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank_hex -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_escaped_mixed -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_escaped_octal -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex_upper -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_null -python-psycopg2/tests.test_types_basic.TypesBasicTests.testAdaptBytearray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testAdaptMemoryview -python-psycopg2/tests.test_types_basic.TypesBasicTests.testArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testArrayEscape -python-psycopg2/tests.test_types_basic.TypesBasicTests.testArrayMalformed -python-psycopg2/tests.test_types_basic.TypesBasicTests.testArrayOfNulls -python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinary -python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryEmptyString -python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryNone -python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryRoundTrip -python-psycopg2/tests.test_types_basic.TypesBasicTests.testBoolean -python-psycopg2/tests.test_types_basic.TypesBasicTests.testByteaHexCheckFalsePositive -python-psycopg2/tests.test_types_basic.TypesBasicTests.testBytesArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testDecimal -python-psycopg2/tests.test_types_basic.TypesBasicTests.testEmptyArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testEmptyArrayNoCast -python-psycopg2/tests.test_types_basic.TypesBasicTests.testEmptyArrayRegression -python-psycopg2/tests.test_types_basic.TypesBasicTests.testFloatInf -python-psycopg2/tests.test_types_basic.TypesBasicTests.testFloatNan -python-psycopg2/tests.test_types_basic.TypesBasicTests.testGenericArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testGenericArrayNull -python-psycopg2/tests.test_types_basic.TypesBasicTests.testIntEnum -python-psycopg2/tests.test_types_basic.TypesBasicTests.testNegNumber -python-psycopg2/tests.test_types_basic.TypesBasicTests.testNestedArrays -python-psycopg2/tests.test_types_basic.TypesBasicTests.testNestedEmptyArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testNetworkArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testNumber -python-psycopg2/tests.test_types_basic.TypesBasicTests.testQuoting -python-psycopg2/tests.test_types_basic.TypesBasicTests.testTextArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testTypeRoundtripBytes -python-psycopg2/tests.test_types_basic.TypesBasicTests.testTypeRoundtripBytesArray -python-psycopg2/tests.test_types_basic.TypesBasicTests.testUnicode -python-psycopg2/tests.test_types_basic.TypesBasicTests.testUnicodeArray -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_cast_composite -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_cast_nested -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_array -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_namespace -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_namespace_path -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_not_found -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_composite_weird_name -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_empty_string -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_from_tables -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_non_dbapi_connection -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_none_fast_path -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_none_in_record -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_register_globally -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_register_on_connection -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_register_on_cursor -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_subclass -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_tokenization -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_wrong_schema -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_adapt_8 -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_adapt_9 -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_array_cast -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_array_cast_oid -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_non_dbapi_connection -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_oid -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_parse -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_register_conn -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_register_curs -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_register_globally -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_roundtrip -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_roundtrip_array -python-psycopg2/tests.test_types_extras.JsonTestCase.test_adapt -python-psycopg2/tests.test_types_extras.JsonTestCase.test_adapt_dumps -python-psycopg2/tests.test_types_extras.JsonTestCase.test_adapt_subclass -python-psycopg2/tests.test_types_extras.JsonTestCase.test_default_cast -python-psycopg2/tests.test_types_extras.JsonTestCase.test_loads -python-psycopg2/tests.test_types_extras.JsonTestCase.test_no_array_oid -python-psycopg2/tests.test_types_extras.JsonTestCase.test_no_conn_curs -python-psycopg2/tests.test_types_extras.JsonTestCase.test_null -python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_default -python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_globally -python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_on_connection -python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_on_cursor -python-psycopg2/tests.test_types_extras.JsonTestCase.test_register_on_dict -python-psycopg2/tests.test_types_extras.JsonTestCase.test_scs -python-psycopg2/tests.test_types_extras.JsonTestCase.test_str -python-psycopg2/tests.test_types_extras.JsonTestCase.test_type_not_available -python-psycopg2/tests.test_types_extras.JsonbTestCase.test_default_cast -python-psycopg2/tests.test_types_extras.JsonbTestCase.test_loads -python-psycopg2/tests.test_types_extras.JsonbTestCase.test_null -python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_default -python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_globally -python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_on_connection -python-psycopg2/tests.test_types_extras.JsonbTestCase.test_register_on_cursor -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_adapt_date_range -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_adapt_number_range -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_adapt_numeric_range -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_date -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_empty -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_inf -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_null -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_numbers -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_timestamp -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_cast_timestamptz -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_rang_weird_name -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_range_escaping -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_range_not_found -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_register_range_adapter -python-psycopg2/tests.test_types_extras.RangeCasterTestCase.test_schema_range -python-psycopg2/tests.test_types_extras.RangeTestCase.test_bad_bounds -python-psycopg2/tests.test_types_extras.RangeTestCase.test_bounds -python-psycopg2/tests.test_types_extras.RangeTestCase.test_empty -python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_hash -python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_subclass -python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_wrong_type -python-psycopg2/tests.test_types_extras.RangeTestCase.test_ge_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_gt_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_in -python-psycopg2/tests.test_types_extras.RangeTestCase.test_keywords -python-psycopg2/tests.test_types_extras.RangeTestCase.test_le_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_lt_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_nobounds -python-psycopg2/tests.test_types_extras.RangeTestCase.test_nonzero -python-psycopg2/tests.test_types_extras.RangeTestCase.test_noparam -python-psycopg2/tests.test_types_extras.RangeTestCase.test_pickling -python-psycopg2/tests.test_types_extras.RangeTestCase.test_str -python-psycopg2/tests.test_types_extras.RangeTestCase.test_str_datetime -python-psycopg2/tests.test_types_extras.TypesExtrasTests.testINET -python-psycopg2/tests.test_types_extras.TypesExtrasTests.testINETARRAY -python-psycopg2/tests.test_types_extras.TypesExtrasTests.testUUID -python-psycopg2/tests.test_types_extras.TypesExtrasTests.testUUIDARRAY -python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_adapt_fail -python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_inet_conform -python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_point_array -python-psycopg2/tests.test_with.WithConnectionTestCase.test_cant_reenter -python-psycopg2/tests.test_with.WithConnectionTestCase.test_subclass_commit -python-psycopg2/tests.test_with.WithConnectionTestCase.test_subclass_rollback -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_autocommit -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_autocommit_pgerror -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_autocommit_pyerror -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_closed -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_connect_idiom -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_error_db -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_error_python -python-psycopg2/tests.test_with.WithConnectionTestCase.test_with_ok -python-psycopg2/tests.test_with.WithCursorTestCase.test_exception_swallow -python-psycopg2/tests.test_with.WithCursorTestCase.test_named_with_noop -python-psycopg2/tests.test_with.WithCursorTestCase.test_subclass -python-psycopg2/tests.test_with.WithCursorTestCase.test_with_error -python-psycopg2/tests.test_with.WithCursorTestCase.test_with_ok diff --git a/ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt b/ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt deleted file mode 100644 index 82c9e871ef30..000000000000 --- a/ydb/tests/postgres_integrations/psycopg2/data/unit-tests.txt +++ /dev/null @@ -1,114 +0,0 @@ -python-psycopg2/tests.test_bugX000.DateTimeAllocationBugTestCase.test_date_time_allocation_bug -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_database_is_a_keyword -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_arguments -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_param -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_empty_string -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_escape -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_no_dsn_munging -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_null_args -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_merging -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_params_validation -python-psycopg2/tests.test_connection.MakeDsnTestCase.test_url_is_cool -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_bad_param -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_parse_dsn_uri -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_str_subclass -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_key -python-psycopg2/tests.test_connection.ParseDsnTestCase.test_unicode_value -python-psycopg2/tests.test_connection.PasswordLeakTestCase.test_url_leak -python-psycopg2/tests.test_connection.TestEncryptPassword.test_encrypt_md5 -python-psycopg2/tests.test_cursor.CursorTests.test_bad_subclass -python-psycopg2/tests.test_cursor.CursorTests.test_column_refcount -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_no_args -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_init_with_timedelta -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_instance_caching -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_pickle -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_name -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_negative_offset -python-psycopg2/tests.test_dates.FixedOffsetTimezoneTests.test_repr_with_positive_offset -python-psycopg2/tests.test_dates.FromTicksTestCase.test_date_value_error_sec_59_99 -python-psycopg2/tests.test_dates.FromTicksTestCase.test_timestamp_value_error_sec_59_99 -python-psycopg2/tests.test_dates.FromTicksTestCase.test_time_value_error_sec_59_99 -python-psycopg2/tests.test_errcodes.ErrocodeTests.test_ambiguous_names -python-psycopg2/tests.test_errcodes.ErrocodeTests.test_lookup_threadsafe -python-psycopg2/tests.test_errors.ErrorsTests.test_connection_exceptions_backwards_compatibility -python-psycopg2/tests.test_errors.ErrorsTests.test_has_base_exceptions -python-psycopg2/tests.test_errors.ErrorsTests.test_lookup -python-psycopg2/tests.test_fast_executemany.TestPaginate.test_paginate -python-psycopg2/tests.test_module.ConnectTestCase.test_async -python-psycopg2/tests.test_module.ConnectTestCase.test_dsn -python-psycopg2/tests.test_module.ConnectTestCase.test_empty_param -python-psycopg2/tests.test_module.ConnectTestCase.test_escape -python-psycopg2/tests.test_module.ConnectTestCase.test_factory -python-psycopg2/tests.test_module.ConnectTestCase.test_generic_keywords -python-psycopg2/tests.test_module.ConnectTestCase.test_int_port_param -python-psycopg2/tests.test_module.ConnectTestCase.test_no_keywords -python-psycopg2/tests.test_module.ConnectTestCase.test_params_merging -python-psycopg2/tests.test_module.ConnectTestCase.test_supported_keywords -python-psycopg2/tests.test_module.ConnectTestCase.test_there_might_be_nothing -python-psycopg2/tests.test_module.TestExtensionModule.test_import_internal -python-psycopg2/tests.test_module.TestVersionDiscovery.test_libpq_version -python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_default -python-psycopg2/tests.test_quote.TestStringAdapter.test_encoding_error -python-psycopg2/tests.test_quote.TestStringAdapter.test_set_encoding -python-psycopg2/tests.test_sql.ComposedTest.test_class -python-psycopg2/tests.test_sql.ComposedTest.test_eq -python-psycopg2/tests.test_sql.ComposedTest.test_iter -python-psycopg2/tests.test_sql.ComposedTest.test_repr -python-psycopg2/tests.test_sql.ComposedTest.test_seq -python-psycopg2/tests.test_sql.IdentifierTests.test_class -python-psycopg2/tests.test_sql.IdentifierTests.test_eq -python-psycopg2/tests.test_sql.IdentifierTests.test_init -python-psycopg2/tests.test_sql.IdentifierTests.test_join -python-psycopg2/tests.test_sql.IdentifierTests.test_repr -python-psycopg2/tests.test_sql.IdentifierTests.test_strings -python-psycopg2/tests.test_sql.LiteralTests.test_class -python-psycopg2/tests.test_sql.LiteralTests.test_eq -python-psycopg2/tests.test_sql.LiteralTests.test_init -python-psycopg2/tests.test_sql.LiteralTests.test_wrapped -python-psycopg2/tests.test_sql.PlaceholderTest.test_bad_name -python-psycopg2/tests.test_sql.PlaceholderTest.test_class -python-psycopg2/tests.test_sql.PlaceholderTest.test_eq -python-psycopg2/tests.test_sql.PlaceholderTest.test_name -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_bad_args_type -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs -python-psycopg2/tests.test_sql.SqlFormatTests.test_compose_badnargs_auto -python-psycopg2/tests.test_sql.SqlFormatTests.test_must_be_composable -python-psycopg2/tests.test_sql.SqlFormatTests.test_no_modifiers -python-psycopg2/tests.test_sql.SQLTests.test_class -python-psycopg2/tests.test_sql.SQLTests.test_eq -python-psycopg2/tests.test_sql.SQLTests.test_init -python-psycopg2/tests.test_sql.SQLTests.test_string -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_most_specific -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_adapt_subtype_3 -python-psycopg2/tests.test_types_basic.AdaptSubclassTest.test_conform_subclass_precedence -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_blank_hex -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_escaped_mixed -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_escaped_octal -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_full_hex_upper -python-psycopg2/tests.test_types_basic.ByteaParserTest.test_null -python-psycopg2/tests.test_types_basic.TypesBasicTests.testBinaryEmptyString -python-psycopg2/tests.test_types_extras.AdaptTypeTestCase.test_tokenization -python-psycopg2/tests.test_types_extras.HstoreTestCase.test_parse -python-psycopg2/tests.test_types_extras.RangeTestCase.test_bad_bounds -python-psycopg2/tests.test_types_extras.RangeTestCase.test_bounds -python-psycopg2/tests.test_types_extras.RangeTestCase.test_empty -python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_hash -python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_subclass -python-psycopg2/tests.test_types_extras.RangeTestCase.test_eq_wrong_type -python-psycopg2/tests.test_types_extras.RangeTestCase.test_ge_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_gt_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_in -python-psycopg2/tests.test_types_extras.RangeTestCase.test_keywords -python-psycopg2/tests.test_types_extras.RangeTestCase.test_le_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_lt_ordering -python-psycopg2/tests.test_types_extras.RangeTestCase.test_nobounds -python-psycopg2/tests.test_types_extras.RangeTestCase.test_nonzero -python-psycopg2/tests.test_types_extras.RangeTestCase.test_noparam -python-psycopg2/tests.test_types_extras.RangeTestCase.test_pickling -python-psycopg2/tests.test_types_extras.RangeTestCase.test_str -python-psycopg2/tests.test_types_extras.RangeTestCase.test_str_datetime -python-psycopg2/tests.test_types_extras.TypesExtrasTests.test_adapt_fail From db7ae4deb7316e329a8b99ea41d46e31b5da8781 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 9 Jul 2024 21:14:35 +0000 Subject: [PATCH 006/261] run tests from container --- .../go-libpq/conftest.py | 20 +++++----- .../go-libpq/data/.gitignore | 1 + .../go-run-separate-tests.bash | 0 .../go-libpq/data/docker-start.bash | 1 + .../go-libpq/docker_wrapper_test.py | 24 +++++++++++- .../postgres_integrations/go-libpq/ya.make | 5 --- .../library/pytest_integration.py | 39 +++++++++++++++---- .../postgres_integrations/library/ya.make | 2 +- 8 files changed, 67 insertions(+), 25 deletions(-) mode change 100644 => 100755 ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash diff --git a/ydb/tests/postgres_integrations/go-libpq/conftest.py b/ydb/tests/postgres_integrations/go-libpq/conftest.py index bdf601c4be56..78b518020bbb 100644 --- a/ydb/tests/postgres_integrations/go-libpq/conftest.py +++ b/ydb/tests/postgres_integrations/go-libpq/conftest.py @@ -1,19 +1,19 @@ -from os import path -from typing import Sequence +# from os import path +# from typing import Sequence -import yatest +# import yatest -import pytest +# import pytest -from ydb.tests.postgres_integrations.library import IntegrationTests +# from ydb.tests.postgres_integrations.library import IntegrationTests -integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) +# integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) -def pytest_generate_tests(metafunc: pytest.Metafunc): - integrations.pytest_generate_tests(metafunc) +# def pytest_generate_tests(metafunc: pytest.Metafunc): +# integrations.pytest_generate_tests(metafunc) -def pytest_deselected(items: Sequence[pytest.Item]): - integrations.pytest_deselected(items) +# def pytest_deselected(items: Sequence[pytest.Item]): +# integrations.pytest_deselected(items) diff --git a/ydb/tests/postgres_integrations/go-libpq/data/.gitignore b/ydb/tests/postgres_integrations/go-libpq/data/.gitignore index f6a04b18adcd..8d846571dd11 100644 --- a/ydb/tests/postgres_integrations/go-libpq/data/.gitignore +++ b/ydb/tests/postgres_integrations/go-libpq/data/.gitignore @@ -1,2 +1,3 @@ +/exchange/ /sources/ /test-result/ \ No newline at end of file diff --git a/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash b/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash old mode 100644 new mode 100755 diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash index 6a12a7a3a470..28aa6e1795d5 100755 --- a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash @@ -34,6 +34,7 @@ echo "Start test" export SKIP_TESTS +mkdir -p /test-result/raw PQTEST_BINARY_PARAMETERS=no /go-run-separate-tests.bash sed -e 's|classname=""|classname="golang-lib-pq"|' -i /test-result/raw/result.xml diff --git a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py index 5779f8b5e91b..c308405fca2b 100644 --- a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py +++ b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py @@ -1,4 +1,26 @@ -from .conftest import integrations +# from .conftest import integrations + + + +from os import path +from typing import Sequence + +import yatest + +import pytest + + +from ydb.tests.postgres_integrations.library import IntegrationTests + +integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) + +def pytest_generate_tests(metafunc: pytest.Metafunc): + integrations.pytest_generate_tests(metafunc) + + +def pytest_deselected(items: Sequence[pytest.Item]): + integrations.pytest_deselected(items) + def test(testname): integrations.execute_test(testname) diff --git a/ydb/tests/postgres_integrations/go-libpq/ya.make b/ydb/tests/postgres_integrations/go-libpq/ya.make index cd9534b5a32a..2fb2ea6e8e7b 100644 --- a/ydb/tests/postgres_integrations/go-libpq/ya.make +++ b/ydb/tests/postgres_integrations/go-libpq/ya.make @@ -24,12 +24,7 @@ DATA( ) PEERDIR( - contrib/python/requests - contrib/python/tornado/tornado-4 - ydb/tests/library ydb/tests/postgres_integrations/library - ydb/tests/oss/ydb_sdk_import - ydb/public/sdk/python ) REQUIREMENTS(ram:10) diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 2bb18b3a7047..1636f256133f 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -1,10 +1,12 @@ from typing import List, Set, Optional +import os from os import path import docker - import pytest +import yatest + class IntegrationTests: _folder: str @@ -49,14 +51,30 @@ def _run_tests_in_docker(self, test_name: Optional[str]): client: docker.Client = docker.from_env() + client.images.build( path = self._folder, tag=self._image_name, network_mode='host', ) - logs = client.containers.run( - self._image_name, - auto_remove=True, + + try: + exchange_folder=path.join(yatest.common.output_path(), "exchange") + os.mkdir(exchange_folder) + except FileExistsError: + pass + + try: + test_result_folder=path.join(yatest.common.output_path(), "test-result") + os.mkdir(test_result_folder) + except FileExistsError: + pass + + container = client.containers.create( + image=self._image_name, + # command="/docker-start.bash", + # detach=True, + # auto_remove=True, environment = [ "PGUSER=root", "PGPASSWORD=1234", @@ -65,23 +83,28 @@ def _run_tests_in_docker(self, test_name: Optional[str]): "PGDATABASE=local", "PQGOSSLTESTS=0", "PQSSLCERTTEST_PATH=certs", - f"YDB_PG_TESTNAME={test_name}", + # f"YDB_PG_TESTNAME={test_name}", ], mounts = [ docker.types.Mount( target="/exchange", - source=path.join(self._folder, "exchange"), + source=exchange_folder, type="bind", ), docker.types.Mount( target="/test-result", - source=path.join(self._folder, "test-result"), + source=test_result_folder, type="bind", ), ], network_mode='host', ) - print(logs) + try: + container.start() + container.wait() + print(container.logs().decode()) + finally: + container.remove() def _read_tests(folder: str) -> Set[str]: with open(path.join(folder, "full-test-list.txt"), "rt") as f: diff --git a/ydb/tests/postgres_integrations/library/ya.make b/ydb/tests/postgres_integrations/library/ya.make index f18b45d46ce8..92e9a54b17e1 100644 --- a/ydb/tests/postgres_integrations/library/ya.make +++ b/ydb/tests/postgres_integrations/library/ya.make @@ -4,7 +4,7 @@ PY3_LIBRARY() ALL_PY_SRCS() PEERDIR( - contrib/docker + contrib/python/docker contrib/python/requests contrib/python/tornado/tornado-4 ydb/tests/library From dc59b1eb441be27f5f2124f8bce655656cc56a2b Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 16 Jul 2024 16:52:02 +0000 Subject: [PATCH 007/261] initial parse test results --- .../library/pytest_integration.py | 82 +++++++++++++++++-- .../library/ut/data/test-results-example.xml | 18 ++++ .../library/ut/integrations_test.py | 17 ++++ .../postgres_integrations/library/ut/ya.make | 18 ++++ .../postgres_integrations/library/ya.make | 13 ++- 5 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml create mode 100644 ydb/tests/postgres_integrations/library/ut/integrations_test.py create mode 100644 ydb/tests/postgres_integrations/library/ut/ya.make diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 1636f256133f..c0f3e19ad21e 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -1,19 +1,36 @@ -from typing import List, Set, Optional +from typing import Dict, List, Set, Optional import os from os import path +from dataclasses import dataclass +from enum import Enum + import docker +import xmltodict import pytest import yatest +class TestState(Enum): + PASSED = 1 + FAILED = 2 + SKIPPED = 3 + +@dataclass +class TestCase: + name: str + state: TestState + log: str + + class IntegrationTests: _folder: str _image_name: str _all_tests: Set[str] _selected_test: Set[str] _docker_executed: bool + _test_results: Dict[str, TestCase] def __init__(self, folder: str, image_name: str = 'ydb-pg-test-image'): self._folder = folder @@ -23,6 +40,7 @@ def __init__(self, folder: str, image_name: str = 'ydb-pg-test-image'): self._selected_test = set(self._all_tests) self._docker_executed = False + self._test_results = dict() def pytest_generate_tests(self, metafunc: pytest.Metafunc): """ @@ -39,7 +57,24 @@ def pytest_deselected(self, items: List[pytest.Item]): self._selected_test.remove(test_name) def execute_test(self, testname: str): - self._run_tests_in_docker(testname) + if not self._docker_executed: + self._run_tests_in_docker(testname) + test_results_file=path.join(self._test_result_folder, "raw", "result.xml") + self._test_results = _read_tests_result(test_results_file) + + self._check_test_results(testname) + + def _check_test_results(self, testname: str): + print(self._test_results) + test = self._test_results[testname] + if test.state == TestState.PASSED: + return + if test.state == TestState.SKIPPED: + pytest.skip(test.log) + if test.state == TestState.FAILED: + pytest.fail(reason=test.log) + + raise Exception(f"Unexpected test state: '{test.state}'") def _run_tests_in_docker(self, test_name: Optional[str]): if self._docker_executed: @@ -65,11 +100,12 @@ def _run_tests_in_docker(self, test_name: Optional[str]): pass try: - test_result_folder=path.join(yatest.common.output_path(), "test-result") - os.mkdir(test_result_folder) + os.mkdir(self._test_result_folder) except FileExistsError: pass + + # TODO: run YDB with scripts/receipt and get connection port/database with runtime container = client.containers.create( image=self._image_name, # command="/docker-start.bash", @@ -93,7 +129,7 @@ def _run_tests_in_docker(self, test_name: Optional[str]): ), docker.types.Mount( target="/test-result", - source=test_result_folder, + source=self._test_result_folder, type="bind", ), ], @@ -106,6 +142,11 @@ def _run_tests_in_docker(self, test_name: Optional[str]): finally: container.remove() + + @property + def _test_result_folder(self): + return path.join(yatest.common.output_path(), "test-result") + def _read_tests(folder: str) -> Set[str]: with open(path.join(folder, "full-test-list.txt"), "rt") as f: all = set(line.strip() for line in f.readlines()) @@ -115,3 +156,34 @@ def _read_tests(folder: str) -> Set[str]: test_list_for_run = all - unit return test_list_for_run + +def _read_tests_result(filepath: str) -> Dict[str, TestCase]: + with open(filepath, "rt") as f: + data = f.read() + d = xmltodict.parse(data) + testsuites = d["testsuites"] + test_suite = testsuites["testsuite"] + test_cases = test_suite["testcase"] + + res: Dict[str, TestCase] = dict() + + for test_case in test_cases: + name = test_case["@classname"] + "/" + test_case["@name"] + test_state = TestState.PASSED + if "failure" in test_case: + test_state = TestState.FAILED + log = test_case["failure"].get("#text", "") + if "skipped" in test_case: + test_state = TestState.SKIPPED + log = test_case["skipped"].get("#text", "") + else: + test_state = TestState.PASSED + log = "" + + res[name] = TestCase( + name=name, + state=test_state, + log=log, + ) + + return res diff --git a/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml b/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml new file mode 100644 index 000000000000..cf02633f318b --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml @@ -0,0 +1,18 @@ + + + + + + fail mess + + + + + + skip message + + + + + + diff --git a/ydb/tests/postgres_integrations/library/ut/integrations_test.py b/ydb/tests/postgres_integrations/library/ut/integrations_test.py new file mode 100644 index 000000000000..e64d72faab06 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/integrations_test.py @@ -0,0 +1,17 @@ +from os import path + +import yatest + +from ydb.tests.postgres_integrations.library import pytest_integration + +TEST_DATA_FOLDER=yatest.common.source_path("ydb/tests/postgres_integrations/library/ut/data") + +def test_read_jtest_results(): + filepath = path.join(TEST_DATA_FOLDER, "test-results-example.xml") + parsed_result = pytest_integration._read_tests_result(filepath) + + expected_result = { + "o/OK": pytest_integration.TestCase(name="o/OK", state=pytest_integration.TestState.PASSED, log=""), + } + + assert expected_result == parsed_result diff --git a/ydb/tests/postgres_integrations/library/ut/ya.make b/ydb/tests/postgres_integrations/library/ut/ya.make new file mode 100644 index 000000000000..7272910853b3 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/ya.make @@ -0,0 +1,18 @@ +PY3TEST() + +TEST_SRCS( + integrations_test.py +) + + +DATA( + arcadia/ydb/tests/postgres_integrations/library/ut/data +) + +PEERDIR( + ydb/tests/postgres_integrations/library +) + +REQUIREMENTS(ram:10) + +END() diff --git a/ydb/tests/postgres_integrations/library/ya.make b/ydb/tests/postgres_integrations/library/ya.make index 92e9a54b17e1..4bd6b33328b6 100644 --- a/ydb/tests/postgres_integrations/library/ya.make +++ b/ydb/tests/postgres_integrations/library/ya.make @@ -5,13 +5,18 @@ ALL_PY_SRCS() PEERDIR( contrib/python/docker - contrib/python/requests - contrib/python/tornado/tornado-4 + contrib/python/xmltodict +# contrib/python/requests +# contrib/python/tornado/tornado-4 ydb/tests/library - ydb/tests/oss/ydb_sdk_import - ydb/public/sdk/python +# ydb/tests/oss/ydb_sdk_import +# ydb/public/sdk/python ) REQUIREMENTS(ram:10) END() + +RECURSE_FOR_TESTS( + ut +) \ No newline at end of file From ccf84f3ea7222f91779d49a195b8be525ec22513 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 31 Jul 2024 10:08:18 +0000 Subject: [PATCH 008/261] fix ya make --- ydb/tests/postgres_integrations/ya.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/tests/postgres_integrations/ya.make b/ydb/tests/postgres_integrations/ya.make index 94d799013f0c..3839f278034e 100644 --- a/ydb/tests/postgres_integrations/ya.make +++ b/ydb/tests/postgres_integrations/ya.make @@ -1,4 +1,4 @@ RECURSE( - psycopg2 + go-libpq library ) From 0b27333308bbc60e9c83b227265861991bd6c198 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 31 Jul 2024 15:19:34 +0000 Subject: [PATCH 009/261] run all test for already runned ydb --- .../library/pytest_integration.py | 49 ++++++++++++++----- .../library/ut/data/test-results-example.xml | 10 ++-- .../library/ut/integrations_test.py | 22 ++++++--- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index c0f3e19ad21e..2c2c98cc7194 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -11,6 +11,8 @@ import yatest +import logging + class TestState(Enum): PASSED = 1 @@ -53,26 +55,41 @@ def pytest_generate_tests(self, metafunc: pytest.Metafunc): def pytest_deselected(self, items: List[pytest.Item]): for item in items: test_name = item.callspec.id - print("rekby-test: removed", test_name) + print(f"rekby-test removed: '{test_name}") self._selected_test.remove(test_name) def execute_test(self, testname: str): if not self._docker_executed: - self._run_tests_in_docker(testname) + if len(self._selected_test) == 1: + execute_test_filter=testname + else: + execute_test_filter="" + + print("rekby-selected-tests: ", self._selected_test) + print(f"rekby-test-name: '{execute_test_filter}'") + + self._run_tests_in_docker(execute_test_filter) test_results_file=path.join(self._test_result_folder, "raw", "result.xml") self._test_results = _read_tests_result(test_results_file) + print(f"rekby result parsed tests:\n\n{self._test_results}") self._check_test_results(testname) def _check_test_results(self, testname: str): - print(self._test_results) - test = self._test_results[testname] + try: + test = self._test_results[testname] + except KeyError: + pytest.fail("test result not found, may be the test was not run") + if test.state == TestState.PASSED: + logging.getLogger().log(logging.INFO, test.log) return if test.state == TestState.SKIPPED: - pytest.skip(test.log) + logging.getLogger().log(logging.INFO, test.log) + pytest.skip() if test.state == TestState.FAILED: - pytest.fail(reason=test.log) + logging.getLogger().log(logging.ERROR, test.log) + pytest.fail() raise Exception(f"Unexpected test state: '{test.state}'") @@ -114,12 +131,12 @@ def _run_tests_in_docker(self, test_name: Optional[str]): environment = [ "PGUSER=root", "PGPASSWORD=1234", - "PGHOST=ydb", + "PGHOST=localhost", "PGPORT=5432", "PGDATABASE=local", "PQGOSSLTESTS=0", "PQSSLCERTTEST_PATH=certs", - # f"YDB_PG_TESTNAME={test_name}", + f"YDB_PG_TESTNAME={test_name}", ], mounts = [ docker.types.Mount( @@ -167,15 +184,23 @@ def _read_tests_result(filepath: str) -> Dict[str, TestCase]: res: Dict[str, TestCase] = dict() + def get_text(test_case, field_name: str)->str: + field_val = test_case[field_name] + if type(field_val) == str: + return field_val + elif type(field_val) == dict: + return field_val.get("#text", "") + raise Exception(f"Unknown field val for field '{field_name}':\n{field_val}") + for test_case in test_cases: name = test_case["@classname"] + "/" + test_case["@name"] - test_state = TestState.PASSED + print("rekby-debug", test_case) if "failure" in test_case: test_state = TestState.FAILED - log = test_case["failure"].get("#text", "") - if "skipped" in test_case: + log = get_text(test_case, "failure") + elif "skipped" in test_case: test_state = TestState.SKIPPED - log = test_case["skipped"].get("#text", "") + log = get_text(test_case, "skipped") else: test_state = TestState.PASSED log = "" diff --git a/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml b/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml index cf02633f318b..36b6a5a0cb5c 100644 --- a/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml +++ b/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml @@ -3,16 +3,16 @@ - fail mess + fail mess - - skip message + + skip message - - + + diff --git a/ydb/tests/postgres_integrations/library/ut/integrations_test.py b/ydb/tests/postgres_integrations/library/ut/integrations_test.py index e64d72faab06..c86da034878d 100644 --- a/ydb/tests/postgres_integrations/library/ut/integrations_test.py +++ b/ydb/tests/postgres_integrations/library/ut/integrations_test.py @@ -1,17 +1,27 @@ from os import path +import pytest import yatest from ydb.tests.postgres_integrations.library import pytest_integration +from ydb.tests.postgres_integrations.library.pytest_integration import TestCase, TestState TEST_DATA_FOLDER=yatest.common.source_path("ydb/tests/postgres_integrations/library/ut/data") -def test_read_jtest_results(): +@pytest.mark.parametrize( + "test", + [ + TestCase(name="o/OK", state=TestState.PASSED, log=""), + TestCase(name="f/failed1", state=TestState.FAILED, log="fail mess"), + TestCase(name="f/failed2", state=TestState.FAILED, log="escaped error"), + TestCase(name="s/skipped1", state=TestState.SKIPPED, log="skip message"), + TestCase(name="s/skipped2", state=TestState.SKIPPED, log="escaped skip message"), + ], + ids=lambda item: item.name +) +def test_read_jtest_results(test): filepath = path.join(TEST_DATA_FOLDER, "test-results-example.xml") parsed_result = pytest_integration._read_tests_result(filepath) - expected_result = { - "o/OK": pytest_integration.TestCase(name="o/OK", state=pytest_integration.TestState.PASSED, log=""), - } - - assert expected_result == parsed_result + parsed_test = parsed_result[test.name] + assert test == parsed_test From 63229272a33cfa66dd8a8024a7cffbd3cf47a22b Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 31 Jul 2024 19:08:47 +0000 Subject: [PATCH 010/261] support pgwire port in command line parameter --- ydb/core/config/init/init_impl.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ydb/core/config/init/init_impl.h b/ydb/core/config/init/init_impl.h index 9da49a0e5584..71468f4b37dd 100644 --- a/ydb/core/config/init/init_impl.h +++ b/ydb/core/config/init/init_impl.h @@ -315,6 +315,7 @@ struct TCommonAppOptions { TString GRpcPublicHost = ""; ui32 GRpcPublicPort = 0; ui32 GRpcsPublicPort = 0; + ui32 PGWirePort = 0; TVector GRpcPublicAddressesV4; TVector GRpcPublicAddressesV6; TString GRpcPublicTargetNameOverride = ""; @@ -326,7 +327,7 @@ struct TCommonAppOptions { bool SysLogEnabled = false; bool TcpEnabled = false; bool SuppressVersionCheck = false; - EWorkload Workload = EWorkload::Hybrid; + EWorkload Workload = EWorkload::Hybrid; void RegisterCliOptions(NLastGetopt::TOpts& opts) { opts.AddLongOption("cluster-name", "which cluster this node belongs to") @@ -383,6 +384,7 @@ struct TCommonAppOptions { opts.AddLongOption("grpc-public-host", "set public gRPC host for discovery").RequiredArgument("HOST").StoreResult(&GRpcPublicHost); opts.AddLongOption("grpc-public-port", "set public gRPC port for discovery").RequiredArgument("PORT").StoreResult(&GRpcPublicPort); opts.AddLongOption("grpcs-public-port", "set public gRPC SSL port for discovery").RequiredArgument("PORT").StoreResult(&GRpcsPublicPort); + opts.AddLongOption("pgwire-port", "set public port for listenin postgres protocol").OptionalArgument("PORT").StoreResult(&PGWirePort); opts.AddLongOption("grpc-public-address-v4", "set public ipv4 address for discovery").RequiredArgument("ADDR").EmplaceTo(&GRpcPublicAddressesV4); opts.AddLongOption("grpc-public-address-v6", "set public ipv6 address for discovery").RequiredArgument("ADDR").EmplaceTo(&GRpcPublicAddressesV6); opts.AddLongOption("grpc-public-target-name-override", "set public hostname override for TLS in discovery").RequiredArgument("HOST").StoreResult(&GRpcPublicTargetNameOverride); @@ -496,7 +498,7 @@ struct TCommonAppOptions { if (offset) { connectorConfig.MutableEndpoint()->Setport(InterconnectPort + offset) ; - // Assign default hostname 'localhost', because + // Assign default hostname 'localhost', because // connector is usually deployed to the same host as the dynamic node. if (connectorConfig.GetEndpoint().host().Empty()) { connectorConfig.MutableEndpoint()->Sethost("localhost"); @@ -597,6 +599,9 @@ struct TCommonAppOptions { } ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::GRpcConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); } + if (PGWirePort) { + appConfig.MutableLocalPgWireConfig()->SetListeningPort(PGWirePort); + } for (const auto& addr : GRpcPublicAddressesV4) { appConfig.MutableGRpcConfig()->AddPublicAddressesV4(addr); } @@ -658,7 +663,7 @@ struct TCommonAppOptions { ApplyDontStartGrpcProxy(*appConfig.MutableGRpcConfig(), ConfigUpdateTracer); break; case EWorkload::Hybrid: - // default, do nothing + // default, do nothing break; } } From af574ac7bd2da40291b77c54551c566b7cd3c24a Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 31 Jul 2024 19:09:18 +0000 Subject: [PATCH 011/261] support pgwire port with port allocator in kikimr cluster runner --- ydb/tests/library/harness/kikimr_config.py | 11 +- .../library/harness/kikimr_port_allocator.py | 17 +- ydb/tests/library/harness/kikimr_runner.py | 2 + .../library/pytest_integration.py | 153 +++++++++++------- 4 files changed, 112 insertions(+), 71 deletions(-) diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index c7e59eb33581..329fdb19bbde 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -163,7 +163,6 @@ def __init__( default_user_sid=None, pg_compatible_expirement=False, generic_connector_config=None, # typing.Optional[TGenericConnectorConfig] - pgwire_port=None, ): if extra_feature_flags is None: extra_feature_flags = [] @@ -260,10 +259,6 @@ def __init__( self.yaml_config["local_pg_wire_config"] = {} self.yaml_config["local_pg_wire_config"]["listening_port"] = os.getenv('PGWIRE_LISTENING_PORT') - if pgwire_port: - self.yaml_config["local_pg_wire_config"] = {} - self.yaml_config["local_pg_wire_config"]["listening_port"] = pgwire_port - if disable_iterator_reads: self.yaml_config["table_service_config"]["enable_kqp_scan_query_source_read"] = False @@ -398,10 +393,8 @@ def __init__( if not "local_pg_wire_config" in self.yaml_config: self.yaml_config["local_pg_wire_config"] = {} - ydb_pg_port=5432 - if 'YDB_PG_PORT' in os.environ: - ydb_pg_port = os.environ['YDB_PG_PORT'] - self.yaml_config['local_pg_wire_config']['listening_port'] = ydb_pg_port + ydb_pgwire_port=self.port_allocator.get_node_port_allocator(node_id).pgwire_port + self.yaml_config['local_pg_wire_config']['listening_port'] = ydb_pgwire_port # https://github.com/ydb-platform/ydb/issues/5152 # self.yaml_config["table_service_config"]["enable_pg_consts_to_params"] = True diff --git a/ydb/tests/library/harness/kikimr_port_allocator.py b/ydb/tests/library/harness/kikimr_port_allocator.py index ea6633cc89c9..b28609f75377 100644 --- a/ydb/tests/library/harness/kikimr_port_allocator.py +++ b/ydb/tests/library/harness/kikimr_port_allocator.py @@ -84,6 +84,7 @@ def __init__(self, port_manager): self.__grpc_ssl_port = None self.__ext_port = None self.__public_http_port = None + self.__pgwire_port = None @property def mon_port(self): @@ -127,6 +128,12 @@ def ext_port(self): self.__ext_port = self.__port_manager.get_port() return self.__ext_port + @property + def pgwire_port(self): + if self.__pgwire_port is None: + self.__pgwire_port = self.__port_manager.get_port() + return self.__pgwire_port + @property def public_http_port(self): if self.__public_http_port is None: @@ -162,7 +169,7 @@ def release_ports(self): class KikimrFixedNodePortAllocator(KikimrNodePortAllocatorInterface): def __init__(self, base_port_offset, mon_port=8765, grpc_port=2135, mbus_port=2134, ic_port=19001, sqs_port=8771, grpc_ssl_port=2137, - ext_port=2237, public_http_port=8766): + ext_port=2237, public_http_port=8766, pgwire_port=5432): super(KikimrFixedNodePortAllocator, self).__init__() self.base_port_offset = base_port_offset @@ -192,6 +199,10 @@ def __init__(self, base_port_offset, mon_port=8765, grpc_port=2135, mbus_port=21 self.__public_http_port = int(os.getenv('PUBLIC_HTTP_PORT')) else: self.__public_http_port = public_http_port + if os.getenv('YDB_PGWIRE_PORT') is not None: + self.__pgwire_port = int(os.getenv('YDB_PGWIRE_PORT')) + else: + self.__pgwire_port = pgwire_port @property def mon_port(self): @@ -224,6 +235,10 @@ def ext_port(self): def public_http_port(self): return self.__public_http_port + self.base_port_offset + @property + def pgwire_port(self): + return self.__pgwire_port + self.base_port_offset + class KikimrFixedPortAllocator(KikimrPortAllocatorInterface): def __init__(self, diff --git a/ydb/tests/library/harness/kikimr_runner.py b/ydb/tests/library/harness/kikimr_runner.py index b6f6c5066037..fbf7f7eb123d 100644 --- a/ydb/tests/library/harness/kikimr_runner.py +++ b/ydb/tests/library/harness/kikimr_runner.py @@ -68,6 +68,7 @@ def __init__(self, node_id, config_path, port_allocator, cluster_name, configura self.mon_port = port_allocator.mon_port self.ic_port = port_allocator.ic_port self.grpc_ssl_port = port_allocator.grpc_ssl_port + self.pgwire_port = port_allocator.pgwire_port self.sqs_port = None if configurator.sqs_service_enabled: self.sqs_port = port_allocator.sqs_port @@ -172,6 +173,7 @@ def __make_run_command(self): "--grpc-port=%s" % self.grpc_port, "--mon-port=%d" % self.mon_port, "--ic-port=%d" % self.ic_port, + "--pgwire-port=%d" % self.pgwire_port, ] ) diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 2c2c98cc7194..61bbf9e8005c 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -13,6 +13,9 @@ import logging +from ydb.tests.library.harness.daemon import DaemonError +from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory +from ydb.tests.library.harness.kikimr_runner import KiKiMR class TestState(Enum): PASSED = 1 @@ -33,6 +36,7 @@ class IntegrationTests: _selected_test: Set[str] _docker_executed: bool _test_results: Dict[str, TestCase] + _kikimr_factory: KiKiMR def __init__(self, folder: str, image_name: str = 'ydb-pg-test-image'): self._folder = folder @@ -43,6 +47,7 @@ def __init__(self, folder: str, image_name: str = 'ydb-pg-test-image'): self._docker_executed = False self._test_results = dict() + self._kikimr_factory = kikimr_cluster_factory() def pytest_generate_tests(self, metafunc: pytest.Metafunc): """ @@ -68,11 +73,16 @@ def execute_test(self, testname: str): print("rekby-selected-tests: ", self._selected_test) print(f"rekby-test-name: '{execute_test_filter}'") - self._run_tests_in_docker(execute_test_filter) + try: + self._run_tests_in_docker(execute_test_filter) + print(f"rekby docker test finished") + except BaseException as e: + print("docker test exception", e) + raise + test_results_file=path.join(self._test_result_folder, "raw", "result.xml") self._test_results = _read_tests_result(test_results_file) print(f"rekby result parsed tests:\n\n{self._test_results}") - self._check_test_results(testname) def _check_test_results(self, testname: str): @@ -93,71 +103,92 @@ def _check_test_results(self, testname: str): raise Exception(f"Unexpected test state: '{test.state}'") - def _run_tests_in_docker(self, test_name: Optional[str]): + def _run_tests_in_docker(self, test_filter: str): if self._docker_executed: return self._docker_executed = True + print("rekby start docker tests") - if test_name is None: - test_name="" - - client: docker.Client = docker.from_env() - - - client.images.build( - path = self._folder, - tag=self._image_name, - network_mode='host', - ) - - try: - exchange_folder=path.join(yatest.common.output_path(), "exchange") - os.mkdir(exchange_folder) - except FileExistsError: - pass - - try: - os.mkdir(self._test_result_folder) - except FileExistsError: - pass - - - # TODO: run YDB with scripts/receipt and get connection port/database with runtime - container = client.containers.create( - image=self._image_name, - # command="/docker-start.bash", - # detach=True, - # auto_remove=True, - environment = [ - "PGUSER=root", - "PGPASSWORD=1234", - "PGHOST=localhost", - "PGPORT=5432", - "PGDATABASE=local", - "PQGOSSLTESTS=0", - "PQSSLCERTTEST_PATH=certs", - f"YDB_PG_TESTNAME={test_name}", - ], - mounts = [ - docker.types.Mount( - target="/exchange", - source=exchange_folder, - type="bind", - ), - docker.types.Mount( - target="/test-result", - source=self._test_result_folder, - type="bind", - ), - ], - network_mode='host', - ) try: - container.start() - container.wait() - print(container.logs().decode()) + print("rekby, start ydbd") + self._kikimr_factory.start() + print("rekby, ydbd started") + print("rekby, ydbd nodes", self._kikimr_factory.nodes) + node = self._kikimr_factory.nodes[1] + pgwire_port = node.pgwire_port + print("rekby, pgwire port: ", pgwire_port) + + + + client: docker.Client = docker.from_env() + + + client.images.build( + path = self._folder, + tag=self._image_name, + network_mode='host', + ) + + try: + exchange_folder=path.join(yatest.common.output_path(), "exchange") + os.mkdir(exchange_folder) + except FileExistsError: + pass + + try: + os.mkdir(self._test_result_folder) + except FileExistsError: + pass + + + # TODO: run YDB with scripts/receipt and get connection port/database with runtime + container = client.containers.create( + image=self._image_name, + # command="/docker-start.bash", + # detach=True, + # auto_remove=True, + environment = [ + "PGUSER=root", + "PGPASSWORD=1234", + "PGHOST=localhost", + f"PGPORT={pgwire_port}", + "PGDATABASE=local", + "PQGOSSLTESTS=0", + "PQSSLCERTTEST_PATH=certs", + f"YDB_PG_TESTNAME={test_filter}", + ], + mounts = [ + docker.types.Mount( + target="/exchange", + source=exchange_folder, + type="bind", + ), + docker.types.Mount( + target="/test-result", + source=self._test_result_folder, + type="bind", + ), + ], + network_mode='host', + ) + try: + container.start() + container.wait() + print(container.logs().decode()) + finally: + container.remove() + except BaseException as e: + print("rekby ydbd exception: ", e) + raise finally: - container.remove() + print("rekby, stop ydbd") + try: + self._kikimr_factory.stop() + except RuntimeError as e: + print("rekby, stop ydbd runtime failed:", e) + pass + # TODO + # handle crash ydbdb as mark in tests @property From c8ea7924af963f4f2f0ed0a43af544ede8fd9517 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Mon, 5 Aug 2024 13:03:00 +0000 Subject: [PATCH 012/261] sync --- ydb/tests/postgres_integrations/library/pytest_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 61bbf9e8005c..2724530dd5d5 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -186,7 +186,6 @@ def _run_tests_in_docker(self, test_filter: str): self._kikimr_factory.stop() except RuntimeError as e: print("rekby, stop ydbd runtime failed:", e) - pass # TODO # handle crash ydbdb as mark in tests From d1e40e445360ce7baa185bd68b5e33caee374313 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 7 Aug 2024 19:19:58 +0000 Subject: [PATCH 013/261] implement tests deselect --- .../go-libpq/conftest.py | 19 ++- .../go-run-separate-tests.bash | 16 +-- .../go-libpq/data/docker-start.bash | 16 +-- .../go-libpq/docker_wrapper_test.py | 29 +++-- .../postgres_integrations/library/__init__.py | 3 +- .../library/pytest_integration.py | 110 +++++++++++------- .../library/ut/data/test-results-example.xml | 3 + .../library/ut/data/test-results-example1.xml | 8 ++ .../library/ut/integrations_test.py | 11 +- 9 files changed, 125 insertions(+), 90 deletions(-) create mode 100644 ydb/tests/postgres_integrations/library/ut/data/test-results-example1.xml diff --git a/ydb/tests/postgres_integrations/go-libpq/conftest.py b/ydb/tests/postgres_integrations/go-libpq/conftest.py index 78b518020bbb..f9bbcb3b4467 100644 --- a/ydb/tests/postgres_integrations/go-libpq/conftest.py +++ b/ydb/tests/postgres_integrations/go-libpq/conftest.py @@ -1,19 +1,14 @@ # from os import path -# from typing import Sequence +from typing import List, Sequence -# import yatest +import yatest -# import pytest +import pytest +selected_items: List[pytest.Function] = [] -# from ydb.tests.postgres_integrations.library import IntegrationTests -# integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) - -# def pytest_generate_tests(metafunc: pytest.Metafunc): -# integrations.pytest_generate_tests(metafunc) - - -# def pytest_deselected(items: Sequence[pytest.Item]): -# integrations.pytest_deselected(items) +def pytest_collection_finish(session: pytest.Session): + global selected_items + selected_items = session.items diff --git a/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash b/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash index a114b7b9419c..f4b5cb533883 100755 --- a/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash +++ b/ydb/tests/postgres_integrations/go-libpq/data/common-go-scripts/go-run-separate-tests.bash @@ -4,29 +4,21 @@ set -eu ONE_TEST_TIMEOUT=5s TEST_BINARY=./test.binary -[ -z "${SKIP_TESTS:-}" ] && SKIP_TESTS='^$' - -GO_LIST_FILTER="^Test" - -if [ -n "${YDB_PG_TESTNAME:-}" ]; then - GO_LIST_FILTER="^${YDB_PG_TESTNAME%%/*}\$" # Cut subtest name: TestAbc/sub -> ^TestAbc$ -fi echo "Get test list" -TESTS=$($TEST_BINARY --test.list "$GO_LIST_FILTER" | sort) +TESTS=$($TEST_BINARY --test.list "^Test" | sort) -echo "Skip tests: '$SKIP_TESTS'" echo "Shell $SHELL" rm -f /test-result/raw/result.txt for TEST_NAME in $TESTS; do echo -n "Test: $TEST_NAME " - if echo "$TEST_NAME" | grep -Eq "$SKIP_TESTS"; then + if echo "$TEST_NAME" | grep -Eq "$YDB_PG_TESTFILTER"; then + echo start + else echo skip continue - else - echo start fi CMD="$TEST_BINARY --test.run '^$TEST_NAME\$' --test.v --test.timeout='$ONE_TEST_TIMEOUT'" echo "$CMD" diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash index 28aa6e1795d5..9e244861733c 100755 --- a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash @@ -20,27 +20,19 @@ fi cd /project/sources/ -export YDB_PG_TESTNAME="${YDB_PG_TESTNAME:-}" # set YDB_PG_TESTNAME to empty string if it not set +export YDB_PG_TESTFILTER="${YDB_PG_TESTFILTER:-}" # set YDB_PG_TESTNAME to empty string if it not set -if [ -n "${YDB_PG_TESTNAME:-}" ]; then - SKIP_TESTS="^\$" -fi - -echo "Run test: '$YDB_PG_TESTNAME'" +echo "Run tests: '$YDB_PG_TESTFILTER'" echo "Start test" -# PQTEST_BINARY_PARAMETERS=no go test -test.skip="$SKIP_TESTS" - -export SKIP_TESTS - mkdir -p /test-result/raw PQTEST_BINARY_PARAMETERS=no /go-run-separate-tests.bash sed -e 's|classname=""|classname="golang-lib-pq"|' -i /test-result/raw/result.xml -if [ -n "${YDB_PG_TESTNAME:-}" ]; then +if [ -n "${YDB_PG_TESTFILTER:-}" ]; then cat /test-result/raw/result.txt fi -chmod -R a+rw /test-result \ No newline at end of file +chmod -R a+rw /test-result diff --git a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py index c308405fca2b..143e66b460e4 100644 --- a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py +++ b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py @@ -1,26 +1,31 @@ # from .conftest import integrations - +import typing from os import path from typing import Sequence -import yatest - import pytest +import yatest -from ydb.tests.postgres_integrations.library import IntegrationTests - -integrations=IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) +from ydb.tests.postgres_integrations.library import IntegrationTests,PgTestWrapper -def pytest_generate_tests(metafunc: pytest.Metafunc): - integrations.pytest_generate_tests(metafunc) +from . import conftest +integrations = IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) -def pytest_deselected(items: Sequence[pytest.Item]): - integrations.pytest_deselected(items) +class TestWrapper(PgTestWrapper): + @classmethod + def setup_class(cls): + print('"rekby setup class') + cls.initialize(integrations, conftest.selected_items) + @classmethod + def filter_format(cls, test_names: typing.List[str])->str: + test_names = [item[len("golang-lib-pq/"):] for item in test_names] + return "^(" + "|".join(test_names) + ")$" -def test(testname): - integrations.execute_test(testname) +def pytest_generate_tests(metafunc: pytest.Metafunc): + if metafunc.definition.name == "test_pg_generated": + integrations.pytest_generate_tests(metafunc) diff --git a/ydb/tests/postgres_integrations/library/__init__.py b/ydb/tests/postgres_integrations/library/__init__.py index 56167a3b5eeb..16e205fd29ff 100644 --- a/ydb/tests/postgres_integrations/library/__init__.py +++ b/ydb/tests/postgres_integrations/library/__init__.py @@ -1,5 +1,6 @@ __all__ = [ - 'IntegrationTests' + 'IntegrationTests', + 'PgTestWrapper', ] from .pytest_integration import * diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 2724530dd5d5..8cc1d4d2fb49 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Set, Optional +from typing import Callable, Dict, List, Set, Optional import os from os import path @@ -28,11 +28,12 @@ class TestCase: state: TestState log: str +_filter_format_function = Callable[[List[str]], str] class IntegrationTests: _folder: str _image_name: str - _all_tests: Set[str] + _all_tests: List[str] _selected_test: Set[str] _docker_executed: bool _test_results: Dict[str, TestCase] @@ -42,47 +43,51 @@ def __init__(self, folder: str, image_name: str = 'ydb-pg-test-image'): self._folder = folder self._image_name = image_name - self._all_tests = _read_tests(folder) - self._selected_test = set(self._all_tests) + self._all_tests = _read_tests(folder) # sorted test list + self._selected_test = [] self._docker_executed = False self._test_results = dict() self._kikimr_factory = kikimr_cluster_factory() + self._filter_formatter: Optional[_filter_format_function] = None + + def tearup(self): + filter = self._filter_formatter(self._selected_test) + self._run_tests_in_docker(filter) + print(f"rekby docker test finished") + + test_results_file=path.join(self._test_result_folder, "raw", "result.xml") + self._test_results = _read_tests_result(test_results_file) + print(f"rekby result parsed tests:\n\n{self._test_results}") + + + def teardown(self): + pass + + def set_filter_formatter(self, f: _filter_format_function): + print("rekby, set filter formatter") + self._filter_formatter = f + + def set_selected_items(self, selected_items: List[pytest.Function]): + print("rekby set selected items: ", selected_items) + selected_tests = [] + for item in selected_items: + print(f"rekby, selected item name: '{item.name}'", ) + if item.name.startswith("test_pg_generated["): + print("rekby selected test item:", item) + test_name = item.callspec.id + print(f"rekby selected test: {test_name}") + selected_tests.append(test_name) + selected_tests.sort() + self._selected_test = selected_tests def pytest_generate_tests(self, metafunc: pytest.Metafunc): """ Return tests for run through pytest. """ - all_tests = list(self._all_tests) - all_tests.sort() - metafunc.parametrize('testname', all_tests, ids=all_tests) - - def pytest_deselected(self, items: List[pytest.Item]): - for item in items: - test_name = item.callspec.id - print(f"rekby-test removed: '{test_name}") - self._selected_test.remove(test_name) + metafunc.parametrize('testname', self._all_tests, ids=self._all_tests) def execute_test(self, testname: str): - if not self._docker_executed: - if len(self._selected_test) == 1: - execute_test_filter=testname - else: - execute_test_filter="" - - print("rekby-selected-tests: ", self._selected_test) - print(f"rekby-test-name: '{execute_test_filter}'") - - try: - self._run_tests_in_docker(execute_test_filter) - print(f"rekby docker test finished") - except BaseException as e: - print("docker test exception", e) - raise - - test_results_file=path.join(self._test_result_folder, "raw", "result.xml") - self._test_results = _read_tests_result(test_results_file) - print(f"rekby result parsed tests:\n\n{self._test_results}") self._check_test_results(testname) def _check_test_results(self, testname: str): @@ -118,11 +123,8 @@ def _run_tests_in_docker(self, test_filter: str): pgwire_port = node.pgwire_port print("rekby, pgwire port: ", pgwire_port) - - client: docker.Client = docker.from_env() - client.images.build( path = self._folder, tag=self._image_name, @@ -155,7 +157,7 @@ def _run_tests_in_docker(self, test_filter: str): "PGDATABASE=local", "PQGOSSLTESTS=0", "PQSSLCERTTEST_PATH=certs", - f"YDB_PG_TESTNAME={test_filter}", + f"YDB_PG_TESTFILTER={test_filter}", ], mounts = [ docker.types.Mount( @@ -201,13 +203,14 @@ def _read_tests(folder: str) -> Set[str]: with open(path.join(folder, "unit-tests.txt"), "rt") as f: unit = set(f.readlines()) - test_list_for_run = all - unit + test_list_for_run = list(all - unit) + test_list_for_run.sort() return test_list_for_run def _read_tests_result(filepath: str) -> Dict[str, TestCase]: with open(filepath, "rt") as f: data = f.read() - d = xmltodict.parse(data) + d = xmltodict.parse(data, force_list=("testcase",)) testsuites = d["testsuites"] test_suite = testsuites["testsuite"] test_cases = test_suite["testcase"] @@ -219,7 +222,10 @@ def get_text(test_case, field_name: str)->str: if type(field_val) == str: return field_val elif type(field_val) == dict: - return field_val.get("#text", "") + prefix=field_val.get("@message", "") + "\n" + if prefix == "\n": + prefix = "" + return prefix + field_val.get("#text", "") raise Exception(f"Unknown field val for field '{field_name}':\n{field_val}") for test_case in test_cases: @@ -228,6 +234,9 @@ def get_text(test_case, field_name: str)->str: if "failure" in test_case: test_state = TestState.FAILED log = get_text(test_case, "failure") + elif "error" in test_case: + test_state = TestState.FAILED + log = get_text(test_case, "error") elif "skipped" in test_case: test_state = TestState.SKIPPED log = get_text(test_case, "skipped") @@ -242,3 +251,26 @@ def get_text(test_case, field_name: str)->str: ) return res + +class PgTestWrapper: + integrations: IntegrationTests + + @classmethod + def initialize(cls, integrations: IntegrationTests, selected_items: List[pytest.Function]): + print("rekby wrapper initialize") + cls.integrations = integrations + integrations.set_selected_items(selected_items) + + print("rekby wrapper settings filter formatter") + integrations.set_filter_formatter(cls.filter_format) + integrations.tearup() + + def test_initialization(self): + # for collect initializations log + pass + + def test_pg_generated(self, testname): + self.integrations.execute_test(testname) + + def filter_format(cls, test_names: List[str])->str: + raise NotImplemented() diff --git a/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml b/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml index 36b6a5a0cb5c..6627aaf29920 100644 --- a/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml +++ b/ydb/tests/postgres_integrations/library/ut/data/test-results-example.xml @@ -8,6 +8,9 @@ + + panic and timeout + skip message diff --git a/ydb/tests/postgres_integrations/library/ut/data/test-results-example1.xml b/ydb/tests/postgres_integrations/library/ut/data/test-results-example1.xml new file mode 100644 index 000000000000..e0c691d72826 --- /dev/null +++ b/ydb/tests/postgres_integrations/library/ut/data/test-results-example1.xml @@ -0,0 +1,8 @@ + + + + + failed-mess + + + diff --git a/ydb/tests/postgres_integrations/library/ut/integrations_test.py b/ydb/tests/postgres_integrations/library/ut/integrations_test.py index c86da034878d..527593d6e43d 100644 --- a/ydb/tests/postgres_integrations/library/ut/integrations_test.py +++ b/ydb/tests/postgres_integrations/library/ut/integrations_test.py @@ -13,8 +13,9 @@ [ TestCase(name="o/OK", state=TestState.PASSED, log=""), TestCase(name="f/failed1", state=TestState.FAILED, log="fail mess"), - TestCase(name="f/failed2", state=TestState.FAILED, log="escaped error"), - TestCase(name="s/skipped1", state=TestState.SKIPPED, log="skip message"), + TestCase(name="f/failed2", state=TestState.FAILED, log="Failed\nescaped error"), + TestCase(name="f/error1", state=TestState.FAILED, log="No test result found\npanic and timeout"), + TestCase(name="s/skipped1", state=TestState.SKIPPED, log="Skipped\nskip message"), TestCase(name="s/skipped2", state=TestState.SKIPPED, log="escaped skip message"), ], ids=lambda item: item.name @@ -25,3 +26,9 @@ def test_read_jtest_results(test): parsed_test = parsed_result[test.name] assert test == parsed_test + +def test_read_jtest_with_one_result(): + filepath = path.join(TEST_DATA_FOLDER, "test-results-example1.xml") + parsed_result = pytest_integration._read_tests_result(filepath) + parsed_test = parsed_result["f/test-failed"] + assert parsed_test == TestCase(name="f/test-failed", state=TestState.FAILED, log="Failed\nfailed-mess") From e2fd2e654fd7bf8eec2bdc3f940245bcb8e9b629 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 21 Aug 2024 21:35:45 +0000 Subject: [PATCH 014/261] refactoring pg test run --- ydb/tests/postgres_integrations/library/docker.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 ydb/tests/postgres_integrations/library/docker.py diff --git a/ydb/tests/postgres_integrations/library/docker.py b/ydb/tests/postgres_integrations/library/docker.py deleted file mode 100644 index b367352b91b0..000000000000 --- a/ydb/tests/postgres_integrations/library/docker.py +++ /dev/null @@ -1,2 +0,0 @@ -from typing import Optional - From 8c42014e4c4b855491adab485e47a5faa23e8606 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 21 Aug 2024 21:56:02 +0000 Subject: [PATCH 015/261] build image --- .../go-libpq/conftest.py | 13 +- .../go-libpq/docker_wrapper_test.py | 27 +- .../postgres_integrations/library/__init__.py | 8 + .../library/pytest_integration.py | 347 ++++++++---------- 4 files changed, 183 insertions(+), 212 deletions(-) diff --git a/ydb/tests/postgres_integrations/go-libpq/conftest.py b/ydb/tests/postgres_integrations/go-libpq/conftest.py index f9bbcb3b4467..a14669deeef2 100644 --- a/ydb/tests/postgres_integrations/go-libpq/conftest.py +++ b/ydb/tests/postgres_integrations/go-libpq/conftest.py @@ -1,14 +1,5 @@ - -# from os import path -from typing import List, Sequence - -import yatest - +import ydb.tests.postgres_integrations.library import pytest -selected_items: List[pytest.Function] = [] - - def pytest_collection_finish(session: pytest.Session): - global selected_items - selected_items = session.items + ydb.tests.postgres_integrations.library.pytest_collection_finish(session) diff --git a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py index 143e66b460e4..587a6a583ba8 100644 --- a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py +++ b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py @@ -9,23 +9,26 @@ import yatest -from ydb.tests.postgres_integrations.library import IntegrationTests,PgTestWrapper +from ydb.tests.postgres_integrations import library as tl from . import conftest -integrations = IntegrationTests(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) +def filter_formatter(test_names: typing.List[str])->str: + test_names = [item[len("golang-lib-pq/"):] for item in test_names] + return "^(" + "|".join(test_names) + ")$" -class TestWrapper(PgTestWrapper): - @classmethod - def setup_class(cls): - print('"rekby setup class') - cls.initialize(integrations, conftest.selected_items) +tl.set_filter_formatter(filter_formatter) +tl.set_tests_folder(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) - @classmethod - def filter_format(cls, test_names: typing.List[str])->str: - test_names = [item[len("golang-lib-pq/"):] for item in test_names] - return "^(" + "|".join(test_names) + ")$" +def setup_module(module: pytest.Module): + tl.setup_module(module) + +def teardown_module(module: pytest.Module): + tl.teardown_module(module) + +def test_pg_generated(testname): + tl.execute_test(testname) def pytest_generate_tests(metafunc: pytest.Metafunc): if metafunc.definition.name == "test_pg_generated": - integrations.pytest_generate_tests(metafunc) + tl.pytest_generate_tests(metafunc) diff --git a/ydb/tests/postgres_integrations/library/__init__.py b/ydb/tests/postgres_integrations/library/__init__.py index 16e205fd29ff..b3a3f0230b4d 100644 --- a/ydb/tests/postgres_integrations/library/__init__.py +++ b/ydb/tests/postgres_integrations/library/__init__.py @@ -1,6 +1,14 @@ __all__ = [ 'IntegrationTests', 'PgTestWrapper', + 'pytest_collection_finish', + 'set_filter_formatter', + 'set_tests_folder', + 'setup_module', + 'teardown_module', + 'execute_test', + 'pytest_generate_tests', + 'execute_test', ] from .pytest_integration import * diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 8cc1d4d2fb49..8439a689689e 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -1,7 +1,8 @@ -from typing import Callable, Dict, List, Set, Optional import os -from os import path +import shutil +from typing import Callable, Dict, List, Set, Optional, Union +from os import path from dataclasses import dataclass from enum import Enum @@ -28,173 +29,164 @@ class TestCase: state: TestState log: str -_filter_format_function = Callable[[List[str]], str] - -class IntegrationTests: - _folder: str - _image_name: str - _all_tests: List[str] - _selected_test: Set[str] - _docker_executed: bool - _test_results: Dict[str, TestCase] - _kikimr_factory: KiKiMR - - def __init__(self, folder: str, image_name: str = 'ydb-pg-test-image'): - self._folder = folder - self._image_name = image_name - - self._all_tests = _read_tests(folder) # sorted test list - self._selected_test = [] - - self._docker_executed = False - self._test_results = dict() - self._kikimr_factory = kikimr_cluster_factory() - self._filter_formatter: Optional[_filter_format_function] = None - - def tearup(self): - filter = self._filter_formatter(self._selected_test) - self._run_tests_in_docker(filter) - print(f"rekby docker test finished") - - test_results_file=path.join(self._test_result_folder, "raw", "result.xml") - self._test_results = _read_tests_result(test_results_file) - print(f"rekby result parsed tests:\n\n{self._test_results}") - - def teardown(self): +_selected_tests_name: pytest.Session = [] +_filter_format_function = Callable[[List[str]], str] +_filter_formatter: Optional[_filter_format_function] = None +_tests_folder: Optional[str] = None +_test_results: Optional[Dict[str, TestCase]] = None +_all_tests: Optional[List[str]] = None +_kikimr_factory: KiKiMR = kikimr_cluster_factory() + +def pytest_collection_finish(session: pytest.Session): + global _selected_tests_name + + print("rekby set selected items: ", session.items) + selected_tests = [] + for item in session.items: + print(f"rekby, selected item name: '{item.name}'", ) + if item.name.startswith("test_pg_generated["): + print("rekby selected test item:", item) + test_name = item.callspec.id + print(f"rekby selected test: {test_name}") + selected_tests.append(test_name) + selected_tests.sort() + print("rekby: result selected tests", selected_tests) + _selected_tests_name = selected_tests + +def set_filter_formatter(f: _filter_format_function): + global _filter_formatter + _filter_formatter = f + + +def set_tests_folder(folder: str): + global _tests_folder, _all_tests + _tests_folder = folder + _all_tests = _read_tests(folder) + +def setup_module(module: pytest.Module): + global _test_results + try: + exchange_folder=path.join(yatest.common.output_path(), "exchange") + os.mkdir(exchange_folder) + except FileExistsError: pass - def set_filter_formatter(self, f: _filter_format_function): - print("rekby, set filter formatter") - self._filter_formatter = f - - def set_selected_items(self, selected_items: List[pytest.Function]): - print("rekby set selected items: ", selected_items) - selected_tests = [] - for item in selected_items: - print(f"rekby, selected item name: '{item.name}'", ) - if item.name.startswith("test_pg_generated["): - print("rekby selected test item:", item) - test_name = item.callspec.id - print(f"rekby selected test: {test_name}") - selected_tests.append(test_name) - selected_tests.sort() - self._selected_test = selected_tests - - def pytest_generate_tests(self, metafunc: pytest.Metafunc): - """ - Return tests for run through pytest. - """ - metafunc.parametrize('testname', self._all_tests, ids=self._all_tests) - - def execute_test(self, testname: str): - self._check_test_results(testname) - - def _check_test_results(self, testname: str): - try: - test = self._test_results[testname] - except KeyError: - pytest.fail("test result not found, may be the test was not run") - - if test.state == TestState.PASSED: - logging.getLogger().log(logging.INFO, test.log) - return - if test.state == TestState.SKIPPED: - logging.getLogger().log(logging.INFO, test.log) - pytest.skip() - if test.state == TestState.FAILED: - logging.getLogger().log(logging.ERROR, test.log) - pytest.fail() - - raise Exception(f"Unexpected test state: '{test.state}'") - - def _run_tests_in_docker(self, test_filter: str): - if self._docker_executed: - return - self._docker_executed = True - print("rekby start docker tests") - - try: - print("rekby, start ydbd") - self._kikimr_factory.start() - print("rekby, ydbd started") - print("rekby, ydbd nodes", self._kikimr_factory.nodes) - node = self._kikimr_factory.nodes[1] - pgwire_port = node.pgwire_port - print("rekby, pgwire port: ", pgwire_port) - - client: docker.Client = docker.from_env() - - client.images.build( - path = self._folder, - tag=self._image_name, - network_mode='host', - ) - - try: - exchange_folder=path.join(yatest.common.output_path(), "exchange") - os.mkdir(exchange_folder) - except FileExistsError: - pass - - try: - os.mkdir(self._test_result_folder) - except FileExistsError: - pass - - - # TODO: run YDB with scripts/receipt and get connection port/database with runtime - container = client.containers.create( - image=self._image_name, - # command="/docker-start.bash", - # detach=True, - # auto_remove=True, - environment = [ - "PGUSER=root", - "PGPASSWORD=1234", - "PGHOST=localhost", - f"PGPORT={pgwire_port}", - "PGDATABASE=local", - "PQGOSSLTESTS=0", - "PQSSLCERTTEST_PATH=certs", - f"YDB_PG_TESTFILTER={test_filter}", - ], - mounts = [ - docker.types.Mount( - target="/exchange", - source=exchange_folder, - type="bind", - ), - docker.types.Mount( - target="/test-result", - source=self._test_result_folder, - type="bind", - ), - ], - network_mode='host', - ) - try: - container.start() - container.wait() - print(container.logs().decode()) - finally: - container.remove() - except BaseException as e: - print("rekby ydbd exception: ", e) - raise - finally: - print("rekby, stop ydbd") - try: - self._kikimr_factory.stop() - except RuntimeError as e: - print("rekby, stop ydbd runtime failed:", e) - # TODO - # handle crash ydbdb as mark in tests - - - @property - def _test_result_folder(self): - return path.join(yatest.common.output_path(), "test-result") + tests_result_folder = path.join(yatest.common.output_path(), "test-result") + shutil.rmtree(tests_result_folder, ignore_errors=True) + os.mkdir(tests_result_folder) + + image = _docker_build(_tests_folder) + + pg_port = _run_ydb() + env = _prepare_docker_env(pg_port, _selected_tests_name) + _run_tests_in_docker(image, env, exchange_folder, tests_result_folder) + + test_results_file=path.join(tests_result_folder, "raw", "result.xml") + _test_results = _read_tests_result(test_results_file) + +def teardown_module(module): + """teardown any state that was previously setup with a setup_module + method. + """ + _stop_ydb() + +def _run_ydb() -> int: + """ + Run YDB cluster and return pgwire port number. + """ + _kikimr_factory.start() + node = _kikimr_factory.nodes[1] + print("rekby: pgwire port", node.pgwire_port) + return node.pgwire_port + +def _stop_ydb(): + _kikimr_factory.stop() + +def _prepare_docker_env(pgwire_port: str, test_names: List[str]) -> List[str]: + test_filter = _filter_formatter(test_names) + return [ + "PGUSER=root", + "PGPASSWORD=1234", + "PGHOST=localhost", + f"PGPORT={pgwire_port}", + "PGDATABASE=local", + "PQGOSSLTESTS=0", + "PQSSLCERTTEST_PATH=certs", + f"YDB_PG_TESTFILTER={test_filter}", + ] + +def _docker_build(folder: str) -> str: + image_name = 'ydb-pg-test-image' + + client: docker.Client = docker.from_env() + client.images.build( + path=folder, + tag=image_name, + rm=True, + network_mode='host', + ) + return image_name + +def _run_tests_in_docker( + image: str, + env: Union[List[str], Dict[str, str]], + exchange_folder: str, + results_folder: str, + ): + # TODO: run YDB with scripts/receipt and get connection port/database with runtime + client: docker.Client = docker.from_env() + + container = client.containers.create( + image=image, + # command="/docker-start.bash", + # detach=True, + # auto_remove=True, + environment = env, + mounts = [ + docker.types.Mount( + target="/exchange", + source=exchange_folder, + type="bind", + ), + docker.types.Mount( + target="/test-result", + source=results_folder, + type="bind", + ), + ], + network_mode='host', + ) + try: + container.start() + container.wait() + print(container.logs().decode()) + finally: + container.remove() + +def pytest_generate_tests(metafunc: pytest.Metafunc): + """ + Return tests for run through pytest. + """ + metafunc.parametrize('testname', _all_tests, ids=_all_tests) + +def execute_test(testname: str): + try: + test = _test_results[testname] + except KeyError: + pytest.fail("test result not found, may be the test was not run") + + if test.state == TestState.PASSED: + logging.getLogger().log(logging.INFO, test.log) + return + if test.state == TestState.SKIPPED: + logging.getLogger().log(logging.INFO, test.log) + pytest.skip() + if test.state == TestState.FAILED: + logging.getLogger().log(logging.ERROR, test.log) + pytest.fail() + + raise Exception(f"Unexpected test state: '{test.state}'") def _read_tests(folder: str) -> Set[str]: with open(path.join(folder, "full-test-list.txt"), "rt") as f: @@ -251,26 +243,3 @@ def get_text(test_case, field_name: str)->str: ) return res - -class PgTestWrapper: - integrations: IntegrationTests - - @classmethod - def initialize(cls, integrations: IntegrationTests, selected_items: List[pytest.Function]): - print("rekby wrapper initialize") - cls.integrations = integrations - integrations.set_selected_items(selected_items) - - print("rekby wrapper settings filter formatter") - integrations.set_filter_formatter(cls.filter_format) - integrations.tearup() - - def test_initialization(self): - # for collect initializations log - pass - - def test_pg_generated(self, testname): - self.integrations.execute_test(testname) - - def filter_format(cls, test_names: List[str])->str: - raise NotImplemented() From 8db8ebc2e06aa780622450f414203ebf876d34ef Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 21 Aug 2024 22:55:00 +0000 Subject: [PATCH 016/261] style --- .../go-libpq/conftest.py | 1 + .../go-libpq/docker_wrapper_test.py | 17 ++++---- .../postgres_integrations/library/__init__.py | 6 +-- .../library/pytest_integration.py | 43 +++++++++++++------ .../library/ut/integrations_test.py | 5 ++- 5 files changed, 44 insertions(+), 28 deletions(-) diff --git a/ydb/tests/postgres_integrations/go-libpq/conftest.py b/ydb/tests/postgres_integrations/go-libpq/conftest.py index a14669deeef2..490e87914143 100644 --- a/ydb/tests/postgres_integrations/go-libpq/conftest.py +++ b/ydb/tests/postgres_integrations/go-libpq/conftest.py @@ -1,5 +1,6 @@ import ydb.tests.postgres_integrations.library import pytest + def pytest_collection_finish(session: pytest.Session): ydb.tests.postgres_integrations.library.pytest_collection_finish(session) diff --git a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py index 587a6a583ba8..96ce0a4c0912 100644 --- a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py +++ b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py @@ -1,34 +1,33 @@ # from .conftest import integrations import typing - -from os import path -from typing import Sequence - import pytest - import yatest from ydb.tests.postgres_integrations import library as tl -from . import conftest -def filter_formatter(test_names: typing.List[str])->str: +def filter_formatter(test_names: typing.List[str]) -> str: test_names = [item[len("golang-lib-pq/"):] for item in test_names] return "^(" + "|".join(test_names) + ")$" -tl.set_filter_formatter(filter_formatter) -tl.set_tests_folder(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) def setup_module(module: pytest.Module): tl.setup_module(module) + def teardown_module(module: pytest.Module): tl.teardown_module(module) + def test_pg_generated(testname): tl.execute_test(testname) + def pytest_generate_tests(metafunc: pytest.Metafunc): if metafunc.definition.name == "test_pg_generated": tl.pytest_generate_tests(metafunc) + + +tl.set_filter_formatter(filter_formatter) +tl.set_tests_folder(yatest.common.source_path("ydb/tests/postgres_integrations/go-libpq/data")) diff --git a/ydb/tests/postgres_integrations/library/__init__.py b/ydb/tests/postgres_integrations/library/__init__.py index b3a3f0230b4d..3d978f45bde1 100644 --- a/ydb/tests/postgres_integrations/library/__init__.py +++ b/ydb/tests/postgres_integrations/library/__init__.py @@ -1,4 +1,4 @@ -__all__ = [ +__all__ = [ # noqa 'IntegrationTests', 'PgTestWrapper', 'pytest_collection_finish', @@ -8,8 +8,6 @@ 'teardown_module', 'execute_test', 'pytest_generate_tests', - 'execute_test', ] -from .pytest_integration import * - +from .pytest_integration import * # noqa diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 8439a689689e..916ed24f9e7d 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -14,15 +14,16 @@ import logging -from ydb.tests.library.harness.daemon import DaemonError from ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory from ydb.tests.library.harness.kikimr_runner import KiKiMR + class TestState(Enum): PASSED = 1 FAILED = 2 SKIPPED = 3 + @dataclass class TestCase: name: str @@ -38,6 +39,7 @@ class TestCase: _all_tests: Optional[List[str]] = None _kikimr_factory: KiKiMR = kikimr_cluster_factory() + def pytest_collection_finish(session: pytest.Session): global _selected_tests_name @@ -54,6 +56,7 @@ def pytest_collection_finish(session: pytest.Session): print("rekby: result selected tests", selected_tests) _selected_tests_name = selected_tests + def set_filter_formatter(f: _filter_format_function): global _filter_formatter _filter_formatter = f @@ -64,10 +67,11 @@ def set_tests_folder(folder: str): _tests_folder = folder _all_tests = _read_tests(folder) + def setup_module(module: pytest.Module): global _test_results try: - exchange_folder=path.join(yatest.common.output_path(), "exchange") + exchange_folder = path.join(yatest.common.output_path(), "exchange") os.mkdir(exchange_folder) except FileExistsError: pass @@ -82,15 +86,17 @@ def setup_module(module: pytest.Module): env = _prepare_docker_env(pg_port, _selected_tests_name) _run_tests_in_docker(image, env, exchange_folder, tests_result_folder) - test_results_file=path.join(tests_result_folder, "raw", "result.xml") + test_results_file = path.join(tests_result_folder, "raw", "result.xml") _test_results = _read_tests_result(test_results_file) + def teardown_module(module): """teardown any state that was previously setup with a setup_module method. """ _stop_ydb() + def _run_ydb() -> int: """ Run YDB cluster and return pgwire port number. @@ -100,9 +106,11 @@ def _run_ydb() -> int: print("rekby: pgwire port", node.pgwire_port) return node.pgwire_port + def _stop_ydb(): _kikimr_factory.stop() + def _prepare_docker_env(pgwire_port: str, test_names: List[str]) -> List[str]: test_filter = _filter_formatter(test_names) return [ @@ -116,6 +124,7 @@ def _prepare_docker_env(pgwire_port: str, test_names: List[str]) -> List[str]: f"YDB_PG_TESTFILTER={test_filter}", ] + def _docker_build(folder: str) -> str: image_name = 'ydb-pg-test-image' @@ -128,12 +137,14 @@ def _docker_build(folder: str) -> str: ) return image_name + def _run_tests_in_docker( - image: str, - env: Union[List[str], Dict[str, str]], - exchange_folder: str, - results_folder: str, - ): + image: str, + env: Union[List[str], Dict[str, str]], + exchange_folder: str, + results_folder: str, + ): + # TODO: run YDB with scripts/receipt and get connection port/database with runtime client: docker.Client = docker.from_env() @@ -142,8 +153,8 @@ def _run_tests_in_docker( # command="/docker-start.bash", # detach=True, # auto_remove=True, - environment = env, - mounts = [ + environment=env, + mounts=[ docker.types.Mount( target="/exchange", source=exchange_folder, @@ -164,12 +175,14 @@ def _run_tests_in_docker( finally: container.remove() + def pytest_generate_tests(metafunc: pytest.Metafunc): """ Return tests for run through pytest. """ metafunc.parametrize('testname', _all_tests, ids=_all_tests) + def execute_test(testname: str): try: test = _test_results[testname] @@ -188,6 +201,7 @@ def execute_test(testname: str): raise Exception(f"Unexpected test state: '{test.state}'") + def _read_tests(folder: str) -> Set[str]: with open(path.join(folder, "full-test-list.txt"), "rt") as f: all = set(line.strip() for line in f.readlines()) @@ -199,6 +213,7 @@ def _read_tests(folder: str) -> Set[str]: test_list_for_run.sort() return test_list_for_run + def _read_tests_result(filepath: str) -> Dict[str, TestCase]: with open(filepath, "rt") as f: data = f.read() @@ -209,12 +224,12 @@ def _read_tests_result(filepath: str) -> Dict[str, TestCase]: res: Dict[str, TestCase] = dict() - def get_text(test_case, field_name: str)->str: + def get_text(test_case, field_name: str) -> str: field_val = test_case[field_name] - if type(field_val) == str: + if type(field_val) is str: return field_val - elif type(field_val) == dict: - prefix=field_val.get("@message", "") + "\n" + elif type(field_val) is dict: + prefix = field_val.get("@message", "") + "\n" if prefix == "\n": prefix = "" return prefix + field_val.get("#text", "") diff --git a/ydb/tests/postgres_integrations/library/ut/integrations_test.py b/ydb/tests/postgres_integrations/library/ut/integrations_test.py index 527593d6e43d..7f03f0d7db06 100644 --- a/ydb/tests/postgres_integrations/library/ut/integrations_test.py +++ b/ydb/tests/postgres_integrations/library/ut/integrations_test.py @@ -6,7 +6,9 @@ from ydb.tests.postgres_integrations.library import pytest_integration from ydb.tests.postgres_integrations.library.pytest_integration import TestCase, TestState -TEST_DATA_FOLDER=yatest.common.source_path("ydb/tests/postgres_integrations/library/ut/data") + +TEST_DATA_FOLDER = yatest.common.source_path("ydb/tests/postgres_integrations/library/ut/data") + @pytest.mark.parametrize( "test", @@ -27,6 +29,7 @@ def test_read_jtest_results(test): parsed_test = parsed_result[test.name] assert test == parsed_test + def test_read_jtest_with_one_result(): filepath = path.join(TEST_DATA_FOLDER, "test-results-example1.xml") parsed_result = pytest_integration._read_tests_result(filepath) From 7befb2054b76c10090180149c454fc544117895e Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Tue, 27 Aug 2024 17:06:47 +0000 Subject: [PATCH 017/261] add support skip tests, adapt tests size for end --- .../postgres_integrations/go-libpq/ya.make | 27 +++++++--- .../library/pytest_integration.py | 54 +++++++++++++++---- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/ydb/tests/postgres_integrations/go-libpq/ya.make b/ydb/tests/postgres_integrations/go-libpq/ya.make index 2fb2ea6e8e7b..462e65f734ae 100644 --- a/ydb/tests/postgres_integrations/go-libpq/ya.make +++ b/ydb/tests/postgres_integrations/go-libpq/ya.make @@ -2,11 +2,28 @@ PY3TEST() FORK_TEST_FILES() TIMEOUT(600) -SIZE(LARGE) -TAG( - ya:fat -) +IF (AUTOCHECK) + # copy from https://docs.yandex-team.ru/devtools/test/environment#docker-compose + INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) + SIZE(LARGE) # Docker можно запускать только в Sandbox, т.е. в LARGE тестах + + TAG( + ya:external + ya:fat + ya:force_sandbox + ) + + REQUIREMENTS( + container:4467981730 # В этом контейнере уже есть Docker и выполнена аутентификация в registry.yandex.net от имени пользователя arcadia-devtools + cpu:all dns:dns64 + ) +ENDIF() + +IF(OPENSOURCE) + SIZE(MEDIUM) # for run per PR +ENDIF() + ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd") DEPENDS( @@ -27,6 +44,4 @@ PEERDIR( ydb/tests/postgres_integrations/library ) -REQUIREMENTS(ram:10) - END() diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 916ed24f9e7d..42d20b43cbee 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -31,17 +31,17 @@ class TestCase: log: str -_selected_tests_name: pytest.Session = [] +_tests_for_run_in_docker: pytest.Session = [] _filter_format_function = Callable[[List[str]], str] _filter_formatter: Optional[_filter_format_function] = None _tests_folder: Optional[str] = None _test_results: Optional[Dict[str, TestCase]] = None -_all_tests: Optional[List[str]] = None _kikimr_factory: KiKiMR = kikimr_cluster_factory() - +_integration_tests: Optional[List[str]] = None +_skip_tests: Dict[str, str] = dict() # [test name: reason] def pytest_collection_finish(session: pytest.Session): - global _selected_tests_name + global _tests_for_run_in_docker print("rekby set selected items: ", session.items) selected_tests = [] @@ -54,7 +54,11 @@ def pytest_collection_finish(session: pytest.Session): selected_tests.append(test_name) selected_tests.sort() print("rekby: result selected tests", selected_tests) - _selected_tests_name = selected_tests + _tests_for_run_in_docker = list() + for test in selected_tests: + if test not in _skip_tests: + _tests_for_run_in_docker.append(test) + print("rekby, tests for run", _tests_for_run_in_docker) def set_filter_formatter(f: _filter_format_function): @@ -63,12 +67,17 @@ def set_filter_formatter(f: _filter_format_function): def set_tests_folder(folder: str): - global _tests_folder, _all_tests + global _tests_folder, _integration_tests, _skip_tests + print("rekby, set_tests_folder called") _tests_folder = folder - _all_tests = _read_tests(folder) + _integration_tests = _read_integration_tests(folder) + _skip_tests = _read_skip_tests(folder) def setup_module(module: pytest.Module): + if len(_tests_for_run_in_docker) == 0: + return + global _test_results try: exchange_folder = path.join(yatest.common.output_path(), "exchange") @@ -83,7 +92,7 @@ def setup_module(module: pytest.Module): image = _docker_build(_tests_folder) pg_port = _run_ydb() - env = _prepare_docker_env(pg_port, _selected_tests_name) + env = _prepare_docker_env(pg_port, _tests_for_run_in_docker) _run_tests_in_docker(image, env, exchange_folder, tests_result_folder) test_results_file = path.join(tests_result_folder, "raw", "result.xml") @@ -180,14 +189,18 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): """ Return tests for run through pytest. """ - metafunc.parametrize('testname', _all_tests, ids=_all_tests) + print("rekby, integration tests:", _integration_tests) + metafunc.parametrize('testname', _integration_tests, ids=_integration_tests) def execute_test(testname: str): + if testname in _skip_tests: + pytest.skip(_skip_tests[testname]) + try: test = _test_results[testname] except KeyError: - pytest.fail("test result not found, may be the test was not run") + pytest.fail("test result not found, may be the test was not runned") if test.state == TestState.PASSED: logging.getLogger().log(logging.INFO, test.log) @@ -202,7 +215,7 @@ def execute_test(testname: str): raise Exception(f"Unexpected test state: '{test.state}'") -def _read_tests(folder: str) -> Set[str]: +def _read_integration_tests(folder: str) -> Set[str]: with open(path.join(folder, "full-test-list.txt"), "rt") as f: all = set(line.strip() for line in f.readlines()) @@ -213,6 +226,25 @@ def _read_tests(folder: str) -> Set[str]: test_list_for_run.sort() return test_list_for_run +def _read_skip_tests(folder: str) -> Dict[str, str]: + res = dict() + try: + fpath = path.join(folder, "skip-tests.txt") + with open(fpath) as f: + for line in f.readlines(): + if "# " in line: + line = line[:line.rindex("# ")] + + line = line.strip() + if line == "": + continue + + res[line] = f"skipped by '{fpath}'" + except FileNotFoundError: + pass + + return res + def _read_tests_result(filepath: str) -> Dict[str, TestCase]: with open(filepath, "rt") as f: From 125e3300d3a27b5e620c1ffd8f2de4a5ab4a8d59 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 28 Aug 2024 14:24:15 +0000 Subject: [PATCH 018/261] skip tests with break ydb --- ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt diff --git a/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt b/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt new file mode 100644 index 000000000000..e1b37a9b2063 --- /dev/null +++ b/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt @@ -0,0 +1,2 @@ +golang-lib-pq/TestBinaryByteSlicetoUUID +golang-lib-pq/TestIssue494 \ No newline at end of file From 22a8d30d520084f54804ec964f08dbfe679fb00e Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 28 Aug 2024 16:12:57 +0000 Subject: [PATCH 019/261] mute tests remove prefix fix test tenant --- .github/config/muted_ya.txt | 67 +++ .../go-libpq/data/docker-start.bash | 2 - .../go-libpq/data/full-test-list.txt | 472 +++++++++--------- .../go-libpq/data/skip-tests.txt | 5 +- .../go-libpq/data/unit-tests.txt | 212 ++++---- .../go-libpq/docker_wrapper_test.py | 1 - .../library/pytest_integration.py | 10 +- 7 files changed, 420 insertions(+), 349 deletions(-) diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 5a15bd723363..07638f7e6183 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -101,3 +101,70 @@ ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.* ydb/tests/functional/tenants test_tenants.py.* ydb/tests/functional/ydb_cli test_ydb_impex.py.TestImpex.test_big_dataset* ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestArrayValueBackend] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestBinaryByteSliceToInt] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestBinaryByteSlicetoUUID] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestBindError] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCommit] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestConnListen] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestConnUnlisten] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestConnUnlistenAll] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestConnectorWithNoticeHandler_Simple] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestConnectorWithNotificationHandler_Simple] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestContextCancelBegin] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyFromError] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyInBinaryError] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyInMultipleValues] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyInRaiseStmtTrigger] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyInStmtAffectedRows] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyInTypes] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyInWrongType] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopyRespLoopConnectionError] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestCopySyntaxError] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestEncodeAndParseTs] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestEncodeDecode] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestErrorClass] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestErrorDuringStartup] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestErrorOnExec] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestErrorOnQuery] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestErrorOnQueryRowSimpleQuery] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestExec] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestHasCorrectRootGroupPermissions] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestInfinityTimestamp] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestIssue1046] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestIssue186] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestListenerFailedQuery] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestListenerListen] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestListenerReconnect] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestListenerUnlisten] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestListenerUnlistenAll] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestNewConnector_Connect] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestNewConnector_Driver] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestNewConnector_WorksWithOpenDB] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestNotifyExtra] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestNullAfterNonNull] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestParseErrorInExtendedQuery] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestPing] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestQueryCancelledReused] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestQueryRowBugWorkaround] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestReconnect] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestReturning] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestRowsResultTag] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestRuntimeParameters] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtExecContext] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtExecContext/context.Background] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtExecContext/context.WithTimeout_exceeded] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtExecContext/context.WithTimeout] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtQueryContext] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtQueryContext/context.Background] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtQueryContext/context.WithTimeout_exceeded] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStmtQueryContext/context.WithTimeout] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestStringWithNul] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTimeWithTimezone] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTimeWithTimezone/24:00Z_=>_0000-01-02T00:00:00Z] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTimeWithTimezone/24:00-04:00_=>_0000-01-02T00:00:00-04:00] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTimeWithTimezone/24:00:00+00_=>_0000-01-02T00:00:00Z] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTimeWithTimezone/24:00:00.0+00_=>_0000-01-02T00:00:00Z] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTimeWithTimezone/24:00:00.000000+00_=>_0000-01-02T00:00:00Z] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTimestampWithTimeZone] +ydb/tests/ydb/tests/postgres_integrations/go-libpq docker_wrapper_test.py::test_pg_generated[TestTxOptions] \ No newline at end of file diff --git a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash index 9e244861733c..5b46ae20492b 100755 --- a/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash +++ b/ydb/tests/postgres_integrations/go-libpq/data/docker-start.bash @@ -29,8 +29,6 @@ echo "Start test" mkdir -p /test-result/raw PQTEST_BINARY_PARAMETERS=no /go-run-separate-tests.bash -sed -e 's|classname=""|classname="golang-lib-pq"|' -i /test-result/raw/result.xml - if [ -n "${YDB_PG_TESTFILTER:-}" ]; then cat /test-result/raw/result.txt fi diff --git a/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt b/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt index 7cc782f5115e..469bfc13475e 100644 --- a/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt +++ b/ydb/tests/postgres_integrations/go-libpq/data/full-test-list.txt @@ -1,236 +1,236 @@ -golang-lib-pq/Test64BitErrorChecking -golang-lib-pq/TestAppendEncodedText -golang-lib-pq/TestAppendEscapedText -golang-lib-pq/TestAppendEscapedTextExistingBuffer -golang-lib-pq/TestArrayScanBackend -golang-lib-pq/TestArrayScanner -golang-lib-pq/TestArrayValueBackend -golang-lib-pq/TestArrayValuer -golang-lib-pq/TestBadConn -golang-lib-pq/TestBinaryByteSliceToInt -golang-lib-pq/TestBinaryByteSlicetoUUID -golang-lib-pq/TestBindError -golang-lib-pq/TestBoolArrayScanBytes -golang-lib-pq/TestBoolArrayScanEmpty -golang-lib-pq/TestBoolArrayScanError -golang-lib-pq/TestBoolArrayScanNil -golang-lib-pq/TestBoolArrayScanString -golang-lib-pq/TestBoolArrayScanUnsupported -golang-lib-pq/TestBoolArrayValue -golang-lib-pq/TestByteSliceToText -golang-lib-pq/TestByteaArrayScanBytes -golang-lib-pq/TestByteaArrayScanEmpty -golang-lib-pq/TestByteaArrayScanError -golang-lib-pq/TestByteaArrayScanNil -golang-lib-pq/TestByteaArrayScanString -golang-lib-pq/TestByteaArrayScanUnsupported -golang-lib-pq/TestByteaArrayValue -golang-lib-pq/TestByteaOutputFormatEncoding -golang-lib-pq/TestByteaOutputFormats -golang-lib-pq/TestCloseBadConn -golang-lib-pq/TestCommit -golang-lib-pq/TestCommitInFailedTransaction -golang-lib-pq/TestCommitInFailedTransactionWithCancelContext -golang-lib-pq/TestConnClose -golang-lib-pq/TestConnExecDeadlock -golang-lib-pq/TestConnListen -golang-lib-pq/TestConnPing -golang-lib-pq/TestConnPrepareContext -golang-lib-pq/TestConnPrepareContext/context.Background -golang-lib-pq/TestConnPrepareContext/context.WithTimeout -golang-lib-pq/TestConnPrepareContext/context.WithTimeout_exceeded -golang-lib-pq/TestConnUnlisten -golang-lib-pq/TestConnUnlistenAll -golang-lib-pq/TestConnectorWithNoticeHandler_Simple -golang-lib-pq/TestConnectorWithNotificationHandler_Simple -golang-lib-pq/TestContextCancelBegin -golang-lib-pq/TestContextCancelExec -golang-lib-pq/TestContextCancelQuery -golang-lib-pq/TestCopyFromError -golang-lib-pq/TestCopyInBinaryError -golang-lib-pq/TestCopyInMultipleValues -golang-lib-pq/TestCopyInRaiseStmtTrigger -golang-lib-pq/TestCopyInSchemaStmt -golang-lib-pq/TestCopyInStmt -golang-lib-pq/TestCopyInStmtAffectedRows -golang-lib-pq/TestCopyInTypes -golang-lib-pq/TestCopyInWrongType -golang-lib-pq/TestCopyOutsideOfTxnError -golang-lib-pq/TestCopyRespLoopConnectionError -golang-lib-pq/TestCopySyntaxError -golang-lib-pq/TestDataType -golang-lib-pq/TestDataTypeLength -golang-lib-pq/TestDataTypeName -golang-lib-pq/TestDataTypePrecisionScale -golang-lib-pq/TestDecodeBool -golang-lib-pq/TestDecodeUUIDBackend -golang-lib-pq/TestDecodeUUIDBinaryError -golang-lib-pq/TestEmptyQuery -golang-lib-pq/TestEmptyResultSetColumns -golang-lib-pq/TestEncodeAndParseTs -golang-lib-pq/TestEncodeDecode -golang-lib-pq/TestErrorClass -golang-lib-pq/TestErrorDuringStartup -golang-lib-pq/TestErrorDuringStartupClosesConn -golang-lib-pq/TestErrorOnExec -golang-lib-pq/TestErrorOnQuery -golang-lib-pq/TestErrorOnQueryRowSimpleQuery -golang-lib-pq/TestErrorSQLState -golang-lib-pq/TestExec -golang-lib-pq/TestFloat32ArrayScanBytes -golang-lib-pq/TestFloat32ArrayScanEmpty -golang-lib-pq/TestFloat32ArrayScanError -golang-lib-pq/TestFloat32ArrayScanNil -golang-lib-pq/TestFloat32ArrayScanString -golang-lib-pq/TestFloat32ArrayScanUnsupported -golang-lib-pq/TestFloat32ArrayValue -golang-lib-pq/TestFloat64ArrayScanBytes -golang-lib-pq/TestFloat64ArrayScanEmpty -golang-lib-pq/TestFloat64ArrayScanError -golang-lib-pq/TestFloat64ArrayScanNil -golang-lib-pq/TestFloat64ArrayScanString -golang-lib-pq/TestFloat64ArrayScanUnsupported -golang-lib-pq/TestFloat64ArrayValue -golang-lib-pq/TestFormatAndParseTimestamp -golang-lib-pq/TestFormatTs -golang-lib-pq/TestFormatTsBackend -golang-lib-pq/TestFullParseURL -golang-lib-pq/TestGenericArrayScanDelimiter -golang-lib-pq/TestGenericArrayScanErrors -golang-lib-pq/TestGenericArrayScanScannerArrayBytes -golang-lib-pq/TestGenericArrayScanScannerArrayString -golang-lib-pq/TestGenericArrayScanScannerSliceBytes -golang-lib-pq/TestGenericArrayScanScannerSliceEmpty -golang-lib-pq/TestGenericArrayScanScannerSliceNil -golang-lib-pq/TestGenericArrayScanScannerSliceString -golang-lib-pq/TestGenericArrayScanUnsupported -golang-lib-pq/TestGenericArrayValue -golang-lib-pq/TestGenericArrayValueErrors -golang-lib-pq/TestGenericArrayValueUnsupported -golang-lib-pq/TestHasCorrectRootGroupPermissions -golang-lib-pq/TestIPv6LoopbackParseURL -golang-lib-pq/TestInfinityTimestamp -golang-lib-pq/TestInt32ArrayScanBytes -golang-lib-pq/TestInt32ArrayScanEmpty -golang-lib-pq/TestInt32ArrayScanError -golang-lib-pq/TestInt32ArrayScanNil -golang-lib-pq/TestInt32ArrayScanString -golang-lib-pq/TestInt32ArrayScanUnsupported -golang-lib-pq/TestInt32ArrayValue -golang-lib-pq/TestInt64ArrayScanBytes -golang-lib-pq/TestInt64ArrayScanEmpty -golang-lib-pq/TestInt64ArrayScanError -golang-lib-pq/TestInt64ArrayScanNil -golang-lib-pq/TestInt64ArrayScanString -golang-lib-pq/TestInt64ArrayScanUnsupported -golang-lib-pq/TestInt64ArrayValue -golang-lib-pq/TestInvalidProtocolParseURL -golang-lib-pq/TestIsUTF8 -golang-lib-pq/TestIssue1046 -golang-lib-pq/TestIssue1062 -golang-lib-pq/TestIssue186 -golang-lib-pq/TestIssue196 -golang-lib-pq/TestIssue282 -golang-lib-pq/TestIssue494 -golang-lib-pq/TestIssue617 -golang-lib-pq/TestListenerClose -golang-lib-pq/TestListenerConnCloseWhileQueryIsExecuting -golang-lib-pq/TestListenerFailedQuery -golang-lib-pq/TestListenerListen -golang-lib-pq/TestListenerPing -golang-lib-pq/TestListenerReconnect -golang-lib-pq/TestListenerUnlisten -golang-lib-pq/TestListenerUnlistenAll -golang-lib-pq/TestMinimalURL -golang-lib-pq/TestMultipleEmptyResult -golang-lib-pq/TestMultipleResult -golang-lib-pq/TestMultipleSimpleQuery -golang-lib-pq/TestNewConnector_Connect -golang-lib-pq/TestNewConnector_Driver -golang-lib-pq/TestNewConnector_WorksWithOpenDB -golang-lib-pq/TestNewListenerConn -golang-lib-pq/TestNoData -golang-lib-pq/TestNotifyExtra -golang-lib-pq/TestNullAfterNonNull -golang-lib-pq/TestOpenURL -golang-lib-pq/TestParameterCountMismatch -golang-lib-pq/TestParseArray -golang-lib-pq/TestParseArrayError -golang-lib-pq/TestParseComplete -golang-lib-pq/TestParseEnviron -golang-lib-pq/TestParseErrorInExtendedQuery -golang-lib-pq/TestParseOpts -golang-lib-pq/TestParseTs -golang-lib-pq/TestParseTsErrors -golang-lib-pq/TestPgpass -golang-lib-pq/TestPing -golang-lib-pq/TestQueryCancelRace -golang-lib-pq/TestQueryCancelledReused -golang-lib-pq/TestQueryRowBugWorkaround -golang-lib-pq/TestQuickClose -golang-lib-pq/TestQuoteIdentifier -golang-lib-pq/TestQuoteLiteral -golang-lib-pq/TestReadFloatPrecision -golang-lib-pq/TestReconnect -golang-lib-pq/TestReturning -golang-lib-pq/TestRowsCloseBeforeDone -golang-lib-pq/TestRowsColumnTypes -golang-lib-pq/TestRowsResultTag -golang-lib-pq/TestRuntimeParameters -golang-lib-pq/TestSNISupport -golang-lib-pq/TestSNISupport/SNI_is_not_passed_when_disabled -golang-lib-pq/TestSNISupport/SNI_is_not_set_for_IPv4 -golang-lib-pq/TestSNISupport/SNI_is_passed_when_asked_for -golang-lib-pq/TestSNISupport/SNI_is_set_by_default -golang-lib-pq/TestSSLClientCertificates -golang-lib-pq/TestSSLConnection -golang-lib-pq/TestSSLRequireWithRootCert -golang-lib-pq/TestSSLVerifyCA -golang-lib-pq/TestSSLVerifyFull -golang-lib-pq/TestScanNilTimestamp -golang-lib-pq/TestScanTimestamp -golang-lib-pq/TestSimpleParseURL -golang-lib-pq/TestSimpleQuery -golang-lib-pq/TestStatment -golang-lib-pq/TestStmtExecContext -golang-lib-pq/TestStmtExecContext/context.Background -golang-lib-pq/TestStmtExecContext/context.WithTimeout -golang-lib-pq/TestStmtExecContext/context.WithTimeout_exceeded -golang-lib-pq/TestStmtQueryContext -golang-lib-pq/TestStmtQueryContext/context.Background -golang-lib-pq/TestStmtQueryContext/context.WithTimeout -golang-lib-pq/TestStmtQueryContext/context.WithTimeout_exceeded -golang-lib-pq/TestStringArrayScanBytes -golang-lib-pq/TestStringArrayScanEmpty -golang-lib-pq/TestStringArrayScanError -golang-lib-pq/TestStringArrayScanNil -golang-lib-pq/TestStringArrayScanString -golang-lib-pq/TestStringArrayScanUnsupported -golang-lib-pq/TestStringArrayValue -golang-lib-pq/TestStringToBytea -golang-lib-pq/TestStringToUUID -golang-lib-pq/TestStringWithNul -golang-lib-pq/TestTextByteSliceToInt -golang-lib-pq/TestTextByteSliceToUUID -golang-lib-pq/TestTextDecodeIntoString -golang-lib-pq/TestTimeWithTimezone -golang-lib-pq/TestTimeWithTimezone/11:59:59+00:00_=>_0000-01-01T11:59:59Z -golang-lib-pq/TestTimeWithTimezone/11:59:59+04:00_=>_0000-01-01T11:59:59+04:00 -golang-lib-pq/TestTimeWithTimezone/11:59:59+04:01:02_=>_0000-01-01T11:59:59+04:01 -golang-lib-pq/TestTimeWithTimezone/11:59:59-04:01:02_=>_0000-01-01T11:59:59-04:01 -golang-lib-pq/TestTimeWithTimezone/24:00+00_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithTimezone/24:00-04:00_=>_0000-01-02T00:00:00-04:00 -golang-lib-pq/TestTimeWithTimezone/24:00:00+00_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithTimezone/24:00:00.0+00_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithTimezone/24:00:00.000000+00_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithTimezone/24:00Z_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithoutTimezone -golang-lib-pq/TestTimeWithoutTimezone/11:59:59_=>_0000-01-01T11:59:59Z -golang-lib-pq/TestTimeWithoutTimezone/24:00:00.000000_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithoutTimezone/24:00:00.0_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithoutTimezone/24:00:00_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimeWithoutTimezone/24:00_=>_0000-01-02T00:00:00Z -golang-lib-pq/TestTimestampWithOutTimezone -golang-lib-pq/TestTimestampWithTimeZone -golang-lib-pq/TestTxOptions -golang-lib-pq/TestXactMultiStmt +Test64BitErrorChecking +TestAppendEncodedText +TestAppendEscapedText +TestAppendEscapedTextExistingBuffer +TestArrayScanBackend +TestArrayScanner +TestArrayValueBackend +TestArrayValuer +TestBadConn +TestBinaryByteSliceToInt +TestBinaryByteSlicetoUUID +TestBindError +TestBoolArrayScanBytes +TestBoolArrayScanEmpty +TestBoolArrayScanError +TestBoolArrayScanNil +TestBoolArrayScanString +TestBoolArrayScanUnsupported +TestBoolArrayValue +TestByteSliceToText +TestByteaArrayScanBytes +TestByteaArrayScanEmpty +TestByteaArrayScanError +TestByteaArrayScanNil +TestByteaArrayScanString +TestByteaArrayScanUnsupported +TestByteaArrayValue +TestByteaOutputFormatEncoding +TestByteaOutputFormats +TestCloseBadConn +TestCommit +TestCommitInFailedTransaction +TestCommitInFailedTransactionWithCancelContext +TestConnClose +TestConnExecDeadlock +TestConnListen +TestConnPing +TestConnPrepareContext +TestConnPrepareContext/context.Background +TestConnPrepareContext/context.WithTimeout +TestConnPrepareContext/context.WithTimeout_exceeded +TestConnUnlisten +TestConnUnlistenAll +TestConnectorWithNoticeHandler_Simple +TestConnectorWithNotificationHandler_Simple +TestContextCancelBegin +TestContextCancelExec +TestContextCancelQuery +TestCopyFromError +TestCopyInBinaryError +TestCopyInMultipleValues +TestCopyInRaiseStmtTrigger +TestCopyInSchemaStmt +TestCopyInStmt +TestCopyInStmtAffectedRows +TestCopyInTypes +TestCopyInWrongType +TestCopyOutsideOfTxnError +TestCopyRespLoopConnectionError +TestCopySyntaxError +TestDataType +TestDataTypeLength +TestDataTypeName +TestDataTypePrecisionScale +TestDecodeBool +TestDecodeUUIDBackend +TestDecodeUUIDBinaryError +TestEmptyQuery +TestEmptyResultSetColumns +TestEncodeAndParseTs +TestEncodeDecode +TestErrorClass +TestErrorDuringStartup +TestErrorDuringStartupClosesConn +TestErrorOnExec +TestErrorOnQuery +TestErrorOnQueryRowSimpleQuery +TestErrorSQLState +TestExec +TestFloat32ArrayScanBytes +TestFloat32ArrayScanEmpty +TestFloat32ArrayScanError +TestFloat32ArrayScanNil +TestFloat32ArrayScanString +TestFloat32ArrayScanUnsupported +TestFloat32ArrayValue +TestFloat64ArrayScanBytes +TestFloat64ArrayScanEmpty +TestFloat64ArrayScanError +TestFloat64ArrayScanNil +TestFloat64ArrayScanString +TestFloat64ArrayScanUnsupported +TestFloat64ArrayValue +TestFormatAndParseTimestamp +TestFormatTs +TestFormatTsBackend +TestFullParseURL +TestGenericArrayScanDelimiter +TestGenericArrayScanErrors +TestGenericArrayScanScannerArrayBytes +TestGenericArrayScanScannerArrayString +TestGenericArrayScanScannerSliceBytes +TestGenericArrayScanScannerSliceEmpty +TestGenericArrayScanScannerSliceNil +TestGenericArrayScanScannerSliceString +TestGenericArrayScanUnsupported +TestGenericArrayValue +TestGenericArrayValueErrors +TestGenericArrayValueUnsupported +TestHasCorrectRootGroupPermissions +TestIPv6LoopbackParseURL +TestInfinityTimestamp +TestInt32ArrayScanBytes +TestInt32ArrayScanEmpty +TestInt32ArrayScanError +TestInt32ArrayScanNil +TestInt32ArrayScanString +TestInt32ArrayScanUnsupported +TestInt32ArrayValue +TestInt64ArrayScanBytes +TestInt64ArrayScanEmpty +TestInt64ArrayScanError +TestInt64ArrayScanNil +TestInt64ArrayScanString +TestInt64ArrayScanUnsupported +TestInt64ArrayValue +TestInvalidProtocolParseURL +TestIsUTF8 +TestIssue1046 +TestIssue1062 +TestIssue186 +TestIssue196 +TestIssue282 +TestIssue494 +TestIssue617 +TestListenerClose +TestListenerConnCloseWhileQueryIsExecuting +TestListenerFailedQuery +TestListenerListen +TestListenerPing +TestListenerReconnect +TestListenerUnlisten +TestListenerUnlistenAll +TestMinimalURL +TestMultipleEmptyResult +TestMultipleResult +TestMultipleSimpleQuery +TestNewConnector_Connect +TestNewConnector_Driver +TestNewConnector_WorksWithOpenDB +TestNewListenerConn +TestNoData +TestNotifyExtra +TestNullAfterNonNull +TestOpenURL +TestParameterCountMismatch +TestParseArray +TestParseArrayError +TestParseComplete +TestParseEnviron +TestParseErrorInExtendedQuery +TestParseOpts +TestParseTs +TestParseTsErrors +TestPgpass +TestPing +TestQueryCancelRace +TestQueryCancelledReused +TestQueryRowBugWorkaround +TestQuickClose +TestQuoteIdentifier +TestQuoteLiteral +TestReadFloatPrecision +TestReconnect +TestReturning +TestRowsCloseBeforeDone +TestRowsColumnTypes +TestRowsResultTag +TestRuntimeParameters +TestSNISupport +TestSNISupport/SNI_is_not_passed_when_disabled +TestSNISupport/SNI_is_not_set_for_IPv4 +TestSNISupport/SNI_is_passed_when_asked_for +TestSNISupport/SNI_is_set_by_default +TestSSLClientCertificates +TestSSLConnection +TestSSLRequireWithRootCert +TestSSLVerifyCA +TestSSLVerifyFull +TestScanNilTimestamp +TestScanTimestamp +TestSimpleParseURL +TestSimpleQuery +TestStatment +TestStmtExecContext +TestStmtExecContext/context.Background +TestStmtExecContext/context.WithTimeout +TestStmtExecContext/context.WithTimeout_exceeded +TestStmtQueryContext +TestStmtQueryContext/context.Background +TestStmtQueryContext/context.WithTimeout +TestStmtQueryContext/context.WithTimeout_exceeded +TestStringArrayScanBytes +TestStringArrayScanEmpty +TestStringArrayScanError +TestStringArrayScanNil +TestStringArrayScanString +TestStringArrayScanUnsupported +TestStringArrayValue +TestStringToBytea +TestStringToUUID +TestStringWithNul +TestTextByteSliceToInt +TestTextByteSliceToUUID +TestTextDecodeIntoString +TestTimeWithTimezone +TestTimeWithTimezone/11:59:59+00:00_=>_0000-01-01T11:59:59Z +TestTimeWithTimezone/11:59:59+04:00_=>_0000-01-01T11:59:59+04:00 +TestTimeWithTimezone/11:59:59+04:01:02_=>_0000-01-01T11:59:59+04:01 +TestTimeWithTimezone/11:59:59-04:01:02_=>_0000-01-01T11:59:59-04:01 +TestTimeWithTimezone/24:00+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00-04:00_=>_0000-01-02T00:00:00-04:00 +TestTimeWithTimezone/24:00:00+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00:00.0+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00:00.000000+00_=>_0000-01-02T00:00:00Z +TestTimeWithTimezone/24:00Z_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone +TestTimeWithoutTimezone/11:59:59_=>_0000-01-01T11:59:59Z +TestTimeWithoutTimezone/24:00:00.000000_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone/24:00:00.0_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone/24:00:00_=>_0000-01-02T00:00:00Z +TestTimeWithoutTimezone/24:00_=>_0000-01-02T00:00:00Z +TestTimestampWithOutTimezone +TestTimestampWithTimeZone +TestTxOptions +TestXactMultiStmt diff --git a/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt b/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt index e1b37a9b2063..aa02c4a274c3 100644 --- a/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt +++ b/ydb/tests/postgres_integrations/go-libpq/data/skip-tests.txt @@ -1,2 +1,3 @@ -golang-lib-pq/TestBinaryByteSlicetoUUID -golang-lib-pq/TestIssue494 \ No newline at end of file +# ydb segfalts +# TestBinaryByteSlicetoUUID +TestIssue494 diff --git a/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt b/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt index 9f62657f9464..e0b282854c6b 100644 --- a/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt +++ b/ydb/tests/postgres_integrations/go-libpq/data/unit-tests.txt @@ -1,106 +1,106 @@ -golang-lib-pq/TestAppendEncodedText -golang-lib-pq/TestAppendEscapedText -golang-lib-pq/TestAppendEscapedTextExistingBuffer -golang-lib-pq/TestArrayScanner -golang-lib-pq/TestArrayValuer -golang-lib-pq/TestBadConn -golang-lib-pq/TestBoolArrayScanBytes -golang-lib-pq/TestBoolArrayScanEmpty -golang-lib-pq/TestBoolArrayScanError -golang-lib-pq/TestBoolArrayScanNil -golang-lib-pq/TestBoolArrayScanString -golang-lib-pq/TestBoolArrayScanUnsupported -golang-lib-pq/TestBoolArrayValue -golang-lib-pq/TestByteaArrayScanBytes -golang-lib-pq/TestByteaArrayScanEmpty -golang-lib-pq/TestByteaArrayScanError -golang-lib-pq/TestByteaArrayScanNil -golang-lib-pq/TestByteaArrayScanString -golang-lib-pq/TestByteaArrayScanUnsupported -golang-lib-pq/TestByteaArrayValue -golang-lib-pq/TestByteaOutputFormatEncoding -golang-lib-pq/TestCloseBadConn -golang-lib-pq/TestConnPrepareContext/context.WithTimeout_exceeded -golang-lib-pq/TestCopyInSchemaStmt -golang-lib-pq/TestCopyInStmt -golang-lib-pq/TestDataType -golang-lib-pq/TestDataTypeLength -golang-lib-pq/TestDataTypeName -golang-lib-pq/TestDataTypePrecisionScale -golang-lib-pq/TestDecodeUUIDBinaryError -golang-lib-pq/TestErrorDuringStartup -golang-lib-pq/TestErrorDuringStartupClosesConn -golang-lib-pq/TestErrorSQLState -golang-lib-pq/TestFloat32ArrayScanBytes -golang-lib-pq/TestFloat32ArrayScanEmpty -golang-lib-pq/TestFloat32ArrayScanError -golang-lib-pq/TestFloat32ArrayScanNil -golang-lib-pq/TestFloat32ArrayScanString -golang-lib-pq/TestFloat32ArrayScanUnsupported -golang-lib-pq/TestFloat32ArrayValue -golang-lib-pq/TestFloat64ArrayScanBytes -golang-lib-pq/TestFloat64ArrayScanEmpty -golang-lib-pq/TestFloat64ArrayScanError -golang-lib-pq/TestFloat64ArrayScanNil -golang-lib-pq/TestFloat64ArrayScanString -golang-lib-pq/TestFloat64ArrayScanUnsupported -golang-lib-pq/TestFloat64ArrayValue -golang-lib-pq/TestFormatAndParseTimestamp -golang-lib-pq/TestFormatTs -golang-lib-pq/TestFullParseURL -golang-lib-pq/TestGenericArrayScanDelimiter -golang-lib-pq/TestGenericArrayScanErrors -golang-lib-pq/TestGenericArrayScanScannerArrayBytes -golang-lib-pq/TestGenericArrayScanScannerArrayString -golang-lib-pq/TestGenericArrayScanScannerSliceBytes -golang-lib-pq/TestGenericArrayScanScannerSliceEmpty -golang-lib-pq/TestGenericArrayScanScannerSliceNil -golang-lib-pq/TestGenericArrayScanScannerSliceString -golang-lib-pq/TestGenericArrayScanUnsupported -golang-lib-pq/TestGenericArrayValue -golang-lib-pq/TestGenericArrayValueErrors -golang-lib-pq/TestGenericArrayValueUnsupported -golang-lib-pq/TestIPv6LoopbackParseURL -golang-lib-pq/TestInt32ArrayScanBytes -golang-lib-pq/TestInt32ArrayScanEmpty -golang-lib-pq/TestInt32ArrayScanError -golang-lib-pq/TestInt32ArrayScanNil -golang-lib-pq/TestInt32ArrayScanString -golang-lib-pq/TestInt32ArrayScanUnsupported -golang-lib-pq/TestInt32ArrayValue -golang-lib-pq/TestInt64ArrayScanBytes -golang-lib-pq/TestInt64ArrayScanEmpty -golang-lib-pq/TestInt64ArrayScanError -golang-lib-pq/TestInt64ArrayScanNil -golang-lib-pq/TestInt64ArrayScanString -golang-lib-pq/TestInt64ArrayScanUnsupported -golang-lib-pq/TestInt64ArrayValue -golang-lib-pq/TestInvalidProtocolParseURL -golang-lib-pq/TestIsUTF8 -golang-lib-pq/TestMinimalURL -golang-lib-pq/TestParseArray -golang-lib-pq/TestParseArrayError -golang-lib-pq/TestParseComplete -golang-lib-pq/TestParseEnviron -golang-lib-pq/TestParseOpts -golang-lib-pq/TestParseTs -golang-lib-pq/TestParseTsErrors -golang-lib-pq/TestQuoteIdentifier -golang-lib-pq/TestQuoteLiteral -golang-lib-pq/TestSNISupport -golang-lib-pq/TestSNISupport/SNI_is_not_passed_when_disabled -golang-lib-pq/TestSNISupport/SNI_is_not_set_for_IPv4 -golang-lib-pq/TestSNISupport/SNI_is_passed_when_asked_for -golang-lib-pq/TestSNISupport/SNI_is_set_by_default -golang-lib-pq/TestScanNilTimestamp -golang-lib-pq/TestScanTimestamp -golang-lib-pq/TestSimpleParseURL -golang-lib-pq/TestStringArrayScanBytes -golang-lib-pq/TestStringArrayScanEmpty -golang-lib-pq/TestStringArrayScanError -golang-lib-pq/TestStringArrayScanNil -golang-lib-pq/TestStringArrayScanString -golang-lib-pq/TestStringArrayScanUnsupported -golang-lib-pq/TestStringArrayValue -golang-lib-pq/TestStringWithNul -golang-lib-pq/TestTextDecodeIntoString +TestAppendEncodedText +TestAppendEscapedText +TestAppendEscapedTextExistingBuffer +TestArrayScanner +TestArrayValuer +TestBadConn +TestBoolArrayScanBytes +TestBoolArrayScanEmpty +TestBoolArrayScanError +TestBoolArrayScanNil +TestBoolArrayScanString +TestBoolArrayScanUnsupported +TestBoolArrayValue +TestByteaArrayScanBytes +TestByteaArrayScanEmpty +TestByteaArrayScanError +TestByteaArrayScanNil +TestByteaArrayScanString +TestByteaArrayScanUnsupported +TestByteaArrayValue +TestByteaOutputFormatEncoding +TestCloseBadConn +TestConnPrepareContext/context.WithTimeout_exceeded +TestCopyInSchemaStmt +TestCopyInStmt +TestDataType +TestDataTypeLength +TestDataTypeName +TestDataTypePrecisionScale +TestDecodeUUIDBinaryError +TestErrorDuringStartup +TestErrorDuringStartupClosesConn +TestErrorSQLState +TestFloat32ArrayScanBytes +TestFloat32ArrayScanEmpty +TestFloat32ArrayScanError +TestFloat32ArrayScanNil +TestFloat32ArrayScanString +TestFloat32ArrayScanUnsupported +TestFloat32ArrayValue +TestFloat64ArrayScanBytes +TestFloat64ArrayScanEmpty +TestFloat64ArrayScanError +TestFloat64ArrayScanNil +TestFloat64ArrayScanString +TestFloat64ArrayScanUnsupported +TestFloat64ArrayValue +TestFormatAndParseTimestamp +TestFormatTs +TestFullParseURL +TestGenericArrayScanDelimiter +TestGenericArrayScanErrors +TestGenericArrayScanScannerArrayBytes +TestGenericArrayScanScannerArrayString +TestGenericArrayScanScannerSliceBytes +TestGenericArrayScanScannerSliceEmpty +TestGenericArrayScanScannerSliceNil +TestGenericArrayScanScannerSliceString +TestGenericArrayScanUnsupported +TestGenericArrayValue +TestGenericArrayValueErrors +TestGenericArrayValueUnsupported +TestIPv6LoopbackParseURL +TestInt32ArrayScanBytes +TestInt32ArrayScanEmpty +TestInt32ArrayScanError +TestInt32ArrayScanNil +TestInt32ArrayScanString +TestInt32ArrayScanUnsupported +TestInt32ArrayValue +TestInt64ArrayScanBytes +TestInt64ArrayScanEmpty +TestInt64ArrayScanError +TestInt64ArrayScanNil +TestInt64ArrayScanString +TestInt64ArrayScanUnsupported +TestInt64ArrayValue +TestInvalidProtocolParseURL +TestIsUTF8 +TestMinimalURL +TestParseArray +TestParseArrayError +TestParseComplete +TestParseEnviron +TestParseOpts +TestParseTs +TestParseTsErrors +TestQuoteIdentifier +TestQuoteLiteral +TestSNISupport +TestSNISupport/SNI_is_not_passed_when_disabled +TestSNISupport/SNI_is_not_set_for_IPv4 +TestSNISupport/SNI_is_passed_when_asked_for +TestSNISupport/SNI_is_set_by_default +TestScanNilTimestamp +TestScanTimestamp +TestSimpleParseURL +TestStringArrayScanBytes +TestStringArrayScanEmpty +TestStringArrayScanError +TestStringArrayScanNil +TestStringArrayScanString +TestStringArrayScanUnsupported +TestStringArrayValue +TestStringWithNul +TestTextDecodeIntoString diff --git a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py index 96ce0a4c0912..08736e56eeda 100644 --- a/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py +++ b/ydb/tests/postgres_integrations/go-libpq/docker_wrapper_test.py @@ -8,7 +8,6 @@ def filter_formatter(test_names: typing.List[str]) -> str: - test_names = [item[len("golang-lib-pq/"):] for item in test_names] return "^(" + "|".join(test_names) + ")$" diff --git a/ydb/tests/postgres_integrations/library/pytest_integration.py b/ydb/tests/postgres_integrations/library/pytest_integration.py index 42d20b43cbee..586bef459671 100644 --- a/ydb/tests/postgres_integrations/library/pytest_integration.py +++ b/ydb/tests/postgres_integrations/library/pytest_integration.py @@ -127,7 +127,7 @@ def _prepare_docker_env(pgwire_port: str, test_names: List[str]) -> List[str]: "PGPASSWORD=1234", "PGHOST=localhost", f"PGPORT={pgwire_port}", - "PGDATABASE=local", + "PGDATABASE=/Root", "PQGOSSLTESTS=0", "PQSSLCERTTEST_PATH=certs", f"YDB_PG_TESTFILTER={test_filter}", @@ -268,7 +268,13 @@ def get_text(test_case, field_name: str) -> str: raise Exception(f"Unknown field val for field '{field_name}':\n{field_val}") for test_case in test_cases: - name = test_case["@classname"] + "/" + test_case["@name"] + class_name = test_case["@classname"] + test_name = test_case["@name"] + if class_name == "": + name = test_name + else: + name = test_case["@classname"] + "/" + test_case["@name"] + print("rekby-debug", test_case) if "failure" in test_case: test_state = TestState.FAILED From 05f7bd1f2f21f00c758fbdc3c67f069c542e4807 Mon Sep 17 00:00:00 2001 From: VPolka <39378135+VPolka@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:03:36 +0200 Subject: [PATCH 020/261] return implicit query params type into feature flag (#8065) --- ydb/core/kqp/session_actor/kqp_query_state.h | 4 +++- ydb/core/protos/feature_flags.proto | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ydb/core/kqp/session_actor/kqp_query_state.h b/ydb/core/kqp/session_actor/kqp_query_state.h index a7de3a81b7f0..8cf59899139f 100644 --- a/ydb/core/kqp/session_actor/kqp_query_state.h +++ b/ydb/core/kqp/session_actor/kqp_query_state.h @@ -67,7 +67,9 @@ class TKqpQueryState : public TNonCopyable { , StartedAt(startedAt) { RequestEv.reset(ev->Release().Release()); - if (tableServiceConfig.GetEnableImplicitQueryParameterTypes() && !RequestEv->GetYdbParameters().empty()) { + bool enableImplicitQueryParameterTypes = tableServiceConfig.GetEnableImplicitQueryParameterTypes() || + AppData()->FeatureFlags.GetEnableImplicitQueryParameterTypes(); + if (enableImplicitQueryParameterTypes && !RequestEv->GetYdbParameters().empty()) { QueryParameterTypes = std::make_shared>(); for (const auto& [name, typedValue] : RequestEv->GetYdbParameters()) { QueryParameterTypes->insert({name, typedValue.Gettype()}); diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index b6a94dc6fcd3..f90cf30970b4 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -109,7 +109,7 @@ message TFeatureFlags { optional bool EnableSeparationComputeActorsFromRead = 90 [default = true]; optional bool EnablePQConfigTransactionsAtSchemeShard = 91 [default = false]; optional bool EnableScriptExecutionOperations = 92 [default = true]; - reserved 93; // optional bool EnableImplicitQueryParameterTypes = 93 [default = true]; + optional bool EnableImplicitQueryParameterTypes = 93 [default = true]; optional bool EnableForceImmediateEffectsExecution = 94 [default = false]; optional bool EnableTopicSplitMerge = 95 [default = false]; optional bool EnableChangefeedDynamoDBStreamsFormat = 96 [default = true]; From 1c39e0a4af7bc0476832aac3584c5e60fc836e81 Mon Sep 17 00:00:00 2001 From: Pisarenko Grigoriy <79596613+GrigoriyPA@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:37:19 +0300 Subject: [PATCH 021/261] YQ-3363 fix internal error for insert without params (#8074) --- .../providers/s3/provider/yql_s3_datasink.cpp | 9 ++++++- ydb/tests/fq/s3/test_insert.py | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp index 4e83e373ce01..c631f86a3c23 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_datasink.cpp @@ -90,12 +90,19 @@ class TS3DataSinkProvider : public TDataProviderBase { TExprNode::TPtr RewriteIO(const TExprNode::TPtr& write, TExprContext& ctx) override { const TS3Write w(write); auto settings = write->Tail().ChildrenList(); + + TExprNode::TPtr format = ExtractFormat(settings); + if (!format) { + ctx.AddError(TIssue(ctx.GetPosition(write->Pos()), "Missing format - please use WITH FORMAT when writing into S3")); + return nullptr; + } + return Build(ctx, w.Pos()) .World(w.World()) .DataSink(w.DataSink()) .Target() .Path(write->Child(2U)->Head().Tail().HeadPtr()) - .Format(ExtractFormat(settings)) + .Format(std::move(format)) .Settings(ctx.NewList(w.Pos(), std::move(settings))) .Build() .Input(write->ChildPtr(3)) diff --git a/ydb/tests/fq/s3/test_insert.py b/ydb/tests/fq/s3/test_insert.py index 9a9946e224d9..45fea6d79092 100644 --- a/ydb/tests/fq/s3/test_insert.py +++ b/ydb/tests/fq/s3/test_insert.py @@ -510,3 +510,28 @@ def test_insert_empty_object(self, kikimr, s3, client, unique_prefix): assert result_set.columns[0].type.type_id == ydb.Type.STRING assert len(result_set.rows) == 1 assert result_set.rows[0].items[0].text_value == "" + + @yq_all + @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) + def test_insert_without_format_error(self, kikimr, s3, client, unique_prefix): + resource = boto3.resource( + "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key" + ) + + bucket = resource.Bucket("insert_bucket") + bucket.create(ACL='public-read-write') + bucket.objects.all().delete() + + storage_connection_name = unique_prefix + "ibucket" + client.create_storage_connection(storage_connection_name, "insert_bucket") + + sql = f''' + insert into `{storage_connection_name}`.`/test/` + select * from AS_TABLE([<|foo:123, bar:"xxx"u|>,<|foo:456, bar:"yyy"u|>]); + ''' + + query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id + client.wait_query_status(query_id, fq.QueryMeta.FAILED) + issues = str(client.describe_query(query_id).result.query.issue) + + assert "Missing format - please use WITH FORMAT when writing into S3" in issues, "Incorrect Issues: " + issues From fa4e02d90c21908f785ee45db65eebf9edce69fd Mon Sep 17 00:00:00 2001 From: kruall Date: Wed, 21 Aug 2024 18:07:06 +0300 Subject: [PATCH 022/261] Change activation queue to variant of queues (#8078) --- ydb/library/actors/core/executor_pool_base.cpp | 15 +++++++++------ ydb/library/actors/core/executor_pool_base.h | 11 +++-------- ydb/library/actors/core/executor_pool_basic.cpp | 9 +++++---- ydb/library/actors/core/executor_pool_io.cpp | 8 +++++--- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/ydb/library/actors/core/executor_pool_base.cpp b/ydb/library/actors/core/executor_pool_base.cpp index 2bc87a17bf89..0c538f37b9e8 100644 --- a/ydb/library/actors/core/executor_pool_base.cpp +++ b/ydb/library/actors/core/executor_pool_base.cpp @@ -65,17 +65,20 @@ namespace NActors { } #endif - TExecutorPoolBase::TExecutorPoolBase(ui32 poolId, ui32 threads, TAffinity* affinity) + TExecutorPoolBase::TExecutorPoolBase(ui32 poolId, ui32 threads, TAffinity* affinity, bool useRingQueue) : TExecutorPoolBaseMailboxed(poolId) , PoolThreads(threads) , ThreadsAffinity(affinity) -#ifdef RING_ACTIVATION_QUEUE - , Activations(threads == 1) -#endif - {} + { + if (useRingQueue) { + Activations.emplace(threads == 1); + } else { + Activations.emplace(); + } + } TExecutorPoolBase::~TExecutorPoolBase() { - while (Activations.Pop(0)) + while (std::visit([](auto &x){return x.Pop(0);}, Activations)) ; } diff --git a/ydb/library/actors/core/executor_pool_base.h b/ydb/library/actors/core/executor_pool_base.h index 7a68d375bb14..323b06595805 100644 --- a/ydb/library/actors/core/executor_pool_base.h +++ b/ydb/library/actors/core/executor_pool_base.h @@ -47,21 +47,16 @@ namespace NActors { class TExecutorPoolBase: public TExecutorPoolBaseMailboxed { protected: - -#ifdef RING_ACTIVATION_QUEUE - using TActivationQueue = TRingActivationQueue; -#else - using TActivationQueue = TUnorderedCache; -#endif + using TUnorderedCacheActivationQueue = TUnorderedCache; const i16 PoolThreads; TIntrusivePtr ThreadsAffinity; TAtomic Semaphore = 0; - TActivationQueue Activations; + std::variant Activations; TAtomic ActivationsRevolvingCounter = 0; std::atomic_bool StopFlag = false; public: - TExecutorPoolBase(ui32 poolId, ui32 threads, TAffinity* affinity); + TExecutorPoolBase(ui32 poolId, ui32 threads, TAffinity* affinity, bool useRingQueue); ~TExecutorPoolBase(); void ScheduleActivation(ui32 activation) override; void SpecificScheduleActivation(ui32 activation) override; diff --git a/ydb/library/actors/core/executor_pool_basic.cpp b/ydb/library/actors/core/executor_pool_basic.cpp index 4a940a9ca37f..98b14b077c2f 100644 --- a/ydb/library/actors/core/executor_pool_basic.cpp +++ b/ydb/library/actors/core/executor_pool_basic.cpp @@ -82,7 +82,7 @@ namespace NActors { } TBasicExecutorPool::TBasicExecutorPool(const TBasicExecutorPoolConfig& cfg, IHarmonizer *harmonizer, TExecutorPoolJail *jail) - : TExecutorPoolBase(cfg.PoolId, cfg.Threads, new TAffinity(cfg.Affinity)) + : TExecutorPoolBase(cfg.PoolId, cfg.Threads, new TAffinity(cfg.Affinity), false) , DefaultSpinThresholdCycles(cfg.SpinThreshold * NHPTimer::GetCyclesPerSecond() * 0.000001) // convert microseconds to cycles , SpinThresholdCycles(DefaultSpinThresholdCycles) , SpinThresholdCyclesPerThread(new NThreading::TPadded>[cfg.Threads]) @@ -235,7 +235,7 @@ namespace NActors { } } else { TInternalActorTypeGuard activityGuard; - if (const ui32 activation = Activations.Pop(++revolvingCounter)) { + if (const ui32 activation = std::visit([&revolvingCounter](auto &x) {return x.Pop(++revolvingCounter);}, Activations)) { if (workerId >= 0) { Threads[workerId].SetWork(); } else { @@ -308,8 +308,9 @@ namespace NActors { void TBasicExecutorPool::ScheduleActivationExCommon(ui32 activation, ui64 revolvingCounter, TAtomic x) { TSemaphore semaphore = TSemaphore::GetSemaphore(x); - - Activations.Push(activation, revolvingCounter); + std::visit([activation, revolvingCounter](auto &x) { + x.Push(activation, revolvingCounter); + }, Activations); bool needToWakeUp = false; bool needToChangeOldSemaphore = true; diff --git a/ydb/library/actors/core/executor_pool_io.cpp b/ydb/library/actors/core/executor_pool_io.cpp index d7b01339671d..2f9edb22b43c 100644 --- a/ydb/library/actors/core/executor_pool_io.cpp +++ b/ydb/library/actors/core/executor_pool_io.cpp @@ -7,7 +7,7 @@ namespace NActors { TIOExecutorPool::TIOExecutorPool(ui32 poolId, ui32 threads, const TString& poolName, TAffinity* affinity) - : TExecutorPoolBase(poolId, threads, affinity) + : TExecutorPoolBase(poolId, threads, affinity, false) , Threads(new TExecutorThreadCtx[threads]) , PoolName(poolName) {} @@ -53,7 +53,7 @@ namespace NActors { } while (!StopFlag.load(std::memory_order_acquire)) { - if (const ui32 activation = Activations.Pop(++revolvingCounter)) { + if (const ui32 activation = std::visit([&revolvingCounter](auto &x){return x.Pop(++revolvingCounter);}, Activations)) { return activation; } SpinLockPause(); @@ -86,7 +86,9 @@ namespace NActors { } void TIOExecutorPool::ScheduleActivationEx(ui32 activation, ui64 revolvingWriteCounter) { - Activations.Push(activation, revolvingWriteCounter); + std::visit([activation, revolvingWriteCounter](auto &x) { + x.Push(activation, revolvingWriteCounter); + }, Activations); const TAtomic x = AtomicIncrement(Semaphore); if (x <= 0) { for (;; ++revolvingWriteCounter) { From 7bfab508c9416df75ccb30ee1eb8b2e33615f6ff Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Wed, 21 Aug 2024 21:50:34 +0300 Subject: [PATCH 023/261] Fixes #8019 (#8109) --- .../yql/parser/pg_wrapper/postgresql/src/timezone/pgtz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/parser/pg_wrapper/postgresql/src/timezone/pgtz.c b/ydb/library/yql/parser/pg_wrapper/postgresql/src/timezone/pgtz.c index 792a42a2b5e7..ca8201cd3b5a 100644 --- a/ydb/library/yql/parser/pg_wrapper/postgresql/src/timezone/pgtz.c +++ b/ydb/library/yql/parser/pg_wrapper/postgresql/src/timezone/pgtz.c @@ -275,7 +275,7 @@ pg_tzset(const char *tzname) /* * "GMT" is always sent to tzparse(), as per discussion above. */ - if (strcmp(uppername, "GMT") == 0) + if (strcmp(uppername, "GMT") == 0 || strcmp(uppername, "Z") == 0) { if (!tzparse(uppername, &tzstate, true)) { From 4f3e2f85c7db4925cb472e7e5a5b9afece28572e Mon Sep 17 00:00:00 2001 From: vporyadke Date: Wed, 21 Aug 2024 22:02:17 +0300 Subject: [PATCH 024/261] improve failsafe on cuttting history (#6918) --- ydb/core/mind/hive/hive_impl.cpp | 22 ++++-- ydb/core/mind/hive/hive_impl.h | 17 +++++ ydb/core/mind/hive/hive_impl_ut.cpp | 75 +++++++++++++++++++ ydb/core/mind/hive/hive_schema.h | 3 +- ydb/core/mind/hive/leader_tablet_info.cpp | 10 ++- ydb/core/mind/hive/leader_tablet_info.h | 10 ++- ydb/core/mind/hive/tx__cut_tablet_history.cpp | 10 +-- ydb/core/mind/hive/tx__load_everything.cpp | 10 ++- ydb/core/mind/hive/tx__start_tablet.cpp | 3 +- .../mind/hive/tx__update_tablet_status.cpp | 15 ++-- ydb/core/protos/config.proto | 2 + ydb/core/protos/counters_hive.proto | 2 + .../flat_hive.schema | 8 +- 13 files changed, 157 insertions(+), 30 deletions(-) diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index b232e14814a1..0d8b59939eb4 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -631,11 +631,8 @@ void THive::BuildLocalConfig() { } void THive::BuildCurrentConfig() { - BLOG_D("THive::BuildCurrentConfig ClusterConfig = " << ClusterConfig.ShortDebugString()); CurrentConfig = ClusterConfig; - BLOG_D("THive::BuildCurrentConfig DatabaseConfig = " << DatabaseConfig.ShortDebugString()); CurrentConfig.MergeFrom(DatabaseConfig); - BLOG_D("THive::BuildCurrentConfig CurrentConfig = " << CurrentConfig.ShortDebugString()); TabletLimit.clear(); for (const auto& tabletLimit : CurrentConfig.GetDefaultTabletLimit()) { TabletLimit.insert_or_assign(tabletLimit.GetType(), tabletLimit); @@ -652,6 +649,22 @@ void THive::BuildCurrentConfig() { } } MakeTabletTypeSet(BalancerIgnoreTabletTypes); + CutHistoryDenyList.clear(); + for (auto name : SplitString(CurrentConfig.GetCutHistoryDenyList(), ",")) { + TTabletTypes::EType type = TTabletTypes::StrToType(Strip(name)); + if (IsValidTabletType(type)) { + CutHistoryDenyList.emplace_back(type); + } + } + MakeTabletTypeSet(CutHistoryDenyList); + CutHistoryAllowList.clear(); + for (auto name : SplitString(CurrentConfig.GetCutHistoryAllowList(), ",")) { + TTabletTypes::EType type = TTabletTypes::StrToType(Strip(name)); + if (IsValidTabletType(type)) { + CutHistoryAllowList.emplace_back(type); + } + } + MakeTabletTypeSet(CutHistoryAllowList); if (!CurrentConfig.GetSpreadNeighbours()) { // SpreadNeighbours can be turned off anytime, but // cannot be safely turned on without Hive restart @@ -2223,9 +2236,8 @@ void THive::Handle(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev) const NKikimrConsole::TConfigNotificationRequest& record = ev->Get()->Record; ClusterConfig = record.GetConfig().GetHiveConfig(); NodeBrokerEpoch = TDuration::MicroSeconds(record.GetConfig().GetNodeBrokerConfig().GetEpochDuration()); - BLOG_D("Received TEvConsole::TEvConfigNotificationRequest with update of cluster config: " << ClusterConfig.ShortDebugString() - << "; " << record.GetConfig().GetNodeBrokerConfig().ShortDebugString()); BuildCurrentConfig(); + BLOG_D("Merged config: " << CurrentConfig); Send(ev->Sender, new NConsole::TEvConsole::TEvConfigNotificationResponse(record), 0, ev->Cookie); } diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h index 3d9b68e2d022..4a8b8003f5bd 100644 --- a/ydb/core/mind/hive/hive_impl.h +++ b/ydb/core/mind/hive/hive_impl.h @@ -453,6 +453,8 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar // normalized to be sorted list of unique values std::vector BalancerIgnoreTabletTypes; // built from CurrentConfig + std::vector CutHistoryDenyList; // built from CurrentConfig + std::vector CutHistoryAllowList; // built from CurrentConfig struct TTabletMoveInfo { TInstant Timestamp; @@ -902,6 +904,21 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId return (found != ignoreList.end()); } + bool IsCutHistoryAllowed(TTabletTypes::EType type) const { + bool allowed = true; + const auto& denyList = CutHistoryDenyList; + if (!denyList.empty()) { + bool found = std::find(denyList.begin(), denyList.end(), type) != denyList.end(); + allowed &= !found; + } + const auto& allowList = CutHistoryAllowList; + if (!allowList.empty()) { + bool found = std::find(allowList.begin(), allowList.end(), type) != allowList.end(); + allowed &= found; + } + return allowed; + } + double GetSpaceUsagePenaltyThreshold() { return CurrentConfig.GetSpaceUsagePenaltyThreshold(); } diff --git a/ydb/core/mind/hive/hive_impl_ut.cpp b/ydb/core/mind/hive/hive_impl_ut.cpp index bf540540a7e0..c7c8be5fc4c1 100644 --- a/ydb/core/mind/hive/hive_impl_ut.cpp +++ b/ydb/core/mind/hive/hive_impl_ut.cpp @@ -218,3 +218,78 @@ Y_UNIT_TEST_SUITE(THiveImplTest) { UNIT_ASSERT_VALUES_EQUAL(stDev1, stDev2); } } + +Y_UNIT_TEST_SUITE(TCutHistoryRestrictions) { + class TTestHive : public THive { + public: + TTestHive(TTabletStorageInfo *info, const TActorId &tablet) : THive(info, tablet) {} + + template + void UpdateConfig(F func) { + func(ClusterConfig); + BuildCurrentConfig(); + } + }; + + Y_UNIT_TEST(BasicTest) { + TIntrusivePtr hiveStorage = new TTabletStorageInfo; + hiveStorage->TabletType = TTabletTypes::Hive; + TTestHive hive(hiveStorage.Get(), TActorId()); + hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) { + config.SetCutHistoryAllowList("DataShard,Coordinator"); + config.SetCutHistoryDenyList("GraphShard"); + }); + UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard)); + UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard)); + UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::Hive)); + } + + Y_UNIT_TEST(EmptyAllowList) { + TIntrusivePtr hiveStorage = new TTabletStorageInfo; + hiveStorage->TabletType = TTabletTypes::Hive; + TTestHive hive(hiveStorage.Get(), TActorId()); + hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) { + config.SetCutHistoryAllowList(""); + config.SetCutHistoryDenyList("GraphShard"); + }); + UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard)); + UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::Hive)); + } + + Y_UNIT_TEST(EmptyDenyList) { + TIntrusivePtr hiveStorage = new TTabletStorageInfo; + hiveStorage->TabletType = TTabletTypes::Hive; + TTestHive hive(hiveStorage.Get(), TActorId()); + hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) { + config.SetCutHistoryAllowList("DataShard,Coordinator"); + config.SetCutHistoryDenyList(""); + }); + UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard)); + UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::GraphShard)); + } + + Y_UNIT_TEST(SameTabletInBothLists) { + TIntrusivePtr hiveStorage = new TTabletStorageInfo; + hiveStorage->TabletType = TTabletTypes::Hive; + TTestHive hive(hiveStorage.Get(), TActorId()); + hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) { + config.SetCutHistoryAllowList("DataShard,Coordinator"); + config.SetCutHistoryDenyList("SchemeShard,DataShard"); + }); + UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::DataShard)); + UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::SchemeShard)); + UNIT_ASSERT(!hive.IsCutHistoryAllowed(TTabletTypes::Hive)); + UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::Coordinator)); + } + + Y_UNIT_TEST(BothListsEmpty) { + TIntrusivePtr hiveStorage = new TTabletStorageInfo; + hiveStorage->TabletType = TTabletTypes::Hive; + TTestHive hive(hiveStorage.Get(), TActorId()); + hive.UpdateConfig([](NKikimrConfig::THiveConfig& config) { + config.SetCutHistoryAllowList(""); + config.SetCutHistoryDenyList(""); + }); + UNIT_ASSERT(hive.IsCutHistoryAllowed(TTabletTypes::DataShard)); + } +} diff --git a/ydb/core/mind/hive/hive_schema.h b/ydb/core/mind/hive/hive_schema.h index 014532e805ee..bd7ffe7a7205 100644 --- a/ydb/core/mind/hive/hive_schema.h +++ b/ydb/core/mind/hive/hive_schema.h @@ -177,9 +177,10 @@ struct Schema : NIceDb::Schema { struct Group : Column<3, NScheme::NTypeIds::Uint64> {}; struct Version : Column<4, NScheme::NTypeIds::Uint64> {}; struct Timestamp : Column<5, NScheme::NTypeIds::Uint64> {}; + struct DeletedAtGeneration : Column<6, NScheme::NTypeIds::Uint64> { static constexpr uint64_t Default = 0; }; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; struct Node : Table<4> { diff --git a/ydb/core/mind/hive/leader_tablet_info.cpp b/ydb/core/mind/hive/leader_tablet_info.cpp index fb7355c8b683..5b30189103fb 100644 --- a/ydb/core/mind/hive/leader_tablet_info.cpp +++ b/ydb/core/mind/hive/leader_tablet_info.cpp @@ -367,12 +367,16 @@ void TLeaderTabletInfo::ActualizeTabletStatistics(TInstant now) { } } -void TLeaderTabletInfo::RestoreDeletedHistory() { - for (const auto& entry : DeletedHistory) { +void TLeaderTabletInfo::RestoreDeletedHistory(TTransactionContext& txc) { + NIceDb::TNiceDb db(txc.DB); + while (!DeletedHistory.empty()) { + const auto& entry = DeletedHistory.front(); if (entry.Channel >= TabletStorageInfo->Channels.size()) { continue; } TabletStorageInfo->Channels[entry.Channel].History.push_back(entry.Entry); + db.Table().Key(Id, entry.Channel, entry.Entry.FromGeneration).Update(0); + DeletedHistory.pop(); } for (auto& channel : TabletStorageInfo->Channels) { @@ -381,8 +385,6 @@ void TLeaderTabletInfo::RestoreDeletedHistory() { return lhs.FromGeneration < rhs.FromGeneration; }); } - - DeletedHistory.clear(); } void TLeaderTabletInfo::SetType(TTabletTypes::EType type) { diff --git a/ydb/core/mind/hive/leader_tablet_info.h b/ydb/core/mind/hive/leader_tablet_info.h index 92a38f926689..d86808090521 100644 --- a/ydb/core/mind/hive/leader_tablet_info.h +++ b/ydb/core/mind/hive/leader_tablet_info.h @@ -53,10 +53,12 @@ struct TLeaderTabletInfo : TTabletInfo { struct TChannelHistoryEntry { ui32 Channel; TTabletChannelInfo::THistoryEntry Entry; + ui32 DeletedAtGeneration; - TChannelHistoryEntry(ui32 channel, const TTabletChannelInfo::THistoryEntry& entry) + TChannelHistoryEntry(ui32 channel, const TTabletChannelInfo::THistoryEntry& entry, ui32 deletedAtGeneration) : Channel(channel) , Entry(entry) + , DeletedAtGeneration(deletedAtGeneration) { } }; @@ -71,8 +73,8 @@ struct TLeaderTabletInfo : TTabletInfo { TIntrusivePtr TabletStorageInfo; TChannelsBindings BoundChannels; std::bitset ChannelProfileNewGroup; - std::vector DeletedHistory; - bool WasAliveSinceCutHistory = false; + std::queue DeletedHistory; + bool WasAliveSinceCutHistory = true; NKikimrHive::TEvReassignTablet::EHiveReassignReason ChannelProfileReassignReason; ui32 KnownGeneration; TTabletCategoryInfo* Category; @@ -352,7 +354,7 @@ struct TLeaderTabletInfo : TTabletInfo { TString GetChannelStoragePoolName(const TChannelProfiles::TProfile::TChannel& channel) const; TString GetChannelStoragePoolName(ui32 channelId) const; TStoragePoolInfo& GetStoragePool(ui32 channelId) const; - void RestoreDeletedHistory(); + void RestoreDeletedHistory(TTransactionContext& txc); void SetType(TTabletTypes::EType type); }; diff --git a/ydb/core/mind/hive/tx__cut_tablet_history.cpp b/ydb/core/mind/hive/tx__cut_tablet_history.cpp index 69875336c5bf..52cac01fc63b 100644 --- a/ydb/core/mind/hive/tx__cut_tablet_history.cpp +++ b/ydb/core/mind/hive/tx__cut_tablet_history.cpp @@ -14,12 +14,12 @@ class TTxCutTabletHistory : public TTransactionBase { TTxType GetTxType() const override { return NHive::TXTYPE_CUT_TABLET_HISTORY; } - bool Execute(TTransactionContext&, const TActorContext&) override { + bool Execute(TTransactionContext& txc, const TActorContext&) override { TEvHive::TEvCutTabletHistory* msg = Event->Get(); auto tabletId = msg->Record.GetTabletID(); BLOG_D("THive::TTxCutTabletHistory::Execute(" << tabletId << ")"); TLeaderTabletInfo* tablet = Self->FindTabletEvenInDeleting(tabletId); - if (tablet != nullptr && tablet->IsReadyToReassignTablet()) { + if (tablet != nullptr && tablet->IsReadyToReassignTablet() && Self->IsCutHistoryAllowed(tablet->Type)) { auto channel = msg->Record.GetChannel(); Y_ABORT_UNLESS(channel < tablet->TabletStorageInfo->Channels.size()); TTabletChannelInfo& channelInfo = tablet->TabletStorageInfo->Channels[channel]; @@ -30,11 +30,11 @@ class TTxCutTabletHistory : public TTransactionBase { channelInfo.History.end(), TTabletChannelInfo::THistoryEntry(fromGeneration, groupId)); if (it != channelInfo.History.end()) { - tablet->DeletedHistory.emplace_back(channel, *it); + Self->TabletCounters->Cumulative()[NHive::COUNTER_HISTORY_CUT].Increment(1); + tablet->DeletedHistory.emplace(channel, *it, tablet->KnownGeneration); channelInfo.History.erase(it); - /* to be safe, don't do it just yet NIceDb::TNiceDb db(txc.DB); - db.Table().Key(tabletId, channel, fromGeneration).Delete();*/ + db.Table().Key(tabletId, channel, fromGeneration).Update(tablet->KnownGeneration); } } return true; diff --git a/ydb/core/mind/hive/tx__load_everything.cpp b/ydb/core/mind/hive/tx__load_everything.cpp index 710a14d2bc38..8017ae554b61 100644 --- a/ydb/core/mind/hive/tx__load_everything.cpp +++ b/ydb/core/mind/hive/tx__load_everything.cpp @@ -533,8 +533,14 @@ class TTxLoadEverything : public TTransactionBase { tablet->TabletStorageInfo->Channels.emplace_back(); tablet->TabletStorageInfo->Channels.back().Channel = tablet->TabletStorageInfo->Channels.size() - 1; } - TTabletChannelInfo& channel = tablet->TabletStorageInfo->Channels[channelId]; - channel.History.emplace_back(generationId, groupId, timestamp); + TTabletChannelInfo::THistoryEntry entry(generationId, groupId, timestamp); + auto deletedAtGeneration = tabletChannelGenRowset.GetValueOrDefault(); + if (deletedAtGeneration) { + tablet->DeletedHistory.emplace(channelId, entry, deletedAtGeneration); + } else { + TTabletChannelInfo& channel = tablet->TabletStorageInfo->Channels[channelId]; + channel.History.push_back(entry); + } } else { ++numMissingTablets; } diff --git a/ydb/core/mind/hive/tx__start_tablet.cpp b/ydb/core/mind/hive/tx__start_tablet.cpp index 034fd6a124bb..4b19a64e3273 100644 --- a/ydb/core/mind/hive/tx__start_tablet.cpp +++ b/ydb/core/mind/hive/tx__start_tablet.cpp @@ -55,7 +55,8 @@ class TTxStartTablet : public TTransactionBase { if (!leader.DeletedHistory.empty()) { if (!leader.WasAliveSinceCutHistory) { BLOG_ERROR("THive::TTxStartTablet::Execute Tablet " << TabletId << " failed to start after cutting history - will restore history"); - leader.RestoreDeletedHistory(); + Self->TabletCounters->Cumulative()[NHive::COUNTER_HISTORY_RESTORED].Increment(leader.DeletedHistory.size()); + leader.RestoreDeletedHistory(txc); } else { leader.WasAliveSinceCutHistory = false; } diff --git a/ydb/core/mind/hive/tx__update_tablet_status.cpp b/ydb/core/mind/hive/tx__update_tablet_status.cpp index c4efbbb9b24f..2cd239fe6d52 100644 --- a/ydb/core/mind/hive/tx__update_tablet_status.cpp +++ b/ydb/core/mind/hive/tx__update_tablet_status.cpp @@ -124,6 +124,14 @@ class TTxUpdateTabletStatus : public TTransactionBase { NIceDb::TUpdate(Generation), NIceDb::TUpdate(tablet->Statistics)); Self->UpdateTabletFollowersNumber(leader, db, SideEffects); + + // tablet booted successfully, we may actually cut history now + while (!leader.DeletedHistory.empty() && leader.DeletedHistory.front().DeletedAtGeneration < leader.KnownGeneration) { + leader.WasAliveSinceCutHistory = true; + const auto& entry = leader.DeletedHistory.front(); + db.Table().Key(TabletId, entry.Channel, entry.Entry.FromGeneration).Delete(); + leader.DeletedHistory.pop(); + } } else { db.Table().Key(TabletId, FollowerId).Update( NIceDb::TUpdate(tablet->AsFollower().FollowerGroup.Id), @@ -160,13 +168,6 @@ class TTxUpdateTabletStatus : public TTransactionBase { db.Table().Key(TabletId).Update(NIceDb::TUpdate(0), NIceDb::TUpdate(leader.KnownGeneration), NIceDb::TUpdate(tablet->Statistics)); - - // tablet booted successfully, we may actually cut history now - leader.WasAliveSinceCutHistory = true; - for (const auto& entry : leader.DeletedHistory) { - db.Table().Key(TabletId, entry.Channel, entry.Entry.FromGeneration).Delete(); - } - leader.DeletedHistory.clear(); } else { db.Table().Key(TabletId, FollowerId).Update( NIceDb::TUpdate(tablet->AsFollower().FollowerGroup.Id), diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index f8f570471b48..fba99342ef41 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1504,6 +1504,8 @@ message THiveConfig { optional double NodeUsageRangeToKick = 75 [default = 0.2]; optional bool LessSystemTabletsMoves = 77 [default = true]; optional uint64 MaxPingsInFlight = 78 [default = 1000]; + optional string CutHistoryDenyList = 76 [default = "ColumnShard,KeyValue,PersQueue,BlobDepot"]; + optional string CutHistoryAllowList = 79 [default = "DataShard"]; } message TBlobCacheConfig { diff --git a/ydb/core/protos/counters_hive.proto b/ydb/core/protos/counters_hive.proto index 7a39f2b3f730..9f58674505ee 100644 --- a/ydb/core/protos/counters_hive.proto +++ b/ydb/core/protos/counters_hive.proto @@ -48,6 +48,8 @@ enum ECumulativeCounters { COUNTER_SUGGESTED_SCALE_DOWN = 11 [(CounterOpts) = {Name: "SuggestedScaleDown"}]; COUNTER_STORAGE_BALANCER_EXECUTED = 12 [(CounterOpts) = {Name: "StorageBalancerExecuted"}]; COUNTER_TABLETS_STORAGE_REASSIGNED = 13 [(CounterOpts) = {Name: "TabletsStorageReassigned"}]; + COUNTER_HISTORY_CUT = 14 [(CounterOpts) = {Name: "TabletHistoryCut"}]; + COUNTER_HISTORY_RESTORED = 15 [(CounterOpts) = {Name: "TabletHistoryRestored"}]; } enum EPercentileCounters { diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema index 8592ac5c5345..861980fd77d7 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema @@ -520,6 +520,11 @@ "ColumnId": 5, "ColumnName": "Timestamp", "ColumnType": "Uint64" + }, + { + "ColumnId": 6, + "ColumnName": "DeletedAtGeneration", + "ColumnType": "Uint64" } ], "ColumnsDropped": [], @@ -531,7 +536,8 @@ 2, 3, 4, - 5 + 5, + 6 ], "RoomID": 0, "Codec": 0, From a39f287c653d95bf6484224a108e0fbed9596da8 Mon Sep 17 00:00:00 2001 From: niksaveliev Date: Thu, 22 Aug 2024 09:49:06 +0500 Subject: [PATCH 025/261] Fix describe consumer (#8095) --- ydb/core/viewer/viewer_describe_consumer.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ydb/core/viewer/viewer_describe_consumer.h b/ydb/core/viewer/viewer_describe_consumer.h index bdc4f0d84516..d7544aae1105 100644 --- a/ydb/core/viewer/viewer_describe_consumer.h +++ b/ydb/core/viewer/viewer_describe_consumer.h @@ -39,6 +39,12 @@ class TJsonDescribeConsumer : public TDescribeConsumerRpc { return ReplyAndPassAway(Viewer->GetHTTPBADREQUEST(Event->Get(), "text/plain", "field 'consumer' is required")); } + if (params.Has("path")) { + Request.set_path(params.Get("path")); + } else { + return ReplyAndPassAway(Viewer->GetHTTPBADREQUEST(Event->Get(), "text/plain", "field 'path' is required")); + } + if (params.Has("include_stats")) { Request.set_include_stats(FromStringWithDefault(params.Get("include_stats"), false)); } From 66030921f448a20cb696b3904d4d8a2acb3965ea Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 22 Aug 2024 07:58:10 +0300 Subject: [PATCH 026/261] multi heads allocations manager (#8126) --- .../compute_actor/kqp_scan_compute_manager.h | 16 +- .../compute_actor/kqp_scan_fetcher_actor.cpp | 21 +- .../compute_actor/kqp_scan_fetcher_actor.h | 3 + .../engines/reader/abstract/read_context.h | 25 +- .../engines/reader/actor/actor.cpp | 2 +- .../reader/plain_reader/iterator/context.cpp | 2 + .../reader/plain_reader/iterator/context.h | 1 + .../reader/plain_reader/iterator/fetching.cpp | 4 +- .../reader/plain_reader/iterator/interval.cpp | 14 +- .../reader/plain_reader/iterator/scanner.cpp | 4 +- .../limiter/grouped_memory/service/actor.cpp | 20 +- .../tx/limiter/grouped_memory/service/actor.h | 4 + .../grouped_memory/service/allocation.cpp | 4 +- .../grouped_memory/service/allocation.h | 9 +- .../limiter/grouped_memory/service/group.cpp | 7 +- .../tx/limiter/grouped_memory/service/group.h | 4 +- .../tx/limiter/grouped_memory/service/ids.cpp | 18 +- .../tx/limiter/grouped_memory/service/ids.h | 34 ++- .../grouped_memory/service/manager.cpp | 38 ++- .../limiter/grouped_memory/service/manager.h | 14 +- .../grouped_memory/service/process.cpp | 98 -------- .../limiter/grouped_memory/service/process.h | 234 ++++++++++++++++-- .../limiter/grouped_memory/usage/abstract.cpp | 29 ++- .../limiter/grouped_memory/usage/abstract.h | 19 +- .../tx/limiter/grouped_memory/usage/events.h | 47 +++- .../tx/limiter/grouped_memory/usage/service.h | 17 +- .../limiter/grouped_memory/ut/ut_manager.cpp | 104 ++++---- 27 files changed, 532 insertions(+), 260 deletions(-) diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h index 1e4d9ac58937..2d684d2f6b09 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h @@ -23,6 +23,7 @@ class TComputeTaskData; class TShardScannerInfo { private: std::optional ActorId; + const ui64 ScanId; const ui64 TabletId; const ui64 Generation; i64 DataChunksInFlightCount = 0; @@ -51,15 +52,16 @@ class TShardScannerInfo { } } public: - TShardScannerInfo(TShardState& state, const IExternalObjectsProvider& externalObjectsProvider) - : TabletId(state.TabletId) + TShardScannerInfo(const ui64 scanId, TShardState& state, const IExternalObjectsProvider& externalObjectsProvider) + : ScanId(scanId) + , TabletId(state.TabletId) , Generation(++state.Generation) { const bool subscribed = std::exchange(state.SubscribedOnTablet, true); const auto& keyColumnTypes = externalObjectsProvider.GetKeyColumnTypes(); auto ranges = state.GetScanRanges(keyColumnTypes); - auto ev = externalObjectsProvider.BuildEvKqpScan(0, Generation, ranges); + auto ev = externalObjectsProvider.BuildEvKqpScan(ScanId, Generation, ranges); AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "start_scanner")("tablet_id", TabletId)("generation", Generation) ("info", state.ToString(keyColumnTypes))("range", DebugPrintRanges(keyColumnTypes, ranges, *AppData()->TypeRegistry)) @@ -250,6 +252,7 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { THashMap ShardsByActorId; bool IsActiveFlag = true; THashMap> ShardScanners; + const ui64 ScanId; const IExternalObjectsProvider& ExternalObjectsProvider; public: @@ -313,7 +316,7 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { AFL_ENSURE(state.TabletId); AFL_ENSURE(!state.ActorId)("actor_id", state.ActorId); state.State = NComputeActor::EShardState::Starting; - auto newScanner = std::make_shared(state, ExternalObjectsProvider); + auto newScanner = std::make_shared(ScanId, state, ExternalObjectsProvider); AFL_ENSURE(ShardScanners.emplace(state.TabletId, newScanner).second); } @@ -356,8 +359,9 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { return nullptr; } - TInFlightShards(const IExternalObjectsProvider& externalObjectsProvider) - : ExternalObjectsProvider(externalObjectsProvider) + TInFlightShards(const ui64 scanId, const IExternalObjectsProvider& externalObjectsProvider) + : ScanId(scanId) + , ExternalObjectsProvider(externalObjectsProvider) { } bool IsActive() const { diff --git a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp index 4d4bfed29c89..89485b792dc1 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp @@ -34,7 +34,7 @@ TKqpScanFetcherActor::TKqpScanFetcherActor(const NKikimrKqp::TKqpSnapshot& snaps , Snapshot(snapshot) , ShardsScanningPolicy(shardsScanningPolicy) , Counters(counters) - , InFlightShards(*this) + , InFlightShards(ScanId, *this) , InFlightComputes(ComputeActorIds) { Y_UNUSED(traceId); @@ -86,7 +86,11 @@ void TKqpScanFetcherActor::Bootstrap() { void TKqpScanFetcherActor::HandleExecute(TEvScanExchange::TEvAckData::TPtr& ev) { Y_ABORT_UNLESS(ev->Get()->GetFreeSpace()); - ALS_DEBUG(NKikimrServices::KQP_COMPUTE) << "EvAckData (" << SelfId() << "): " << ev->Sender; + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "AckDataFromCompute")("self_id", SelfId())("scan_id", ScanId) + ("packs_to_send", InFlightComputes.GetPacksToSendCount()) + ("from", ev->Sender)("shards remain", PendingShards.size()) + ("in flight scans", InFlightShards.GetScansCount()) + ("in flight shards", InFlightShards.GetShardsCount()); InFlightComputes.OnComputeAck(ev->Sender, ev->Get()->GetFreeSpace()); CheckFinish(); } @@ -458,12 +462,13 @@ void TKqpScanFetcherActor::ProcessPendingScanDataItem(TEvKqpCompute::TEvScanData state->LastKey = std::move(msg.LastKey); const ui64 rowsCount = msg.GetRowsCount(); - CA_LOG_D("action=got EvScanData;rows=" << rowsCount << ";finished=" << msg.Finished << ";exceeded=" << msg.RequestedBytesLimitReached - << ";from=" << ev->Sender << ";shards remain=" << PendingShards.size() - << ";in flight scans=" << InFlightShards.GetScansCount() - << ";in flight shards=" << InFlightShards.GetShardsCount() - << ";delayed_for=" << latency.SecondsFloat() << " seconds by ratelimiter" - << ";tablet_id=" << state->TabletId); + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("action","got EvScanData")("rows", rowsCount)("finished", msg.Finished)("exceeded", msg.RequestedBytesLimitReached) + ("scan", ScanId)("packs_to_send", InFlightComputes.GetPacksToSendCount()) + ("from", ev->Sender)("shards remain", PendingShards.size()) + ("in flight scans", InFlightShards.GetScansCount()) + ("in flight shards", InFlightShards.GetShardsCount()) + ("delayed_for_seconds_by_ratelimiter", latency.SecondsFloat()) + ("tablet_id", state->TabletId); auto shardScanner = InFlightShards.GetShardScannerVerified(state->TabletId); auto tasksForCompute = shardScanner->OnReceiveData(msg, shardScanner); AFL_ENSURE(tasksForCompute.size() == 1 || tasksForCompute.size() == 0 || tasksForCompute.size() == ComputeActorIds.size())("size", tasksForCompute.size())("compute_size", ComputeActorIds.size()); diff --git a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h index 962aee326470..98c839be624c 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h @@ -168,6 +168,9 @@ class TKqpScanFetcherActor: public NActors::TActorBootstrapped PendingShards; std::deque PendingResolveShards; + static inline TAtomicCounter ScanIdCounter = 0; + const ui64 ScanId = ScanIdCounter.Inc(); + TInFlightShards InFlightShards; TInFlightComputes InFlightComputes; ui32 TotalRetries = 0; diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h index 94f108350e69..c596c7e46026 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h @@ -1,10 +1,12 @@ #pragma once #include "read_metadata.h" + #include -#include -#include #include +#include #include +#include + #include namespace NKikimr::NOlap::NReader { @@ -13,6 +15,7 @@ class TComputeShardingPolicy { private: YDB_READONLY(ui32, ShardsCount, 0); YDB_READONLY_DEF(std::vector, ColumnNames); + public: TString DebugString() const { return TStringBuilder() << "shards_count:" << ShardsCount << ";columns=" << JoinSeq(",", ColumnNames) << ";"; @@ -42,10 +45,12 @@ class TReadContext { const NColumnShard::TConcreteScanCounters Counters; TReadMetadataBase::TConstPtr ReadMetadata; NResourceBroker::NSubscribe::TTaskContext ResourcesTaskContext; + const ui64 ScanId; const TActorId ScanActorId; const TActorId ResourceSubscribeActorId; const TActorId ReadCoordinatorActorId; const TComputeShardingPolicy ComputeShardingPolicy; + public: template std::shared_ptr GetReadMetadataPtrVerifiedAs() const { @@ -74,6 +79,10 @@ class TReadContext { return ScanActorId; } + ui64 GetScanId() const { + return ScanId; + } + const TReadMetadataBase::TConstPtr& GetReadMetadata() const { return ReadMetadata; } @@ -86,17 +95,18 @@ class TReadContext { return ResourcesTaskContext; } - TReadContext(const std::shared_ptr& storagesManager, const NColumnShard::TConcreteScanCounters& counters, const TReadMetadataBase::TConstPtr& readMetadata, - const TActorId& scanActorId, const TActorId& resourceSubscribeActorId, const TActorId& readCoordinatorActorId, const TComputeShardingPolicy& computeShardingPolicy) + TReadContext(const std::shared_ptr& storagesManager, const NColumnShard::TConcreteScanCounters& counters, + const TReadMetadataBase::TConstPtr& readMetadata, const TActorId& scanActorId, const TActorId& resourceSubscribeActorId, + const TActorId& readCoordinatorActorId, const TComputeShardingPolicy& computeShardingPolicy, const ui64 scanId) : StoragesManager(storagesManager) , Counters(counters) , ReadMetadata(readMetadata) , ResourcesTaskContext("CS::SCAN_READ", counters.ResourcesSubscriberCounters) + , ScanId(scanId) , ScanActorId(scanActorId) , ResourceSubscribeActorId(resourceSubscribeActorId) , ReadCoordinatorActorId(readCoordinatorActorId) - , ComputeShardingPolicy(computeShardingPolicy) - { + , ComputeShardingPolicy(computeShardingPolicy) { Y_ABORT_UNLESS(ReadMetadata); } }; @@ -111,6 +121,7 @@ class IDataReader { virtual bool DoIsFinished() const = 0; virtual std::vector> DoExtractReadyResults(const int64_t maxRowsInBatch) = 0; virtual TConclusion DoReadNextInterval() = 0; + public: IDataReader(const std::shared_ptr& context); virtual ~IDataReader() = default; @@ -171,4 +182,4 @@ class IDataReader { } }; -} +} // namespace NKikimr::NOlap::NReader diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp index 96de37e50599..351ecca82294 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp @@ -80,7 +80,7 @@ void TColumnShardScan::Bootstrap(const TActorContext& ctx) { ReadCoordinatorActorId = ctx.Register(new NBlobOperations::NRead::TReadCoordinatorActor(TabletId, SelfId())); std::shared_ptr context = std::make_shared(StoragesManager, ScanCountersPool, - ReadMetadataRange, SelfId(), ResourceSubscribeActorId, ReadCoordinatorActorId, ComputeShardingPolicy); + ReadMetadataRange, SelfId(), ResourceSubscribeActorId, ReadCoordinatorActorId, ComputeShardingPolicy, ScanId); ScanIterator = ReadMetadataRange->StartScan(context); auto startResult = ScanIterator->Start(); StartInstant = TMonotonic::Now(); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp index f759fca64f74..d04188a4d9d6 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp @@ -286,6 +286,8 @@ TSpecialReadContext::TSpecialReadContext(const std::shared_ptr& co }; ProcessMemoryGuard = NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(CommonContext->GetReadMetadata()->GetTxId(), stages); + ProcessScopeGuard = + NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildScopeGuard(CommonContext->GetReadMetadata()->GetTxId(), GetCommonContext()->GetScanId()); auto readSchema = ReadMetadata->GetResultSchema(); SpecColumns = std::make_shared(TIndexInfo::GetSnapshotColumnIdsSet(), readSchema); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h index 9f6952dd323b..211f837074a0 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h @@ -15,6 +15,7 @@ class TSpecialReadContext { private: YDB_READONLY_DEF(std::shared_ptr, CommonContext); YDB_READONLY_DEF(std::shared_ptr, ProcessMemoryGuard); + YDB_READONLY_DEF(std::shared_ptr, ProcessScopeGuard); YDB_READONLY_DEF(std::shared_ptr, SpecColumns); YDB_READONLY_DEF(std::shared_ptr, MergeColumns); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp index ecc0ca754479..90e5b5a7b28c 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp @@ -222,8 +222,8 @@ TConclusion TAllocateMemoryStep::DoExecuteInplace( const std::shared_ptr& source, const TFetchingScriptCursor& step) const { auto allocation = std::make_shared(source, GetProcessingDataSize(source), step); - NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation( - source->GetContext()->GetProcessMemoryControlId(), source->GetFirstIntervalId(), { allocation }, (ui32)StageIndex); + NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation(source->GetContext()->GetProcessMemoryControlId(), + source->GetContext()->GetCommonContext()->GetScanId(), source->GetFirstIntervalId(), { allocation }, (ui32)StageIndex); return false; } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp index 1dee839137c3..9da043a366c1 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp @@ -22,8 +22,8 @@ void TFetchingInterval::ConstructResult() { auto task = std::make_shared(MergingContext, Context, std::move(Sources)); task->SetPriority(NConveyor::ITask::EPriority::High); - NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation( - Context->GetProcessMemoryControlId(), GetIntervalId(), { task }, (ui32)EStageFeaturesIndexes::Merge); + NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation(Context->GetProcessMemoryControlId(), + Context->GetCommonContext()->GetScanId(), GetIntervalId(), { task }, (ui32)EStageFeaturesIndexes::Merge); } } @@ -41,14 +41,16 @@ TFetchingInterval::TFetchingInterval(const NArrow::NMerger::TSortableBatchPositi , TaskGuard(Context->GetCommonContext()->GetCounters().GetResourcesAllocationTasksGuard()) , Sources(sources) , IntervalIdx(intervalIdx) - , IntervalGroupGuard(NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildGroupGuard(Context->GetProcessMemoryControlId())) + , IntervalGroupGuard(NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildGroupGuard( + Context->GetProcessMemoryControlId(), context->GetCommonContext()->GetScanId())) , IntervalStateGuard(Context->GetCommonContext()->GetCounters().CreateIntervalStateGuard()) { AFL_VERIFY(Sources.size()); for (auto&& [_, i] : Sources) { if (!i->IsDataReady()) { ++WaitSourcesCount; } else { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "ready_source")("interval_idx", IntervalIdx)("interval_id", GetIntervalId()); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "ready_source")("interval_idx", IntervalIdx)( + "interval_id", GetIntervalId()); } AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "register_source")("interval_idx", IntervalIdx)("interval_id", GetIntervalId()); i->RegisterInterval(*this, i); @@ -81,8 +83,8 @@ void TFetchingInterval::OnPartSendingComplete() { auto task = std::make_shared(MergingContext, Context, std::move(Merger)); task->SetPriority(NConveyor::ITask::EPriority::High); - NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation( - Context->GetProcessMemoryControlId(), GetIntervalId(), { task }, (ui32)EStageFeaturesIndexes::Merge); + NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation(Context->GetProcessMemoryControlId(), + Context->GetCommonContext()->GetScanId(), GetIntervalId(), { task }, (ui32)EStageFeaturesIndexes::Merge); } } // namespace NKikimr::NOlap::NReader::NPlain diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp index d8204201837c..9062602df80f 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/scanner.cpp @@ -38,11 +38,11 @@ void TScanHead::OnIntervalResult(std::shared_ptrHasMerger())("intervalId", interval->GetIntervalId()); + "merger", interval->HasMerger())("interval_id", interval->GetIntervalId()); break; } else { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "interval_result")("interval_idx", intervalIdx)("count", - it->second ? it->second->GetRecordsCount() : 0)("merger", interval->HasMerger())("intervalId", interval->GetIntervalId()); + it->second ? it->second->GetRecordsCount() : 0)("merger", interval->HasMerger())("interval_id", interval->GetIntervalId()); } auto result = it->second; ReadyIntervals.erase(it); diff --git a/ydb/core/tx/limiter/grouped_memory/service/actor.cpp b/ydb/core/tx/limiter/grouped_memory/service/actor.cpp index 1112c7f7aaef..e7573c23612e 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/actor.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/actor.cpp @@ -9,24 +9,26 @@ void TMemoryLimiterActor::Bootstrap() { void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvStartTask::TPtr& ev) { for (auto&& i : ev->Get()->GetAllocations()) { - Manager->RegisterAllocation(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalGroupId(), i, ev->Get()->GetStageFeaturesIdx()); + Manager->RegisterAllocation(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalScopeId(), ev->Get()->GetExternalGroupId(), i, + ev->Get()->GetStageFeaturesIdx()); } } void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvFinishTask::TPtr& ev) { - Manager->UnregisterAllocation(ev->Get()->GetExternalProcessId(), ev->Get()->GetAllocationId()); + Manager->UnregisterAllocation(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalScopeId(), ev->Get()->GetAllocationId()); } void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvUpdateTask::TPtr& ev) { - Manager->UpdateAllocation(ev->Get()->GetExternalProcessId(), ev->Get()->GetAllocationId(), ev->Get()->GetVolume()); + Manager->UpdateAllocation( + ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalScopeId(), ev->Get()->GetAllocationId(), ev->Get()->GetVolume()); } void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvFinishGroup::TPtr& ev) { - Manager->UnregisterGroup(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalGroupId()); + Manager->UnregisterGroup(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalScopeId(), ev->Get()->GetExternalGroupId()); } void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvStartGroup::TPtr& ev) { - Manager->RegisterGroup(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalGroupId()); + Manager->RegisterGroup(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalScopeId(), ev->Get()->GetExternalGroupId()); } void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvFinishProcess::TPtr& ev) { @@ -37,4 +39,12 @@ void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvStartProcess::TPtr& ev Manager->RegisterProcess(ev->Get()->GetExternalProcessId(), ev->Get()->GetStages()); } +void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvFinishProcessScope::TPtr& ev) { + Manager->UnregisterProcessScope(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalScopeId()); +} + +void TMemoryLimiterActor::Handle(NEvents::TEvExternal::TEvStartProcessScope::TPtr& ev) { + Manager->RegisterProcessScope(ev->Get()->GetExternalProcessId(), ev->Get()->GetExternalScopeId()); +} + } // namespace NKikimr::NOlap::NGroupedMemoryManager diff --git a/ydb/core/tx/limiter/grouped_memory/service/actor.h b/ydb/core/tx/limiter/grouped_memory/service/actor.h index c653c8c6e94a..4b4506ba5b99 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/actor.h +++ b/ydb/core/tx/limiter/grouped_memory/service/actor.h @@ -34,6 +34,8 @@ class TMemoryLimiterActor: public NActors::TActorBootstrappedGetTypeName()); } diff --git a/ydb/core/tx/limiter/grouped_memory/service/allocation.cpp b/ydb/core/tx/limiter/grouped_memory/service/allocation.cpp index a9c5b3585a4c..2d04be2c9cef 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/allocation.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/allocation.cpp @@ -3,12 +3,14 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { -TAllocationInfo::TAllocationInfo(const ui64 processId, const ui64 allocationInternalGroupId, const std::shared_ptr& allocation, +TAllocationInfo::TAllocationInfo(const ui64 processId, const ui64 scopeId, const ui64 allocationInternalGroupId, + const std::shared_ptr& allocation, const std::shared_ptr& stage) : Allocation(allocation) , AllocationInternalGroupId(allocationInternalGroupId) , Identifier(TValidator::CheckNotNull(Allocation)->GetIdentifier()) , ProcessId(processId) + , ScopeId(scopeId) , Stage(stage) { AFL_VERIFY(Stage); AFL_VERIFY(Allocation); diff --git a/ydb/core/tx/limiter/grouped_memory/service/allocation.h b/ydb/core/tx/limiter/grouped_memory/service/allocation.h index 8c8c45092a0a..47d5043188d0 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/allocation.h +++ b/ydb/core/tx/limiter/grouped_memory/service/allocation.h @@ -16,6 +16,7 @@ class TAllocationInfo { ui64 AllocatedVolume = 0; YDB_READONLY(ui64, Identifier, 0); YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, ScopeId, 0); const std::shared_ptr Stage; bool AllocationFailed = false; @@ -25,7 +26,7 @@ class TAllocationInfo { Stage->Free(AllocatedVolume, GetAllocationStatus() == EAllocationStatus::Allocated); } - AFL_INFO(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "destroy")("allocation_id", Identifier)("stage", Stage->GetName()); + AFL_TRACE(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "destroy")("allocation_id", Identifier)("stage", Stage->GetName()); } bool IsAllocatable(const ui64 additional) const { @@ -43,11 +44,11 @@ class TAllocationInfo { } [[nodiscard]] bool Allocate(const NActors::TActorId& ownerId) { - AFL_INFO(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "allocated")("allocation_id", Identifier)("stage", Stage->GetName()); + AFL_TRACE(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "allocated")("allocation_id", Identifier)("stage", Stage->GetName()); AFL_VERIFY(Allocation)("status", GetAllocationStatus())("volume", AllocatedVolume)("id", Identifier)("stage", Stage->GetName())( "allocation_internal_group_id", AllocationInternalGroupId); const bool result = Allocation->OnAllocated( - std::make_shared(ProcessId, Allocation->GetIdentifier(), ownerId, Allocation->GetMemory()), Allocation); + std::make_shared(ProcessId, ScopeId, Allocation->GetIdentifier(), ownerId, Allocation->GetMemory()), Allocation); if (result) { Stage->Allocate(AllocatedVolume); } else { @@ -68,7 +69,7 @@ class TAllocationInfo { } } - TAllocationInfo(const ui64 processId, const ui64 allocationInternalGroupId, const std::shared_ptr& allocation, + TAllocationInfo(const ui64 processId, const ui64 scopeId, const ui64 allocationInternalGroupId, const std::shared_ptr& allocation, const std::shared_ptr& stage); }; diff --git a/ydb/core/tx/limiter/grouped_memory/service/group.cpp b/ydb/core/tx/limiter/grouped_memory/service/group.cpp index 288ca98d31d1..3bf671ff76b9 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/group.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/group.cpp @@ -19,13 +19,16 @@ std::vector> TGrouppedAllocations::AllocatePoss return result; } -bool TAllocationGroups::Allocate(TProcessMemory& process, const ui32 allocationsLimit) { +bool TAllocationGroups::Allocate(const bool isPriorityProcess, TProcessMemoryScope& process, const ui32 allocationsLimit) { + AFL_DEBUG(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "try_allocation")("limit", allocationsLimit)( + "external_process_id", process.ExternalProcessId)("forced_internal_group_id", process.GroupIds.GetMinInternalIdOptional())( + "external_scope_id", process.ExternalScopeId)("forced_external_group_id", process.GroupIds.GetMinExternalIdOptional()); ui32 allocationsCount = 0; while (true) { std::vector toRemove; for (auto it = Groups.begin(); it != Groups.end();) { const ui64 internalGroupId = it->first; - const bool forced = process.IsPriorityProcess() && internalGroupId == process.GroupIds.GetMinInternalIdVerified(); + const bool forced = isPriorityProcess && internalGroupId == process.GroupIds.GetMinInternalIdVerified(); std::vector> allocated; if (forced) { allocated = it->second.ExtractAllocationsToVector(); diff --git a/ydb/core/tx/limiter/grouped_memory/service/group.h b/ydb/core/tx/limiter/grouped_memory/service/group.h index 52d2665ce254..8f4434c886b8 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/group.h +++ b/ydb/core/tx/limiter/grouped_memory/service/group.h @@ -3,7 +3,7 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { -class TProcessMemory; +class TProcessMemoryScope; class TGrouppedAllocations { private: @@ -48,7 +48,7 @@ class TAllocationGroups { return Groups.empty(); } - [[nodiscard]] bool Allocate(TProcessMemory& process, const ui32 allocationsLimit); + [[nodiscard]] bool Allocate(const bool isPriorityProcess, TProcessMemoryScope& process, const ui32 allocationsLimit); [[nodiscard]] std::vector> ExtractGroup(const ui64 id) { auto it = Groups.find(id); diff --git a/ydb/core/tx/limiter/grouped_memory/service/ids.cpp b/ydb/core/tx/limiter/grouped_memory/service/ids.cpp index 5185230efad0..163c90efcf12 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/ids.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/ids.cpp @@ -7,7 +7,7 @@ ui64 TIdsControl::ExtractInternalIdVerified(const ui64 externalId) { auto it = ExternalIdIntoInternalId.find(externalId); AFL_VERIFY(it != ExternalIdIntoInternalId.end())("external_id", externalId); const ui64 result = it->second; - InternalIds.erase(result); + InternalIdIntoExternalId.erase(result); ExternalIdIntoInternalId.erase(it); return result; } @@ -21,8 +21,8 @@ std::optional TIdsControl::GetInternalIdOptional(const ui64 externalId) co } ui64 TIdsControl::GetMinInternalIdVerified() const { - AFL_VERIFY(InternalIds.size()); - return *InternalIds.begin(); + AFL_VERIFY(InternalIdIntoExternalId.size()); + return InternalIdIntoExternalId.begin()->first; } ui64 TIdsControl::GetInternalIdVerified(const ui64 externalId) const { @@ -33,7 +33,7 @@ ui64 TIdsControl::GetInternalIdVerified(const ui64 externalId) const { ui64 TIdsControl::RegisterExternalId(const ui64 externalId) { AFL_VERIFY(ExternalIdIntoInternalId.emplace(externalId, ++CurrentInternalId).second); - InternalIds.emplace(CurrentInternalId); + InternalIdIntoExternalId.emplace(CurrentInternalId, externalId); return CurrentInternalId; } @@ -43,7 +43,7 @@ ui64 TIdsControl::RegisterExternalIdOrGet(const ui64 externalId) { return it->second; } AFL_VERIFY(ExternalIdIntoInternalId.emplace(externalId, ++CurrentInternalId).second); - InternalIds.emplace(CurrentInternalId); + InternalIdIntoExternalId.emplace(CurrentInternalId, externalId); return CurrentInternalId; } @@ -52,9 +52,15 @@ bool TIdsControl::UnregisterExternalId(const ui64 externalId) { if (it == ExternalIdIntoInternalId.end()) { return false; } - AFL_VERIFY(InternalIds.erase(it->second)); + AFL_VERIFY(InternalIdIntoExternalId.erase(it->second)); ExternalIdIntoInternalId.erase(it); return true; } +ui64 TIdsControl::GetExternalIdVerified(const ui64 internalId) const { + auto it = InternalIdIntoExternalId.find(internalId); + AFL_VERIFY(it != InternalIdIntoExternalId.end()); + return it->second; +} + } // namespace NKikimr::NOlap::NGroupedMemoryManager diff --git a/ydb/core/tx/limiter/grouped_memory/service/ids.h b/ydb/core/tx/limiter/grouped_memory/service/ids.h index 19ce04f95367..acaa700411ab 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/ids.h +++ b/ydb/core/tx/limiter/grouped_memory/service/ids.h @@ -1,5 +1,8 @@ #pragma once #include + +#include + #include #include @@ -7,19 +10,28 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { class TIdsControl { private: - std::map ExternalIdIntoInternalId; - YDB_READONLY_DEF(std::set, InternalIds); + THashMap ExternalIdIntoInternalId; + std::map InternalIdIntoExternalId; ui64 CurrentInternalId = 0; public: void Clear() { ExternalIdIntoInternalId.clear(); - InternalIds.clear(); + InternalIdIntoExternalId.clear(); + } + + const std::map& GetInternalIdToExternalIds() const { + return InternalIdIntoExternalId; + } + + ui64 GetSize() const { + return InternalIdIntoExternalId.size(); } [[nodiscard]] ui64 ExtractInternalIdVerified(const ui64 externalId); ui64 GetMinInternalIdVerified() const; + ui64 GetExternalIdVerified(const ui64 internalId) const; std::optional GetInternalIdOptional(const ui64 externalId) const; @@ -31,16 +43,24 @@ class TIdsControl { [[nodiscard]] bool UnregisterExternalId(const ui64 externalId); std::optional GetMinInternalIdOptional() const { - if (InternalIds.size()) { - return *InternalIds.begin(); + if (InternalIdIntoExternalId.size()) { + return InternalIdIntoExternalId.begin()->first; + } else { + return std::nullopt; + } + } + + std::optional GetMinExternalIdOptional() const { + if (InternalIdIntoExternalId.size()) { + return InternalIdIntoExternalId.begin()->second; } else { return std::nullopt; } } ui64 GetMinInternalIdDef(const ui64 def) const { - if (InternalIds.size()) { - return *InternalIds.begin(); + if (InternalIdIntoExternalId.size()) { + return InternalIdIntoExternalId.begin()->first; } else { return def; } diff --git a/ydb/core/tx/limiter/grouped_memory/service/manager.cpp b/ydb/core/tx/limiter/grouped_memory/service/manager.cpp index a6d3ef0d94cf..96fe8bcefc17 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/manager.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/manager.cpp @@ -12,23 +12,27 @@ TProcessMemory* TManager::GetProcessMemoryByExternalIdOptional(const ui64 extern return GetProcessMemoryOptional(*internalId); } -void TManager::RegisterGroup(const ui64 externalProcessId, const ui64 externalGroupId) { +void TManager::RegisterGroup(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId) { + AFL_DEBUG(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "register_group")("external_process_id", externalProcessId)( + "external_group_id", externalGroupId)("size", ProcessIds.GetSize())("external_scope_id", externalScopeId); if (auto* process = GetProcessMemoryByExternalIdOptional(externalProcessId)) { - process->RegisterGroup(externalGroupId); + process->RegisterGroup(externalScopeId, externalGroupId); } RefreshSignals(); } -void TManager::UnregisterGroup(const ui64 externalProcessId, const ui64 externalGroupId) { +void TManager::UnregisterGroup(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId) { + AFL_DEBUG(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "unregister_group")("external_process_id", externalProcessId)( + "external_group_id", externalGroupId)("size", ProcessIds.GetSize()); if (auto* process = GetProcessMemoryByExternalIdOptional(externalProcessId)) { - process->UnregisterGroup(externalGroupId); + process->UnregisterGroup(externalScopeId, externalGroupId); } RefreshSignals(); } -void TManager::UpdateAllocation(const ui64 externalProcessId, const ui64 allocationId, const ui64 volume) { +void TManager::UpdateAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId, const ui64 volume) { TProcessMemory& process = GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)); - if (process.UpdateAllocation(allocationId, volume)) { + if (process.UpdateAllocation(externalScopeId, allocationId, volume)) { TryAllocateWaiting(); } @@ -56,21 +60,21 @@ void TManager::TryAllocateWaiting() { RefreshSignals(); } -void TManager::UnregisterAllocation(const ui64 externalProcessId, const ui64 allocationId) { +void TManager::UnregisterAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId) { if (auto* process = GetProcessMemoryByExternalIdOptional(externalProcessId)) { - if (process->UnregisterAllocation(allocationId)) { + if (process->UnregisterAllocation(externalScopeId, allocationId)) { TryAllocateWaiting(); } } RefreshSignals(); } -void TManager::RegisterAllocation( - const ui64 externalProcessId, const ui64 externalGroupId, const std::shared_ptr& task, const std::optional& stageIdx) { +void TManager::RegisterAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId, + const std::shared_ptr& task, const std::optional& stageIdx) { if (auto* process = GetProcessMemoryByExternalIdOptional(externalProcessId)) { - process->RegisterAllocation(externalGroupId, task, stageIdx); + process->RegisterAllocation(externalScopeId, externalGroupId, task, stageIdx); } else { - AFL_VERIFY(!task->OnAllocated(std::make_shared(externalProcessId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))( + AFL_VERIFY(!task->OnAllocated(std::make_shared(externalProcessId, externalScopeId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))( "ext_group", externalGroupId)("stage_idx", stageIdx); } RefreshSignals(); @@ -105,4 +109,14 @@ void TManager::UnregisterProcess(const ui64 externalProcessId) { RefreshSignals(); } +void TManager::RegisterProcessScope(const ui64 externalProcessId, const ui64 externalProcessScopeId) { + GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)).RegisterScope(externalProcessScopeId); + RefreshSignals(); +} + +void TManager::UnregisterProcessScope(const ui64 externalProcessId, const ui64 externalProcessScopeId) { + GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)).UnregisterScope(externalProcessScopeId); + RefreshSignals(); +} + } // namespace NKikimr::NOlap::NGroupedMemoryManager diff --git a/ydb/core/tx/limiter/grouped_memory/service/manager.h b/ydb/core/tx/limiter/grouped_memory/service/manager.h index 7e64b83e1167..fd641a3f69b1 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/manager.h +++ b/ydb/core/tx/limiter/grouped_memory/service/manager.h @@ -56,16 +56,20 @@ class TManager { { } - void RegisterGroup(const ui64 externalProcessId, const ui64 externalGroupId); - void UnregisterGroup(const ui64 externalProcessId, const ui64 externalGroupId); + void RegisterGroup(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId); + void UnregisterGroup(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId); + + void RegisterProcessScope(const ui64 externalProcessId, const ui64 externalScopeId); + void UnregisterProcessScope(const ui64 externalProcessId, const ui64 externalScopeId); void RegisterProcess(const ui64 externalProcessId, const std::vector>& stages); void UnregisterProcess(const ui64 externalProcessId); - void RegisterAllocation(const ui64 externalProcessId, const ui64 externalGroupId, const std::shared_ptr& task, + void RegisterAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId, + const std::shared_ptr& task, const std::optional& stageIdx); - void UnregisterAllocation(const ui64 externalProcessId, const ui64 allocationId); - void UpdateAllocation(const ui64 externalProcessId, const ui64 allocationId, const ui64 volume); + void UnregisterAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId); + void UpdateAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId, const ui64 volume); bool IsEmpty() const { return Processes.empty(); diff --git a/ydb/core/tx/limiter/grouped_memory/service/process.cpp b/ydb/core/tx/limiter/grouped_memory/service/process.cpp index c928cbc56849..bcde6532e797 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/process.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/process.cpp @@ -2,102 +2,4 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { -void TProcessMemory::RegisterAllocation( - const ui64 externalGroupId, const std::shared_ptr& task, const std::optional& stageIdx) { - AFL_VERIFY(task); - std::shared_ptr stage; - if (Stages.empty()) { - AFL_VERIFY(!stageIdx); - stage = DefaultStage; - } else { - AFL_VERIFY(stageIdx); - AFL_VERIFY(*stageIdx < Stages.size()); - stage = Stages[*stageIdx]; - } - AFL_VERIFY(stage); - const std::optional internalGroupIdOptional = GroupIds.GetInternalIdOptional(externalGroupId); - if (!internalGroupIdOptional) { - AFL_VERIFY(!task->OnAllocated(std::make_shared(ExternalProcessId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))("ext_group", - externalGroupId)("min_group", GroupIds.GetMinInternalIdOptional())( - "stage", stage->GetName()); - AFL_VERIFY(!AllocationInfo.contains(task->GetIdentifier())); - } else { - const ui64 internalGroupId = *internalGroupIdOptional; - auto allocationInfo = RegisterAllocationImpl(internalGroupId, task, stage); - - if (allocationInfo->GetAllocationStatus() != EAllocationStatus::Waiting) { - } else if (WaitAllocations.GetMinGroupId().value_or(internalGroupId) < internalGroupId) { - WaitAllocations.AddAllocation(internalGroupId, allocationInfo); - } else if (allocationInfo->IsAllocatable(0) || (IsPriorityProcess() && internalGroupId == GroupIds.GetMinInternalIdVerified())) { - Y_UNUSED(WaitAllocations.RemoveAllocation(internalGroupId, allocationInfo)); - if (!allocationInfo->Allocate(OwnerActorId)) { - UnregisterAllocation(allocationInfo->GetIdentifier()); - } - } else { - WaitAllocations.AddAllocation(internalGroupId, allocationInfo); - } - } -} - -const std::shared_ptr& TProcessMemory::RegisterAllocationImpl( - const ui64 internalGroupId, const std::shared_ptr& task, const std::shared_ptr& stage) { - auto it = AllocationInfo.find(task->GetIdentifier()); - if (it == AllocationInfo.end()) { - it = AllocationInfo.emplace(task->GetIdentifier(), std::make_shared(ExternalProcessId, internalGroupId, task, stage)).first; - } - return it->second; -} - -bool TProcessMemory::UnregisterAllocation(const ui64 allocationId) { - ui64 memoryAllocated = 0; - auto it = AllocationInfo.find(allocationId); - AFL_VERIFY(it != AllocationInfo.end()); - bool waitFlag = false; - const ui64 internalGroupId = it->second->GetAllocationInternalGroupId(); - switch (it->second->GetAllocationStatus()) { - case EAllocationStatus::Allocated: - case EAllocationStatus::Failed: - AFL_VERIFY(!WaitAllocations.RemoveAllocation(internalGroupId, it->second)); - break; - case EAllocationStatus::Waiting: - AFL_VERIFY(WaitAllocations.RemoveAllocation(internalGroupId, it->second)); - waitFlag = true; - break; - } - AFL_DEBUG(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "allocation_unregister")("allocation_id", allocationId)("wait", waitFlag)( - "internal_group_id", internalGroupId)("allocation_status", it->second->GetAllocationStatus()); - memoryAllocated = it->second->GetAllocatedVolume(); - AllocationInfo.erase(it); - return !!memoryAllocated; -} - -void TProcessMemory::UnregisterGroupImpl(const ui64 internalGroupId) { - auto data = WaitAllocations.ExtractGroup(internalGroupId); - for (auto&& allocation : data) { - AFL_VERIFY(!allocation->Allocate(OwnerActorId)); - } -} - -void TProcessMemory::UnregisterGroup(const ui64 externalGroupId) { - const ui64 internalGroupId = GroupIds.ExtractInternalIdVerified(externalGroupId); - AFL_INFO(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "remove_group")("external_group_id", externalGroupId)( - "internal_group_id", internalGroupId); - UnregisterGroupImpl(internalGroupId); - if (IsPriorityProcess() && (internalGroupId < GroupIds.GetMinInternalIdDef(internalGroupId))) { - Y_UNUSED(TryAllocateWaiting(0)); - } -} - -void TProcessMemory::RegisterGroup(const ui64 externalGroupId) { - Y_UNUSED(GroupIds.RegisterExternalId(externalGroupId)); -} - -void TProcessMemory::Unregister() { - for (auto&& i : GroupIds.GetInternalIds()) { - UnregisterGroupImpl(i); - } - GroupIds.Clear(); - AllocationInfo.clear(); -} - } // namespace NKikimr::NOlap::NGroupedMemoryManager diff --git a/ydb/core/tx/limiter/grouped_memory/service/process.h b/ydb/core/tx/limiter/grouped_memory/service/process.h index 9b7b5e522c6d..a1c13e091d59 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/process.h +++ b/ydb/core/tx/limiter/grouped_memory/service/process.h @@ -2,37 +2,231 @@ #include "group.h" #include "ids.h" +#include + namespace NKikimr::NOlap::NGroupedMemoryManager { -class TProcessMemory { +class TProcessMemoryScope { private: const ui64 ExternalProcessId; + const ui64 ExternalScopeId; TAllocationGroups WaitAllocations; THashMap> AllocationInfo; TIdsControl GroupIds; + ui32 Links = 1; const NActors::TActorId OwnerActorId; - bool PriorityProcessFlag = false; - - const std::shared_ptr& RegisterAllocationImpl( - const ui64 internalGroupId, const std::shared_ptr& task, const std::shared_ptr& stage); - void UnregisterGroupImpl(const ui64 internalGroupId); TAllocationInfo& GetAllocationInfoVerified(const ui64 allocationId) const { auto it = AllocationInfo.find(allocationId); AFL_VERIFY(it != AllocationInfo.end()); return *it->second; } + + void UnregisterGroupImpl(const ui64 internalGroupId) { + auto data = WaitAllocations.ExtractGroup(internalGroupId); + for (auto&& allocation : data) { + AFL_VERIFY(!allocation->Allocate(OwnerActorId)); + } + } + + const std::shared_ptr& RegisterAllocationImpl( + const ui64 internalGroupId, const std::shared_ptr& task, const std::shared_ptr& stage) { + auto it = AllocationInfo.find(task->GetIdentifier()); + if (it == AllocationInfo.end()) { + it = AllocationInfo + .emplace(task->GetIdentifier(), + std::make_shared(ExternalProcessId, ExternalScopeId, internalGroupId, task, stage)) + .first; + } + return it->second; + } + friend class TAllocationGroups; +public: + TProcessMemoryScope(const ui64 externalProcessId, const ui64 externalScopeId, const NActors::TActorId& ownerActorId) + : ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) + , OwnerActorId(ownerActorId) { + } + + void Register() { + ++Links; + } + + [[nodiscard]] bool Unregister() { + if (--Links) { + return false; + } + for (auto&& [i, _] : GroupIds.GetInternalIdToExternalIds()) { + UnregisterGroupImpl(i); + } + GroupIds.Clear(); + AllocationInfo.clear(); + return true; + } + + void RegisterAllocation(const bool isPriorityProcess, const ui64 externalGroupId, const std::shared_ptr& task, + const std::shared_ptr& stage) { + AFL_VERIFY(task); + AFL_VERIFY(stage); + const std::optional internalGroupIdOptional = GroupIds.GetInternalIdOptional(externalGroupId); + if (!internalGroupIdOptional) { + AFL_VERIFY(!task->OnAllocated(std::make_shared(ExternalProcessId, ExternalScopeId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))( + "ext_group", externalGroupId)( + "min_group", GroupIds.GetMinInternalIdOptional())("stage", stage->GetName()); + AFL_VERIFY(!AllocationInfo.contains(task->GetIdentifier())); + } else { + const ui64 internalGroupId = *internalGroupIdOptional; + auto allocationInfo = RegisterAllocationImpl(internalGroupId, task, stage); + + if (allocationInfo->GetAllocationStatus() != EAllocationStatus::Waiting) { + } else if (WaitAllocations.GetMinGroupId().value_or(internalGroupId) < internalGroupId) { + WaitAllocations.AddAllocation(internalGroupId, allocationInfo); + } else if (allocationInfo->IsAllocatable(0) || (isPriorityProcess && internalGroupId == GroupIds.GetMinInternalIdVerified())) { + Y_UNUSED(WaitAllocations.RemoveAllocation(internalGroupId, allocationInfo)); + if (!allocationInfo->Allocate(OwnerActorId)) { + UnregisterAllocation(allocationInfo->GetIdentifier()); + } + } else { + WaitAllocations.AddAllocation(internalGroupId, allocationInfo); + } + } + } + + bool UpdateAllocation(const ui64 allocationId, const ui64 volume) { + GetAllocationInfoVerified(allocationId).SetAllocatedVolume(volume); + return true; + } + + bool TryAllocateWaiting(const bool isPriorityProcess, const ui32 allocationsCountLimit) { + return WaitAllocations.Allocate(isPriorityProcess, *this, allocationsCountLimit); + } + + bool UnregisterAllocation(const ui64 allocationId) { + ui64 memoryAllocated = 0; + auto it = AllocationInfo.find(allocationId); + AFL_VERIFY(it != AllocationInfo.end()); + bool waitFlag = false; + const ui64 internalGroupId = it->second->GetAllocationInternalGroupId(); + switch (it->second->GetAllocationStatus()) { + case EAllocationStatus::Allocated: + case EAllocationStatus::Failed: + AFL_VERIFY(!WaitAllocations.RemoveAllocation(internalGroupId, it->second)); + break; + case EAllocationStatus::Waiting: + AFL_VERIFY(WaitAllocations.RemoveAllocation(internalGroupId, it->second)); + waitFlag = true; + break; + } + AFL_DEBUG(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "allocation_unregister")("allocation_id", allocationId)("wait", waitFlag)( + "internal_group_id", internalGroupId)("allocation_status", it->second->GetAllocationStatus()); + memoryAllocated = it->second->GetAllocatedVolume(); + AllocationInfo.erase(it); + return !!memoryAllocated; + } + + void UnregisterGroup(const bool isPriorityProcess, const ui64 externalGroupId) { + const ui64 internalGroupId = GroupIds.ExtractInternalIdVerified(externalGroupId); + AFL_INFO(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "remove_group")("external_group_id", externalGroupId)( + "internal_group_id", internalGroupId); + UnregisterGroupImpl(internalGroupId); + if (isPriorityProcess && (internalGroupId < GroupIds.GetMinInternalIdDef(internalGroupId))) { + Y_UNUSED(TryAllocateWaiting(isPriorityProcess, 0)); + } + } + + void RegisterGroup(const ui64 externalGroupId) { + Y_UNUSED(GroupIds.RegisterExternalId(externalGroupId)); + } +}; + +class TProcessMemory { +private: + const ui64 ExternalProcessId; + + const NActors::TActorId OwnerActorId; + bool PriorityProcessFlag = false; + YDB_ACCESSOR(ui32, LinksCount, 1); YDB_READONLY_DEF(std::vector>, Stages); const std::shared_ptr DefaultStage; + THashMap> AllocationScopes; + + TProcessMemoryScope* GetAllocationScopeOptional(const ui64 externalScopeId) const { + auto it = AllocationScopes.find(externalScopeId); + if (it == AllocationScopes.end()) { + return nullptr; + } + return it->second.get(); + } + + TProcessMemoryScope& GetAllocationScopeVerified(const ui64 externalScopeId) const { + return *TValidator::CheckNotNull(GetAllocationScopeOptional(externalScopeId)); + } public: bool IsPriorityProcess() const { return PriorityProcessFlag; } + bool UpdateAllocation(const ui64 externalScopeId, const ui64 allocationId, const ui64 volume) { + return GetAllocationScopeVerified(externalScopeId).UpdateAllocation(allocationId, volume); + } + + void RegisterAllocation( + const ui64 externalScopeId, const ui64 externalGroupId, const std::shared_ptr& task, const std::optional& stageIdx) { + AFL_VERIFY(task); + std::shared_ptr stage; + if (Stages.empty()) { + AFL_VERIFY(!stageIdx); + stage = DefaultStage; + } else { + AFL_VERIFY(stageIdx); + AFL_VERIFY(*stageIdx < Stages.size()); + stage = Stages[*stageIdx]; + } + AFL_VERIFY(stage); + auto& scope = GetAllocationScopeVerified(externalScopeId); + scope.RegisterAllocation(IsPriorityProcess(), externalGroupId, task, stage); + } + + bool UnregisterAllocation(const ui64 externalScopeId, const ui64 allocationId) { + if (auto* scope = GetAllocationScopeOptional(externalScopeId)) { + return scope->UnregisterAllocation(allocationId); + } + return false; + } + + void UnregisterGroup(const ui64 externalScopeId, const ui64 externalGroupId) { + if (auto* scope = GetAllocationScopeOptional(externalScopeId)) { + scope->UnregisterGroup(IsPriorityProcess(), externalGroupId); + } + } + + void RegisterGroup(const ui64 externalScopeId, const ui64 externalGroupId) { + GetAllocationScopeVerified(externalScopeId).RegisterGroup(externalGroupId); + } + + void UnregisterScope(const ui64 externalScopeId) { + auto it = AllocationScopes.find(externalScopeId); + AFL_VERIFY(it != AllocationScopes.end()); + if (it->second->Unregister()) { + AllocationScopes.erase(it); + } + + } + + void RegisterScope(const ui64 externalScopeId) { + auto it = AllocationScopes.find(externalScopeId); + if (it == AllocationScopes.end()) { + AFL_VERIFY(AllocationScopes.emplace(externalScopeId, std::make_shared(ExternalProcessId, externalScopeId, OwnerActorId)).second); + } else { + it->second->Register(); + } + + } + void SetPriorityProcess() { AFL_VERIFY(!PriorityProcessFlag); PriorityProcessFlag = true; @@ -44,24 +238,24 @@ class TProcessMemory { , OwnerActorId(ownerActorId) , PriorityProcessFlag(isPriority) , Stages(stages) - , DefaultStage(defaultStage) - { + , DefaultStage(defaultStage) { } - void RegisterAllocation(const ui64 externalGroupId, const std::shared_ptr& task, const std::optional& stageIdx); - bool UnregisterAllocation(const ui64 allocationId); - bool UpdateAllocation(const ui64 allocationId, const ui64 volume) { - GetAllocationInfoVerified(allocationId).SetAllocatedVolume(volume); - return true; + bool TryAllocateWaiting(const ui32 allocationsCountLimit) { + bool allocated = false; + for (auto&& i : AllocationScopes) { + if (i.second->TryAllocateWaiting(IsPriorityProcess(), allocationsCountLimit)) { + allocated = true; + } + } + return allocated; } - void Unregister(); - - void UnregisterGroup(const ui64 externalGroupId); - void RegisterGroup(const ui64 externalGroupId); - - bool TryAllocateWaiting(const ui32 allocationsCountLimit) { - return WaitAllocations.Allocate(*this, allocationsCountLimit); + void Unregister() { + for (auto&& i : AllocationScopes) { + Y_UNUSED(i.second->Unregister()); + } + AllocationScopes.clear(); } }; diff --git a/ydb/core/tx/limiter/grouped_memory/usage/abstract.cpp b/ydb/core/tx/limiter/grouped_memory/usage/abstract.cpp index 67910b9625d3..2d72f0039846 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/abstract.cpp +++ b/ydb/core/tx/limiter/grouped_memory/usage/abstract.cpp @@ -7,7 +7,8 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { TAllocationGuard::~TAllocationGuard() { if (TlsActivationContext && !Released) { - NActors::TActivationContext::AsActorContext().Send(ActorId, std::make_unique(ProcessId, AllocationId)); + NActors::TActivationContext::AsActorContext().Send( + ActorId, std::make_unique(ProcessId, ScopeId, AllocationId)); } } @@ -16,7 +17,7 @@ void TAllocationGuard::Update(const ui64 newVolume) { Memory = newVolume; if (TlsActivationContext) { NActors::TActivationContext::AsActorContext().Send( - ActorId, std::make_unique(ProcessId, AllocationId, newVolume)); + ActorId, std::make_unique(ProcessId, ScopeId, AllocationId, newVolume)); } } @@ -30,16 +31,19 @@ bool IAllocation::OnAllocated(std::shared_ptr&& guard, const s TGroupGuard::~TGroupGuard() { if (TlsActivationContext) { - NActors::TActivationContext::AsActorContext().Send(ActorId, std::make_unique(ProcessId, GroupId)); + NActors::TActivationContext::AsActorContext().Send( + ActorId, std::make_unique(ProcessId, ExternalScopeId, GroupId)); } } -TGroupGuard::TGroupGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 groupId) +TGroupGuard::TGroupGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 externalScopeId, const ui64 groupId) : ActorId(actorId) , ProcessId(processId) + , ExternalScopeId(externalScopeId) , GroupId(groupId) { if (TlsActivationContext) { - NActors::TActivationContext::AsActorContext().Send(ActorId, std::make_unique(ProcessId, GroupId)); + NActors::TActivationContext::AsActorContext().Send( + ActorId, std::make_unique(ProcessId, ExternalScopeId, GroupId)); } } @@ -57,4 +61,19 @@ TProcessGuard::TProcessGuard(const NActors::TActorId& actorId, const ui64 proces } } +TScopeGuard::~TScopeGuard() { + if (TlsActivationContext) { + NActors::TActivationContext::AsActorContext().Send(ActorId, std::make_unique(ProcessId, ScopeId)); + } +} + +TScopeGuard::TScopeGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 scopeId) + : ActorId(actorId) + , ProcessId(processId) + , ScopeId(scopeId) { + if (TlsActivationContext) { + NActors::TActivationContext::AsActorContext().Send(ActorId, std::make_unique(ProcessId, ScopeId)); + } +} + } // namespace NKikimr::NOlap::NGroupedMemoryManager diff --git a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h index c65be421b377..d92120f46fb6 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h @@ -14,10 +14,11 @@ class TGroupGuard { private: const NActors::TActorId ActorId; YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, ExternalScopeId, 0); YDB_READONLY(ui64, GroupId, 0); public: - TGroupGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 groupId); + TGroupGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 externalScopeId, const ui64 groupId); ~TGroupGuard(); }; @@ -33,18 +34,32 @@ class TProcessGuard { ~TProcessGuard(); }; +class TScopeGuard { +private: + const NActors::TActorId ActorId; + YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, ScopeId, 0); + +public: + TScopeGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 scopeId); + + ~TScopeGuard(); +}; + class TAllocationGuard { private: const NActors::TActorId ActorId; YDB_READONLY(ui64, ProcessId, 0) + YDB_READONLY(ui64, ScopeId, 0) YDB_READONLY(ui64, AllocationId, 0) YDB_READONLY(ui64, Memory, 0) bool Released = false; public: - TAllocationGuard(const ui64 processId, const ui64 allocationId, const NActors::TActorId actorId, const ui64 memory) + TAllocationGuard(const ui64 processId, const ui64 scopeId, const ui64 allocationId, const NActors::TActorId actorId, const ui64 memory) : ActorId(actorId) , ProcessId(processId) + , ScopeId(scopeId) , AllocationId(allocationId) , Memory(memory) { } diff --git a/ydb/core/tx/limiter/grouped_memory/usage/events.h b/ydb/core/tx/limiter/grouped_memory/usage/events.h index 3829f9a03d4d..d3a8200c584c 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/events.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/events.h @@ -17,6 +17,8 @@ struct TEvExternal { EvUpdateAllocationTask, EvStartAllocationProcess, EvFinishAllocationProcess, + EvStartAllocationProcessScope, + EvFinishAllocationProcessScope, EvEnd }; @@ -25,15 +27,17 @@ struct TEvExternal { YDB_READONLY_DEF(std::vector>, Allocations); YDB_READONLY_DEF(std::optional, StageFeaturesIdx); YDB_READONLY(ui64, ExternalProcessId, 0); + YDB_READONLY(ui64, ExternalScopeId, 0); YDB_READONLY(ui64, ExternalGroupId, 0); public: - explicit TEvStartTask(const ui64 externalProcessId, + explicit TEvStartTask(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId, const std::vector>& allocations, const std::optional& stageFeaturesIdx) : Allocations(allocations) , StageFeaturesIdx(stageFeaturesIdx) , ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) , ExternalGroupId(externalGroupId) { AFL_VERIFY(Allocations.size()); } @@ -42,11 +46,13 @@ struct TEvExternal { class TEvFinishTask: public NActors::TEventLocal { private: YDB_READONLY(ui64, ExternalProcessId, 0); + YDB_READONLY(ui64, ExternalScopeId, 0); YDB_READONLY(ui64, AllocationId, 0); public: - explicit TEvFinishTask(const ui64 externalProcessId, const ui64 allocationId) + explicit TEvFinishTask(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId) : ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) , AllocationId(allocationId) { } }; @@ -54,12 +60,14 @@ struct TEvExternal { class TEvUpdateTask: public NActors::TEventLocal { private: YDB_READONLY(ui64, ExternalProcessId, 0); + YDB_READONLY(ui64, ExternalScopeId, 0); YDB_READONLY(ui64, AllocationId, 0); YDB_READONLY(ui64, Volume, 0); public: - explicit TEvUpdateTask(const ui64 externalProcessId, const ui64 allocationId, const ui64 volume) + explicit TEvUpdateTask(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId, const ui64 volume) : ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) , AllocationId(allocationId) , Volume(volume) { } @@ -68,11 +76,13 @@ struct TEvExternal { class TEvFinishGroup: public NActors::TEventLocal { private: YDB_READONLY(ui64, ExternalProcessId, 0); + YDB_READONLY(ui32, ExternalScopeId, 0); YDB_READONLY(ui64, ExternalGroupId, 0); public: - explicit TEvFinishGroup(const ui64 externalProcessId, const ui64 externalGroupId) + explicit TEvFinishGroup(const ui64 externalProcessId, const ui32 externalScopeId, const ui64 externalGroupId) : ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) , ExternalGroupId(externalGroupId) { } }; @@ -80,11 +90,13 @@ struct TEvExternal { class TEvStartGroup: public NActors::TEventLocal { private: YDB_READONLY(ui64, ExternalProcessId, 0); + YDB_READONLY(ui32, ExternalScopeId, 0); YDB_READONLY(ui64, ExternalGroupId, 0); public: - explicit TEvStartGroup(const ui64 externalProcessId, const ui64 externalGroupId) + explicit TEvStartGroup(const ui64 externalProcessId, const ui32 externalScopeId, const ui64 externalGroupId) : ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) , ExternalGroupId(externalGroupId) { } }; @@ -109,5 +121,30 @@ struct TEvExternal { , Stages(stages) { } }; + + class TEvFinishProcessScope: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, ExternalProcessId, 0); + YDB_READONLY(ui64, ExternalScopeId, 0); + + public: + explicit TEvFinishProcessScope(const ui64 externalProcessId, const ui64 externalScopeId) + : ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) + { + } + }; + + class TEvStartProcessScope: public NActors::TEventLocal { + private: + YDB_READONLY(ui64, ExternalProcessId, 0); + YDB_READONLY(ui64, ExternalScopeId, 0); + + public: + explicit TEvStartProcessScope(const ui64 externalProcessId, const ui64 externalScopeId) + : ExternalProcessId(externalProcessId) + , ExternalScopeId(externalScopeId) { + } + }; }; } // namespace NKikimr::NOlap::NGroupedMemoryManager::NEvents diff --git a/ydb/core/tx/limiter/grouped_memory/usage/service.h b/ydb/core/tx/limiter/grouped_memory/usage/service.h index 166f3c1827e4..8192743218b1 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/service.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/service.h @@ -44,11 +44,17 @@ class TServiceOperatorImpl { return Singleton()->DefaultStageFeatures; } - static std::shared_ptr BuildGroupGuard(const ui64 processId) { + static std::shared_ptr BuildGroupGuard(const ui64 processId, const ui32 scopeId) { static TAtomicCounter counter = 0; auto& context = NActors::TActorContext::AsActorContext(); const NActors::TActorId& selfId = context.SelfID; - return std::make_shared(MakeServiceId(selfId.NodeId()), processId, counter.Inc()); + return std::make_shared(MakeServiceId(selfId.NodeId()), processId, scopeId, counter.Inc()); + } + + static std::shared_ptr BuildScopeGuard(const ui64 processId, const ui32 scopeId) { + auto& context = NActors::TActorContext::AsActorContext(); + const NActors::TActorId& selfId = context.SelfID; + return std::make_shared(MakeServiceId(selfId.NodeId()), processId, scopeId); } static std::shared_ptr BuildProcessGuard(const ui64 processId, const std::vector>& stages) { @@ -57,17 +63,18 @@ class TServiceOperatorImpl { return std::make_shared(MakeServiceId(selfId.NodeId()), processId, stages); } - static bool SendToAllocation(const ui64 processId, const ui64 groupId, const std::vector>& tasks, + static bool SendToAllocation(const ui64 processId, const ui64 scopeId, const ui64 groupId, + const std::vector>& tasks, const std::optional& stageIdx) { auto& context = NActors::TActorContext::AsActorContext(); const NActors::TActorId& selfId = context.SelfID; if (TSelf::IsEnabled()) { - context.Send(MakeServiceId(selfId.NodeId()), new NEvents::TEvExternal::TEvStartTask(processId, groupId, tasks, stageIdx)); + context.Send(MakeServiceId(selfId.NodeId()), new NEvents::TEvExternal::TEvStartTask(processId, scopeId, groupId, tasks, stageIdx)); return true; } else { for (auto&& i : tasks) { if (!i->IsAllocated()) { - AFL_VERIFY(i->OnAllocated(std::make_shared(0, 0, NActors::TActorId(), i->GetMemory()), i)); + AFL_VERIFY(i->OnAllocated(std::make_shared(0, 0, 0, NActors::TActorId(), i->GetMemory()), i)); } } return false; diff --git a/ydb/core/tx/limiter/grouped_memory/ut/ut_manager.cpp b/ydb/core/tx/limiter/grouped_memory/ut/ut_manager.cpp index ac9abcdf5b48..277d62903205 100644 --- a/ydb/core/tx/limiter/grouped_memory/ut/ut_manager.cpp +++ b/ydb/core/tx/limiter/grouped_memory/ut/ut_manager.cpp @@ -42,24 +42,26 @@ Y_UNIT_TEST_SUITE(GroupedMemoryLimiter) { { auto alloc1 = std::make_shared(50); manager->RegisterProcess(0, {}); - manager->RegisterGroup(0, 1); - manager->RegisterAllocation(0, 1, alloc1, {}); + manager->RegisterProcessScope(0, 0); + manager->RegisterGroup(0, 0, 1); + manager->RegisterAllocation(0, 0, 1, alloc1, {}); AFL_VERIFY(alloc1->IsAllocated()); auto alloc1_1 = std::make_shared(50); - manager->RegisterAllocation(0, 1, alloc1_1, {}); + manager->RegisterAllocation(0, 0, 1, alloc1_1, {}); AFL_VERIFY(alloc1_1->IsAllocated()); - manager->RegisterGroup(0, 2); + manager->RegisterGroup(0, 0, 2); auto alloc2 = std::make_shared(50); - manager->RegisterAllocation(0, 2, alloc2, {}); + manager->RegisterAllocation(0, 0, 2, alloc2, {}); AFL_VERIFY(!alloc2->IsAllocated()); - manager->UnregisterAllocation(0, alloc1->GetIdentifier()); + manager->UnregisterAllocation(0, 0, alloc1->GetIdentifier()); AFL_VERIFY(alloc2->IsAllocated()); - manager->UnregisterAllocation(0, alloc2->GetIdentifier()); - manager->UnregisterAllocation(0, alloc1_1->GetIdentifier()); - manager->UnregisterGroup(0, 1); - manager->UnregisterGroup(0, 2); + manager->UnregisterAllocation(0, 0, alloc2->GetIdentifier()); + manager->UnregisterAllocation(0, 0, alloc1_1->GetIdentifier()); + manager->UnregisterGroup(0, 0, 1); + manager->UnregisterGroup(0, 0, 2); + manager->UnregisterProcessScope(0, 0); manager->UnregisterProcess(0); } AFL_VERIFY(!stage->GetUsage().Val()); @@ -80,36 +82,38 @@ Y_UNIT_TEST_SUITE(GroupedMemoryLimiter) { auto manager = std::make_shared(NActors::TActorId(), config, "test", counters, stage); { manager->RegisterProcess(0, {}); + manager->RegisterProcessScope(0, 0); auto alloc1 = std::make_shared(10); - manager->RegisterGroup(0, 1); - manager->RegisterAllocation(0, 1, alloc1, {}); + manager->RegisterGroup(0, 0, 1); + manager->RegisterAllocation(0, 0, 1, alloc1, {}); AFL_VERIFY(alloc1->IsAllocated()); auto alloc2 = std::make_shared(1000); - manager->RegisterGroup(0, 2); - manager->RegisterAllocation(0, 2, alloc2, {}); + manager->RegisterGroup(0, 0, 2); + manager->RegisterAllocation(0, 0, 2, alloc2, {}); AFL_VERIFY(!alloc2->IsAllocated()); auto alloc3 = std::make_shared(1000); - manager->RegisterGroup(0, 3); - manager->RegisterAllocation(0, 3, alloc3, {}); + manager->RegisterGroup(0, 0, 3); + manager->RegisterAllocation(0, 0, 3, alloc3, {}); AFL_VERIFY(alloc1->IsAllocated()); AFL_VERIFY(!alloc2->IsAllocated()); AFL_VERIFY(!alloc3->IsAllocated()); auto alloc1_1 = std::make_shared(1000); - manager->RegisterAllocation(0, 1, alloc1_1, {}); + manager->RegisterAllocation(0, 0, 1, alloc1_1, {}); AFL_VERIFY(alloc1_1->IsAllocated()); AFL_VERIFY(!alloc2->IsAllocated()); - manager->UnregisterAllocation(0, alloc1_1->GetIdentifier()); + manager->UnregisterAllocation(0, 0, alloc1_1->GetIdentifier()); AFL_VERIFY(!alloc2->IsAllocated()); - manager->UnregisterGroup(0, 1); + manager->UnregisterGroup(0, 0, 1); AFL_VERIFY(alloc2->IsAllocated()); - manager->UnregisterAllocation(0, alloc1->GetIdentifier()); + manager->UnregisterAllocation(0, 0, alloc1->GetIdentifier()); AFL_VERIFY(!alloc3->IsAllocated()); - manager->UnregisterGroup(0, 2); - manager->UnregisterAllocation(0, alloc2->GetIdentifier()); + manager->UnregisterGroup(0, 0, 2); + manager->UnregisterAllocation(0, 0, alloc2->GetIdentifier()); AFL_VERIFY(alloc3->IsAllocated()); - manager->UnregisterGroup(0, 3); - manager->UnregisterAllocation(0, alloc3->GetIdentifier()); + manager->UnregisterGroup(0, 0, 3); + manager->UnregisterAllocation(0, 0, alloc3->GetIdentifier()); + manager->UnregisterProcessScope(0, 0); manager->UnregisterProcess(0); } AFL_VERIFY(!stage->GetUsage().Val()); @@ -131,44 +135,45 @@ Y_UNIT_TEST_SUITE(GroupedMemoryLimiter) { auto manager = std::make_shared(NActors::TActorId(), config, "test", counters, stage); { manager->RegisterProcess(0, {}); - manager->RegisterGroup(0, 1); + manager->RegisterProcessScope(0, 0); + manager->RegisterGroup(0, 0, 1); auto alloc0 = std::make_shared(1000); - manager->RegisterAllocation(0, 1, alloc0, {}); + manager->RegisterAllocation(0, 0, 1, alloc0, {}); auto alloc1 = std::make_shared(1000); - manager->RegisterAllocation(0, 1, alloc1, {}); + manager->RegisterAllocation(0, 0, 1, alloc1, {}); AFL_VERIFY(alloc0->IsAllocated()); AFL_VERIFY(alloc1->IsAllocated()); - manager->RegisterGroup(0, 2); + manager->RegisterGroup(0, 0, 2); auto alloc2 = std::make_shared(1000); - manager->RegisterAllocation(0, 2, alloc0, {}); - manager->RegisterAllocation(0, 2, alloc2, {}); + manager->RegisterAllocation(0, 0, 2, alloc0, {}); + manager->RegisterAllocation(0, 0, 2, alloc2, {}); AFL_VERIFY(alloc0->IsAllocated()); AFL_VERIFY(!alloc2->IsAllocated()); auto alloc3 = std::make_shared(1000); - manager->RegisterGroup(0, 3); - manager->RegisterAllocation(0, 3, alloc0, {}); - manager->RegisterAllocation(0, 3, alloc3, {}); + manager->RegisterGroup(0, 0, 3); + manager->RegisterAllocation(0, 0, 3, alloc0, {}); + manager->RegisterAllocation(0, 0, 3, alloc3, {}); AFL_VERIFY(alloc0->IsAllocated()); AFL_VERIFY(alloc1->IsAllocated()); AFL_VERIFY(!alloc2->IsAllocated()); AFL_VERIFY(!alloc3->IsAllocated()); - manager->UnregisterGroup(0, 1); - manager->UnregisterAllocation(0, alloc1->GetIdentifier()); + manager->UnregisterGroup(0, 0, 1); + manager->UnregisterAllocation(0, 0, alloc1->GetIdentifier()); AFL_VERIFY(alloc0->IsAllocated()); AFL_VERIFY(alloc2->IsAllocated()); AFL_VERIFY(!alloc3->IsAllocated()); - manager->UnregisterGroup(0, 2); - manager->UnregisterAllocation(0, alloc2->GetIdentifier()); + manager->UnregisterGroup(0, 0, 2); + manager->UnregisterAllocation(0, 0, alloc2->GetIdentifier()); AFL_VERIFY(alloc0->IsAllocated()); AFL_VERIFY(alloc3->IsAllocated()); - manager->UnregisterGroup(0, 3); - manager->UnregisterAllocation(0, alloc3->GetIdentifier()); - manager->UnregisterAllocation(0, alloc0->GetIdentifier()); + manager->UnregisterGroup(0, 0, 3); + manager->UnregisterAllocation(0, 0, alloc3->GetIdentifier()); + manager->UnregisterAllocation(0, 0, alloc0->GetIdentifier()); manager->UnregisterProcess(0); } AFL_VERIFY(!stage->GetUsage().Val()); @@ -190,23 +195,24 @@ Y_UNIT_TEST_SUITE(GroupedMemoryLimiter) { auto manager = std::make_shared(NActors::TActorId(), config, "test", counters, stage); { manager->RegisterProcess(0, {}); + manager->RegisterProcessScope(0, 0); auto alloc1 = std::make_shared(1000); - manager->RegisterGroup(0, 1); - manager->RegisterAllocation(0, 1, alloc1, {}); + manager->RegisterGroup(0, 0, 1); + manager->RegisterAllocation(0, 0, 1, alloc1, {}); AFL_VERIFY(alloc1->IsAllocated()); auto alloc2 = std::make_shared(10); - manager->RegisterGroup(0, 3); - manager->RegisterAllocation(0, 3, alloc2, {}); + manager->RegisterGroup(0, 0, 3); + manager->RegisterAllocation(0, 0, 3, alloc2, {}); AFL_VERIFY(!alloc2->IsAllocated()); - manager->UpdateAllocation(0, alloc1->GetIdentifier(), 10); + manager->UpdateAllocation(0, 0, alloc1->GetIdentifier(), 10); AFL_VERIFY(alloc2->IsAllocated()); - manager->UnregisterGroup(0, 3); - manager->UnregisterAllocation(0, alloc2->GetIdentifier()); + manager->UnregisterGroup(0, 0, 3); + manager->UnregisterAllocation(0, 0, alloc2->GetIdentifier()); - manager->UnregisterGroup(0, 1); - manager->UnregisterAllocation(0, alloc1->GetIdentifier()); + manager->UnregisterGroup(0, 0, 1); + manager->UnregisterAllocation(0, 0, alloc1->GetIdentifier()); manager->UnregisterProcess(0); } AFL_VERIFY(!stage->GetUsage().Val()); From c5f3c3a25a3adaf91723331aec3cfae722fc99b8 Mon Sep 17 00:00:00 2001 From: Ivan Katkov <44121163+Pseudolukian@users.noreply.github.com> Date: Thu, 22 Aug 2024 07:57:08 +0200 Subject: [PATCH 027/261] Fix YQL optional types note in YT DOC (#8057) Co-authored-by: Ivan Blinkov --- .../en/core/yql/reference/yql-core/types/_includes/optional.md | 2 ++ .../ru/core/yql/reference/yql-core/types/_includes/optional.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ydb/docs/en/core/yql/reference/yql-core/types/_includes/optional.md b/ydb/docs/en/core/yql/reference/yql-core/types/_includes/optional.md index 0af28f13806a..d8c5f442e773 100644 --- a/ydb/docs/en/core/yql/reference/yql-core/types/_includes/optional.md +++ b/ydb/docs/en/core/yql/reference/yql-core/types/_includes/optional.md @@ -13,12 +13,14 @@ The following operations are most often performed on optional data types: `Optional` (nullable) isn't a property of a data type or column, but a container type where [containers](../containers.md) can be arbitrarily nested into each other. For example, a column with the type `Optional>` can accept 4 values: `NULL` of the whole container, `NULL` of the inner container, `TRUE`, and `FALSE`. The above-declared type differs from `List>`, because it uses `NULL` as an empty list, and you can't put more than one non-null element in it. In addition, `Optional>` type values are returned as results when searching by the key in the `Dict(k,v)` dictionary with `Optional` type values. Using this type of result data, you can distinguish between a `NULL` value in the dictionary and a situation when the key is missing. +{% if backend_name == "YDB" %} {% note info %} Container types (including `Optional` containers and more complex types derived from them) can't currently be used as column data types when creating {{ ydb-short-name }} tables. YQL queries can return values of container types and accept them as input parameters. {% endnote %} +{% endif %} **Example** diff --git a/ydb/docs/ru/core/yql/reference/yql-core/types/_includes/optional.md b/ydb/docs/ru/core/yql/reference/yql-core/types/_includes/optional.md index 4e461900bf3d..367b125c2db0 100644 --- a/ydb/docs/ru/core/yql/reference/yql-core/types/_includes/optional.md +++ b/ydb/docs/ru/core/yql/reference/yql-core/types/_includes/optional.md @@ -13,6 +13,7 @@ `Optional` (nullable) является не свойством типа данных или колонки, а одним из видов [контейнеров](../containers.md), которые могут быть произвольным образом вложены друг в друга. Так, например, столбец с типом `Optional>` может принимать 4 значения - `NULL` всего контейнера, `NULL` внутреннего контейнера, `TRUE` и `FALSE`. Описанный тип отличается от `List>` тем, что роль пустого списка в нем играет `NULL` и отсутствует возможность положить больше одного содержательного элемента. Также значения типа `Optional>` возвращаются в качестве результата поиска по ключу в словаре `Dict(k,v)` со значениями типа `Optional`. Такой тип данных результата позволяет отличать лежащий в словаре `NULL` от ситуации отсутствия ключа. +{% if backend_name == "YDB" %} {% note info %} По умолчанию, при указании примитивного типа `T` как типа колонки таблицы {{ ydb-short-name }} в базе данных создается соответсвующий контейнерный тип `Optional`. Колонку с типом `T` можно создать, используя ключевое слово `NOT NULL`. @@ -21,6 +22,7 @@ Запросы YQL могут возвращать значения контейнерных типов, а также принимать их в качестве входных параметров. {% endnote %} +{% endif %} **Пример** From ffdb879a2c114d95f0e6c9b375cb23ace9fb6f0e Mon Sep 17 00:00:00 2001 From: SloNN Date: Thu, 22 Aug 2024 08:58:29 +0300 Subject: [PATCH 028/261] pg (#7381) Co-authored-by: Alex Dmitriev Co-authored-by: Ivan Blinkov --- ydb/docs/en/core/changelog-cli.md | 4 +- .../core/postgresql/{pg-dump.md => import.md} | 0 ydb/docs/en/core/postgresql/toc_i.yaml | 2 +- ydb/docs/redirects.yaml | 18 +- ydb/docs/ru/core/changelog-cli.md | 6 +- ydb/docs/ru/core/dev/getting-started.md | 7 +- .../postgresql/_includes/alert_preview.md | 6 +- ydb/docs/ru/core/postgresql/connect.md | 49 +++++ ydb/docs/ru/core/postgresql/docker-connect.md | 144 ------------ .../core/postgresql/{pg-dump.md => import.md} | 14 ++ .../ru/core/postgresql/interoperability.md | 169 ++++++++++++++ ydb/docs/ru/core/postgresql/intro.md | 17 +- ydb/docs/ru/core/postgresql/toc_i.yaml | 15 +- ydb/docs/ru/core/reference/toc_p.yaml | 3 +- .../reference/ydb-cli/_includes/commands.md | 2 +- .../yql-core/syntax/_includes/lexer.md | 6 + .../yql-core/udf/list/_includes/frompg.md | 15 ++ .../yql-core/udf/list/_includes/pgliterals.md | 95 ++++++++ .../yql-core/udf/list/_includes/topg.md | 35 +++ .../reference/yql-core/udf/list/postgres.md | 206 ++++++++++++++++++ .../reference/yql-core/udf/list/toc_base.yaml | 1 + 21 files changed, 635 insertions(+), 179 deletions(-) rename ydb/docs/en/core/postgresql/{pg-dump.md => import.md} (100%) create mode 100644 ydb/docs/ru/core/postgresql/connect.md delete mode 100644 ydb/docs/ru/core/postgresql/docker-connect.md rename ydb/docs/ru/core/postgresql/{pg-dump.md => import.md} (80%) create mode 100644 ydb/docs/ru/core/postgresql/interoperability.md create mode 100644 ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/frompg.md create mode 100644 ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/pgliterals.md create mode 100644 ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/topg.md create mode 100644 ydb/docs/ru/core/yql/reference/yql-core/udf/list/postgres.md diff --git a/ydb/docs/en/core/changelog-cli.md b/ydb/docs/en/core/changelog-cli.md index 0e021091ee4d..af9923053ecd 100644 --- a/ydb/docs/en/core/changelog-cli.md +++ b/ydb/docs/en/core/changelog-cli.md @@ -73,7 +73,7 @@ Released on January 12, 2024. To update to version **2.8.0**, select the [Downlo * Fixed an error displaying tables in `pretty` format with [Unicode](https://en.wikipedia.org/wiki/Unicode) characters. -* Fixed an error substituting the wrong primary key in the command [ydb tools pg-convert](postgresql/pg-dump.md#pg-convert). +* Fixed an error substituting the wrong primary key in the command [ydb tools pg-convert](postgresql/import.md#pg-convert). ## Version 2.7.0 {#2-7-0} @@ -81,7 +81,7 @@ Released on October 23, 2023. To update to version **2.7.0**, select the [Downlo **Features:** -* Added the [ydb tools pg-convert](postgresql/pg-dump.md#pg-convert) command, which prepares a dump obtained by the [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html) utility for loading into the YDB postgres-compatible layer. +* Added the [ydb tools pg-convert](postgresql/import.md#pg-convert) command, which prepares a dump obtained by the [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html) utility for loading into the YDB postgres-compatible layer. * Added the `ydb workload query` load testing command, which loads the database with [script execution queries](reference/ydb-cli/yql.md) in multiple threads. diff --git a/ydb/docs/en/core/postgresql/pg-dump.md b/ydb/docs/en/core/postgresql/import.md similarity index 100% rename from ydb/docs/en/core/postgresql/pg-dump.md rename to ydb/docs/en/core/postgresql/import.md diff --git a/ydb/docs/en/core/postgresql/toc_i.yaml b/ydb/docs/en/core/postgresql/toc_i.yaml index e85bc5275924..d1d05b2aed27 100644 --- a/ydb/docs/en/core/postgresql/toc_i.yaml +++ b/ydb/docs/en/core/postgresql/toc_i.yaml @@ -6,4 +6,4 @@ items: - name: PostgreSQL functions href: functions.md - name: Data dump from PostgreSQL - href: pg-dump.md \ No newline at end of file + href: import.md diff --git a/ydb/docs/redirects.yaml b/ydb/docs/redirects.yaml index 80d306e16857..7a355ee2195f 100644 --- a/ydb/docs/redirects.yaml +++ b/ydb/docs/redirects.yaml @@ -33,6 +33,10 @@ common: to: /public-materials/videos.md - from: /concepts/databases.md to: /concepts/glossary.md + - from: /postgresql/pg-dump.md + to: /postgresql/import.md + - from: /postgresql/docker-connect.md + to: /postgresql/connect.md # DevOps-related redirects - from: /getting_started/kubernetes.md @@ -278,21 +282,21 @@ common: - from: reference/ydb-sdk/recipes/upsert.md to: recipes/ydb-sdk/upsert.md -# Redirects from export_import to export-import +# Redirects from export_import to export-import - from: /reference/ydb-cli/export_import/s3_conn.md to: /reference/ydb-cli/export-import/auth-s3.md - from: /reference/ydb-cli/export_import/s3_export.md - to: /reference/ydb-cli/export-import/export-s3.md + to: /reference/ydb-cli/export-import/export-s3.md - from: /reference/ydb-cli/export_import/file_structure.md - to: /reference/ydb-cli/export-import/file-structure.md + to: /reference/ydb-cli/export-import/file-structure.md - from: /reference/ydb-cli/export_import/import-file.md - to: /reference/ydb-cli/export-import/import-file.md + to: /reference/ydb-cli/export-import/import-file.md - from: /reference/ydb-cli/export_import/s3_import.md - to: /reference/ydb-cli/export-import/import-s3.md + to: /reference/ydb-cli/export-import/import-s3.md - from: /reference/ydb-cli/export_import/index.md - to: /reference/ydb-cli/export-import/index.md + to: /reference/ydb-cli/export-import/index.md - from: /reference/ydb-cli/export_import/tools_dump.md - to: /reference/ydb-cli/export-import/tools-dump.md + to: /reference/ydb-cli/export-import/tools-dump.md - from: /reference/ydb-cli/export_import/tools_restore.md to: /reference/ydb-cli/export-import/tools-restore.md diff --git a/ydb/docs/ru/core/changelog-cli.md b/ydb/docs/ru/core/changelog-cli.md index d6c52fb6be04..2a0d6b91ddf5 100644 --- a/ydb/docs/ru/core/changelog-cli.md +++ b/ydb/docs/ru/core/changelog-cli.md @@ -15,7 +15,7 @@ * Добавлена поддержка типов big datetime: `Date32`, `Datetime64`, `Timestamp64`, `Interval64`. * Переработана команда `ydb workload`: * Добавлена опция `--clear` в подкоманде `init`, позволяющая удалить все существующие таблицы перед созданием новых. - * Добавлена команда `ydb workload * import` для заполнения таблиц начальным контентом перед началом нагрузки. + * Добавлена команда `ydb workload * import` для заполнения таблиц начальным контентом перед началом нагрузки. **Изменения с потерей обратной совместимости:** * Переработана команда `ydb workload`: @@ -73,7 +73,7 @@ * Исправлена ошибка вывода таблиц в `pretty` формате с [Unicode](https://ru.wikipedia.org/wiki/Юникод) символами. -* Исправлена ошибка подстановки неправильного первичного ключа в команде [ydb tools pg-convert](postgresql/pg-dump.md#pg-convert). +* Исправлена ошибка подстановки неправильного первичного ключа в команде [ydb tools pg-convert](postgresql/import.md#pg-convert). ## Версия 2.7.0 {#2-7-0} @@ -81,7 +81,7 @@ **Функциональность:** -* Добавлена команда [ydb tools pg-convert](postgresql/pg-dump.md#pg-convert), выполняющая подготовку дампа, полученного утилитой [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html), к загрузке в postgres-совместимую прослойку YDB. +* Добавлена команда [ydb tools pg-convert](postgresql/import.md#pg-convert), выполняющая подготовку дампа, полученного утилитой [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html), к загрузке в postgres-совместимую прослойку YDB. * Добавлена команда нагрузочного тестирования `ydb workload query`, которая нагружает базу [запросами выполнения скрипта](reference/ydb-cli/yql.md) в несколько потоков. diff --git a/ydb/docs/ru/core/dev/getting-started.md b/ydb/docs/ru/core/dev/getting-started.md index d563d1abf0dc..ebfecd295e88 100644 --- a/ydb/docs/ru/core/dev/getting-started.md +++ b/ydb/docs/ru/core/dev/getting-started.md @@ -14,7 +14,7 @@ - Альтернативно, {{ ydb-short-name }} предоставляет [PostgreSQL-совместимый API](../postgresql/intro.md). Он предназначен для упрощения миграции существующих приложений, которые переросли PostgreSQL. Однако это также полезно для экзотических языков программирования, для которых есть клиентская библиотека PostgreSQL, но нет {{ ydb-short-name }} SDK. Обратитесь к документации по совместимости PostgreSQL, чтобы проверить, соответствует ли ее полнота вашим потребностям. - Если вас интересует функция [{{ ydb-short-name }} топиков](../concepts/topic.md), стоит отметить, что они предоставляют [Kafka-совместимый API](../reference/kafka-api/index.md). Перейдите по этой ссылке, если этот вариант использования для вас актуален. - В крайнем случае, собственный API {{ ydb-short-name }} основан на протоколе [gRPC](https://grpc.io/), вокруг которого есть экосистема, включая генерацию кода клиентов. [gRPC спецификации {{ ydb-short-name}} API](https://github.com/ydb-platform/ydb/tree/main/ydb/public/api/grpc) размещены на GitHub, и вы можете использовать их в своём приложении. Сгенерированные клиенты являются низкоуровневыми и потребуют дополнительной работы для правильной обработки таких аспектов, как ретраи и таймауты. Следуйте по этому пути только если другие варианты невозможны, и вы знаете, что делаете. - + ## Установите пререквизиты Выберите конкретный язык программирования, который вы будете использовать. [Установите соответствующий {{ ydb-short-name }} SDK](../reference/ydb-sdk/index.md) или [драйвер PostgreSQL](https://wiki.postgresql.org/wiki/List_of_drivers) в зависимости от выбранного выше пути. @@ -34,11 +34,10 @@ - Изучите [примеры приложений](example-app/index.md), чтобы узнать, как выглядит работа с SDK. - Ознакомьтесь с [рецептами SDK](../recipes/ydb-sdk/index.md) для типичных случаев использования SDK, к которым вы сможете обратиться позже. - Ориентируйтесь на возможности вашего IDE по навигации по коду при работе с SDK. - + ### Для пути совместимости с PostgreSQL -- Узнайте, как [подключить PostgreSQL драйвер к кластеру {{ ydb-short-name }}](../postgresql/docker-connect.md). -- Остальное должно быть похоже на использование самого PostgreSQL. Используйте свой опыт или обратитесь к любым ресурсам по PostgreSQL. Однако сверяйтесь со списком поддерживаемых [функций](../postgresql/functions.md) и операторов, чтобы скорректировать свои ожидания. +- Используйте свой опыт или обратитесь к любым ресурсам по PostgreSQL. Однако сверяйтесь со списком поддерживаемых [функций](../postgresql/functions.md) и операторов, чтобы скорректировать свои ожидания. ## Тестирование diff --git a/ydb/docs/ru/core/postgresql/_includes/alert_preview.md b/ydb/docs/ru/core/postgresql/_includes/alert_preview.md index 8223725c000d..cbeadcf63c2c 100644 --- a/ydb/docs/ru/core/postgresql/_includes/alert_preview.md +++ b/ydb/docs/ru/core/postgresql/_includes/alert_preview.md @@ -1,5 +1,7 @@ {% note warning %} -На данный момент совместимость YDB с PostgreSQL **находится в разработке**, поэтому пока поддерживаются не все PostgreSQL конструкции и [функции](../functions.md). PostgreSQL совместимость доступна для тестирования в виде Docker-контейнера, который можно развернуть, следуя данной [инструкции](../docker-connect.md). +Поддержка синтаксиса PostgreSQL в {{ ydb-name }} **находится в разработке**. Использовать её в production окружениях **не рекомендуется**. Запросы в PostgreSQL могут исполняться до нескольких раз медленнее по сравнению с аналогичными запросами на YQL. -{% endnote %} \ No newline at end of file +Основной сценарий, который можно тестировать — выполнение аналитических запросов к хранимым в {{ ydb-name }} данным. + +{% endnote %} diff --git a/ydb/docs/ru/core/postgresql/connect.md b/ydb/docs/ru/core/postgresql/connect.md new file mode 100644 index 000000000000..a0ef958152dd --- /dev/null +++ b/ydb/docs/ru/core/postgresql/connect.md @@ -0,0 +1,49 @@ +# Выполнение запросов + +{% include [./_includes/alert.md](./_includes/alert_preview.md) %} + +Существует несколько способов запуска запросов на диалекте PostgreSQL: +1. Использование нативных PostgreSQL-инструментов. {{ydb-name}} обладает сервисом совместимости для поддержки нативного сетевого протокола PostgreSQL - [pgwire](https://www.postgresql.org/docs/current/protocol.html), поэтому к {{ydb-name}} можно подключаться стандартными клиентами PostgreSQL. +1. Использование нативных {{ ydb-short-name}} инструментов и явного указания, что запрос выполняется в синтаксисе PostgreSQL с помощью параметров (настроек) этих инструментов при выполнении запросов. +1. Использование любых инструментов, взаимодействующих с {{ ydb-short-name }} и явного указания, что запрос выполняется в синтаксисе PostgreSQL с помощью специального комментария в начале тела запроса `--!syntax_pg`. + +## Нативный сетевой протокол PostgreSQL {#pgwire} + +Для подключения к {{ ydb-name }} через PostgreSQL-протокол можно использовать любые клиентские библиотеки и приложения, поддерживающие этот протокол. Ниже приведен пример подключения с помощью утилиты `psql`, входящей в комплект поставки PostgreSQL. + +Для подключения необходимо выполнить предварительные шаги: +1. Получить адрес кластера, к которому производится подключение. +1. Получить название базы данных в кластере, к которому производится подключение. + +Выполнить запрос в {{ydb-name}}: + +``` +psql -h -U +``` + +Где: +- `` - адрес кластера {{ ydb-short-name }}, к которому выполняется подключение. +- `` - название базы данных в кластере. Может быть сложным именем, например, `mycluster/tenant1/database`. +- `` - логин пользователя. + + +## Использование стандартных инструментов {{ydb-name}} {#syntaxpg} + +Выполнять запросы на PostgreSQL-диалекте можно с помощью любых инструментов {{ydb-name}}, явно указав в начале тела запроса признак диалекта с помощью специального комментария: `--!syntax_pg`. + +Для подключения необходимо выполнить предварительные шаги: +1. Получить адрес [кластера](../concepts/glossary.md#cluster), к которому производится подключение. +1. Получить название [базы данных](../concepts/glossary.md#database) в кластере, к которому производится подключение. + +Выполнить запрос к {{ ydb-name }}: + +``` +ydb -e -d --user sql -s '--!syntax_pg +SELECT 1; +' +``` + +Где: +- `` — адрес кластера {{ ydb-short-name }}, к которому выполняется подключение. +- `` — название базы данных в кластере. Может быть сложным именем, например, `mycluster/tenant1/database/`. +- `` — логин пользователя. diff --git a/ydb/docs/ru/core/postgresql/docker-connect.md b/ydb/docs/ru/core/postgresql/docker-connect.md deleted file mode 100644 index 0afe4739ed19..000000000000 --- a/ydb/docs/ru/core/postgresql/docker-connect.md +++ /dev/null @@ -1,144 +0,0 @@ -# Подключение через PostgreSQL-протокол - -## Запуск {{ ydb-short-name }} с включенным PostgreSQL - -Сейчас функционал postgres-совместимости доступен в образе: `ghcr.io/ydb-platform/local-ydb:nightly`. - -Команды для запуска локального докер-контейнера с YDB и открытыми портами postgres и Web-UI. - -{% note tip %} - -В этом примере контейнеры намеренно создаются так, чтобы их состояние удалялось после завершения работы. Это упрощает инструкцию и позволяет многократно запускать тесты в известном (чистом) окружении, не думая о поломках. - -Для того чтобы состояние контейнера сохранялось: нужно убрать переменную окружения YDB_USE_IN_MEMORY_PDISKS. - -{% endnote %} - -{% list tabs %} - -- Docker-compose - - Для запуска через конфигурационный файл docker-compose он уже должен быть [установлен в системе](https://docs.docker.com/compose/install/standalone/) - - docker-compose.yaml: - ``` - services: - ydb: - image: ghcr.io/ydb-platform/local-ydb:nightly - ports: - - "5432:5432" - - "8765:8765" - environment: - - "YDB_USE_IN_MEMORY_PDISKS=true" - - "POSTGRES_USER=${YDB_PG_USER:-root}" - - "POSTGRES_PASSWORD=${YDB_PG_PASSWORD:-1234}" - - "YDB_EXPERIMENTAL_PG=1" - ``` - - запуск: - ```bash - docker-compose up -d --pull=always - ``` - -- Docker-команда - - ```bash - docker run --name ydb-postgres -d --pull always -p 5432:5432 -p 8765:8765 -e POSTGRES_USER=root -e POSTGRES_PASSWORD=1234 -e YDB_EXPERIMENTAL_PG=1 -e YDB_USE_IN_MEMORY_PDISKS=true ghcr.io/ydb-platform/local-ydb:nightly - ``` - -{% endlist %} - - -После запуска контейнера можно подключаться к нему через postgres-клиенты на порт 5432, база local или открыть [веб-интерфейс](http://localhost:8765) на порту 8765. - -## Подключение к запущенному контейнеру через psql - -При выполнении этой команды запустится интерактивный консольный клиент postgres. Все последующие запросы нужно вводить внутри этого клиента. - -```bash -docker run --rm -it --network=host postgres:14 psql postgresql://root:1234@localhost:5432/local -``` - -### Первый Hello world - -```sql -SELECT 'Hello, world!'; -``` - -Вывод: -``` - column0 ---------------- - Hello, world! -(1 row) -``` - -### Создание таблицы -Основная цель существования систем управления базами данных - сохранение данных для последующего извлечения. Как система, базирующаяся на SQL, основной абстракцией для хранения данных является таблица. Чтобы создать нашу первую таблицу, выполните следующий запрос: - -```sql - -CREATE TABLE example -( - key int4, - value text, - PRIMARY KEY (key) -); -``` - -### Добавление тестовых данных -Теперь давайте заполним нашу таблицу первыми данными. Самый простой способ - использовать литералы. - -```sql -INSERT INTO example (key, value) -VALUES (123, 'hello'), - (321, 'world'); -``` - -### Запрос к тестовым данным - -```sql -SELECT COUNT(*) FROM example; -``` - -Вывод: -``` - column0 ---------- - 2 -(1 row) -``` - - -## Остановка контейнера - -Эта команда остановит запущенный контейнер и удалит все хранящиеся в нём данные. - -{% list tabs %} - -- Docker-compose - - В папке с исходным файлом docker-compose.yaml выполнить команду, которая остановит контейнер и удалит его данные: - - ```bash - docker-compose down -vt 1 - ``` - {% note info %} - - Для остановки контейнера с сохранением данных уберите из конфига переменную YDB_USE_IN_MEMORY_PDISKS и используйте команду остановки: - - ```bash - docker-compose stop - ``` - - {% endnote %} - -- Docker-команда - - Эта команда остановит и удалит данные: - - ```bash - docker rm -f ydb-postgres - ``` - -{% endlist %} diff --git a/ydb/docs/ru/core/postgresql/pg-dump.md b/ydb/docs/ru/core/postgresql/import.md similarity index 80% rename from ydb/docs/ru/core/postgresql/pg-dump.md rename to ydb/docs/ru/core/postgresql/import.md index 899f6142a642..8efb6633d53b 100644 --- a/ydb/docs/ru/core/postgresql/pg-dump.md +++ b/ydb/docs/ru/core/postgresql/import.md @@ -1,5 +1,19 @@ # Импорт данных из PostgreSQL +Данные из PostgreSQL в {{ydb-name}} можно импортировать различными способами: +- С помощью [pg-dump](#pg-dump). +- С помощью импорта данных [из файлов](#file-import). +- С помощью утилиты [ydb-importer](../integrations/import-jdbc.md). + +|Способ импорта|Способ работы|Сценарии использования| +|--------------|------------------|------------| +|[pg-dump](#pg-dump)|Создание всей необходимой структуры таблиц и данных|Импорт баз данных PostgreSQL целиком| +|Импорт данных [из файлов](../reference/ydb-cli/export-import/import-file.md)|Импорт файлов с данными в заранее созданные таблицы базы данных {{ydb-name}}|Импорт данных из баз данных Greenplum или любых других баз данных со сменой структуры хранения данных| +|[ydb-importer](../integrations/import-jdbc.md)|Импорт данных из другой базы данных в заранее созданные таблицы базы данных {{ydb-name}}|Импорт данных из любых баз данных, поддерживающих [JDBC-протокол](https://ru.wikipedia.org/wiki/Java_Database_Connectivity)| + + +## pg-dump {#pg-dump} + Данные из PostgreSQL в YDB можно перенести c помощью утилит [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html), [psql](https://www.postgresql.org/docs/current/app-psql.html) и [YDB CLI](../reference/ydb-cli/index.md). Утилиты [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html) и [psql](https://www.postgresql.org/docs/current/app-psql.html) устанавливаются вместе с PostgreSQL, [YDB CLI](../reference/ydb-cli/index.md) — консольный клиент YDB, который [устанавливается отдельно](../reference/ydb-cli/install.md). Для этого нужно: diff --git a/ydb/docs/ru/core/postgresql/interoperability.md b/ydb/docs/ru/core/postgresql/interoperability.md new file mode 100644 index 000000000000..b2dd52a6972c --- /dev/null +++ b/ydb/docs/ru/core/postgresql/interoperability.md @@ -0,0 +1,169 @@ +# Интероперабельность PostgreSQL и {{ydb-name}} + +{% include [./_includes/alert.md](./_includes/alert_preview.md) %} + +Поддержка исполнения запросов в {{ydb-name}} в синтаксисе PostgreSQL реализована с помощью слоя совместимости: +1. Программа отправляет запросы в {{ ydb-short-name }}, где их обрабатывает компонент под названием `pgwire`. Pgwire реализует [сетевой протокол](https://www.postgresql.org/docs/16/protocol.html) PostgreSQL и передает команды в query processor. +2. Query processor транслирует PostgreSQL запросы в YQL AST. +3. После обработки запросов результаты собираются и отправляются обратно в программу, отправившую запрос, по сетевому протоколу PostgreSQL. При обработке запроса он может распараллеливаться и исполняться на произвольном количестве узлов {{ydb-name}}. + +Графически работу PostgreSQL совместимости можно представить так: +![Схема работы PostgreSQL совместимости](./_includes/ydb_pg_scheme.png) + +Такая архитектура интеграции с PostgreSQL позволяет выполнять запросы на PostgreSQL над {{ydb-name}} типами данных и наоборот, выполнять YQL-запросы над типами данных PostgreSQL, обеспечивая интероперабельность работы с данными. + +Проиллюстрируем это с помощью следующего сценария: +1. Создадим таблицу {{ydb-name}} с помощью YQL-синтаксиса + ```sql + CREATE TABLE `test_table`(col INT, PRIMARY KEY(col)); + ``` +1. Добавим туда тестовые данные + ```sql + INSERT INTO test_table(col) VALUES(1) + ``` +1. Прочитаем эти данные с помощью PostgreSQL-синтаксиса + ``` + psql -h -d -U -c "SELECT * FROM test_table" + col + --- + 1 + (1 row) + ``` + + Где: + - `` - адрес кластера ydb, к которому выполняется подключение. + - `` - название базы данных в кластере. Может быть сложным именем, например, `mycluster/tenant1/database`. + - `` - логин пользователя. + + +## Соответствие типов данных {#supported_types} + +Система типов данных {{ydb-name}} и PostgreSQL похожи, но при этом не совпадают. + +### Использование данных таблиц, созданных в YQL синтаксисе, в синтаксисе PostgreSQL {#topg} +Типы данных из {{ydb-name}} автоматически преобразовываются в соответствующие им типы PostgreSQL. Преобразование выполняется неявно с помощью команды [`ToPg`](../yql/reference/udf/list/postgres.md#topg). + +Пример: +1. Создадим таблицу {{ydb-name}} с помощью YQL-синтаксиса + ```sql + CREATE TABLE `test_table`(col INT, PRIMARY KEY(col)); + ``` +1. Добавим туда тестовые данные + ```sql + INSERT INTO test_table(col) VALUES(1) + ``` +1. Прочитаем эти данные с помощью PostgreSQL-синтаксиса. YQL-тип данных `INT` был автоматически переведен в PostgreSQL-тип `int4`, над которым была выполнена операция инкремента. + ``` + psql -c "SELECT col+1 AS col FROM test_table" + col + --- + 2 + (1 row) + ``` + +Так как все вычисления выполняются внутри YDB, то для для каждого PostgreSQL типа создан "зеркальный тип" в YDB. Например, тип `text` из PostgreSQL при обработке внутри YDB будет иметь тип `pgtext`. Это сделано, чтобы обеспечить точную семантику работы типов PostgreSQL внутри YDB. При преобразовании типа из PostgreSQL в YDB применяется правило, что для каждого такого типа добавляется префикс `pg`, после чего используется оригинальное имя типа из PostgreSQL. + +Таблица соответствия типов данных YQL, при их использовании в PostgreSQL запросах: + +{% include [topg](../yql/reference/udf/list/_includes/topg.md) %} + + +### Использование данных таблиц, созданных в PostgreSQL синтаксисе, в синтаксисе YQL {#frompg} + +При использовании данных из таблиц, созданных в PostgreSQL синтаксисе, YQL интерпретирует данные в этих колонках, как специальные типы из семейства `pg*`. + +Пример: +1. Создадим таблицу {{ydb-name}} с помощью PostgreSQL-синтаксиса + ```sql + CREATE TABLE test_table_pg(numeric INT, PRIMARY KEY(col)); + ``` +1. Добавим туда тестовые данные + ```sql + INSERT INTO test_table_pg(col) VALUES(10) + ``` +1. Прочитаем эти данные с помощью YQL-синтаксиса + ``` + ydb sql -s "SELECT col+1 AS col FROM test_table_pg" + col + --- + "11" -- pgnumeric + (1 row) + ``` + +Правила преобразования типов PostgreSQL в типы YQL приведены в таблице: + +|PostgreSQL | YQL| +|---|---| +| `bool` |`pgbool` | +| `int2` |`pgint2` | +| `int4` |`pgint4` | +| `int8` |`pgint8` | +|`numeric` |`pgnumeric` | +| `float4` |`pgfloat4` | +| `float8` |`pgfloat8` | +| `bytea` |`pgbytea` | +| `text` |`pgtext` | +| `bytea` |`pgbytea` | +| `json` |`pgjson` | +| `uuid` |`pguuid` | +| `jsonb` |`pgjsonb` | +| `date` |`pgdate` | +| `timestamp` |`pgtimestamp` | +| `interval` | `pginterval` | +| `text` |`pgtext` | +| `date` | `pgdate`| +| `timestamp` |`pgtimestamp` | +| `interval` |`pginterval` | +| `numeric` |`pgnumeric` | + +Встроенные функции YQL ориентированы на работу с собственными типами данных, например, `Ip::FromString` получает на вход типы данных `Utf8` или `String`. Поэтому встроенные функции YQL не могут работать с типами данных PostgreSQL. Для решения задачи конвертации типов существует функция [`FromPg`](../yql/reference/udf/list/postgres.md#frompg), выполняющая преобразование данных из типов PostgreSQL в типы YQL. + +Пример: +1. Создадим таблицу {{ydb-name}} с помощью PostgreSQL-синтаксиса + ```sql + CREATE TABLE test_table_pg_ip(col text, PRIMARY KEY(col)); + ``` +1. Добавим туда тестовые данные + ```sql + INSERT INTO test_table_pg_ip(col) VALUES('::ffff:77.75.155.3') + ``` +1. Прочитаем эти данные с помощью YQL-синтаксиса: + ``` + ydb sql -s "SELECT Ip::ToString(Ip::GetSubnet(Ip::FromString(col))) AS subnet + FROM test_table_pg_ip" + Status: GENERIC_ERROR + Issues: +
: Error: Type annotation, code: 1030 +
:1:1: Error: At function: RemovePrefixMembers, At function: PersistableRepr, At function: SqlProject, At function: SqlProjectItem +
:1:12: Error: At function: Apply, Callable is produced by Udf: Ip.FromString +
:1:23: Error: Mismatch type argument #1, type diff: String!=pgtext + ``` +1. Для работы функции `Ip::FromString` необходимо предварительно выполнить преобразование типов данных с помощью функции `PgFrom`: + ``` + ydb sql -s "SELECT Ip::ToString(Ip::GetSubnet(Ip::FromString(FromPg(col))) AS subnet + FROM test_table_pg_ip" + ┌────────┐ + │ subnet │ + ├────────┤ + │ "::" │ + └────────┘ + ``` + +Кроме этого, при работе с PostgreSQL типами данных [можно использовать функции самого PostgreSQL](../yql/reference/udf/list/postgres.md#callpgfunction) непосредственно из YQL-синтаксиса: + +1. Создадим таблицу {{ydb-name}} с помощью PostgreSQL-синтаксиса + ```sql + CREATE TABLE test_table_array(col INT, col2 _INT, PRIMARY KEY(col)); + ``` +1. Добавим туда тестовые данные + ```sql + INSERT INTO test_table_array(col, col2) VALUES(1, '{1,2,3}') + ``` +1. Вызовем встроенную функцию PostgreSQL `array_length`: + ``` + ydb sql -s "select Pg::array_length(col2, 1) FROM test_table_array" + col + --- + "3" -- число элементов в массиве + (1 row) + ``` diff --git a/ydb/docs/ru/core/postgresql/intro.md b/ydb/docs/ru/core/postgresql/intro.md index 9992741169a4..4ce15ec3c21f 100644 --- a/ydb/docs/ru/core/postgresql/intro.md +++ b/ydb/docs/ru/core/postgresql/intro.md @@ -1,15 +1,16 @@ -## Совместимость {{ ydb-short-name }} с PostgreSQL +# Совместимость {{ ydb-short-name }} с PostgreSQL {% include [./_includes/alert.md](./_includes/alert_preview.md) %} [PostgreSQL](https://www.postgresql.org) совместимость – это механизм выполнения SQL запросов в PostgreSQL диалекте на инфраструктуре YDB с использованием сетевого протокола PostgreSQL. Благодаря этой возможности можно использовать привычные инструменты работы с PostgreSQL, такие, как [psql](https://www.postgresql.org/docs/14/app-psql.html) и драйвера (например, [pq](https://github.com/lib/pq) для Golang и [psycopg2](https://pypi.org/project/psycopg2/) для Python). Можно разрабатывать запросы на привычном PostgreSQL синтаксисе и получать такие преимущества YDB, как горизонтальная масштабируемость и отказоустойчивость. -PostgreSQL совместимость упрощает миграцию приложений, ранее работавших в экосистеме PostgreSQL. Сейчас поддерживается ограниченное количество инструкций и функций PostgreSQL 14. PostgreSQL совместимость позволяет переключаться с PostgreSQL на YDB без изменения кода проекта (в случае полной поддержки совместимостью используемых в проекте SQL-конструкций), просто изменив параметры подключения. +PostgreSQL совместимость упрощает миграцию приложений, ранее работавших в экосистеме PostgreSQL. Сейчас поддерживается ограниченное количество инструкций и функций PostgreSQL 16. PostgreSQL совместимость позволяет переключаться с PostgreSQL на {{ydb-name}} без изменения кода проекта (в случае полной поддержки совместимостью используемых в проекте SQL-конструкций), просто изменив параметры подключения. -Принцип работы PostgreSQL совместимости можно описать так: -1. Программа отправляет запросы в YDB, где их обрабатывает компонент под названием pgwire. Pgwire реализует [сетевой протокол](https://postgrespro.ru/docs/postgresql/14/protocol) PostgreSQL и передает команды в query processor. -2. Query processor транслирует PostgreSQL запросы в YQL AST. -3. После обработки запросов результаты собираются и отправляются обратно в программу, отправившую запрос, по сетевому протоколу PostgreSQL. При обработке запроса он может распараллеливаться и исполняться на произвольном количестве узлов YDB. -Графически работу PostgreSQL совместимости можно представить так: -![Схема работы PostgreSQL совместимости](./_includes/ydb_pg_scheme.png) \ No newline at end of file + + +В этом разделе описываются: +- [{#T}](connect.md) +- [{#T}](interoperability.md) +- [{#T}](functions.md) +- [{#T}](import.md) diff --git a/ydb/docs/ru/core/postgresql/toc_i.yaml b/ydb/docs/ru/core/postgresql/toc_i.yaml index a472d07bf565..d2794f94a0ea 100644 --- a/ydb/docs/ru/core/postgresql/toc_i.yaml +++ b/ydb/docs/ru/core/postgresql/toc_i.yaml @@ -1,11 +1,16 @@ items: -- name: Подключение через PostgreSQL-протокол - href: docker-connect.md -- name: Синтаксис PostgreSQL +- name: Описание + href: ../postgresql/intro.md +- name: Выполнение запросов + href: connect.md +- name: Интероперабельность PostgreSQL и {{ydb-name}} + href: interoperability.md +- name: Синтаксис PostgreSQL include: { mode: link, path: ./statements/toc_m.yaml } -- name: Механизмы PostgreSQL +- name: Механизмы PostgreSQL include: { mode: link, path: ./mechanisms/toc_m.yaml } - name: Функции PostgreSQL href: functions.md - name: Импорт данных из PostgreSQL - href: pg-dump.md + href: import.md + diff --git a/ydb/docs/ru/core/reference/toc_p.yaml b/ydb/docs/ru/core/reference/toc_p.yaml index df3a125ef8f9..9fdeb0b90296 100644 --- a/ydb/docs/ru/core/reference/toc_p.yaml +++ b/ydb/docs/ru/core/reference/toc_p.yaml @@ -1,11 +1,10 @@ items: - name: YQL href: ../yql/reference/index.md - include: + include: mode: link path: ../yql/toc_p.yaml - name: Совместимость с PostgreSQL - href: ../postgresql/intro.md include: mode: link path: ../postgresql/toc_p.yaml diff --git a/ydb/docs/ru/core/reference/ydb-cli/_includes/commands.md b/ydb/docs/ru/core/reference/ydb-cli/_includes/commands.md index e412e7fab15d..90edf088bc1b 100644 --- a/ydb/docs/ru/core/reference/ydb-cli/_includes/commands.md +++ b/ydb/docs/ru/core/reference/ydb-cli/_includes/commands.md @@ -66,7 +66,7 @@ table attribute drop | Удаление атрибута таблицы [tools copy](../tools-copy.md) | Копирование таблиц [tools dump](../export-import/tools-dump.md) | Выгрузка директории или таблицы в файловую систему {% if ydb-cli == "ydb" %} -[tools pg-convert](../../../postgresql/pg-dump.md#pg-convert) | Конвертация дампа PostgreSQL, полученного утилитой pg_dump, в формат, понятный YDB +[tools pg-convert](../../../postgresql/import.md#pg-convert) | Конвертация дампа PostgreSQL, полученного утилитой pg_dump, в формат, понятный YDB {% endif %} [tools rename](../commands/tools/rename.md) | Переименование таблиц [tools restore](../export-import/tools-restore.md) | Восстановление из файловой системы diff --git a/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/lexer.md b/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/lexer.md index aded5aaec838..67961e905f16 100644 --- a/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/lexer.md +++ b/ydb/docs/ru/core/yql/reference/yql-core/syntax/_includes/lexer.md @@ -205,3 +205,9 @@ SELECT 456s AS `Int16`, 1.2345f AS `Float`; ``` + +## Литералы PostgreSQL {#pgliterals} + +Строковые и числовые литералы Pg типов можно создавать с помощью специальных суффиксов (аналогично простым [строковым](#string-literals) и [числовым](#literal-numbers) литералам). + +{% include [pgliterals](../../udf/list/_includes/pgliterals.md) %} diff --git a/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/frompg.md b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/frompg.md new file mode 100644 index 000000000000..2c0413b7b6e3 --- /dev/null +++ b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/frompg.md @@ -0,0 +1,15 @@ +| PostgreSQL | YQL | +|---|---| +| `bool` | `bool` | +| `int2` | `Int16` | +| `int4` | `Int32` | +| `int8` | `Int64` | +| `float4` | `Float` | +| `float8` | `Double` | +| `bytea` | `String` | +| `varchar` | `Utf8` | +| `text` | `Utf8` | +| `cstring` | `Utf8` | +| `uuid` | `Uuid` | +| `date` | `Date32` | +| `timestamp` | `Timestamp64` | diff --git a/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/pgliterals.md b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/pgliterals.md new file mode 100644 index 000000000000..ad5a1fee537a --- /dev/null +++ b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/pgliterals.md @@ -0,0 +1,95 @@ +### Целочисленные литералы {#intliterals} + +Суффикс | Тип | Комментарий +----- | ----- | ----- +`p` | `PgInt4` | 32-битное знаковое целое (в PostgreSQL нет беззнаковых типов) +`ps`| `PgInt2` | 16-битное знаковое целое +`pi`| `PgInt4` | +`pb`| `PgInt8` | 64-битное знаковое цело +`pn`| `PgNumeric` | знаковое целое произвольной точности (до 131072 цифр) + +### Литералы с плавающей точкой {#floatliterals} + +Суффикс | Тип | Комментарий +----- | ----- | ----- +`p` | `PgFloat8` | число с плавающей точкой (64 бит double) +`pf4`| `PgFloat4` | число с плавающей точкой (32 бит float) +`pf8`| `PgFloat8` | +`pn` | `PgNumeric` | число с плавающей точкой произвольной точности (до 131072 цифр перед запятой, до 16383 цифр после запятой) + +### Строковые литералы {#stringliterals} + +Суффикс | Тип | Комментарий +----- | ----- | ----- +`p` | `PgText` | текстовая строка +`pt`| `PgText` | +`pv`| `PgVarchar` | текстовая строка +`pb`| `PgBytea` | бинарная строка + +{% note warning "Внимание" %} + +Значения строковых/числовых литералов (т.е. то что идет перед суффиксом) должны быть валидной строкой/числом с точки зрения YQL. +В частности, должны соблюдаться правила эскейпинга YQL, а не PostgreSQL. + +{% endnote %} + +Пример: + +``` sql +SELECT + 1234p, -- pgint4 + 0x123pb, -- pgint8 + "тест"pt, -- pgtext + 123e-1000pn; -- pgnumeric +; +``` + +### Литерал массива + +Для построения литерала массива используется функция `PgArray`: + +``` sql +SELECT + PgArray(1p, NULL ,2p) -- {1,NULL,2}, тип _int4 +; +``` + + +### Конструктор литералов произвольного типа {#literals_constructor} + +Литералы всех типов (в том числе и Pg типов) могут создаваться с помощью конструктора литералов со следующей сигнатурой: +`Имя_типа(<строковая константа>)`. + +Напрмер: +``` sql +DECLARE $foo AS String; +SELECT + PgInt4("1234"), -- то же что и 1234p + PgInt4(1234), -- в качестве аргумента можно использовать литеральные константы + PgInt4($foo), -- и declare параметры + PgBool(true), + PgInt8(1234), + PgDate("1932-01-07"), +; +``` + +Также поддерживается встроенная функция `PgConst` со следующей сигнатурой: `PgConst(<строковое значение>, <тип>)`. +Такой способ более удобен для кодогенерации. + +Например: + +``` sql +SELECT + PgConst("1234", PgInt4), -- то же что и 1234p + PgConst("true", PgBool) +; +``` + +Для некоторых типов в функции `PgConst` можно указать дополнительные модификаторы. Возможные модификаторы для типа `pginterval` перечислены в [документации PostgreSQL](https://www.postgresql.org/docs/16/datatype-datetime.html). + +``` sql +SELECT + PgConst(90, pginterval, "day"), -- 90 days + PgConst(13.45, pgnumeric, 10, 1); -- 13.5 +; +``` diff --git a/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/topg.md b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/topg.md new file mode 100644 index 000000000000..9fc1ce17d32d --- /dev/null +++ b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/_includes/topg.md @@ -0,0 +1,35 @@ +| YQL | PostgreSQL | Название PostgreSQL-типа в {{ ydb-short-name }} | +|---|---|---| +| `Bool` | `bool` |`pgbool` | +| `Int8` | `int2` |`pgint2` | +| `Uint8` | `int2` |`pgint2` | +| `Int16` | `int2` |`pgint2` | +| `Uint16` | `int4` |`pgint4` | +| `Int32` | `int4` |`pgint4` | +| `Uint32` | `int8` |`pgint8` | +| `Int64` | `int8` |`pgint8` | +| `Uint64` | `numeric` |`pgnumeric` | +| `Float` | `float4` |`pgfloat4` | +| `Double` | `float8` |`pgfloat8` | +| `String` | `bytea` |`pgbytea` | +| `Utf8` | `text` |`pgtext` | +| `Yson` | `bytea` |`pgbytea` | +| `Json` | `json` |`pgjson` | +| `Uuid` | `uuid` |`pguuid` | +| `JsonDocument` | `jsonb` |`pgjsonb` | +| `Date` | `date` |`pgdate` | +| `Datetime` | `timestamp` |`pgtimestamp` | +| `Timestamp` | `timestamp` |`pgtimestamp` | +| `Interval` | `interval` | `pginterval` | +| `TzDate` | `text` |`pgtext` | +| `TzDatetime` | `text` |`pgtext` | +| `TzTimestamp` | `text` |`pgtext` | +| `Date32` | `date` | `pgdate`| +| `Datetime64` | `timestamp` |`pgtimestamp` | +| `Timestamp64` | `timestamp` |`pgtimestamp` | +| `Interval64`| `interval` |`pginterval` | +| `TzDate32` | `text` | |`pgtext` | +| `TzDatetime64` | `text` | |`pgtext` | +| `TzTimestamp64` | `text` | |`pgtext` | +| `Decimal` | `numeric` |`pgnumeric` | +| `DyNumber` | `numeric` |`pgnumeric` | diff --git a/ydb/docs/ru/core/yql/reference/yql-core/udf/list/postgres.md b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/postgres.md new file mode 100644 index 000000000000..f1ba145b619b --- /dev/null +++ b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/postgres.md @@ -0,0 +1,206 @@ +# PostgreSQL UDF + +[YQL](../../index.md) предоставляет возможность доступа к [функциям](https://www.postgresql.org/docs/16/functions.html) и [типам данных](https://www.postgresql.org/docs/16/datatype.html) PostgreSQL. + +Имена PostgreSQL типов в YQL получаются добавлением префикса `Pg` к исходному имени типа. +Например `PgVarchar`, `PgInt4`, `PgText`. Имена pg типов (как и вообще всех типов) в YQL являются case-insensitive. На данный момент поддерживаются все простые типы данных из PostgreSQL, а также массивы. + +Если исходный тип является типом массива (в PostgreSQL такие типы начинаются с подчеркивания: `_int4` - массив 32-битных целых), то имя типа в YQL тоже начинается с подчеркивания – `_PgInt4`. + +## Литералы {#literals} + +Строковые и числовые литералы Pg типов можно создавать с помощью специальных суффиксов (аналогично простым [строковым](../yql/reference/yql-core/syntax/lexer#string-literals) и [числовым](../yql/reference/yql-core/syntax/lexer#literal-numbers) литералам). + +{% include [pgliterals](./_includes/pgliterals.md) %} + +## Операторы {#operators} + +Операторы PostgreSQL (унарные и бинарные) доступны через встроенную функцию `PgOp(<оператор>, <операнды>)`: + +``` sql +SELECT + PgOp("*", 123456789987654321pn, 99999999999999999999pn), -- 12345678998765432099876543210012345679 + PgOp('|/', 10000.0p), -- 100.0p (квадратный корень) + PgOp("-", 1p), -- -1p + -1p, -- унарный минус для литералов работает и без PgOp +; +``` + +## Оператор приведения типа {#cast_operator} + +Для приведения значения одного Pg типа к другому используется встроенная функция `PgCast(<исходное значение>, <желаемый тип>)`: + +``` sql +SELECT + PgCast(123p, PgText), -- преобразуем число в строку +; +``` + +При преобразовании из строковых Pg типов в некоторые целевые типы можно указать дополнительные модификаторы. Возможные модификаторы для типа `pginterval` перечислены в [документации](https://www.postgresql.org/docs/16/datatype-datetime.html). + +``` sql +SELECT + PgCast('90'p, pginterval, "day"), -- 90 days + PgCast('13.45'p, pgnumeric, 10, 1); -- 13.5 +; +``` + +## Преобразование значений Pg типов в значения YQL типов и обратно {#frompgtopg} + +Для некоторых Pg типов возможна конвертация в YQL типы и обратно. Конвертация осуществляется с помощью встроенных функций +`FromPg(<значение Pg типа>)` и `ToPg(<значение YQL типа>)`: + +``` sql +SELECT + FromPg("тест"pt), -- Just(Utf8("тест")) - pg типы всегда nullable + ToPg(123.45), -- 123.45pf8 +; +``` + +### Список псевдонимов типов PostgreSQL при их использовании в YQL {#pgyqltypes} + +Ниже приведены типы данных YQL, соответствующие им логические типы PostgreSQL и названия типов PostgreSQL при их использовании в YQL: + +| YQL | PostgreSQL | Название PostgreSQL-типа в YQL| +|---|---|---| +| `Bool` | `bool` |`pgbool` | +| `Int8` | `int2` |`pgint2` | +| `Uint8` | `int2` |`pgint2` | +| `Int16` | `int2` |`pgint2` | +| `Uint16` | `int4` |`pgint4` | +| `Int32` | `int4` |`pgint4` | +| `Uint32` | `int8` |`pgint8` | +| `Int64` | `int8` |`pgint8` | +| `Uint64` | `numeric` |`pgnumeric` | +| `Float` | `float4` |`pgfloat4` | +| `Double` | `float8` |`pgfloat8` | +| `String` | `bytea` |`pgbytea` | +| `Utf8` | `text` |`pgtext` | +| `Yson` | `bytea` |`pgbytea` | +| `Json` | `json` |`pgjson` | +| `Uuid` | `uuid` |`pguuid` | +| `JsonDocument` | `jsonb` |`pgjsonb` | +| `Date` | `date` |`pgdate` | +| `Datetime` | `timestamp` |`pgtimestamp` | +| `Timestamp` | `timestamp` |`pgtimestamp` | +| `Interval` | `interval` | `pginterval` | +| `TzDate` | `text` |`pgtext` | +| `TzDatetime` | `text` |`pgtext` | +| `TzTimestamp` | `text` |`pgtext` | +| `Date32` | `date` | `pgdate`| +| `Datetime64` | `timestamp` |`pgtimestamp` | +| `Timestamp64` | `timestamp` |`pgtimestamp` | +| `Interval64`| `interval` |`pginterval` | +| `TzDate32` | `text` | |`pgtext` | +| `TzDatetime64` | `text` | |`pgtext` | +| `TzTimestamp64` | `text` | |`pgtext` | +| `Decimal` | `numeric` |`pgnumeric` | +| `DyNumber` | `numeric` |`pgnumeric` | + + +### Таблица соответствия типов `ToPg` {#topg} + +Таблица соответствия типов данных YQL и PostgreSQL при использовании функции `ToPg`: + +{% include [topg](_includes/topg.md) %} + +### Таблица соответствия типов `FromPg` {#frompg} + +Таблица соответствия типов данных PostgreSQL и YQL при использовании функции `FromPg`: + +{% include [frompg](_includes/frompg.md) %} + +## Вызов PostgreSQL функций {#callpgfunction} + +Чтобы вызвать PostgreSQL функцию, необходимо добавить префикс `Pg::` к ее имени: + +``` sql +SELECT + Pg::extract('isodow'p,PgCast('1961-04-12'p,PgDate)), -- 3pn (среда) - работа с датами до 1970 года + Pg::generate_series(1p,5p), -- [1p,2p,3p,4p,5p] - для функций-генераторов возвращается ленивый список +; +``` + +Существует также альтернативный способ вызова функций через встроенную функцию `PgCall(<имя функции>, <операнды>)`: + +``` sql +SELECT + PgCall('lower', 'Test'p), -- 'test'p +; +``` + +При вызове функции, возвращающей набор `pgrecord`, можно распаковать результат в список структур, используя функцию `PgRangeCall(<имя функции>, <операнды>)`: + +``` sql +SELECT * FROM + AS_TABLE(PgRangeCall("json_each", pgjson('{"a":"foo", "b":"bar"}'))); + --- 'a'p,pgjson('"foo"') + --- 'b'p,pgjson('"bar"') +; +``` + + + +## Вызов агрегационных PostgreSQL функций {#pgaggrfunction} + +Чтобы вызвать агрегационную PostgreSQL функцию, необходимо добавить префикс `Pg::` к ее имени: + +``` sql +SELECT +Pg::string_agg(x,','p) +FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'a,b,c'p + +SELECT +Pg::string_agg(x,','p) OVER (ORDER BY x) +FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'a'p,'a,b'p,'a,b,c'p +; +``` + +Также можно использовать агрегационную PostgreSQL функцию для построения фабрики агрегационных функций с последующим применением в `AGGREGATE_BY`: + +``` sql +$agg_max = AggregationFactory("Pg::max"); + +SELECT +AGGREGATE_BY(x,$agg_max) +FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'c'p + +SELECT +AGGREGATE_BY(x,$agg_max) OVER (ORDER BY x), +FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'a'p,'b'p,'c'p +``` + +В этом случае вызов `AggregationFactory` принимает только имя функции с префиксом `Pg::`, а все аргументы функции передаются в `AGGREGATE_BY`. + +Если в агрегационной функции не один аргумент, а ноль или два и более, необходимо использовать кортеж при вызове `AGGREGATE_BY`: + +``` sql +$agg_string_agg = AggregationFactory("Pg::string_agg"); + +SELECT +AGGREGATE_BY((x,','p),$agg_string_agg) +FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'a,b,c'p + +SELECT +AGGREGATE_BY((x,','p),$agg_string_agg) OVER (ORDER BY x) +FROM (VALUES ('a'p),('b'p),('c'p)) as a(x); -- 'a'p,'a,b'p,'a,b,c'p +``` + +{% note warning "Внимание" %} + +Не поддерживается режим `DISTINCT` над аргументами при вызове агрегационных PostgreSQL функций, а также использование `MULTI_AGGREGATE_BY`. + +{% endnote %} + +## Логические операции + +Для выполнения логических операций используются функции `PgAnd`, `PgOr`, `PgNot`: + +``` sql +SELECT + PgAnd(PgBool(true), PgBool(true)), -- PgBool(true) + PgOr(PgBool(false), null), -- PgCast(null, pgbool) + PgNot(PgBool(true)), -- PgBool(false) +; +``` + diff --git a/ydb/docs/ru/core/yql/reference/yql-core/udf/list/toc_base.yaml b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/toc_base.yaml index a33b7bd78d4d..9e2acf73c4b5 100644 --- a/ydb/docs/ru/core/yql/reference/yql-core/udf/list/toc_base.yaml +++ b/ydb/docs/ru/core/yql/reference/yql-core/udf/list/toc_base.yaml @@ -10,6 +10,7 @@ items: - { name: Math, href: math.md } - { name: Pcre, href: pcre.md } - { name: Pire, href: pire.md } +- { name: PostgreSQL, href: postgres.md } - { name: Re2, href: re2.md } - { name: String, href: string.md } - { name: Unicode, href: unicode.md } From c190b9c208ed823ca664de56e02de31f15faf59a Mon Sep 17 00:00:00 2001 From: Maksim Zinal Date: Thu, 22 Aug 2024 09:18:14 +0300 Subject: [PATCH 029/261] Moved cluster topology information to the topology overview section (#7144) Co-authored-by: Ivan Blinkov --- ydb/docs/en/core/concepts/glossary.md | 18 ++- ydb/docs/en/core/concepts/topology.md | 104 ++++++++++++++++-- .../en/core/deploy/configuration/config.md | 87 +-------------- ydb/docs/ru/core/concepts/glossary.md | 18 ++- ydb/docs/ru/core/concepts/topology.md | 104 +++++++++++++++--- .../ru/core/deploy/configuration/config.md | 89 +-------------- 6 files changed, 220 insertions(+), 200 deletions(-) diff --git a/ydb/docs/en/core/concepts/glossary.md b/ydb/docs/en/core/concepts/glossary.md index 91c85980f9ff..f390544ed07a 100644 --- a/ydb/docs/en/core/concepts/glossary.md +++ b/ydb/docs/en/core/concepts/glossary.md @@ -68,6 +68,10 @@ A static group might require special attention during major maintenance, such as Regular storage groups that are not [static](#static-group) are called **dynamic groups**. They are called dynamic because they can be created and decommissioned on the fly during [cluster](#cluster) operation. +### Storage pool {#storage-pool} + +**Storage pool** is a collection of data storage devices with similar characteristics. Each storage pool is assigned a unique name within a {{ ydb-short-name }} cluster. Technically, each storage pool consists of multiple [PDisks](#pdisk). Each [storage group](#storage-group) is created in a particular storage pool, which determines the performance characteristics of the storage group through the selection of appropriate storage devices. It is typical to have separate storage pools for NVMe, SSD, and HDD devices or particular models of those devices with different capacities and speeds. + ### Actor {#actor} The [actor model](https://en.wikipedia.org/wiki/Actor_model) is one of the main approaches for concurrent programming, which is employed by {{ ydb-short-name }}. In this model, **actors** are lightweight user-space processes that may have and modify their private state but can only affect each other indirectly through message passing. {{ ydb-short-name }} has its own implementation of this model, which is covered [below](#actor-implementation). @@ -357,7 +361,7 @@ The **cluster management system** or **CMS** is a system tablet responsible for A **slot** in {{ ydb-short-name }} can be used in two contexts: * **Slot** is a portion of a server's resources allocated to running a single {{ ydb-short-name }} [node](#node). A common slot size is 10 CPU cores and 50 GB of RAM. Slots are used if a {{ ydb-short-name }} cluster is deployed on servers or virtual machines with sufficient resources to host multiple slots. -* [VDisk](#slot) **slot** or **VSlot** is a fraction of PDisk that can be allocated to one of the [VDisks](#vdisk). +* [VDisk](#slot) **slot** or **VSlot** is a fraction of [PDisk](#pdisk) that can be allocated to one of the [VDisks](#vdisk). ### State storage {#state-storage} @@ -416,7 +420,7 @@ A **LogoBlobID** is the [LogoBlob](#logoblob) identifier in the [Distributed sto **PDisk** or **Physical disk** is a component that controls a physical disk drive (block device). In other words, PDisk is a subsystem that implements an abstraction similar to a specialized file system on top of block devices (or files simulating a block device for testing purposes). PDisk provides data integrity controls (including [erasure encoding](#erasure-coding) of sector groups for data recovery on single bad sectors, integrity control with checksums), transparent data-at-rest encryption of all disk data, and transactional guarantees of disk operations (write confirmation strictly after `fsync`). PDisk contains a scheduler that provides device bandwidth sharing between several clients ([VDisks](#vdisk)). PDisk divides a block device into chunks called [slots](#slot) (about 128 megabytes in size; smaller chunks are allowed). No more than 1 VDisk can own each slot at a time. PDisk also supports a recovery log shared between PDisk service records and all VDisks. - + #### VDisk {#vdisk} **VDisk** or **Virtual disk** is a component that implements the persistence of [distributed storage](#distributed-storage) [LogoBlobs](#logoblob) on [PDisks](#pdisk). VDisk stores all its data on PDisks. One VDisk corresponds to one PDisk, but usually, several VDisks are linked to one PDisk. Unlike PDisk, which hides chunks and logs behind it, VDisk provides an interface at the LogoBlob and [LogoBlobID](#logoblobid) level, like writing LogoBlob, reading LogoBlobID data, and deleting a set of LogoBlob using a special command. VDisk is a member of a [storage group](#storage-group). VDisk itself is local, but many VDisks in a given group provide reliable data storage. The VDisks in a group synchronize the data with each other and replicate the data in case of loss. A set of VDisks in a storage group forms a distributed RAID. @@ -449,11 +453,17 @@ Technically, DS Proxy is implemented as an [actor service](#actor-service) launc #### Fail realm {#fail-realm} -A [storage group](#storage-group) is a set of [VDisks](#vdisk) combined into one or more **Fail realms**. A Fail realm contains one or more [Fail domains](#fail-domain). The correlated failure of two VDisks within the same Fail realm is more likely than that of two VDisks from different Fail realms. A Fail realm usually corresponds to the physical concept of a data center or Availability Zone (AZ). +A **fail realm** is a set of [fail domains](#fail-domain) that are likely to fail simultaneously. The correlated failure of two [VDisks](#vdisk) within the same fail realm is more probable than that of two VDisks from different fail realms. + +An example of a fail realm is a set of hardware located in the same [data center or availability zone](#regions-az) that can all fail together due to a natural disaster, major power outage, or similar event. #### Fail domain {#fail-domain} -A [Fail realm](#fail-realm) contains one or more **Fail domains**. The correlated failure of two [VDisks](#vdisk) within the same Fail domain is more likely than that of two VDisks from different Fail domains. A Fail domain usually corresponds to a [server rack](#rack) concept. +A **fail domain** is a set of hardware that may fail simultaneously. The correlated failure of two [VDisks](#vdisk) within the same fail domain is more probable than the failure of two VDisks from different fail domains. In the case of different fail domains, this probability is also affected by whether these domains belong to the same [fail realm](#fail-realm) or not. + +For example, a fail domain includes disks on the same server, as all server disks may become unavailable if the server's PSU or network controller fails. A fail domain also typically includes servers located in the same server rack, as all the hardware in the rack may become unavailable if there is a power outage or an issue with the network hardware in the same rack. Thus, the typical fail domain corresponds to a server rack if the [cluster](#cluster) is configured to be rack-aware, or to a server otherwise. + +Domain failures are handled automatically by {{ ydb-short-name }} without shutting down the cluster. #### Distributed storage channel {#channel} diff --git a/ydb/docs/en/core/concepts/topology.md b/ydb/docs/en/core/concepts/topology.md index db89b79f0dd4..780733a6cddf 100644 --- a/ydb/docs/en/core/concepts/topology.md +++ b/ydb/docs/en/core/concepts/topology.md @@ -1,25 +1,105 @@ -# Topology +# {{ ydb-short-name }} cluster topology -A {{ ydb-short-name }} cluster consists of static and dynamic nodes. +A {{ ydb-short-name }} cluster consists of [storage](glossary.md#storage-node) and [database](glossary.md#database-node) nodes. As the data stored in {{ ydb-short-name }} is available only via queries and API calls, both types of nodes are essential for [database availability](#database-availability). However, [distributed storage](glossary.md#distributed-storage) consisting of storage nodes has the most impact on the cluster's fault tolerance and ability to persist data reliably. During the initial cluster deployment, an appropriate distributed storage [operating mode](#cluster-config) needs to be chosen according to the expected workload and [database availability](#database-availability) requirements. -* Static nodes enable data storage, implementing one of the supported redundancy schemes depending on the established operating mode. -* Dynamic nodes enable query execution, transaction coordination, and other data control functionality. +## Cluster operating modes {#cluster-config} -Cluster topology is determined by the fault tolerance requirements. The following operating modes are available: +Cluster topology is based on the chosen operating mode, which needs to be determined according to the fault tolerance requirements. {{ ydb-short-name }}'s failure model is based on the concepts of [fail domain](glossary.md#fail-domain) and [fail realm](glossary.md#fail-realm). -| Mode | Storage
volume multiplier | Minimum
number
of nodes | Description | ---- | --- | --- | --- -| `none` | 1 | 1 | There is no redundancy.
Any hardware failure causes the storage pool to become unavailable.
This mode is only recommended for functional testing. | -| `block-4-2` | 1.5 | 8 | [Erasure coding](https://en.wikipedia.org/wiki/Erasure_code) with two blocks of redundancy added to the four blocks of source data is applied. Storage nodes are placed in at least 8 failure domains (usually racks).
The storage pool is available if any two domains fail, continuing to record all 6 data parts in the remaining domains.
This mode is recommended for storage pools within a single availability zone (usually a data processing center). | -| `mirror-3-dc` | 3 | 9 | Data is replicated to 3 availability zones using 3 failure domains (usually racks) within each zone.
The storage pool is available if one availability zone and one failure domain fail in the remaining zones.
This mode is recommended for multi-data center installations. | -| `mirror-3dc-3-nodes` | 3 | 3 | It is a simplified version of `mirror-3dc`. This mode requires at least 3 servers with 3 disks each. Each server must be located in an independent data center in order to provide the best fault tolerance.
Health in this mode is maintained if no more than 1 node fails.
This mode is only recommended for functional testing. | +The following {{ ydb-short-name }} distributed storage operating modes are available: + +- `none`. There is no redundancy. Any hardware failure causes data to become unavailable or permanently lost. This mode is only recommended for development and functional testing. +- `block-4-2`. [Erasure coding](https://en.wikipedia.org/wiki/Erasure_code) is applied with two blocks of redundancy added to the four blocks of source data. Storage nodes are placed in at least 8 failure domains (usually racks). A [storage pool](glossary.md#storage-pool) remains available if any two domains fail, continuing to record all 6 data parts in the remaining domains. This mode is recommended for clusters deployed within a single availability zone or data center. +- `mirror-3-dc`. Data is replicated to 3 availability zones using 3 failure domains (usually racks) within each zone. {{ ydb-short-name }} cluster remains available even if one availability zone completely fails; additionally, one failure domain in the remaining zones can fail at the same time. This mode is recommended for multi-datacenter clusters with high availability requirements. +- `mirror-3-dc-3-nodes`. A simplified version of `mirror-3-dc`. This mode requires at least 3 servers with 3 disks each. Each server must be located in an independent data center to provide reasonable fault tolerance. System health in this mode is maintained if no more than one server fails. This mode is only recommended for functional testing or building prototypes. {% note info %} Node failure means both its total and partial unavailability, for example, failure of a single disk on a node. -The storage volume multiplier specified above only applies to the fault tolerance factor. Other influencing factors (for example, slot fragmentation and granularity) must be taken into account for storage size planning. +{% endnote %} + +The table below describes the requirements and fault tolerance levels for different operating modes: + +| Mode | Storage
volume multiplier | Minimum
number
of nodes | Fail
domain | Fail
realm | Number of
data centers | Number of
server racks | +| --- | --- | --- | --- | --- | --- | --- | +| `none`, no fault tolerance | 1 | 1 | Node | Node | 1 | 1 | +| `block-4-2`, can stand a failure of 2 racks | 1.5 | 8 (10 recommended) | Rack | Data center | 1 | 8 | +| `mirror-3-dc`, can stand a failure of a data center and 1 rack in one of the remaining data centers | 3 | 9 (12 recommended) | Rack | Data center | 3 | 3 in each data center | +| `block-4-2` *(reduced)*, can stand a failure of 1 rack | 1.5 | 10 | ½ a rack | Data center | 1 | 5 | +| `mirror-3-dc` *(reduced)*, can stand a failure of a data center and 1 server in one of the two other data centers | 3 | 12 | ½ a rack | Data center | 3 | 6 | +| `mirror-3-dc` *(3 nodes)*, can stand a failure of a single server, or a failure of a data center | 3 | 3 | Server | Data center | 3 | Doesn't matter | + +{% note info %} + +The storage volume multiplier specified above only applies to the fault tolerance factor. Other influencing factors (for example, [slot](glossary.md#slot) fragmentation and granularity) must be taken into account for storage size planning. {% endnote %} +When creating a [storage group](glossary.md#storage-group), which is a basic allocation unit for storage management, {{ ydb-short-name }} selects [VDisks](glossary.md#vdisk) that are located on [PDisks](glossary.md#pdisk) from different fail domains. For `block-4-2` mode, a storage group should be distributed across at least 8 fail domains, while for `mirror-3-dc` mode, it should be distributed across 3 fail realms, with at least 3 fail domains in each realm. + For information about how to set the {{ ydb-short-name }} cluster topology, see [{#T}](../deploy/configuration/config.md#domains-blob). + +### Reduced configurations {#reduced} + +If it is impossible to use the [recommended amount](#cluster-config) of hardware, you can divide servers within a single rack into two dummy fail domains. In this configuration, the failure of one rack results in the failure of two domains instead of just one. In such reduced configurations, {{ ydb-short-name }} will continue to operate if two domains fail. The minimum number of racks in a cluster is five for `block-4-2` mode and two per data center (e.g., six in total) for `mirror-3-dc` mode. + +The minimal fault-tolerant configuration of a {{ ydb-short-name }} cluster uses the `mirror-3-dc-3-nodes` operating mode, which requires only three servers. In this configuration, each server acts as both a fail domain and a fail realm, and the cluster can withstand the failure of only a single server. + +## Redundancy recovery {#rebuild} + +If a disk fails, {{ ydb-short-name }} can automatically reconfigure a storage group. Whether the disk failure is caused by the whole server failure or not is irrelevant in this context. Auto reconfiguration of storage groups reduces the risk of data loss in the event of a sequence of failures, provided these failures occur with sufficient time intervals to recover redundancy. By default, reconfiguration begins one hour after {{ ydb-short-name }} detects a failure. + +Disk group reconfiguration replaces the VDisk located on the failed hardware with a new VDisk, and the system tries to place it on operational hardware. The same rules apply as when creating a storage group: + +* The new VDisk is created in a fail domain that is different from any other VDisks in the group. +* In the `mirror-3-dc` mode, it is created within the same fail realm as the failed VDisk. + +To ensure reconfiguration is possible, a cluster should have free slots available for creating VDisks in different fail domains. When determining the number of slots to keep free, consider the risk of hardware failure, the time required to replicate data, and the time needed to replace the failed hardware. + +The disk group reconfiguration process increases the load on other VDisks in the group as well as on the network. The total data replication speed is limited on both the source and target VDisks to minimize the impact of redundancy recovery on system performance. + +The time required to restore redundancy depends on the amount of data and hardware performance. For example, replication on fast NVMe SSDs may take an hour, while it could take more than 24 hours on large HDDs. + +Disk group reconfiguration is limited or totally impossible when a cluster's hardware is distributed across the minimum required number of fail domains: + +* If an entire fail domain is down, reconfiguration becomes impractical, as a new VDisk can only be placed in the fail domain that is down. +* Reconfiguration only happens when part of a fail domain is down. However, the load previously handled by the failed hardware will be redistributed across the surviving hardware, remaining in the same fail domain. + +The load can be redistributed across all the hardware that is still running if the number of fail domains in a cluster exceeds the minimum amount required for creating storage groups by at least one. This means having 9 fail domains for `block-4-2` and 4 fail domains in each fail realm for `mirror-3-dc`, which is recommended. + +## Capacity and performance considerations {#capacity} + +The system can function with fail domains of any size. However, if there are few domains with varying numbers of disks, the number of storage groups that can be created will be limited. In such cases, hardware in overly large fail domains may be underutilized. If all hardware is fully utilized, significant differences in domain sizes may prevent reconfiguration. + +For example, consider a cluster in `block-4-2` mode with 15 racks. The first rack contains 20 servers, while the other 14 racks each contain 10 servers. To fully utilize the 20 servers from the first rack, {{ ydb-short-name }} will create groups that include 1 disk from this largest fail domain in each group. Consequently, if any other fail domain's hardware fails, the load cannot be redistributed to the hardware in the first rack. + +{{ ydb-short-name }} can group disk drives of different vendors, capacities, and speeds. The resulting characteristics of a group depend on the set of the worst characteristics of the hardware serving the group. Generally, the best results can be achieved by using homogeneous hardware. + +{% note info %} + +Hardware from the same batch is more likely to have similar defects and may fail simultaneously. It is essential to consider this when building large-scale {{ ydb-short-name }} clusters. + +{% endnote %} + +Therefore, the optimal initial hardware configurations for production {{ ydb-short-name }} clusters are as follows: + +* **A cluster hosted in one availability zone**: This setup uses the `block-4-2` mode and consists of nine or more racks, each with an identical number of servers. +* **A cluster hosted in three availability zones**: This setup uses the `mirror-3-dc` mode and is distributed across three data centers, with four or more racks in each, all containing an identical number of servers. + +## Database availability {#database-availability} + +A [database](glossary.md#database) within a {{ ydb-short-name }} cluster is available if both its storage and compute resources are operational: + +- All [storage groups](glossary.md#storage-group) allocated for the database must be operational, i.e., stay within the allowed level of failures. +- The compute resources of the currently available [database nodes](glossary.md#database-node) (primarily the amount of main memory) must be sufficient to start all the [tablets](glossary.md#tablet) managing objects like [tables](glossary.md#table) or [topics](glossary.md#topic) within the database and to handle user sessions. + +To survive an entire data center outage at the database level, assuming a cluster configured with the `mirror-3-dc` operating mode: + +- The [storage nodes](glossary.md#storage-node) need to have at least double the I/O bandwidth and disk capacity compared to what is required for normal operation. In the worst case, the load on the remaining nodes during the maximum allowed outage might triple, but that's only temporary until self-heal restores failed disks in operating data centers. +- The [database nodes](glossary.md#database-node) must be evenly distributed between all 3 data centers and include sufficient resources to handle the entire workload when running in just 2 of the 3 data centers. To achieve this, database nodes in each datacenter need at least 35% extra spare CPU and RAM resources when running normally without ongoing failures. If database nodes are typically utilized above this threshold, consider adding more of them or moving them to servers with more resources. + +## See also + +* [Documentation for DevOps Engineers](../devops/index.md) +* [Example cluster configuration files](https://github.com/ydb-platform/ydb/tree/main/ydb/deploy/yaml_config_examples/) diff --git a/ydb/docs/en/core/deploy/configuration/config.md b/ydb/docs/en/core/deploy/configuration/config.md index 9a8a8eb3415d..b649a1785dd5 100644 --- a/ydb/docs/en/core/deploy/configuration/config.md +++ b/ydb/docs/en/core/deploy/configuration/config.md @@ -601,90 +601,11 @@ blob_storage_config: For a configuration located in 3 availability zones, specify 3 rings. For a configuration within a single availability zone, specify exactly one ring. -## BlobStorage production configurations +## Enabling stable node names {#node-broker-config} -To ensure the required fault tolerance of {{ ydb-short-name }}, configure the [cluster disk subsystem](../../concepts/cluster/distributed_storage.md) properly: select the appropriate [fault tolerance mode](#fault-tolerance) and [hardware configuration](#requirements) for your cluster. - -### Fault tolerance modes {#fault-tolerance} - -We recommend using the following [fault tolerance modes](../../concepts/topology.md) for {{ ydb-short-name }} production installations: - -* `block-4-2`: For a cluster hosted in a single availability zone. -* `mirror-3-dc`: For a cluster hosted in three availability zones. - -A fail model of {{ ydb-short-name }} is based on concepts such as fail domain and fail realm. - -Fail domain - -: A set of hardware that may fail concurrently. - - For example, a fail domain includes disks of the same server (as all server disks may be unavailable if the server PSU or network controller is down). A fail domain also includes servers located in the same server rack (as the entire hardware in the rack may be unavailable if there is a power outage or some issue with the network hardware in the same rack). - - Any domain fail is handled automatically, without shutting down the system. - -Fail realm - -: A set of fail domains that may fail concurrently. - - An example of a fail realm is hardware located in the same data center that may fail as a result of a natural disaster. - -Usually a fail domain is a server rack, while a fail realm is a data center. - -When creating a [storage group](../../concepts/glossary.md#storage-groups), {{ ydb-short-name }} groups VDisks that are located on PDisks from different fail domains. For `block-4-2` mode, a PDisk should be distributed across at least 8 fail domains, and for `mirror-3-dc` mode, across 3 fail realms with at least 3 fail domains in each of them. - -### Hardware configuration {#requirements} - -If a disk fails, {{ ydb-short-name }} may automatically reconfigure a storage group so that, instead of the VDisk located on the failed hardware, a new VDisk is used that the system tries to place on hardware that is running while the group is being reconfigured. In this case, the same rule applies as when creating a group: a VDisk is created in a fail domain that is different from the fail domains of any other VDisk in this group (and in the same fail realm as that of the failed VDisk for `mirror-3-dc`). - -This causes some issues when a cluster's hardware is distributed across the minimum required amount of fail domains: - -* If the entire fail domain is down, reconfiguration no longer makes sense, since a new VDisk can only be located in the fail domain that is down. -* If part of a fail domain is down, reconfiguration is possible, but the load that was previously handled by the failed hardware will only be redistributed across hardware in the same fail domain. - -If the number of fail domains in a cluster exceeds the minimum amount required for creating storage groups at least by one (that is, 9 domains for `block-4-2` and 4 domains in each fail realm for `mirror-3-dc)`, in case some hardware fails, the load can be redistributed across all the hardware that is still running. - -The system can work with fail domains of any size. However, if there are few domains and a different number of disks in different domains, the number of storage groups that you can create will be limited. In this case, some hardware in fail domains that are too large may be underutilized. If the hardware is used in full, significant distortions in domain sizes may make reconfiguration impossible. - -For example, there are 15 racks in a cluster with `block-4-2` fault tolerance mode. The first of the 15 racks hosts 20 servers and the other 14 racks host 10 servers each. To fully utilize all the 20 servers from the first rack, {{ ydb-short-name }} will create groups so that 1 disk from this largest fail domain is used in each group. As a result, if any other fail domain's hardware is down, the load can't be distributed to the hardware in the first rack. - -{{ ydb-short-name }} can group disks of different vendors, capacity, and speed. The resulting characteristics of a group depend on a set of the worst characteristics of the hardware that is serving the group. Usually the best results can be achieved if you use same-type hardware. When creating large clusters, keep in mind that hardware from the same batch is more likely to have the same defect and fail simultaneously. - -Therefore, we recommend the following optimal hardware configurations for production installations: - -* **A cluster hosted in 1 availability zone**: It uses `block4-2` fault tolerance mode and consists of 9 or more racks with the same amount of identical servers in each rack. -* **A cluster hosted in 3 availability zones**: It uses `mirror3-dc` fault tolerance mode and is distributed across 3 data centers with 4 or more racks in each of them, the racks being equipped with the same amount of identical servers. - -See also [{#T}](#reduced). - -### Redundancy recovery {#rebuild} - -Auto reconfiguration of storage groups reduces the risk of data loss in the event of multiple failures that occur within intervals sufficient to recover the redundancy. By default, reconfiguration is done one hour after {{ ydb-short-name }} detects a failure. - -Once a group is reconfigured, a new VDisk is automatically populated with data to restore the required storage redundancy in the group. This increases the load on other VDisks in the group and the network. To reduce the impact of redundancy recovery on the system performance, the total data replication speed is limited both on the source and target VDisks. - -The time it takes to restore the redundancy depends on the amount of data and hardware performance. For example, replication on fast NVMe SSDs may take an hour, while on large HDDs more than 24 hours. To make reconfiguration possible in general, a cluster should have free slots for creating VDisks in different fail domains. When determining the number of slots to be kept free, factor in the risk of hardware failure, the time it takes to replicate data and replace the failed hardware. - -### Simplified hardware configurations {#reduced} - -If it's not possible to use the [recommended amount](#requirements) of hardware, you can divide servers within a single rack into two dummy fail domains. In this configuration, a failure of 1 rack means a failure of 2 domains and not a single one. In [both fault tolerance modes](#fault-tolerance), {{ ydb-short-name }} will keep running if 2 domains fail. If you use the configuration with dummy fail domains, the minimum number of racks in a cluster is 5 for `block-4-2` mode and 2 in each data center for `mirror-3-dc` mode. - -### Fault tolerance level {#reliability} - -The table below describes fault tolerance levels for different fault tolerance modes and hardware configurations of a {{ ydb-short-name }} cluster: - -Fault tolerance
mode | Fail
domain | Fail
realm | Number of
data centers | Number of
server racks | Fault tolerance
level -:--- | :---: | :---: | :---: | :---: | :--- -`block-4-2` | Rack | Data center | 1 | 9 or more | Can stand a failure of 2 racks -`block-4-2` | ½ a rack | Data center | 1 | 5 or more | Can stand a failure of 1 rack -`block-4-2` | Server | Data center | 1 | Doesn't matter | Can stand a failure of 2 servers -`mirror-3-dc` | Rack | Data center | 3 | 4 in each data center | Can stand a failure of a data center and 1 rack in one of the two other data centers -`mirror-3-dc` | Server | Data center | 3 | Doesn't matter | Can stand a failure of a data center and 1 server in one of the two other data centers - -## Node Broker Configuration {#node-broker-config} - -Node Broker is a system tablet that registers dynamic nodes in the {{ ydb-short-name }} cluster. - -Node broker gives names to dynamic nodes when they register. By default, a node name consists of the hostname and the port on which the node is running. +Node names are assigned through the Node Broker, which is a system tablet that registers dynamic nodes in the {{ ydb-short-name }} cluster. + +Node Broker assigns names to dynamic nodes when they register in the cluster. By default, a node name consists of the hostname and the port on which the node is running. In a dynamic environment where hostnames often change, such as in Kubernetes, using hostname and port leads to an uncontrollable increase in the number of unique node names. This is true even for a database with a handful of dynamic nodes. Such behavior may be undesirable for a time series monitoring system as the number of metrics grows uncontrollably. To solve this problem, the system administrator can set up *stable* node names. diff --git a/ydb/docs/ru/core/concepts/glossary.md b/ydb/docs/ru/core/concepts/glossary.md index 7f3162354a01..6c5d3ceb5ba0 100644 --- a/ydb/docs/ru/core/concepts/glossary.md +++ b/ydb/docs/ru/core/concepts/glossary.md @@ -68,6 +68,10 @@ Обычные группы хранения, которые не являются [статическими](#static-group), называются **динамическими группами** или **dynamic group**. Их называют динамическими, потому что они могут быть созданы и удалены на лету во время работы [кластера](#cluster). +### Пул хранения {#storage-pool} + +**Пул хранения** или **storage pool** — это набор устройств хранения данных со схожими характеристиками. Каждому пулу хранения присваивается уникальное имя в рамках кластера {{ ydb-short-name }}. Технически, каждый пул хранения состоит из множества физических дисков ([PDisk](#pdisk)). Каждая [группа хранения](#storage-group) создаётся в определённом пуле хранения, который определяет характеристики производительности группы хранения через выбор соответствующих устройств хранения. Обычно отдельные пулы хранения создают для устройств разных типов (например, NVMe, SSD и HDD) или для конкретных моделей этих устройств, обладающих различной ёмкостью и скоростью доступа. + ### Актор {#actor} [Модель акторов](https://ru.wikipedia.org/wiki/%D0%9C%D0%BE%D0%B4%D0%B5%D0%BB%D1%8C_%D0%B0%D0%BA%D1%82%D0%BE%D1%80%D0%BE%D0%B2) — это один из основных подходов к параллелизму выполнения, используемый в {{ ydb-short-name }}. В этой модели **акторы** или **actor** — это легковесные процессы в пользовательском пространстве, которые могут иметь и изменять своё частное состояние, но могут влиять друг на друга только косвенно через обмен сообщениями. {{ ydb-short-name }} имеет собственную реализацию этой модели, которая описана [ниже](#actor-implementation). @@ -364,7 +368,7 @@ **Слот** в {{ ydb-short-name }} может использоваться в двух контекстах: * **Слот** — это часть ресурсов сервера, выделенная для запуска одного [узла](#node) {{ ydb-short-name }}. Обычный размер слота — 10 процессорных ядер и 50 ГБ оперативной памяти. Слоты используются, если кластер {{ ydb-short-name }} развернут на серверах или виртуальных машинах с достаточными ресурсами для размещения нескольких слотов. -* **Слот VDisk** или **VSlot** — это доля PDisk, которая может быть выделена одному из [VDisk](#vdisk). +* **Слот VDisk** или **VSlot** — это доля [PDisk](#pdisk), которая может быть выделена одному из [VDisk](#vdisk). ### Хранилище состояния {#state-storage} @@ -423,7 +427,7 @@ **PDisk**, **physical disk** или **физический диск** — это компонент, который контролирует физический дисковый накопитель (блочное устройство). Другими словами, PDisk — это подсистема, которая реализует абстракцию, аналогичную специализированной файловой системе поверх блочных устройств (или файлов, эмулирующих блочное устройство для целей тестирования). PDisk обеспечивает контроль целостности данных (включая [кодирование с исправлением ошибок](#erasure-coding) групп секторов для восстановления данных на отдельных повреждённых секторах, контроль целостности с помощью контрольных сумм), прозрачное шифрование всех данных на диске и транзакционные гарантии дисковых операций (подтверждение записи строго после `fsync`). PDisk содержит планировщик, который обеспечивает совместное использование пропускной способности устройства между несколькими клиентами ([VDisk](#vdisk)). PDisk делит блочное устройство на блоки, называемые [слотами](#slot) (размером около 128 мегабайт; разрешены и меньшие блоки). В каждый момент времени не более одного VDisk может владеть каждым слотом. PDisk также поддерживает журнал восстановления, общий для записей службы PDisk и всех VDisk. - + #### VDisk {#vdisk} **VDisk**, **virtual disk** или **виртуальный диск** — это компонент, который реализует хранение данных [распределённого хранилища](#distributed-storage) [LogoBlob](#logoblob) на [PDisk](#pdisk). VDisk хранит все свои данные на PDisk. Один VDisk соответствует одному PDisk, но обычно несколько VDisk связаны с одним PDisk. В отличие от PDisk, который скрывает блоки и журналы за собой, VDisk предоставляет интерфейс на уровне LogoBlob и [LogoBlobID](#logoblobid), например запись LogoBlob, чтение данных LogoBlobID и удаление набора LogoBlob с помощью специальной команды. VDisk является членом [группы хранения](#storage-group). Сам VDisk является локальным, но многие VDisk в данной группе обеспечивают надёжное хранение данных. VDisk в группе синхронизируют данные друг с другом и реплицируют данные в случае потерь. Набор VDisk в группе хранения образует распределённый RAID. @@ -456,11 +460,17 @@ PDisk содержит планировщик, который обеспечив #### Область отказа {#fail-realm} -[Группа хранения](#storage-group) — это набор [VDisk](#vdisk), объединённых в одну или несколько **областей отказа** или **fail realm**. Область отказа содержит один или несколько [доменов отказа](#fail-domain). Коррелированный отказ двух VDisk в одной и той же области отказа более вероятен, чем отказ двух VDisk из разных областей отказа. Область отказа обычно соответствует физическому понятию датацентра или зоны доступности (AZ). +**Область отказа** или **fail realm** — это набор [доменов отказа](#fail-domain), которые могут выйти из строя одновременно по общей причине. Коррелированный сбой двух [VDisks](#vdisk) в одной и той же области отказа более вероятен, чем сбой двух VDisk из разных областей отказа. + +Примером области отказа является набор оборудования, размещённый в одном [дата-центре, или зоне доступности](#regions-az), который может выйти из строя целиком вследствие природной катастрофы, масштабного отключения электропитания или другого подобного события. #### Домен отказа {#fail-domain} -[Fail realm](#fail-realm) содержит одну или несколько **доменов отказа** или **fail domains**. Коррелированная ошибка двух [VDisks](#vdisk) в пределах одного Fail domain более вероятна, чем ошибка двух VDisks из разных Fail domains. Fail domain обычно соответствует понятию [серверной стойки](#rack). +**Домен отказа** или **fail domain** — это набор оборудования, который может выйти из строя одновременно. Коррелированный сбой двух [VDisk](#vdisk) в пределах одного домена отказа более вероятен, чем сбой двух VDisk из разных доменов отказа. В случае разных доменов отказа вероятность одновременного сбоя также зависит от того, принадлежат ли рассматриваемые домены одной области отказа или разным. + +Примером домена отказа является набор дисков, подключённых к одному серверу, поскольку все диски конкретного сервера могут оказаться недоступными при отказе блока питания сервера или сетевого контроллера. Обычно к общему домену отказа относят все серверы, размещённые в одной [серверной стойке](#rack), поскольку проблемы с электропитанием или сетевыми подключениями на уровне стойки приводят к недоступности всего размещаемого в ней оборудования. Таким образом, типичный домен отказа соответствует серверной стойке (если [cluster](#cluster) настроен с учётом топологии размещения оборудования в стойках) или отдельному серверу. + +Сбои уровня домена отказа автоматически обрабатываются {{ ydb-short-name }} без остановки кластера. #### Канал распределённого хранилища {#channel} diff --git a/ydb/docs/ru/core/concepts/topology.md b/ydb/docs/ru/core/concepts/topology.md index 32ffa0b24dff..0a3d85d0b7c3 100644 --- a/ydb/docs/ru/core/concepts/topology.md +++ b/ydb/docs/ru/core/concepts/topology.md @@ -1,25 +1,103 @@ # Топология кластера {{ ydb-short-name }} -Кластер {{ ydb-short-name }} состоит из статических и динамических узлов: +Кластер {{ ydb-short-name }} состоит из [узлов хранения](glossary.md#storage-node) и [узлов баз данных](glossary.md#database-node). Работоспособность обоих типов узлов важна для обеспечения доступности баз данных {{ ydb-short-name }}: узлы баз данных реализуют логику управления данными, а узлы хранения обеспечивают их сохранность. При этом наибольшее влияние на отказоустойчивость кластера и его способность обеспечивать надёжное хранение данных оказывает подсистема [распределённого хранилища](glossary.md#distributed-storage), состоящая из набора узлов хранения. При развёртывании кластера необходимо выбрать [режим работы](#cluster-config) распределённого хранилища в соответствии с ожидаемой нагрузкой и требованиями к [доступности баз данных](#database-availability). -* статические узлы обеспечивают хранение данных, реализуя одну из поддерживаемых схем избыточности в зависимости от установленного режима работы; -* динамические узлы обеспечивают выполнение запросов, координацию транзакций и другие функции управления данными. +## Режимы работы кластера {#cluster-config} -Топология кластера определяется требованиями к отказоустойчивости. Доступны следующие режимы работы: +Топология кластера строится в соответствии с его режимом работы, который должен быть выбран на основе требований к уровню отказоустойчивости. Модель отказа, используемая в {{ ydb-short-name }}, основана на концепциях [домена отказа](glossary.md#fail-domain) и [области отказа](glossary.md#fail-realm). {{ ydb-short-name }} предоставляет следующие режимы работы распределённого хранилища: -Режим | Множитель
объема хранения | Минимальное
количество
узлов | Описание ---- | --- | --- | --- -`none` | 1 | 1 | Избыточность отсутствует.
Любой сбой оборудования приводит к недоступности пула хранения.
Режим рекомендуется использовать только для функционального тестирования. -`block-4-2` | 1,5 | 8 | Применяется [Erasure coding](https://ru.wikipedia.org/wiki/Стирающий_код) с двумя блоками избыточности, добавляемыми к четырем блокам исходных данных. Узлы хранилища размещаются в не менее чем 8 доменах отказа (обычно стойках).
Пул хранения доступен при потере любых двух доменов, продолжая запись всех 6 частей данных в оставшихся доменах.
Режим рекомендуется для пулов хранения в пределах одной зоны доступности (обычно центра обработки данных). -`mirror-3-dc` | 3 | 9 | Данные реплицируются в 3 зоны доступности, использующие 3 домена отказа (обычно стойки) внутри каждой зоны.
Пул хранения доступен при сбое одной зоны доступности и одного домена отказа в оставшихся зонах.
Режим рекомендуется для мультидатацентровых инсталляций. -`mirror-3dc-3-nodes` | 3 | 3 | Является упрощенным вариантом `mirror-3dc`. Для данного режима необходимо минимум 3 сервера по 3 диска в каждом. Для обеспечения наибольшей отказоустойчивости каждый сервер должен находиться в независимом датацентре.
Работоспособность в данном режиме сохраняется при выходе из строя не более 1 узла.
Режим рекомендуется использовать только для функционального тестирования. +- `none`. Избыточность отсутствует. Любой отказ приводит к временной недоступности или потере всех или части хранимых данных. Этот режим рекомендуется использовать только для разработки приложений или проведения функциональных тестов. +- `block-4-2`. Избыточность по схеме [стирающего кода](https://ru.wikipedia.org/wiki/Стирающий_код). На каждые 4 блока исходных данных формируются 2 дополнительных блока с кодами избыточности. Узлы хранения размещаются как минимум в 8 доменах отказа (обычно серверных стойках). [Пул хранения](glossary.md#storage-pool) остаётся полностью доступным при недоступности двух любых доменов отказа, продолжая записывать все 6 частей данных в оставшихся доменах. Этот режим рекомендуется для кластеров, размещённых в одном дата-центре или зоне доступности. +- `mirror-3-dc`. Данные реплицируются между 3 зонами доступности (обычно разными дата-центрами) с использованием как минимум 3 доменов отказа (обычно серверных стоек) в каждой зоне доступности. Кластер {{ ydb-short-name }} остаётся доступным при выходе из строя любой зоны доступности; кроме того, дополнительно может выйти из строя ещё один домен отказа в любой из 2 работоспособных зон доступности без прекращения работы кластера. Этот режим рекомендуется для кластеров с высокими требованиями к отказоустойчивости, размещённых в нескольких дата-центрах. +- `mirror-3-dc-3-nodes`. Упрощённый вариант режима `mirror-3-dc`. В этом режиме требуется не менее 3 серверов, каждый из которых должен содержать как минимум 3 диска для хранения данных. Для обеспечения гарантий отказоустойчивости каждый сервер должен размещаться в своём независимом дата-центре. Доступность кластера обеспечивается при условии отказа не более чем одного сервера. Этот режим рекомендуется для функционального тестирования или прототипирования. {% note info %} -Под выходом из строя узла подразумевается как полная, так и частичная его недоступность, например выход из строя одного диска на узле. +Под выходом из строя сервера подразумевается как полная, так и частичная его недоступность, например выход из строя одного диска на сервере. -Приведенный выше множитель объема хранения относится только к фактору обеспечения отказоустойчивости. Для планирования размера хранилища необходимо учитывать другие влияющие факторы (например, фрагментацию и гранулярность слотов). +{% endnote %} + +Приведённая ниже таблица описывает требования и обеспечиваемый уровень устойчивости к отказам для различных режимов работы кластера: + +| Режим | Множитель
объёма
хранения | Минимальное
количество
узлов | Домен
отказа | Область
отказа | Кол-во
дата-центров | Кол-во
серверных
стоек | +| --- | --- | --- | --- | --- | --- | --- | +| `none`, избыточность отсутствует | 1 | 1 | Узел | Узел | 1 | 1 | +| `block-4-2`, переживает отказ 2 стоек | 1.5 | 8 (10 рекомендовано) | Стойка | Дата-центр | 1 | 8 | +| `mirror-3-dc`, переживает отказ дата-центра и ещё 1 стойки в оставшихся дата-центрах | 3 | 9 (12 рекомендовано) | Стойка | Дата-центр | 3 | 3 в каждом дата-центре | +| `block-4-2` *(упрощённый)*, переживает отказ 1 стойки | 1.5 | 10 | ½ стойки | Дата-центр | 1 | 5 | +| `mirror-3-dc` *(упрощённый)*, переживает отказ дата-центра и ещё 1 сервера в оставшихся дата-центрах | 3 | 12 | ½ стойки | Дата-центр | 3 | 6 | +| `mirror-3-dc` *(3 узла)*, переживает отказ 1 узла или 1 дата-центра | 3 | 3 | Сервер | Дата-центр | 3 | Не важно | + +{% note info %} + +Приведённый выше множитель объёма хранения относится только к фактору обеспечения отказоустойчивости. Для планирования размера хранилища необходимо учитывать и другие влияющие на него факторы, такие как фрагментация и гранулярность [слотов](glossary.md#slot). + +{% endnote %} + +Базовой единицей выделения ресурсов хранения данных в кластере {{ ydb-short-name }} является [группа хранения](glossary.md#storage-group). При создании группы хранения {{ ydb-short-name }} её части — [VDisk-и](glossary.md#vdisk) — размещаются на [физических дисках](glossary.md#pdisk), принадлежащих разным доменам отказа. Для режима `block-4-2` каждая группа хранения распределена между 8 доменами отказа, а в режиме `mirror-3-dc` группа хранения распределяется между 3 областями отказа, в каждой из которых используются 3 домена отказа. + +О том, как задать топологию кластера {{ ydb-short-name }}, читайте в разделе [{#T}](../deploy/configuration/config.md#domains-blob). + +### Упрощённые конфигурации {#reduced} + +В случаях, когда невозможно использовать [рекомендованное количество](#cluster-config) оборудования, можно разделить серверы одной стойки на 2 фиктивных домена отказа. В такой конфигурации отказ одной стойки будет означать отказ не одного, а сразу двух доменов. При использовании таких упрощённых конфигураций {{ ydb-short-name }} сохраняет работоспособность при отказе сразу двух доменов. Минимальное количество стоек в кластере для режима `block-4-2` составляет 5, а для `mirror-3-dc` — по 2 в каждом дата-центре (т.е. суммарно 6 стоек). + +В минимальной отказоустойчивой конфигурации {{ ydb-short-name }} используется режим `mirror-3-dc-3-nodes`, и кластер состоит из 3 серверов. В такой конфигурации каждый сервер одновременно является доменом отказа и областью отказа, и кластер может выдержать сбой только одного сервера. + +## Восстановление избыточности {#rebuild} + +В случае отказа физического диска {{ ydb-short-name }} может автоматически реконфигурировать связанные с ним группы хранения. При этом не важно, связана ли недоступность диска с выходом из строя сервера целиком или нет. Автоматическая реконфигурация групп хранения снижает вероятность потери данных при возникновении последовательности множественных отказов, при условии, что между отказами проходит достаточно времени для завершения восстановления избыточности. По умолчанию реконфигурация начинается через 1 час после того, как {{ ydb-short-name }} выявляет отказ. + +Реконфигурация дисковой группы заменяет VDisk, размещённый на отказавшем оборудовании, новым VDisk, который система старается разместить на работоспособном оборудовании. Действуют те же правила, что и при создании новой группы хранения: + +* новый VDisk создаётся в домене отказа, отличающемся от доменов отказа всех остальных VDisk в группе; +* в режиме `mirror-3-dc` новый VDisk размещается в той же области отказа, что и отказавший VDisk. + +Для того чтобы реконфигурация была возможной, в кластере должны быть свободные слоты для создания VDisk в разных доменах отказа. При расчёте количества оставляемых свободными слотов следует учитывать вероятность отказа оборудования, длительность репликации, а также время, необходимое на замену отказавшего оборудования. + +Процесс реконфигурации дисковой группы создаёт повышенную нагрузку на остальные VDisk группы и на сеть. Чтобы уменьшить влияние восстановления избыточности на производительность системы, суммарная скорость репликации данных ограничивается как на стороне VDisk-источника, так и на стороне VDisk-получателя. + +Время восстановления избыточности зависит от объёма данных и производительности оборудования. Например, репликация на быстрых NVMe SSD может завершиться за час, а на больших HDD репликация может длиться более суток. + +Реконфигурация дисковых групп может быть ограничена или оказаться полностью невозможной, если оборудование кластера состоит из минимально необходимого количества доменов отказа: + +* при отказе всего домена реконфигурация теряет смысл, так как новый VDisk может быть размещён только в уже отказавшем домене отказа; +* при отказе части домена реконфигурация возможна, но нагрузка, ранее обрабатываемая отказавшим оборудованием, будет перераспределена только по оставшемуся оборудованию в том же домене отказа. + +Если в кластере доступен хотя бы на один домен отказа больше, чем минимально необходимо для создания групп хранения (9 доменов для `block-4-2` и по 4 домена в каждой области отказа для `mirror-3-dc`), то при отказе части оборудования нагрузка может быть перераспределена по всему оставшемуся в строю оборудованию. + +## Доступный объём ресурсов и производительность {#capacity} + +Система может работать с доменами отказа любого размера, однако при малом количестве доменов и неодинаковом количестве дисков в разных доменах количество групп хранения, которые можно будет создать, окажется ограничено. В таких условиях часть оборудования в слишком больших доменах отказа может оказаться недоиспользованной. В случае полной утилизации оборудования значительный перекос в размерах доменов может сделать невозможной реконфигурацию. + +Например, в кластере с режимом отказоустойчивости `block-4-2` имеется 15 стоек. В первой из них расположено 20 серверов, а в остальных 14 стойках — по 10 серверов. Для полноценной утилизации всех 20 серверов из первой стойки {{ ydb-short-name }} будет создавать группы так, что в каждой из них будет участвовать 1 диск из этого самого большого домена отказа. В результате, при выходе из строя оборудования в любом другом домене отказа нагрузка не сможет быть распределена на оборудование в первой стойке. + +{{ ydb-short-name }} может объединять в группу хранения диски разных производителей, с различной ёмкостью и скоростью. Результирующие характеристики всей группы будут ограничены наихудшими характеристиками оборудования, которое в неё входит. Обычно лучших результатов удаётся добиться при использовании однотипного оборудования. + +{% note info %} + +При создании больших кластеров следует учитывать, что оборудование из одной партии с большей вероятностью может иметь одинаковый дефект и отказать одновременно. {% endnote %} -О том, как задать топологию кластера {{ ydb-short-name }} читайте в разделе [{#T}](../deploy/configuration/config.md#domains-blob). +Таким образом, в качестве оптимальных аппаратных конфигураций кластеров {{ ydb-short-name }} для промышленного применения рекомендуются следующие: + +* **Кластер в одной зоне доступности**: использует режим отказоустойчивости `block-4-2` и состоит из 9 или более стоек с равным количеством одинаковых серверов в каждой. +* **Кластер в трёх зонах доступности**: использует режим отказоустойчивости `mirror-3-dc` и расположен в трёх дата-центрах с четырьмя или более стойками в каждом. Стойки укомплектованы равным количеством одинаковых серверов. + +## Обеспечение доступности баз данных {#database-availability} + +[База данных](glossary.md#database) в кластере {{ ydb-short-name }} доступна, если её хранилище и вычислительные ресурсы находятся в рабочем состоянии: + +- Все выделенные базе данных [группы хранения](glossary.md#storage-group) должны быть доступны, что означает соблюдение допустимого уровня отказов для каждой группы. +- Вычислительные ресурсы доступных в данный момент [узлов базы данных](glossary.md#database-node) (в первую очередь объём оперативной памяти) должны быть достаточны для запуска всех [таблеток](glossary.md#tablet), управляющих пользовательскими объектами, такими как [таблицы](glossary.md#table) или [топики](glossary.md#topic), и для обработки пользовательских сессий. + +Чтобы база данных могла пережить отказ одного из дата-центров в кластере, использующем режим работы `mirror-3-dc`, должны быть выполнены следующие условия: + +- [Узлы хранения](glossary.md#storage-node) должны обеспечивать как минимум двойной запас пропускной способности ввода-вывода и дисковой ёмкости по сравнению с необходимыми для работы в обычных условиях. В худшем случае нагрузка на сохранившиеся узлы при длительном отключении одного из дата-центров может утроиться, но лишь на ограниченный период времени — до завершения восстановления ставших недоступными дисков в оставшихся дата-центрах. +- [Узлы базы данных](glossary.md#database-node) должны быть равномерно распределены между всеми тремя дата-центрами и содержать достаточно ресурсов для обработки полной нагрузки при сохранении двух из трёх дата-центров. Это означает наличие как минимум 35% запаса по ресурсам CPU и оперативной памяти в обычных условиях, то есть при отсутствии каких-либо отказов. Если узлы базы данных обычно нагружены более чем на 65%, следует рассмотреть возможность добавления дополнительных узлов либо увеличения вычислительной мощности каждого узла. + +## Дополнительная информация + +* [Документация для DevOps-инженеров](../devops/index.md) +* [Примеры конфигурационных файлов кластера](https://github.com/ydb-platform/ydb/tree/main/ydb/deploy/yaml_config_examples/) diff --git a/ydb/docs/ru/core/deploy/configuration/config.md b/ydb/docs/ru/core/deploy/configuration/config.md index 8d38bda60444..c7fe4a671632 100644 --- a/ydb/docs/ru/core/deploy/configuration/config.md +++ b/ydb/docs/ru/core/deploy/configuration/config.md @@ -376,7 +376,7 @@ domains_config: {% endlist %} -## Акторная система {#actor-system} +## Настройка акторной системы {#actor-system} Основной потребитель CPU — акторная система. Все акторы, в зависимости от своего типа, выполняются в одном из пулов (параметр `name`). Конфигурирование заключается в распределении ядер процессора узла по пулам акторной системы. При выделении процессорных ядер пулам нужно учитывать, что PDisk и gRPC API работают вне акторной системы и требуют отдельных ресурсов. @@ -642,92 +642,13 @@ auth_config: `ldap_authentication_domain` | Идентификатор, прикрепляемый к имени пользователя, позволяющий отличать пользователей из LDAP каталога от пользователей аутентифицируемых с помощью других провайдеров. Значение по умолчанию `ldap` `refresh_time` | Определяет время, когда будет попытка обновить информацию о пользователе. Конкретное время обновления будет лежать в интервале от `refresh_time/2` до `refresh_time` -## Промышленные конфигурации BlobStorage +## Настройка стабильных имен узлов кластера {#node-broker-config} -Для обеспечения необходимой отказоустойчивости {{ ydb-short-name }} нужно правильно сконфигурировать [дисковую подсистему кластера](../../concepts/cluster/distributed_storage.md): выбрать подходящий [режим отказоустойчивости](#fault-tolerance) и [аппаратную конфигурацию](#requirements) кластера. +Присвоение имен узлам осуществляет Node Broker — системная таблетка, которая отвечает за регистрацию динамических узлов в кластере {{ ydb-short-name }}. -### Режимы отказоустойчивости {#fault-tolerance} +Node Broker присваивает имена динамическим узлам при их регистрации в кластере. По умолчанию имя узла состоит из имени хоста и номера порта, на котором работает узел. -Для промышленных инсталляций {{ ydb-short-name }} рекомендуется использовать следующие [режимы отказоустойчивости](../concepts /topology.md): - -* `block-4-2` — если кластер расположен в одной зоне доступности. -* `mirror-3-dc` — если кластер расположен в трех зонах доступности. - -В модели отказа {{ ydb-short-name }} различаются понятия домена отказа и области отказа. - -Домен отказа (fail domain) - -: Набор оборудования, для которого вероятен одновременный отказ. - - Например, доменом отказа можно считать диски одного сервера (в связи с вероятной недоступностью всех дисков сервера при отказе блока питания или сетевого контроллера сервера). Также доменом отказа можно считать серверы, расположенные в одной серверной стойке (в связи с вероятной недоступностью всего оборудования в стойке при проблемах с питанием или сетевым оборудованием, расположенным в той же стойке). - - Обработка любых отказов домена осуществляется автоматически, без остановки работы системы. - -Область отказа (fail realm) - -: Набор доменов отказа, для которого вероятен одновременный отказ. - - Например, областью отказа можно считать оборудование, расположенное в одном центре обработки данных (ЦОД), который может выйти из строя в результате стихийного бедствия. - -Обычно под доменом отказа подразумевается серверная стойка, а под областью отказа – ЦОД. - -При создании [групп хранения](../../concepts/glossary.md#storage-groups) {{ ydb-short-name }} объединяет в группы VDisk, расположенные на PDisk из разных доменов отказа. Таким образом, для режима `block-4-2` необходимо распределение PDisk по не менее чем 8 доменам отказа, а для режима `mirror-3-dc` — по 3 областям отказа с не менее чем 3 доменами отказа в каждой. - -### Аппаратная конфигурация {#requirements} - -При отказе диска {{ ydb-short-name }} может автоматически переконфигурировать группу хранения так, чтобы вместо расположенного на отказавшем оборудовании VDisk использовался новый VDisk, который система пытается расположить на работающем в момент реконфигурации оборудовании. При этом соблюдается то же правило, что и при создании группы — VDisk создается в домене отказа, отличном от доменов отказа всех остальных VDisk этой группы (и в той же области отказа, что и отказавший VDisk для `mirror-3-dc`). - -Это вызывает ряд проблем в случе, когда оборудование кластера распределено по минимально необходимому количеству доменов отказа: - -* при отказе всего домена реконфигурация теряет смысл т.к. новый VDisk может быть расположен только в отказавшем домене отказа; -* при отказе части домена реконфигурация возможна, но при этом нагрузка, которая раньше обрабатывалась отказавшим оборудованием будет перераспределена только по оборудованию в том же домене отказа. - -Если же в кластере доступно хотя бы на 1 домен отказа больше, чем минимально необходимо для создания групп хранения (9 доменов для `block-4-2` и по 4 домена в каждой области отказа для `mirror-3-dc)`, то при отказе части оборудования нагрузка может быть перераспределена по всему оставшемуся в строю оборудованию. - -Система может работать с доменами отказа любого размера, однако при малом количестве доменов и неодинаковом количестве дисков в разных доменах количество групп хранения, которые возможно будет создать, окажется ограничено. При этом часть оборудования в слишком больших доменах отказа может оказаться недоутилизированной. В случае полной утилизации оборудования существенный перекос в размерах доменов может сделать невозможной реконфигурацию. - -Например, в кластере с режимом отказоустойчивости `block-4-2` имеется 15 стоек. В первой из 15 стоек расположено 20 серверов, а в остальных 14 стойках — по 10. Для полноценной утилизации всех 20 серверов из первой стойки {{ ydb-short-name }} будет создавать группы так, что в каждой из них будет участвовать 1 диск из этого самого большого домена отказа. В результате при выходе из строя оборудования в любом другом домене отказа нагрузка не сможет быть распределена на оборудование в первой стойке. - -{{ ydb-short-name }} может объединять в группу диски разных производителей, различной емкости и скорости. Результирующие характеристики группы определяются набором наихудших характеристик оборудования, которое ее обслуживает. Обычно лучших результатов удается добиться при использовании однотипного оборудования. При создании больших кластеров следует учитывать и то, что оборудование из одной партии с большей вероятностью может иметь одинаковый дефект и отказать одновременно. - -Таким образом, в качестве оптимальных аппаратных конфигураций для промышленного применения рекомендуются следующие: - -* **Кластер в 1 зоне доступности** — использует режим отказоустойчивости `block4-2` и состоит из 9 или более стоек с равным количеством одинаковых серверов в каждой. -* **кластер в 3 зонах доступности** — использует режим отказоустойчивости `mirror3-dc` и расположен в 3 ЦОД с 4 или более стойками в каждом, стойки укомплектованы равным количеством одинаковых серверов. - -Смотрите также [{#T}](#reduced). - -### Восстановление избыточности {#rebuild} - -Автоматическая реконфигурация групп хранения снижает вероятность потери данных при возникновении множественных отказов, возникающих с достаточными для завершения восстановления избыточности интервалами. По умолчанию реконфигурация происходит через 1 час после того, как {{ ydb-short-name }} выявляет отказ. - -После реконфигурации новый VDisk автоматически наполняется данными для восстановления требуемой избыточности хранения в группе. При этом возникает повышенная нагрузка на остальные VDisk группы и на сеть. Для снижения влияния восстановления избыточности на производительность системы суммарная скорость репликации данных ограничивается как на стороне VDisk-источника так и на стороне VDisk-получателя. - -Время восстановления избыточности при этом зависит от объема данных и от производительности оборудования. Например, репликация на быстрых NVMe SSD может завершиться за час, а на больших HDD репликация может длиться более суток. Для того, чтобы реконфигурация в принципе была возможной, в кластере должны быть свободные слоты для создания VDisk в разных доменах отказа. При расчете количества оставляемых свободными слотов следует учитывать вероятность отказа оборудования, длительность репликации, время, необходимое на замену отказавшего оборудования. - -### Упрощенные аппаратные конфигурации {#reduced} - -В случаях, когда невозможно использовать [рекомендованное количество](#requirements) оборудования, можно разделить серверы одной стойки на 2 фиктивных домена отказа. В такой конфигурации отказ 1 стойки будет означать отказ не 1, а сразу 2 доменов. В [обоих режимах](#fault-tolerance) отказоустойчивости {{ ydb-short-name }} сохранит работоспособность при отказе 2 доменов. При использовании конфигурации с фиктивными доменами отказа минимальное количество стоек в кластере для режима `block-4-2` составляет 5, для `mirror-3-dc` — по 2 в каждом ЦОД. - -### Уровень отказоустойчивости {#reliability} - -В таблице приведены уровни отказоустойчивости для различных режимов отказоустойчивости и аппаратных конфигураций кластера {{ ydb-short-name }}: - -Режим
отказоустойчивости | Домен
отказа | Область
отказа | Количество
ЦОД | Количество
серверных стоек | Уровень
отказоустойчивости -:--- | :---: | :---: | :---: | :---: | :--- -`block-4-2` | Стойка | ЦОД | 1 | 9 или более | Переживает отказ 2 стоек -`block-4-2` | ½ стойки | ЦОД | 1 | 5 или более | Переживает отказ 1 стойки -`block-4-2` | Сервер | ЦОД | 1 | Не важно | Переживает отказ 2 серверов -`mirror-3-dc` | Стойка | ЦОД | 3 | По 4 в каждом ЦОД | Переживает отказ ЦОД и 1 стойки в одном из двух других ЦОД -`mirror-3-dc` | Сервер | ЦОД | 3 | Не важно | Переживает отказ ЦОД и 1 сервера в одном из двух других ЦОД - -## Конфигурация Node Broker {#node-broker-config} - -Node Broker — это системная таблетка, которая отвечает за регистрацию динамических узлов в кластере {{ ydb-short-name }}. - -Node Broker присваивает имена динамическим узлам при их регистрации. По умолчанию, имя узла состоит из имени хоста и порта, на котором работает узел. - -В динамической среде, где имена хостов часто меняются, например в Kubernetes, использование имени хоста и порта приводит к неконтролируемому росту количества уникальных имен узлов. Даже для базы данных из единиц динамических узлов. Подобное поведение может быть нежелательным для системы time series мониторинга — количество метрик растет некотролируемо. Для решения этой проблемы администратор системы может настроить *стабильные* имена узлов. +В динамической среде, где имена хостов часто меняются, например в Kubernetes, использование имени хоста и порта приводит к неконтролируемому росту количества уникальных имен узлов, даже для базы данных с небольшим количеством динамических узлов. Подобное поведение может быть нежелательным для системы time series мониторинга — количество метрик растет некотролируемо. Для решения этой проблемы администратор системы может настроить *стабильные* имена узлов. Стабильное имя идентифицирует узел в рамках тенанта. Оно состоит из префикса и порядкового номера узла в рамках тенанта. Если динамический узел был выключен, то по истечении таймаута его стабильное имя может быть занято новым динамическим узлом, обслуживающим того же тенанта. From 6ae083c1209992fb167717c2a9f582f1dd3d8ba8 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Thu, 22 Aug 2024 14:18:36 +0800 Subject: [PATCH 030/261] [docs] update create_table.md (#8071) --- ydb/docs/en/core/postgresql/statements/create_table.md | 4 ++-- ydb/docs/ru/core/postgresql/statements/create_table.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ydb/docs/en/core/postgresql/statements/create_table.md b/ydb/docs/en/core/postgresql/statements/create_table.md index 6da1f6b626b5..7a222dc0e469 100644 --- a/ydb/docs/en/core/postgresql/statements/create_table.md +++ b/ydb/docs/en/core/postgresql/statements/create_table.md @@ -11,7 +11,7 @@ The `CREATE TABLE` statement is used to create an empty table in the current dat When creating a table, you can specify: -1. **Table Type**: {% include [temp-table-description.md](../../_includes/temp-table-description.md) %} +1. **Table Type**: `TEMPORARY` / `TEMP` – a temporary table that is automatically deleted at the end of the session. If this parameter is not set (left empty), a permanent table is created. Any indexes created on a temporary table will also be deleted at the end of the session, which means that they are temporary as well. A temporary table and a permanent table with the same name are allowed, in which case a temporary table will be selected. 2. **Table Name**: `` – you can use English letters in lowercase, numbers, underscores and dollar signs ($). For example, the table name "People" will be stored as "people". For more information, see [Identifiers and Key Words](https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS). 3. **Column Name**: `` – the same naming rules apply as for table names. 4. **Data Type**: `` – [standard PostgreSQL data types](https://www.postgresql.org/docs/14/datatype.html) are specified. @@ -62,4 +62,4 @@ The temporary table is defined using the `TEMPORARY` or `TEMP` keywords. In this example, the "name" and "lastname" columns use sorting with `en_US` localization. -{% include [../_includes/alert_locks](../_includes/alert_locks.md) %} \ No newline at end of file +{% include [../_includes/alert_locks](../_includes/alert_locks.md) %} diff --git a/ydb/docs/ru/core/postgresql/statements/create_table.md b/ydb/docs/ru/core/postgresql/statements/create_table.md index 5f3f9f7c71bb..1eda161c3b8e 100644 --- a/ydb/docs/ru/core/postgresql/statements/create_table.md +++ b/ydb/docs/ru/core/postgresql/statements/create_table.md @@ -15,7 +15,7 @@ CREATE [TEMPORARY | TEMP] TABLE
( ); ``` При создании таблицы можно задать: -1. **Тип таблицы**: {% include [temp-table-description.md](../../_includes/temp-table-description.md) %} +1. **Тип таблицы**: `TEMPORARY` / `TEMP` – временная таблица, которая автоматически удаляется при завершении сессии. Если параметр не задан (оставлен пустым) – создается постоянная таблица. Любые индексы, созданные во временных таблицах, также будут удалены при завершении сессиии, следовательно они тоже являются временными. Допускается существование временной таблицы и постоянной таблицы с одинаковым именем, в этом случае будет выбрана временная таблица. 2. **Имя таблицы**: `
` – можно использовать английские буквы в нижнем регистре, цифры, нижнее подчёркивание и знак доллара ($). Например, название таблицы "People" будет сохранено как "people"; 3. **Имя столбца/колонки**: – действую такие же правила нейминга как и для имен таблиц; 4. **Тип данных**: – указываются [стандартные типы](https://www.postgresql.org/docs/current/datatype.html) данных PostgreSQL; @@ -117,4 +117,4 @@ CREATE TABLE people ( В этом примере колонки "name" и "lastname" используют сортировку с `en_US` локализацией. -{% include [../_includes/alert_locks.md](../_includes/alert_locks.md) %} \ No newline at end of file +{% include [../_includes/alert_locks.md](../_includes/alert_locks.md) %} From ea6b219574f9a1ad4c93bfbbac853bc07903ede3 Mon Sep 17 00:00:00 2001 From: Semyon Date: Thu, 22 Aug 2024 10:48:52 +0300 Subject: [PATCH 031/261] Add unit-test for data eviction in tiering (#7971) --- ydb/core/kqp/ut/common/columnshard.cpp | 2 +- ydb/core/kqp/ut/olap/tiering_ut.cpp | 97 ++++++++++++++++++++++++++ ydb/core/kqp/ut/olap/ya.make | 1 + 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 ydb/core/kqp/ut/olap/tiering_ut.cpp diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index 267c4103adef..dd9314d90643 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -22,7 +22,7 @@ namespace NKqp { } SecretableSecretKey: { Value: { - Data: "secretSecretKey" + Data: "fakeSecret" } } } diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp new file mode 100644 index 000000000000..f10684a2af30 --- /dev/null +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -0,0 +1,97 @@ +#include "helpers/get_value.h" +#include "helpers/local.h" +#include "helpers/query_executor.h" +#include "helpers/typed_local.h" +#include "helpers/writer.h" + +#include +#include +#include +#include +#include + +namespace NKikimr::NKqp { + +Y_UNIT_TEST_SUITE(KqpOlapTiering) { + Y_UNIT_TEST(Eviction) { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + + TKikimrSettings runnerSettings; + runnerSettings.WithSampleTables = false; + TTestHelper testHelper(runnerSettings); + TLocalHelper localHelper(testHelper.GetKikimr()); + NYdb::NTable::TTableClient tableClient = testHelper.GetKikimr().GetTableClient(); + Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).Initialize(); + Singleton()->SetSecretKey("fakeSecret"); + + localHelper.CreateTestOlapTable(); + testHelper.CreateTier("tier1"); + const TString tieringRule = testHelper.CreateTieringRule("tier1", "timestamp"); + + for (ui64 i = 0; i < 100; ++i) { + WriteTestData(testHelper.GetKikimr(), "/Root/olapStore/olapTable", 0, i * 10000, 1000); + } + + csController->WaitActualization(TDuration::Seconds(5)); + + ui64 columnRawBytes = 0; + { + auto selectQuery = TString(R"( + SELECT + TierName, SUM(ColumnRawBytes) As RawBytes + FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` + WHERE Activity == 1 + GROUP BY TierName + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); + + columnRawBytes = GetUint64(rows[0].at("RawBytes")); + UNIT_ASSERT_GT(columnRawBytes, 0); + } + + testHelper.SetTiering("/Root/olapStore/olapTable", tieringRule); + csController->WaitActualization(TDuration::Seconds(5)); + + { + auto selectQuery = TString(R"( + SELECT + TierName, SUM(ColumnRawBytes) As RawBytes + FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` + WHERE Activity == 1 + GROUP BY TierName + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "tier1"); + UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, + TStringBuilder() << "RawBytes changed after eviction: before=" << columnRawBytes + << " after=" << GetUint64(rows[0].at("RawBytes"))); + } + + testHelper.ResetTiering("/Root/olapStore/olapTable"); + csController->WaitCompactions(TDuration::Seconds(5)); + + { + auto selectQuery = TString(R"( + SELECT + TierName, SUM(ColumnRawBytes) As RawBytes + FROM `/Root/olapStore/olapTable/.sys/primary_index_portion_stats` + WHERE Activity == 1 + GROUP BY TierName + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(GetUtf8(rows[0].at("TierName")), "__DEFAULT"); + UNIT_ASSERT_VALUES_EQUAL_C(GetUint64(rows[0].at("RawBytes")), columnRawBytes, + TStringBuilder() << "RawBytes changed after resetting tiering: before=" << columnRawBytes + << " after=" << GetUint64(rows[0].at("RawBytes"))); + } + } +} + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/ut/olap/ya.make b/ydb/core/kqp/ut/olap/ya.make index 06deb96569b9..acf719f7cba2 100644 --- a/ydb/core/kqp/ut/olap/ya.make +++ b/ydb/core/kqp/ut/olap/ya.make @@ -24,6 +24,7 @@ SRCS( aggregations_ut.cpp write_ut.cpp sparsed_ut.cpp + tiering_ut.cpp ) PEERDIR( From 338b6260ae8d290a468c6879aa8117ffcff13c1e Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 22 Aug 2024 11:09:59 +0300 Subject: [PATCH 032/261] Update github.com/ydb-platform/fq-connector-go to 0.5.5-rc.1 (#8127) --- .../connector/tests/datasource/clickhouse/docker-compose.yml | 2 +- .../connector/tests/datasource/ms_sql_server/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/mysql/docker-compose.yml | 2 +- .../connector/tests/datasource/oracle/docker-compose.yml | 2 +- .../connector/tests/datasource/postgresql/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/ydb/docker-compose.yml | 2 +- .../providers/generic/connector/tests/join/docker-compose.yml | 2 +- ydb/tests/fq/generic/docker-compose.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml index af157639929f..f5c85822772f 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-ch-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml index 5ad72dc4ecdb..f3f0d8528f42 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mssql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml index a21a51757299..d828f70ea5cf 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mysql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml index c077f34d6b8e..ed29ad70d07c 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-oracle-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml index cfcf50f16c1b..74c98a3ad1a5 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-pg-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml index 58e75a12917f..a3c3303a74e0 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml @@ -5,7 +5,7 @@ services: echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: fq-tests-ydb-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml index 09e3e239e058..b824f638c024 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-join-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - 2130 volumes: diff --git a/ydb/tests/fq/generic/docker-compose.yml b/ydb/tests/fq/generic/docker-compose.yml index ef7348e9d42b..915a7ae0fdec 100644 --- a/ydb/tests/fq/generic/docker-compose.yml +++ b/ydb/tests/fq/generic/docker-compose.yml @@ -15,7 +15,7 @@ services: echo \"$$(dig tests-fq-generic-ydb +short) tests-fq-generic-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.3@sha256:b79727ebf2d3ce20cf25dbf8509ad533c4696d59c165af03b445ef7a97301028 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 ports: - "2130" postgresql: From c53ec8026990aaef8e76ec79f42ab27828111718 Mon Sep 17 00:00:00 2001 From: azevaykin <145343289+azevaykin@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:50:17 +0300 Subject: [PATCH 033/261] Statistics: AnalyzeStatus should use the same sender to preserve event order (#8136) --- ydb/core/statistics/aggregator/aggregator_impl.cpp | 3 +++ .../statistics/aggregator/ut/ut_analyze_columnshard.cpp | 6 +++--- ydb/core/statistics/ut_common/ut_common.cpp | 3 +-- ydb/core/statistics/ut_common/ut_common.h | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 2a087684293f..98dde3778cc3 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -450,6 +450,9 @@ void TStatisticsAggregator::Handle(TEvStatistics::TEvAnalyzeStatus::TPtr& ev) { outRecord.SetStatus(NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); } } + + SA_LOG_D("[" << TabletID() << "] Send TEvStatistics::TEvAnalyzeStatusResponse. Status " << outRecord.GetStatus()); + Send(ev->Sender, response.release(), 0, ev->Cookie); } diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp index 0c4f1e5615b0..6b5beea5ace7 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -51,19 +51,19 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { const TString operationId = "operationId"; - AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); + AnalyzeStatus(runtime, sender, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); auto analyzeRequest = MakeAnalyzeRequest({{tableInfo.PathId, {1, 2}}}, operationId); runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest.release()); - AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); + AnalyzeStatus(runtime, sender, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED); schemeShardStatsBlocker.Remove(); auto analyzeResonse = runtime.GrabEdgeEventRethrow(sender); UNIT_ASSERT_VALUES_EQUAL(analyzeResonse->Get()->Record.GetOperationId(), operationId); - AnalyzeStatus(runtime, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); + AnalyzeStatus(runtime, sender, tableInfo.SaTabletId, operationId, NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION); } Y_UNIT_TEST(AnalyzeSameOperationId) { diff --git a/ydb/core/statistics/ut_common/ut_common.cpp b/ydb/core/statistics/ut_common/ut_common.cpp index d73928baca8d..ae92cb1885fc 100644 --- a/ydb/core/statistics/ut_common/ut_common.cpp +++ b/ydb/core/statistics/ut_common/ut_common.cpp @@ -404,10 +404,9 @@ void AnalyzeTable(TTestActorRuntime& runtime, ui64 shardTabletId, const TAnalyze runtime.GrabEdgeEventRethrow(sender); } -void AnalyzeStatus(TTestActorRuntime& runtime, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus) { +void AnalyzeStatus(TTestActorRuntime& runtime, TActorId sender, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus) { auto analyzeStatusRequest = std::make_unique(); analyzeStatusRequest->Record.SetOperationId(operationId); - auto sender = runtime.AllocateEdgeActor(); runtime.SendToPipe(saTabletId, sender, analyzeStatusRequest.release()); auto analyzeStatusResponse = runtime.GrabEdgeEventRethrow(sender); diff --git a/ydb/core/statistics/ut_common/ut_common.h b/ydb/core/statistics/ut_common/ut_common.h index 7939bcf8f7a2..a07a799e4bcc 100644 --- a/ydb/core/statistics/ut_common/ut_common.h +++ b/ydb/core/statistics/ut_common/ut_common.h @@ -103,7 +103,7 @@ std::unique_ptr MakeAnalyzeRequest(const std::vector< void Analyze(TTestActorRuntime& runtime, ui64 saTabletId, const std::vector& table, const TString operationId = "operationId"); void AnalyzeTable(TTestActorRuntime& runtime, ui64 shardTabletId, const TAnalyzedTable& table); -void AnalyzeStatus(TTestActorRuntime& runtime, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus); +void AnalyzeStatus(TTestActorRuntime& runtime, TActorId sender, ui64 saTabletId, const TString operationId, const NKikimrStat::TEvAnalyzeStatusResponse::EStatus expectedStatus); } // namespace NStat From bb3aabc1ad169348a3a37d3f412eb9d81cbd253d Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Thu, 22 Aug 2024 12:00:46 +0300 Subject: [PATCH 034/261] Show a nicer demangled name in block/unblock messages (#8138) --- ydb/core/testlib/actors/block_events.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ydb/core/testlib/actors/block_events.h b/ydb/core/testlib/actors/block_events.h index a267d4f3cc33..5cbf01ca000e 100644 --- a/ydb/core/testlib/actors/block_events.h +++ b/ydb/core/testlib/actors/block_events.h @@ -36,7 +36,10 @@ namespace NActors { } ui32 nodeId = ev->GetRecipientRewrite().NodeId(); ui32 nodeIdx = nodeId - Runtime.GetFirstNodeId(); - Cerr << "TBlockEvents::Unblock " << typeid(TEvType).name() << " from " << Runtime.FindActorName(ev->Sender) << " to " << Runtime.FindActorName(ev->GetRecipientRewrite()) << Endl; + Cerr << "... unblocking " << (ev->HasEvent() ? TypeName(*ev->GetBase()) : TypeName()) + << " from " << Runtime.FindActorName(ev->Sender) + << " to " << Runtime.FindActorName(ev->GetRecipientRewrite()) + << Endl; Runtime.Send(ev.Release(), nodeIdx, /* viaActorSystem */ true); this->pop_front(); --count; @@ -68,7 +71,10 @@ namespace NActors { return; } - Cerr << "TBlockEvents::Block " << typeid(TEvType).name() << " from " << Runtime.FindActorName(ev->Sender) << " to " << Runtime.FindActorName(ev->GetRecipientRewrite()) << Endl; + Cerr << "... blocking " << (ev->HasEvent() ? TypeName(*ev->GetBase()) : TypeName()) + << " from " << Runtime.FindActorName(ev->Sender) + << " to " << Runtime.FindActorName(ev->GetRecipientRewrite()) + << Endl; this->emplace_back(std::move(ev)); } From e26721c941bccf6139f3ed8567c5bdace3ef38f6 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Thu, 22 Aug 2024 12:06:08 +0300 Subject: [PATCH 035/261] Add docs dir into RECURSE ydb (#8092) --- .github/workflows/docs_build.yaml | 12 ------------ .github/workflows/pr_check.yml | 2 -- ydb/ya.make | 1 + 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/docs_build.yaml b/.github/workflows/docs_build.yaml index 7af18434b746..1a2ef9a9d77d 100644 --- a/.github/workflows/docs_build.yaml +++ b/.github/workflows/docs_build.yaml @@ -12,12 +12,6 @@ jobs: cancel-in-progress: true runs-on: ubuntu-latest steps: - - name: Update integrated status - shell: bash - run: | - curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \ - -d '{"state":"pending","description":"Waiting for relevant checks to complete","context":"checks_integrated"}' - name: Checkout uses: actions/checkout@v3 with: @@ -27,9 +21,3 @@ jobs: with: revision: "pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}" src-root: "./ydb/docs" - - name: Update integrated status - shell: bash - run: | - curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \ - -d '{"state":"success","description":"All checks completed","context":"checks_integrated"}' diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index 3cf94deb814b..6b8cc8f57dda 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -6,8 +6,6 @@ on: - 'stable-*' - 'stream-nb-*' - '*-stable-*' - paths-ignore: - - 'ydb/docs/**' types: - 'opened' - 'synchronize' diff --git a/ydb/ya.make b/ydb/ya.make index c06bda61e084..623a89c5fd47 100644 --- a/ydb/ya.make +++ b/ydb/ya.make @@ -1,5 +1,6 @@ RECURSE( apps + docs core library mvp From 33d8ce90ba5ba61d48a154fbb0e43d621bc5767d Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Thu, 22 Aug 2024 12:37:42 +0300 Subject: [PATCH 036/261] streaming grace join test (#3862) --- ydb/tests/fq/s3/test_streaming_join.py | 97 ++++++++++++++++++++++++++ ydb/tests/fq/s3/ya.make | 1 + 2 files changed, 98 insertions(+) create mode 100644 ydb/tests/fq/s3/test_streaming_join.py diff --git a/ydb/tests/fq/s3/test_streaming_join.py b/ydb/tests/fq/s3/test_streaming_join.py new file mode 100644 index 000000000000..61332fbbc235 --- /dev/null +++ b/ydb/tests/fq/s3/test_streaming_join.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import boto3 +import logging +import os +import pytest + +from ydb.tests.tools.datastreams_helpers.test_yds_base import TestYdsBase +import ydb.public.api.protos.ydb_value_pb2 as ydb_value +import ydb.public.api.protos.draft.fq_pb2 as fq +from ydb.tests.tools.fq_runner.kikimr_utils import yq_v1 + + +class TestStreamingJoin(TestYdsBase): + @yq_v1 + @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) + def test_grace_join(self, kikimr, s3, client, unique_prefix): + self.init_topics("pq_test_grace_join") + + # S3 + resource = boto3.resource( + "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key" + ) + + bucket = resource.Bucket("test_grace_join") + bucket.create(ACL='public-read') + + s3_client = boto3.client( + "s3", endpoint_url=s3.s3_url, aws_access_key_id="key", aws_secret_access_key="secret_key" + ) + + fruits = R'''Time,Fruit,Price + 0,Banana,3 + 1,Apple,2 + 2,Pear,15''' + s3_client.put_object(Body=fruits, Bucket='test_grace_join', Key='fruits.csv', ContentType='text/plain') + + kikimr.control_plane.wait_bootstrap(1) + + storage_connection_name = unique_prefix + "test_grace_join" + client.create_storage_connection(storage_connection_name, "test_grace_join") + + connection_response = client.create_yds_connection( + "myyds", os.getenv("YDB_DATABASE"), os.getenv("YDB_ENDPOINT") + ) + keyColumn = ydb_value.Column(name="Data", type=ydb_value.Type(type_id=ydb_value.Type.PrimitiveTypeId.STRING)) + client.create_yds_binding( + name="my_binding_in", + stream=self.input_topic, + format="raw", + connection_id=connection_response.result.connection_id, + columns=[keyColumn], + ) + client.create_yds_binding( + name="my_binding_out", + stream=self.output_topic, + format="raw", + connection_id=connection_response.result.connection_id, + columns=[keyColumn], + ) + + sql = fR''' + pragma dq.HashJoinMode="grace"; + + $stream = SELECT `Data` FROM bindings.`my_binding_in`; + + $s3 = SELECT *, + FROM `{storage_connection_name}`.`fruits.csv` + WITH (format=csv_with_names, SCHEMA ( + Time UInt64 NOT NULL, + Fruit String NOT NULL, + Price Int NOT NULL + )); + + $my_join = SELECT * FROM $stream AS a LEFT JOIN $s3 AS b ON a.Data = b.Fruit; + + INSERT INTO bindings.`my_binding_out` + SELECT Data AS data FROM $my_join WHERE Data LIKE '%ahaha%' + ''' + + query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.STREAMING).result.query_id + client.wait_query_status(query_id, fq.QueryMeta.RUNNING) + kikimr.compute_plane.wait_zero_checkpoint(query_id) + + data = ['{{"event_class": "{}", "time": "{}"}}'.format("ahaha", "2024-01-03T00:00:00Z")] + self.write_stream(data, self.input_topic) + + read_data = self.read_stream(1) + logging.info("Data was read: {}".format(read_data)) + + # This condition will be failed after the fix grace join for streaming queries + assert len(read_data) == 0 + + client.abort_query(query_id) + + client.wait_query(query_id) diff --git a/ydb/tests/fq/s3/ya.make b/ydb/tests/fq/s3/ya.make index ddb3347e17ca..21b6cc6fc4c3 100644 --- a/ydb/tests/fq/s3/ya.make +++ b/ydb/tests/fq/s3/ya.make @@ -35,6 +35,7 @@ TEST_SRCS( test_push_down.py test_s3_0.py test_s3_1.py + test_streaming_join.py test_size_limit.py test_statistics.py test_test_connection.py From 09f4427f4a63e97958bf9bf260dc8c60125f00a1 Mon Sep 17 00:00:00 2001 From: nikita kozlovsky Date: Thu, 22 Aug 2024 12:43:37 +0300 Subject: [PATCH 037/261] ci: download the latest github runner on a VM start (#7987) --- .../src/scaler/cloud_config.py | 3 ++- .../src/scaler/controller.py | 7 ++++--- .../src/scaler/scripts/install_runner.sh | 20 +++++++++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ydb/ci/gh-runner-controller/src/scaler/cloud_config.py b/ydb/ci/gh-runner-controller/src/scaler/cloud_config.py index d3f7c3341401..cd0f8791e170 100644 --- a/ydb/ci/gh-runner-controller/src/scaler/cloud_config.py +++ b/ydb/ci/gh-runner-controller/src/scaler/cloud_config.py @@ -3,7 +3,7 @@ import yaml -def create_userdata(repo_url, gh_token, runner_name, runner_labels, ssh_keys): +def create_userdata(repo_url, gh_token, runner_name, runner_labels, ssh_keys, agent_mirror_url_prefix): runner_username = "runner" install_script = pkgutil.get_data(__name__, "scripts/install_runner.sh") @@ -16,6 +16,7 @@ def create_userdata(repo_url, gh_token, runner_name, runner_labels, ssh_keys): GITHUB_TOKEN="{gh_token}" RUNNER_NAME="{runner_name}" RUNNER_LABELS="{runner_labels}" +AGENT_MIRROR_URL_PREFIX="{agent_mirror_url_prefix}" """.strip().encode( "utf8" ) diff --git a/ydb/ci/gh-runner-controller/src/scaler/controller.py b/ydb/ci/gh-runner-controller/src/scaler/controller.py index ed48c506c379..916120385f63 100644 --- a/ydb/ci/gh-runner-controller/src/scaler/controller.py +++ b/ydb/ci/gh-runner-controller/src/scaler/controller.py @@ -189,7 +189,8 @@ def start_runner(self, preset_name: str): labels.extend(preset['additional_labels']) vm_labels = {self.prefix: runner_name} - user_data = create_userdata(self.gh.html_url, new_runner_token, runner_name, labels, self.cfg.ssh_keys) + user_data = create_userdata(self.gh.html_url, new_runner_token, runner_name, labels, self.cfg.ssh_keys, + self.cfg.agent_mirror_url_prefix) placement = random.choice(self.cfg.yc_zones) zone_id = placement['zone_id'] @@ -198,7 +199,7 @@ def start_runner(self, preset_name: str): self.logger.info("start runner %s in %s (%s)", runner_name, zone_id, labels) try: - instance = self.yc.start_vm(zone_id, subnet_id, runner_name, preset_name, user_data, vm_labels) + response = self.yc.start_vm(zone_id, subnet_id, runner_name, preset_name, user_data, vm_labels) except grpc.RpcError as rpc_error: # noinspection PyUnresolvedReferences if rpc_error.code() == grpc.StatusCode.RESOURCE_EXHAUSTED: @@ -206,7 +207,7 @@ def start_runner(self, preset_name: str): else: raise - self.logger.info("instance %s started", instance.id) + self.logger.info("starting, operation_id=%s", response.id) return runner_name def delete_runner(self, runner): diff --git a/ydb/ci/gh-runner-controller/src/scaler/scripts/install_runner.sh b/ydb/ci/gh-runner-controller/src/scaler/scripts/install_runner.sh index f2f6d0cd2fb4..d41755f9fd17 100644 --- a/ydb/ci/gh-runner-controller/src/scaler/scripts/install_runner.sh +++ b/ydb/ci/gh-runner-controller/src/scaler/scripts/install_runner.sh @@ -11,7 +11,7 @@ function fail() { } function get_instance_id { - curl -H Metadata-Flavor:Google 169.254.169.254/computeMetadata/v1/instance/vendor/identity/document | jq -r '.instanceId' + curl -sS --retry 8 -H Metadata-Flavor:Google 169.254.169.254/computeMetadata/v1/instance/vendor/identity/document | jq -r '.instanceId' } vars_to_check=("REPO_URL" "GITHUB_TOKEN" "RUNNER_NAME" "RUNNER_LABELS" "RUNNER_USERNAME") @@ -24,15 +24,23 @@ done H=/home/$RUNNER_USERNAME +mkdir "$H"/actions_runner +cd "$_" + +{ + agent_latest_version=$(curl -sS --retry 8 "$AGENT_MIRROR_URL_PREFIX/latest") && \ + agent_download_url="$AGENT_MIRROR_URL_PREFIX/$agent_latest_version" && \ + curl -sS --retry 8 "$agent_download_url" | tar -xz +} || { + # use bundled agent + cp -rT /opt/cache/actions-runner/latest/ "$H"/actions_runner +} -cp -r /opt/cache/actions-runner/latest "$H"/actions_runner -chown "$RUNNER_USERNAME":"$RUNNER_USERNAME" "$H"/actions_runner - -cd "$H"/actions_runner +chown -R "$RUNNER_USERNAME":"$RUNNER_USERNAME" "$H"/actions_runner instance_id=$(get_instance_id) -sudo -u "$RUNNER_USERNAME" ./config.sh --unattended --url "${REPO_URL}" --token "${GITHUB_TOKEN}" --name "${RUNNER_NAME}" --labels "${RUNNER_LABELS},instance:${instance_id}" +sudo -u "$RUNNER_USERNAME" ./config.sh --unattended --disableupdate --url "${REPO_URL}" --token "${GITHUB_TOKEN}" --name "${RUNNER_NAME}" --labels "${RUNNER_LABELS},instance:${instance_id}" ./svc.sh install "${RUNNER_USERNAME}" || fail "failed to install service" From 75100de91033f125e6b902f6b0587534ce359b09 Mon Sep 17 00:00:00 2001 From: azevaykin <145343289+azevaykin@users.noreply.github.com> Date: Thu, 22 Aug 2024 12:57:00 +0300 Subject: [PATCH 038/261] Debug logging in WaiFor (#8137) --- ydb/core/testlib/actors/test_runtime.h | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/core/testlib/actors/test_runtime.h b/ydb/core/testlib/actors/test_runtime.h index de30fec4c030..b8a1d2e46aa6 100644 --- a/ydb/core/testlib/actors/test_runtime.h +++ b/ydb/core/testlib/actors/test_runtime.h @@ -119,6 +119,7 @@ namespace NActors { this->DispatchEvents(options, simTimeout); Y_ABORT_UNLESS(condition(), "Timeout while waiting for %s", description.c_str()); + Cerr << "... waiting for " << description << " (done)" << Endl; } } From 6dcdcc0edc641c6b46fada1e94771838eeeb8398 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 22 Aug 2024 13:07:13 +0300 Subject: [PATCH 039/261] correct portions index usage for control memory scan intervals (#8135) --- .../engines/column_engine_logs.cpp | 24 ++++---- .../engines/storage/granule/granule.cpp | 6 +- .../storage/granule/portions_index.cpp | 8 +-- .../engines/storage/granule/portions_index.h | 60 ++++++++++++------- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 70cc62226ed9..63c3289f9c9d 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -510,20 +510,18 @@ std::shared_ptr TColumnEngineForLogs::Select(ui64 pathId, TSnapshot return out; } - for (const auto& [indexKey, keyPortions] : spg->GetPortionsIndex().GetPoints()) { - for (auto&& [_, portionInfo] : keyPortions.GetStart()) { - if (!portionInfo->IsVisible(snapshot)) { - continue; - } - Y_ABORT_UNLESS(portionInfo->Produced()); - const bool skipPortion = !pkRangesFilter.IsPortionInUsage(*portionInfo); - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", skipPortion ? "portion_skipped" : "portion_selected") - ("pathId", pathId)("portion", portionInfo->DebugString()); - if (skipPortion) { - continue; - } - out->PortionsOrderedPK.emplace_back(portionInfo); + for (const auto& [_, portionInfo] : spg->GetPortions()) { + if (!portionInfo->IsVisible(snapshot)) { + continue; + } + Y_ABORT_UNLESS(portionInfo->Produced()); + const bool skipPortion = !pkRangesFilter.IsPortionInUsage(*portionInfo); + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", skipPortion ? "portion_skipped" : "portion_selected")("pathId", pathId)( + "portion", portionInfo->DebugString()); + if (skipPortion) { + continue; } + out->PortionsOrderedPK.emplace_back(portionInfo); } return out; diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp index 5266e6738fde..ebb2b9acde63 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp @@ -47,10 +47,9 @@ bool TGranuleMeta::ErasePortion(const ui64 portion) { void TGranuleMeta::OnAfterChangePortion(const std::shared_ptr portionAfter, NStorageOptimizer::IOptimizerPlanner::TModificationGuard* modificationGuard) { if (portionAfter) { - PortionsIndex.AddPortion(portionAfter); - PortionInfoGuard.OnNewPortion(portionAfter); if (!portionAfter->HasRemoveSnapshot()) { + PortionsIndex.AddPortion(portionAfter); if (modificationGuard) { modificationGuard->AddPortion(portionAfter); } else { @@ -74,10 +73,9 @@ void TGranuleMeta::OnAfterChangePortion(const std::shared_ptr port void TGranuleMeta::OnBeforeChangePortion(const std::shared_ptr portionBefore) { if (portionBefore) { - PortionsIndex.RemovePortion(portionBefore); - PortionInfoGuard.OnDropPortion(portionBefore); if (!portionBefore->HasRemoveSnapshot()) { + PortionsIndex.RemovePortion(portionBefore); OptimizerPlanner->StartModificationGuard().RemovePortion(portionBefore); ActualizationIndex->RemovePortion(portionBefore); } diff --git a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp index f1ef4d9edcfe..e56487e5f8ef 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.cpp @@ -55,11 +55,11 @@ void TPortionsIndex::RemovePortion(const std::shared_ptr& p) { auto itTo = Points.find(p->IndexKeyEnd()); AFL_VERIFY(itTo != Points.end()); { - const TPortionInfoStat stat(p->GetMinMemoryForReadColumns({}), p->GetTotalBlobBytes()); + const TPortionInfoStat stat(p); auto it = itFrom; while (true) { RemoveFromMemoryUsageControl(it->second.GetIntervalStats()); - it->second.RemoveContained(p->GetPortionId(), stat); + it->second.RemoveContained(stat); RawMemoryUsage.Add(it->second.GetIntervalStats().GetMinRawBytes()); BlobMemoryUsage.Add(it->second.GetIntervalStats().GetBlobBytes()); if (it == itTo) { @@ -98,10 +98,10 @@ void TPortionsIndex::AddPortion(const std::shared_ptr& p) { itTo->second.AddFinish(p); auto it = itFrom; - const TPortionInfoStat stat(p->GetMinMemoryForReadColumns({}), p->GetTotalBlobBytes()); + const TPortionInfoStat stat(p); while (true) { RemoveFromMemoryUsageControl(it->second.GetIntervalStats()); - it->second.AddContained(p->GetPortionId(), stat); + it->second.AddContained(stat); RawMemoryUsage.Add(it->second.GetIntervalStats().GetMinRawBytes()); BlobMemoryUsage.Add(it->second.GetIntervalStats().GetBlobBytes()); if (it == itTo) { diff --git a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h index 68503629e26a..981943dc4dab 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/portions_index.h @@ -10,29 +10,41 @@ namespace NKikimr::NOlap::NGranule::NPortionsIndex { class TPortionInfoStat { private: + std::shared_ptr PortionInfo; YDB_READONLY(ui64, MinRawBytes, 0); YDB_READONLY(ui64, BlobBytes, 0); public: - TPortionInfoStat() = default; - - TPortionInfoStat(const ui64 rawBytes, const ui64 blobBytes) - : MinRawBytes(rawBytes) - , BlobBytes(blobBytes) + TPortionInfoStat(const std::shared_ptr& portionInfo) + : PortionInfo(portionInfo) + , MinRawBytes(PortionInfo->GetMinMemoryForReadColumns({})) + , BlobBytes(PortionInfo->GetTotalBlobBytes()) { } + const TPortionInfo& GetPortionInfoVerified() const { + AFL_VERIFY(PortionInfo); + return *PortionInfo; + } +}; + +class TIntervalInfoStat { +private: + YDB_READONLY(ui64, MinRawBytes, 0); + YDB_READONLY(ui64, BlobBytes, 0); + +public: void Add(const TPortionInfoStat& source) { - MinRawBytes += source.MinRawBytes; - BlobBytes += source.BlobBytes; + MinRawBytes += source.GetMinRawBytes(); + BlobBytes += source.GetBlobBytes(); } void Sub(const TPortionInfoStat& source) { - AFL_VERIFY(MinRawBytes >= source.MinRawBytes); - MinRawBytes -= source.MinRawBytes; - AFL_VERIFY(BlobBytes >= source.BlobBytes); - BlobBytes -= source.BlobBytes; + AFL_VERIFY(MinRawBytes >= source.GetMinRawBytes()); + MinRawBytes -= source.GetMinRawBytes(); + AFL_VERIFY(BlobBytes >= source.GetBlobBytes()); + BlobBytes -= source.GetBlobBytes(); AFL_VERIFY(!!BlobBytes == !!MinRawBytes); } @@ -46,7 +58,7 @@ class TPortionsPKPoint { THashMap> Start; THashMap> Finish; THashMap PortionIds; - YDB_READONLY_DEF(TPortionInfoStat, IntervalStats); + YDB_READONLY_DEF(TIntervalInfoStat, IntervalStats); public: const THashMap>& GetStart() const { @@ -54,12 +66,12 @@ class TPortionsPKPoint { } void ProvidePortions(const TPortionsPKPoint& source) { - IntervalStats = TPortionInfoStat(); + IntervalStats = TIntervalInfoStat(); for (auto&& [i, stat] : source.PortionIds) { if (source.Finish.contains(i)) { continue; } - AddContained(i, stat); + AddContained(stat); } } @@ -71,17 +83,19 @@ class TPortionsPKPoint { return Start.empty() && Finish.empty(); } - void AddContained(const ui32 portionId, const TPortionInfoStat& stat) { - IntervalStats.Add(stat); - AFL_VERIFY(PortionIds.emplace(portionId, stat).second); + void AddContained(const TPortionInfoStat& stat) { + if (!stat.GetPortionInfoVerified().HasRemoveSnapshot()) { + IntervalStats.Add(stat); + } + AFL_VERIFY(PortionIds.emplace(stat.GetPortionInfoVerified().GetPortionId(), stat).second); } - void RemoveContained(const ui32 portionId, const TPortionInfoStat& stat) { - IntervalStats.Sub(stat); - AFL_VERIFY(PortionIds.erase(portionId)); - if (PortionIds.empty()) { - AFL_VERIFY(!IntervalStats); + void RemoveContained(const TPortionInfoStat& stat) { + if (!stat.GetPortionInfoVerified().HasRemoveSnapshot()) { + IntervalStats.Sub(stat); } + AFL_VERIFY(PortionIds.erase(stat.GetPortionInfoVerified().GetPortionId())); + AFL_VERIFY(PortionIds.size() || !IntervalStats); } void RemoveStart(const std::shared_ptr& p) { @@ -162,7 +176,7 @@ class TPortionsIndex { return it; } - void RemoveFromMemoryUsageControl(const TPortionInfoStat& stat) { + void RemoveFromMemoryUsageControl(const TIntervalInfoStat& stat) { RawMemoryUsage.Remove(stat.GetMinRawBytes()); BlobMemoryUsage.Remove(stat.GetBlobBytes()); } From 8a590a9e40dfb19ba801db41ec84d88ab7211e30 Mon Sep 17 00:00:00 2001 From: Semyon Date: Thu, 22 Aug 2024 13:08:17 +0300 Subject: [PATCH 040/261] Don't skip null values on COUNT(*) in column shards (#8024) --- ydb/core/formats/arrow/custom_registry.cpp | 7 ++ ydb/core/formats/arrow/program.cpp | 27 ++++++ ydb/core/formats/arrow/program.h | 14 ++- .../formats/arrow/ssa_program_optimizer.cpp | 5 +- ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 90 +++++++++++++++++++ .../tx/columnshard/engines/ut/ut_program.cpp | 60 +++++++++++++ ydb/core/tx/program/program.cpp | 5 +- .../AggregateFunctionNumRows.h | 79 ++++++++++++++++ .../AggregateFunctions/IAggregateFunction.cpp | 3 + .../AggregateFunctions/IAggregateFunction.h | 18 ++-- ydb/library/arrow_kernels/func_num_rows.h | 20 +++++ ydb/library/arrow_kernels/functions.h | 1 + 12 files changed, 317 insertions(+), 12 deletions(-) create mode 100644 ydb/library/arrow_clickhouse/AggregateFunctions/AggregateFunctionNumRows.h create mode 100644 ydb/library/arrow_kernels/func_num_rows.h diff --git a/ydb/core/formats/arrow/custom_registry.cpp b/ydb/core/formats/arrow/custom_registry.cpp index 13e8dc6150a3..9d61c8bf6476 100644 --- a/ydb/core/formats/arrow/custom_registry.cpp +++ b/ydb/core/formats/arrow/custom_registry.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #endif namespace cp = ::arrow::compute; @@ -62,6 +63,10 @@ static void RegisterYdbCast(cp::FunctionRegistry* registry) { Y_ABORT_UNLESS(registry->AddFunction(std::make_shared()).ok()); } +static void RegisterCustomAggregates(cp::FunctionRegistry* registry) { + Y_ABORT_UNLESS(registry->AddFunction(std::make_shared(GetFunctionName(EAggregate::NumRows))).ok()); +} + static void RegisterHouseAggregates(cp::FunctionRegistry* registry) { #ifndef WIN32 try { @@ -71,6 +76,7 @@ static void RegisterHouseAggregates(cp::FunctionRegistry* registry) { Y_ABORT_UNLESS(registry->AddFunction(std::make_shared(GetHouseFunctionName(EAggregate::Max))).ok()); Y_ABORT_UNLESS(registry->AddFunction(std::make_shared(GetHouseFunctionName(EAggregate::Sum))).ok()); //Y_ABORT_UNLESS(registry->AddFunction(std::make_shared(GetHouseFunctionName(EAggregate::Avg))).ok()); + Y_ABORT_UNLESS(registry->AddFunction(std::make_shared(GetHouseFunctionName(EAggregate::NumRows))).ok()); Y_ABORT_UNLESS(registry->AddFunction(std::make_shared(GetHouseGroupByName())).ok()); } catch (const std::exception& /*ex*/) { @@ -88,6 +94,7 @@ static std::unique_ptr CreateCustomRegistry() { RegisterRound(registry.get()); RegisterArithmetic(registry.get()); RegisterYdbCast(registry.get()); + RegisterCustomAggregates(registry.get()); RegisterHouseAggregates(registry.get()); return registry; } diff --git a/ydb/core/formats/arrow/program.cpp b/ydb/core/formats/arrow/program.cpp index 60e59749bb7a..a741a3002e89 100644 --- a/ydb/core/formats/arrow/program.cpp +++ b/ydb/core/formats/arrow/program.cpp @@ -398,6 +398,8 @@ const char * GetFunctionName(EAggregate op) { return "min_max"; case EAggregate::Sum: return "sum"; + case EAggregate::NumRows: + return "num_rows"; #if 0 // TODO case EAggregate::Avg: return "mean"; @@ -424,6 +426,8 @@ const char * GetHouseFunctionName(EAggregate op) { case EAggregate::Avg: return "ch.avg"; #endif + case EAggregate::NumRows: + return "ch.num_rows"; default: break; } @@ -448,6 +452,8 @@ CH::AggFunctionId GetHouseFunction(EAggregate op) { case EAggregate::Avg: return CH::AggFunctionId::AGG_AVG; #endif + case EAggregate::NumRows: + return CH::AggFunctionId::AGG_NUM_ROWS; default: break; } @@ -678,6 +684,27 @@ IStepFunction::TPtr TAggregateAssign::GetFunction(arrow::compu return std::make_shared(ctx); } +TString TAggregateAssign::DebugString() const { + TStringBuilder sb; + sb << "{"; + if (Operation != EAggregate::Unspecified) { + sb << "op=" << GetFunctionName(Operation) << ";"; + } + if (Arguments.size()) { + sb << "arguments=["; + for (auto&& i : Arguments) { + sb << i.DebugString() << ";"; + } + sb << "];"; + } + sb << "options=" << ScalarOpts.ToString() << ";"; + if (KernelFunction) { + sb << "kernel=" << KernelFunction->name() << ";"; + } + sb << "column=" << Column.DebugString() << ";"; + sb << "}"; + return sb; +} arrow::Status TProgramStep::ApplyAssignes(TDatumBatch& batch, arrow::compute::ExecContext* ctx) const { if (Assignes.empty()) { diff --git a/ydb/core/formats/arrow/program.h b/ydb/core/formats/arrow/program.h index e3f9943e6c13..2b953b55e070 100644 --- a/ydb/core/formats/arrow/program.h +++ b/ydb/core/formats/arrow/program.h @@ -21,6 +21,7 @@ enum class EAggregate { Max = 4, Sum = 5, //Avg = 6, + NumRows = 7, }; } @@ -323,6 +324,7 @@ class TAggregateAssign { const arrow::compute::ScalarAggregateOptions* GetOptions() const { return &ScalarOpts; } IStepFunction::TPtr GetFunction(arrow::compute::ExecContext* ctx) const; + TString DebugString() const; private: TColumnInfo Column; @@ -372,10 +374,18 @@ class TProgramStep { sb << "];"; } if (GroupBy.size()) { - sb << "group_by_count=" << GroupBy.size() << "; "; + sb << "group_by_assignes=["; + for (auto&& i : GroupBy) { + sb << i.DebugString() << ";"; + } + sb << "];"; } if (GroupByKeys.size()) { - sb << "group_by_keys_count=" << GroupByKeys.size() << ";"; + sb << "group_by_keys=["; + for (auto&& i : GroupByKeys) { + sb << i.DebugString() << ";"; + } + sb << "];"; } sb << "projections=["; diff --git a/ydb/core/formats/arrow/ssa_program_optimizer.cpp b/ydb/core/formats/arrow/ssa_program_optimizer.cpp index f55f63110f7b..ff1e5a5cb38c 100644 --- a/ydb/core/formats/arrow/ssa_program_optimizer.cpp +++ b/ydb/core/formats/arrow/ssa_program_optimizer.cpp @@ -1,5 +1,7 @@ #include "ssa_program_optimizer.h" +#include + namespace NKikimr::NSsa { namespace { @@ -11,7 +13,8 @@ void ReplaceCountAll(TProgram& program) { Y_ABORT_UNLESS(step); for (auto& groupBy : step->MutableGroupBy()) { - if (groupBy.GetOperation() == EAggregate::Count && groupBy.GetArguments().empty()) { + if (groupBy.GetOperation() == EAggregate::NumRows) { + AFL_VERIFY(groupBy.GetArguments().empty()); if (step->GetGroupByKeys().size()) { groupBy.MutableArguments().push_back(step->GetGroupByKeys()[0]); } else { diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index e3c94dd2804e..090f54891054 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -2761,6 +2761,96 @@ Y_UNIT_TEST_SUITE(KqpOlap) { } } + Y_UNIT_TEST(CountWhereColumnIsNull) { + auto settings = TKikimrSettings() + .SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_DEBUG); + + TLocalHelper(kikimr).CreateTestOlapTable(); + + WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000, 300, true); + + auto client = kikimr.GetTableClient(); + + Tests::NCommon::TLoggerInit(kikimr).Initialize(); + + { + auto it = client.StreamExecuteScanQuery(R"( + --!syntax_v1 + + SELECT COUNT(*), COUNT(level) + FROM `/Root/olapStore/olapTable` + WHERE level IS NULL + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + TString result = StreamResultToYson(it); + Cout << result << Endl; + CompareYson("[[100u;0u]]", result); + } + + { + auto it = client.StreamExecuteScanQuery(R"( + --!syntax_v1 + + SELECT COUNT(*), COUNT(level) + FROM `/Root/olapStore/olapTable` + WHERE level IS NULL AND uid IS NOT NULL + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + TString result = StreamResultToYson(it); + Cout << result << Endl; + CompareYson("[[100u;0u]]", result); + } + + { + auto it = client.StreamExecuteScanQuery(R"( + --!syntax_v1 + + SELECT COUNT(*), COUNT(level) + FROM `/Root/olapStore/olapTable` + WHERE level IS NULL + GROUP BY level + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + TString result = StreamResultToYson(it); + Cout << result << Endl; + CompareYson("[[100u;0u]]", result); + } + } + + Y_UNIT_TEST(SimpleCount) { + auto settings = TKikimrSettings() + .SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_DEBUG); + + TLocalHelper(kikimr).CreateTestOlapTable(); + + WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000, 300, true); + + auto client = kikimr.GetTableClient(); + + Tests::NCommon::TLoggerInit(kikimr).Initialize(); + + { + auto it = client.StreamExecuteScanQuery(R"( + --!syntax_v1 + + SELECT COUNT(level) + FROM `/Root/olapStore/olapTable` + WHERE StartsWith(uid, "uid_") + )").GetValueSync(); + + UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); + TString result = StreamResultToYson(it); + Cout << result << Endl; + CompareYson("[[200u]]", result); + } + } } } diff --git a/ydb/core/tx/columnshard/engines/ut/ut_program.cpp b/ydb/core/tx/columnshard/engines/ut/ut_program.cpp index 798ba6ec505d..f957cfea5592 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_program.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_program.cpp @@ -852,4 +852,64 @@ Y_UNIT_TEST_SUITE(TestProgram) { auto expected = result.BuildArrow(); UNIT_ASSERT_VALUES_EQUAL(batch->ToString(), expected->ToString()); } + + Y_UNIT_TEST(CountWithNulls) { + TIndexInfo indexInfo = BuildTableInfo(testColumns, testKey); + ; + NReader::NPlain::TIndexColumnResolver columnResolver(indexInfo); + + NKikimrSSA::TProgram programProto; + { + auto* command = programProto.AddCommand(); + auto* functionProto = command->MutableAssign()->MutableFunction(); + auto* column = command->MutableAssign()->MutableColumn(); + column->SetName("0"); + auto* funcArg = functionProto->AddArguments(); + funcArg->SetName("uid"); + functionProto->SetId(NKikimrSSA::TProgram::TAssignment::EFunction::TProgram_TAssignment_EFunction_FUNC_IS_NULL); + } + { + auto* command = programProto.AddCommand(); + auto* filter = command->MutableFilter(); + auto* predicate = filter->MutablePredicate(); + predicate->SetName("0"); + } + { + auto* command = programProto.AddCommand(); + auto* groupBy = command->MutableGroupBy(); + auto* aggregate = groupBy->AddAggregates(); + aggregate->MutableFunction()->SetId(static_cast(NArrow::EAggregate::Count)); + aggregate->MutableColumn()->SetName("1"); + } + { + auto* command = programProto.AddCommand(); + auto* projectionProto = command->MutableProjection(); + auto* column = projectionProto->AddColumns(); + column->SetName("1"); + } + const auto programSerialized = SerializeProgram(programProto); + + TProgramContainer program; + TString errors; + UNIT_ASSERT_C( + program.Init(columnResolver, NKikimrSchemeOp::EOlapProgramType::OLAP_PROGRAM_SSA_PROGRAM_WITH_PARAMETERS, programSerialized, errors), + errors); + + TTableUpdatesBuilder updates(NArrow::MakeArrowSchema({ std::make_pair("uid", TTypeInfo(NTypeIds::Utf8)) })); + updates.AddRow().Add("a"); + updates.AddRow().AddNull(); + updates.AddRow().Add("bbb"); + updates.AddRow().AddNull(); + updates.AddRow().AddNull(); + + auto batch = updates.BuildArrow(); + auto res = program.ApplyProgram(batch); + UNIT_ASSERT_C(res.ok(), res.ToString()); + + TTableUpdatesBuilder result(NArrow::MakeArrowSchema({ std::make_pair("1", TTypeInfo(NTypeIds::Uint64)) })); + result.AddRow().Add(3); + + auto expected = result.BuildArrow(); + UNIT_ASSERT_VALUES_EQUAL(batch->ToString(), expected->ToString()); + } } diff --git a/ydb/core/tx/program/program.cpp b/ydb/core/tx/program/program.cpp index a6dbce7fed75..a4ada441cd74 100644 --- a/ydb/core/tx/program/program.cpp +++ b/ydb/core/tx/program/program.cpp @@ -330,7 +330,7 @@ NSsa::TAggregateAssign TProgramBuilder::MakeAggregate(const NSsa::TColumnInfo& n } } else if (func.ArgumentsSize() == 0 && func.GetId() == TId::AGG_COUNT) { // COUNT(*) case - return TAggregateAssign(name, EAggregate::Count); + return TAggregateAssign(name, EAggregate::NumRows); } return TAggregateAssign(name); // !ok() } @@ -483,7 +483,7 @@ bool TProgramContainer::Init(const IColumnResolver& columnResolver, const NKikim if (IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD)) { TString out; ::google::protobuf::TextFormat::PrintToString(programProto, &out); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("program", out); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "parse_program")("program", out); } if (programProto.HasKernels()) { @@ -496,6 +496,7 @@ bool TProgramContainer::Init(const IColumnResolver& columnResolver, const NKikim } return false; } + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "program_parsed")("result", DebugString()); return true; } diff --git a/ydb/library/arrow_clickhouse/AggregateFunctions/AggregateFunctionNumRows.h b/ydb/library/arrow_clickhouse/AggregateFunctions/AggregateFunctionNumRows.h new file mode 100644 index 000000000000..80e5ff270b34 --- /dev/null +++ b/ydb/library/arrow_clickhouse/AggregateFunctions/AggregateFunctionNumRows.h @@ -0,0 +1,79 @@ +// The code in this file is based on original ClickHouse source code +// which is licensed under Apache license v2.0 +// See: https://github.com/ClickHouse/ClickHouse/ + +#pragma once +#include "arrow_clickhouse_types.h" + +#include +#include +#include + +#include + +namespace CH +{ + + +struct AggregateFunctionNumRowsData +{ + UInt64 count = 0; +}; + + +/// Count rows. +class AggregateFunctionNumRows final + : public IAggregateFunctionDataHelper +{ +public: + AggregateFunctionNumRows(const DataTypes & argument_types_) + : IAggregateFunctionDataHelper(argument_types_, {}) + {} + + DataTypePtr getReturnType() const override + { + return std::make_shared(); + } + + bool allocatesMemoryInArena() const override { return false; } + + void add(AggregateDataPtr __restrict place, const IColumn **, size_t, Arena *) const override + { + ++data(place).count; + } + + void addBatchSinglePlace( + size_t row_begin, + size_t row_end, + AggregateDataPtr __restrict place, + const IColumn ** /*columns*/, + Arena *) const override + { + data(place).count += row_end - row_begin; + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override + { + data(place).count += data(rhs).count; + } + + void insertResultInto(AggregateDataPtr __restrict place, MutableColumn & to, Arena *) const override + { + assert_cast(to).Append(data(place).count).ok(); + } +}; + +class WrappedNumRows final : public ArrowAggregateFunctionWrapper +{ +public: + WrappedNumRows(std::string name) + : ArrowAggregateFunctionWrapper(std::move(name)) + {} + + AggregateFunctionPtr getHouseFunction(const DataTypes & argument_types) const override + { + return std::make_shared(argument_types); + } +}; + +} diff --git a/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.cpp b/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.cpp index 87eccca5e419..9442b239798a 100644 --- a/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.cpp +++ b/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace CH { @@ -22,6 +23,8 @@ AggregateFunctionPtr GetAggregateFunction(AggFunctionId id, const DataTypes & ar return WrappedSum("").getHouseFunction(argument_types); case AggFunctionId::AGG_AVG: return WrappedAvg("").getHouseFunction(argument_types); + case AggFunctionId::AGG_NUM_ROWS: + return WrappedNumRows("").getHouseFunction(argument_types); default: break; } diff --git a/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.h b/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.h index f4f21463a10e..8638da084af0 100644 --- a/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.h +++ b/ydb/library/arrow_clickhouse/AggregateFunctions/IAggregateFunction.h @@ -181,7 +181,7 @@ class IAggregateFunction : public std::enable_shared_from_this +template class IAggregateFunctionHelper : public IAggregateFunction { private: @@ -204,7 +204,7 @@ class IAggregateFunctionHelper : public IAggregateFunction const IColumn ** columns, Arena * arena) const override { - if (columns && columns[0]->null_bitmap_data()) + if (skip_nulls && columns && columns[0]->null_bitmap_data()) { for (size_t i = row_begin; i < row_end; ++i) { @@ -240,7 +240,7 @@ class IAggregateFunctionHelper : public IAggregateFunction const IColumn ** columns, Arena * arena) const override { - if (columns && columns[0]->null_bitmap_data()) + if (skip_nulls && columns && columns[0]->null_bitmap_data()) { for (size_t i = row_begin; i < row_end; ++i) { @@ -340,9 +340,12 @@ class IAggregateFunctionHelper : public IAggregateFunction /// Implements several methods for manipulation with data. T - type of structure with data for aggregation. -template -class IAggregateFunctionDataHelper : public IAggregateFunctionHelper +template +class IAggregateFunctionDataHelper : public IAggregateFunctionHelper { +private: + using Base = IAggregateFunctionHelper; + protected: using Data = T; @@ -354,7 +357,7 @@ class IAggregateFunctionDataHelper : public IAggregateFunctionHelper static constexpr bool DateTime64Supported = true; IAggregateFunctionDataHelper(const DataTypes & argument_types_, const Array & parameters_) - : IAggregateFunctionHelper(argument_types_, parameters_) {} + : Base(argument_types_, parameters_) {} void create(AggregateDataPtr __restrict place) const override /// NOLINT { @@ -397,7 +400,7 @@ class IAggregateFunctionDataHelper : public IAggregateFunctionHelper if (func.allocatesMemoryInArena() || sizeof(Data) > 16 || func.sizeOfData() != sizeof(Data)) { - IAggregateFunctionHelper::addBatchLookupTable8(row_begin, row_end, map, place_offset, init, key, columns, arena); + Base::addBatchLookupTable8(row_begin, row_end, map, place_offset, init, key, columns, arena); return; } @@ -493,6 +496,7 @@ enum class AggFunctionId { //AGG_QUANTILES = 14, //AGG_TOP_COUNT = 15, //AGG_TOP_SUM = 16, + AGG_NUM_ROWS = 17, }; struct GroupByOptions : public arrow::compute::ScalarAggregateOptions { diff --git a/ydb/library/arrow_kernels/func_num_rows.h b/ydb/library/arrow_kernels/func_num_rows.h new file mode 100644 index 000000000000..f9d353232ec9 --- /dev/null +++ b/ydb/library/arrow_kernels/func_num_rows.h @@ -0,0 +1,20 @@ +#pragma once +#include "clickhouse_type_traits.h" +#include "func_common.h" + +namespace NKikimr::NKernels { + +struct TNumRows: public arrow::compute::MetaFunction { +public: + TNumRows(const TString name) + : arrow::compute::MetaFunction(name.data(), arrow::compute::Arity::Unary(), nullptr) { + } + + arrow::Result ExecuteImpl(const std::vector& args, const arrow::compute::FunctionOptions* /*options*/, + arrow::compute::ExecContext* /*ctx*/) const override { + Y_ABORT_UNLESS(args.size() == 1); + return arrow::Datum(std::make_shared(args[0].make_array()->length())); + } +}; + +} // namespace NKikimr::NKernels diff --git a/ydb/library/arrow_kernels/functions.h b/ydb/library/arrow_kernels/functions.h index 2f4523a4fe20..10d45bfc7eec 100644 --- a/ydb/library/arrow_kernels/functions.h +++ b/ydb/library/arrow_kernels/functions.h @@ -7,3 +7,4 @@ #include "func_modulo_or_zero.h" #include "func_math.h" #include "func_round.h" +#include "func_num_rows.h" From ae26bfc58fba121c25f33c9d72fecd0528fdc41e Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Thu, 22 Aug 2024 13:13:54 +0300 Subject: [PATCH 041/261] Use granular updates for mediator timecast when available (#8090) --- ydb/core/base/domain.cpp | 4 +- ydb/core/base/domain.h | 30 +- ydb/core/base/tx_processing.cpp | 2 +- ydb/core/persqueue/pq_impl.h | 2 +- ydb/core/protos/feature_flags.proto | 1 + ydb/core/protos/tx_mediator_timecast.proto | 4 + ydb/core/testlib/basics/feature_flags.h | 1 + ydb/core/testlib/tablet_helpers.cpp | 6 +- ydb/core/testlib/test_client.cpp | 6 +- ydb/core/testlib/test_client.h | 2 + ydb/core/tx/columnshard/columnshard_impl.h | 2 +- ydb/core/tx/datashard/datashard_impl.h | 2 +- ydb/core/tx/datashard/datashard_ut_order.cpp | 4 + .../datashard/datashard_ut_read_iterator.cpp | 6 + .../tx/datashard/datashard_ut_snapshot.cpp | 16 + ydb/core/tx/mediator/mediator_impl.h | 2 +- ydb/core/tx/time_cast/time_cast.cpp | 746 ++++++++++++++---- ydb/core/tx/time_cast/time_cast.h | 48 +- ydb/core/tx/time_cast/time_cast_ut.cpp | 344 +++++++- 19 files changed, 1041 insertions(+), 187 deletions(-) diff --git a/ydb/core/base/domain.cpp b/ydb/core/base/domain.cpp index 3fe01adabb31..a8cd4bf14554 100644 --- a/ydb/core/base/domain.cpp +++ b/ydb/core/base/domain.cpp @@ -5,7 +5,8 @@ namespace NKikimr { TDomainsInfo::TDomain::TDomain(const TString &name, ui32 domainUid, ui64 schemeRootId, TVectorUi64 coordinators, TVectorUi64 mediators, TVectorUi64 allocators, - ui64 domainPlanResolution, const TStoragePoolKinds *poolTypes) + ui64 domainPlanResolution, ui32 timecastBucketsPerMediator, + const TStoragePoolKinds *poolTypes) : DomainUid(domainUid) , SchemeRoot(schemeRootId) , Name(name) @@ -13,6 +14,7 @@ namespace NKikimr { , Mediators(std::move(mediators)) , TxAllocators(std::move(allocators)) , DomainPlanResolution(domainPlanResolution) + , TimecastBucketsPerMediator(timecastBucketsPerMediator) , StoragePoolTypes(poolTypes ? *poolTypes : TStoragePoolKinds()) {} diff --git a/ydb/core/base/domain.h b/ydb/core/base/domain.h index 0dcb8e6740ac..0ac95cd54895 100644 --- a/ydb/core/base/domain.h +++ b/ydb/core/base/domain.h @@ -67,15 +67,18 @@ struct TDomainsInfo : public TThrRefBase { const TVector Mediators; const TVector TxAllocators; const ui64 DomainPlanResolution; + const ui32 TimecastBucketsPerMediator; const TStoragePoolKinds StoragePoolTypes; - static constexpr ui32 TimecastBucketsPerMediator = 2; // <- any sense in making this configurable? may be for debug?.. + static constexpr ui64 DefaultPlanResolution = 500; + static constexpr ui32 DefaultTimecastBucketsPerMediator = 2; private: //don't reinterpret any data TDomain(const TString &name, ui32 domainUid, ui64 schemeRootId, TVectorUi64 coordinators, TVectorUi64 mediators, TVectorUi64 allocators, - ui64 domainPlanResolution, const TStoragePoolKinds *poolTypes); + ui64 domainPlanResolution, ui32 timecastBucketsPerMediator, + const TStoragePoolKinds *poolTypes); public: ~TDomain(); @@ -84,8 +87,23 @@ struct TDomainsInfo : public TThrRefBase { static TDomain::TPtr ConstructEmptyDomain(const TString &name, ui32 domainId = 0) { const ui64 schemeRoot = 0; - ui64 planResolution = 500; - return new TDomain(name, domainId, schemeRoot, {}, {}, {}, planResolution, nullptr); + return new TDomain(name, domainId, schemeRoot, {}, {}, {}, + DefaultPlanResolution, DefaultTimecastBucketsPerMediator, nullptr); + } + + template + static TDomain::TPtr ConstructDomainWithExplicitTabletIds(const TString &name, ui32 domainUid, ui64 schemeRoot, + ui64 planResolution, ui32 timecastBucketsPerMediator, + const TUidsContainerUi64 &coordinatorUids, + const TUidsContainerUi64 &mediatorUids, + const TUidsContainerUi64 &allocatorUids, + const TStoragePoolKinds &poolTypes) + { + return new TDomain(name, domainUid, schemeRoot, + TVectorUi64(coordinatorUids.begin(), coordinatorUids.end()), + TVectorUi64(mediatorUids.begin(), mediatorUids.end()), + TVectorUi64(allocatorUids.begin(), allocatorUids.end()), + planResolution, timecastBucketsPerMediator, &poolTypes); } template @@ -100,7 +118,7 @@ struct TDomainsInfo : public TThrRefBase { TVectorUi64(coordinatorUids.begin(), coordinatorUids.end()), TVectorUi64(mediatorUids.begin(), mediatorUids.end()), TVectorUi64(allocatorUids.begin(), allocatorUids.end()), - planResolution, &poolTypes); + planResolution, DefaultTimecastBucketsPerMediator, &poolTypes); } template @@ -114,7 +132,7 @@ struct TDomainsInfo : public TThrRefBase { TVectorUi64(coordinatorUids.begin(), coordinatorUids.end()), TVectorUi64(mediatorUids.begin(), mediatorUids.end()), TVectorUi64(allocatorUids.begin(), allocatorUids.end()), - planResolution, nullptr); + planResolution, DefaultTimecastBucketsPerMediator, nullptr); } ui32 DomainRootTag() const { diff --git a/ydb/core/base/tx_processing.cpp b/ydb/core/base/tx_processing.cpp index 1ae6299aab34..f34f47a7e159 100644 --- a/ydb/core/base/tx_processing.cpp +++ b/ydb/core/base/tx_processing.cpp @@ -69,7 +69,7 @@ NKikimrSubDomains::TProcessingParams NKikimr::ExtractProcessingParams(const NKik } NKikimr::TTimeCastBuckets::TTimeCastBuckets() - : TimecastBucketsPerMediator(TDomainsInfo::TDomain::TimecastBucketsPerMediator) + : TimecastBucketsPerMediator(TDomainsInfo::TDomain::DefaultTimecastBucketsPerMediator) {} NKikimr::TTimeCastBuckets::TTimeCastBuckets(ui32 timecastBuckets) diff --git a/ydb/core/persqueue/pq_impl.h b/ydb/core/persqueue/pq_impl.h index ce85e20394bb..dbd6296fb05a 100644 --- a/ydb/core/persqueue/pq_impl.h +++ b/ydb/core/persqueue/pq_impl.h @@ -424,7 +424,7 @@ class TPersQueue : public NKeyValue::TKeyValueFlat { void MediatorTimeCastUnregisterTablet(const TActorContext& ctx); void Handle(TEvMediatorTimecast::TEvRegisterTabletResult::TPtr& ev, const TActorContext& ctx); - TIntrusivePtr MediatorTimeCastEntry; + TMediatorTimecastEntry::TCPtr MediatorTimeCastEntry; void DeleteExpiredTransactions(const TActorContext& ctx); void Handle(TEvPersQueue::TEvCancelTransactionProposal::TPtr& ev, const TActorContext& ctx); diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index f90cf30970b4..364aadd6533f 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -153,4 +153,5 @@ message TFeatureFlags { optional bool EnableChangefeedsOnIndexTables = 134 [default = false]; optional bool EnableResourcePoolsCounters = 135 [default = false]; optional bool EnableOptionalColumnsInColumnShard = 136 [default = false]; + optional bool EnableGranularTimecast = 137 [default = true]; } diff --git a/ydb/core/protos/tx_mediator_timecast.proto b/ydb/core/protos/tx_mediator_timecast.proto index 298232e4c78c..3f495be0532c 100644 --- a/ydb/core/protos/tx_mediator_timecast.proto +++ b/ydb/core/protos/tx_mediator_timecast.proto @@ -29,6 +29,10 @@ message TEvGranularWatch { // The list of tablets the client is currently tracking repeated fixed64 Tablets = 3 [packed = true]; + + // The maximum LatestStep observed in the past + // Mediator may avoid sending updates with LatestStep less than MinStep + optional uint64 MinStep = 4; } // sent from local timecast to mediator to change the list of tablets diff --git a/ydb/core/testlib/basics/feature_flags.h b/ydb/core/testlib/basics/feature_flags.h index 8553a4f861da..95662f200d35 100644 --- a/ydb/core/testlib/basics/feature_flags.h +++ b/ydb/core/testlib/basics/feature_flags.h @@ -63,6 +63,7 @@ class TTestFeatureFlagsHolder { FEATURE_FLAG_SETTER(EnableResourcePools) FEATURE_FLAG_SETTER(EnableChangefeedsOnIndexTables) FEATURE_FLAG_SETTER(EnableBackupService) + FEATURE_FLAG_SETTER(EnableGranularTimecast) #undef FEATURE_FLAG_SETTER }; diff --git a/ydb/core/testlib/tablet_helpers.cpp b/ydb/core/testlib/tablet_helpers.cpp index dea9f26c4d72..883082995330 100644 --- a/ydb/core/testlib/tablet_helpers.cpp +++ b/ydb/core/testlib/tablet_helpers.cpp @@ -88,14 +88,14 @@ namespace NKikimr { const ui64 tabletId = ev->Get()->TabletId; auto& entry = Entries[tabletId]; if (!entry) { - entry = new TMediatorTimecastEntry(); + entry = new TMediatorTimecastSharedEntry(); } - ctx.Send(ev->Sender, new TEvMediatorTimecast::TEvRegisterTabletResult(tabletId, entry)); + ctx.Send(ev->Sender, new TEvMediatorTimecast::TEvRegisterTabletResult(tabletId, new TMediatorTimecastEntry(entry, entry))); } private: - THashMap> Entries; + THashMap> Entries; }; void SetupMediatorTimecastProxy(TTestActorRuntime& runtime, ui32 nodeIndex, bool useFake = false) diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index a12f7a293948..05aee2c04386 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -503,8 +503,12 @@ namespace Tests { if (!planResolution) { planResolution = Settings->UseRealThreads ? 7 : 500; } + ui32 timecastBuckets = Settings->DomainTimecastBuckets; + if (!timecastBuckets) { + timecastBuckets = TDomainsInfo::TDomain::DefaultTimecastBucketsPerMediator; + } auto domain = TDomainsInfo::TDomain::ConstructDomainWithExplicitTabletIds(Settings->DomainName, domainId, ChangeStateStorage(SchemeRoot, domainId), - planResolution, + planResolution, timecastBuckets, TVector{TDomainsInfo::MakeTxCoordinatorIDFixed(1)}, TVector{TDomainsInfo::MakeTxMediatorIDFixed(1)}, TVector{TDomainsInfo::MakeTxAllocatorIDFixed(1)}, diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 69de2759a9ba..19c8420ace81 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -145,6 +145,7 @@ namespace Tests { NKikimrConfig::TCompactionConfig CompactionConfig; TMap NodeKeys; ui64 DomainPlanResolution = 0; + ui32 DomainTimecastBuckets = 0; std::shared_ptr PersQueueGetReadSessionsInfoWorkerFactory; std::shared_ptr DataStreamsAuthFactory; std::shared_ptr PersQueueMirrorReaderFactory = std::make_shared(); @@ -193,6 +194,7 @@ namespace Tests { TServerSettings& SetEnableKqpSpilling(bool value) { EnableKqpSpilling = value; return *this; } TServerSettings& SetEnableForceFollowers(bool value) { EnableForceFollowers = value; return *this; } TServerSettings& SetDomainPlanResolution(ui64 resolution) { DomainPlanResolution = resolution; return *this; } + TServerSettings& SetDomainTimecastBuckets(ui32 buckets) { DomainTimecastBuckets = buckets; return *this; } TServerSettings& SetFeatureFlags(const NKikimrConfig::TFeatureFlags& value) { FeatureFlags = value; return *this; } TServerSettings& SetCompactionConfig(const NKikimrConfig::TCompactionConfig& value) { CompactionConfig = value; return *this; } TServerSettings& SetEnableDbCounters(bool value) { FeatureFlags.SetEnableDbCounters(value); return *this; } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index c0014c156eed..e81c021c0f31 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -436,7 +436,7 @@ class TColumnShard ui64 StatsReportRound = 0; TString OwnerPath; - TIntrusivePtr MediatorTimeCastEntry; + TMediatorTimecastEntry::TCPtr MediatorTimeCastEntry; bool MediatorTimeCastRegistered = false; TSet MediatorTimeCastWaitingSteps; const TDuration PeriodicWakeupActivationPeriod; diff --git a/ydb/core/tx/datashard/datashard_impl.h b/ydb/core/tx/datashard/datashard_impl.h index 36fabeea3af3..18c07fc3a342 100644 --- a/ydb/core/tx/datashard/datashard_impl.h +++ b/ydb/core/tx/datashard/datashard_impl.h @@ -2698,7 +2698,7 @@ class TDataShard { } }; - TIntrusivePtr MediatorTimeCastEntry; + TMediatorTimecastEntry::TCPtr MediatorTimeCastEntry; TSet MediatorTimeCastWaitingSteps; TMultiMap MediatorDelayedReplies; diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp index a2b4acf86292..f3c5856d6879 100644 --- a/ydb/core/tx/datashard/datashard_ut_order.cpp +++ b/ydb/core/tx/datashard/datashard_ut_order.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -3877,6 +3878,9 @@ Y_UNIT_TEST(TestUnprotectedReadsThenWriteVisibility) { }; THashMap mediatorState; + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(runtime); + bool mustWaitForSteps[2] = { false, false }; auto captureTimecast = [&](TAutoPtr& ev) -> auto { diff --git a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp index d3b6b88ec5e7..02228e4db679 100644 --- a/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp +++ b/ydb/core/tx/datashard/datashard_ut_read_iterator.cpp @@ -2887,6 +2887,9 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { TTestHelper helper(serverSettings); + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(*helper.Server->GetRuntime()); + auto waitFor = [&](const auto& condition, const TString& description) { if (!condition()) { Cerr << "... waiting for " << description << Endl; @@ -3018,6 +3021,9 @@ Y_UNIT_TEST_SUITE(DataShardReadIterator) { TTestHelper helper(serverSettings); + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(*helper.Server->GetRuntime()); + auto waitFor = [&](const auto& condition, const TString& description) { if (!condition()) { Cerr << "... waiting for " << description << Endl; diff --git a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp index 6f25050cb429..52b7f9e7d779 100644 --- a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp +++ b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include // Y_UNIT_TEST_(TWIN|QUAD) #include @@ -807,6 +808,9 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { auto &runtime = *server->GetRuntime(); auto sender = runtime.AllocateEdgeActor(); + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(runtime); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG); @@ -3732,6 +3736,9 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { auto &runtime = *server->GetRuntime(); auto sender = runtime.AllocateEdgeActor(); + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(runtime); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG); @@ -4150,6 +4157,9 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { auto &runtime = *server->GetRuntime(); auto sender = runtime.AllocateEdgeActor(); + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(runtime); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_TRACE); @@ -4265,6 +4275,9 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { auto &runtime = *server->GetRuntime(); auto sender = runtime.AllocateEdgeActor(); + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(runtime); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_TRACE); @@ -4402,6 +4415,9 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { auto &runtime = *server->GetRuntime(); auto sender = runtime.AllocateEdgeActor(); + // Don't allow granular timecast side-stepping mediator time hacks in this test + TBlockEvents blockGranularUpdate(runtime); + runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::TX_COORDINATOR, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::TX_MEDIATOR_TIMECAST, NLog::PRI_TRACE); diff --git a/ydb/core/tx/mediator/mediator_impl.h b/ydb/core/tx/mediator/mediator_impl.h index 628a6cb1f317..928c180a56b1 100644 --- a/ydb/core/tx/mediator/mediator_impl.h +++ b/ydb/core/tx/mediator/mediator_impl.h @@ -388,7 +388,7 @@ class TTxMediator : public TActor, public NTabletFlatExecutor::TTab struct DomainConfiguration : Table<2> { struct Version : Column<1, NScheme::NTypeIds::Uint64> {}; struct Coordinators : Column<2, NScheme::NTypeIds::String> { using Type = TVector; }; - struct TimeCastBuckets : Column<3, NScheme::NTypeIds::Uint32> { static constexpr ui32 Default = TDomainsInfo::TDomain::TimecastBucketsPerMediator; }; + struct TimeCastBuckets : Column<3, NScheme::NTypeIds::Uint32> { static constexpr ui32 Default = TDomainsInfo::TDomain::DefaultTimecastBucketsPerMediator; }; using TKey = TableKey; using TColumns = TableColumns; diff --git a/ydb/core/tx/time_cast/time_cast.cpp b/ydb/core/tx/time_cast/time_cast.cpp index e11b8de22920..fa93e556c765 100644 --- a/ydb/core/tx/time_cast/time_cast.cpp +++ b/ydb/core/tx/time_cast/time_cast.cpp @@ -20,20 +20,43 @@ namespace NKikimr { // We will unsubscribe from idle coordinators after 5 minutes static constexpr TDuration MaxIdleCoordinatorSubscriptionTime = TDuration::Minutes(5); -ui64 TMediatorTimecastEntry::Get(ui64 tabletId) const { +ui64 TMediatorTimecastSharedEntry::Get() const noexcept { + return Step.load(std::memory_order_acquire); +} + +void TMediatorTimecastSharedEntry::Set(ui64 step) noexcept { + ui64 current = Step.load(std::memory_order_relaxed); + Step.store(Max(current, step), std::memory_order_release); +} + +TMediatorTimecastEntry::TMediatorTimecastEntry( + const TMediatorTimecastSharedEntry::TPtr& safeStep, + const TMediatorTimecastSharedEntry::TPtr& latestStep) noexcept + : SafeStep(safeStep) + , LatestStep(latestStep) +{ + Y_ABORT_UNLESS(SafeStep); + Y_ABORT_UNLESS(LatestStep); +} + +TMediatorTimecastEntry::~TMediatorTimecastEntry() noexcept { + // nothing +} + +ui64 TMediatorTimecastEntry::Get(ui64 tabletId) const noexcept { Y_UNUSED(tabletId); - return AtomicGet(Step); + ui64 latest = LatestStep->Get(); + ui64 frozen = FrozenStep.load(std::memory_order_relaxed); + ui64 safe = SafeStep->Get(); + return Max(Min(latest, frozen), safe); } -void TMediatorTimecastEntry::Update(ui64 step, ui64 *exemption, ui64 exsz) { - Y_ABORT_UNLESS(exsz == 0, "exemption lists not supported yet"); - Y_UNUSED(exemption); - Y_UNUSED(exsz); +ui64 TMediatorTimecastEntry::GetFrozenStep() const noexcept { + return FrozenStep.load(std::memory_order_relaxed); +} - // Mediator time shouldn't go back while shards are running - if (Get(0) < step) { - AtomicSet(Step, step); - } +void TMediatorTimecastEntry::SetFrozenStep(ui64 step) noexcept { + FrozenStep.store(step, std::memory_order_relaxed); } class TMediatorTimecastProxy : public TActor { @@ -64,27 +87,88 @@ class TMediatorTimecastProxy : public TActor { , PlanStep(planStep) { } - // An inverse based on (PlanStep, TabletId) tuple - inline bool operator<(const TWaiter& rhs) const { - return rhs.PlanStep < PlanStep || - (rhs.PlanStep == PlanStep && rhs.TabletId < TabletId); + struct TCompare { + constexpr bool operator()(const TWaiter& a, const TWaiter& b) const noexcept { + return a.PlanStep < b.PlanStep; + } + }; + + struct TInverseCompare { + constexpr bool operator()(const TWaiter& a, const TWaiter& b) const noexcept { + return b.PlanStep < a.PlanStep; + } + }; + }; + + using TWaiterSet = std::multiset; + using TWaiterQueue = std::priority_queue, TWaiter::TInverseCompare>; + + struct TTabletWaiter { + ui64 PlanStep; + ui64 TabletId; + + TTabletWaiter() = default; + TTabletWaiter(ui64 planStep, ui64 tabletId) + : PlanStep(planStep) + , TabletId(tabletId) + {} + + inline bool operator<(const TTabletWaiter& rhs) const noexcept { + return PlanStep < rhs.PlanStep || + (PlanStep == rhs.PlanStep && TabletId < rhs.TabletId); } }; + using TTabletWaiterSet = std::set; + + struct TMediator; + struct TMediatorBucket; + + struct TTabletInfo : public TIntrusiveListItem { + const ui64 TabletId; + TMediator* Mediator = nullptr; + ui32 BucketId = 0; + ui32 RefCount = 0; + TMediatorTimecastEntry::TPtr Entry; + TWaiterSet Waiters; + ui64 SubscriptionId = 0; + ui64 FrozenStep = Max(); + ui64 MinStep = 0; + bool Synced = false; + + explicit TTabletInfo(ui64 tabletId) + : TabletId(tabletId) + {} + }; + struct TMediatorBucket { - TIntrusivePtr Entry; - std::priority_queue Waiters; + const TMediatorTimecastSharedEntry::TPtr SafeStep = new TMediatorTimecastSharedEntry; + const TMediatorTimecastSharedEntry::TPtr LatestStep = new TMediatorTimecastSharedEntry; + THashSet Tablets; + TIntrusiveList PendingSubscribe; + TIntrusiveList PendingUpdate; + TWaiterQueue Waiters; + TTabletWaiterSet TabletWaiters; + ui64 SubscriptionId = 0; + bool WatchSent = false; + bool WatchSynced = false; + + bool AddNewTablet(ui64 tabletId) { + return Tablets.insert(tabletId).second; + } }; struct TMediator { + const ui64 TabletId; const ui32 BucketsSz; TArrayHolder Buckets; std::vector Coordinators; TActorId PipeClient; - TMediator(ui32 bucketsSz) - : BucketsSz(bucketsSz) + TMediator(ui64 tabletId, ui32 bucketsSz) + : TabletId(tabletId) + , BucketsSz(bucketsSz) , Buckets(new TMediatorBucket[bucketsSz]) {} }; @@ -95,13 +179,6 @@ class TMediatorTimecastProxy : public TActor { bool Subscribed = false; }; - struct TTabletInfo { - ui64 MediatorTabletId; - TMediator* Mediator; - ui32 BucketId; - ui32 RefCount = 0; - }; - struct TCoordinatorSubscriber { TSet Coordinators; }; @@ -125,55 +202,176 @@ class TMediatorTimecastProxy : public TActor { ui64 LastSeqNo = 0; THashMap Coordinators; THashMap CoordinatorSubscribers; + ui64 NextWatchSubscriptionId = 1; - TMediator& MediatorInfo(ui64 mediator, const NKikimrSubDomains::TProcessingParams &processing) { - auto pr = Mediators.try_emplace(mediator, processing.GetTimeCastBucketsPerMediator()); - if (!pr.second) { - Y_ABORT_UNLESS(pr.first->second.BucketsSz == processing.GetTimeCastBucketsPerMediator()); + TMediator& MediatorInfo(ui64 mediator, const NKikimrSubDomains::TProcessingParams& processing) { + auto it = Mediators.find(mediator); + if (it == Mediators.end()) { + auto res = Mediators.emplace( + std::piecewise_construct, + std::forward_as_tuple(mediator), + std::forward_as_tuple(mediator, processing.GetTimeCastBucketsPerMediator())); + it = res.first; + } else { + Y_ABORT_UNLESS(it->second.BucketsSz == processing.GetTimeCastBucketsPerMediator()); } - if (pr.first->second.Coordinators.empty()) { - pr.first->second.Coordinators.assign( + + // Note: some older tablets may have an empty list of coordinators + // We fill the list of coordinators the first time we observe a non-empty list + if (it->second.Coordinators.empty()) { + it->second.Coordinators.assign( processing.GetCoordinators().begin(), processing.GetCoordinators().end()); } - return pr.first->second; + + return it->second; + } + + const TActorId& MediatorPipe(TMediator& mediator, const TActorContext& ctx) { + if (!mediator.PipeClient) { + mediator.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, mediator.TabletId)); + } + return mediator.PipeClient; } - void RegisterMediator(ui64 mediatorTabletId, TMediator &mediator, const TActorContext &ctx) { - TAutoPtr ev(new TEvMediatorTimecast::TEvWatch()); + void SendGranularWatch(TMediator& mediator, ui32 bucketId, const TActorContext& ctx) { + auto& bucket = mediator.Buckets[bucketId]; + + while (!bucket.PendingSubscribe.Empty()) { + bucket.PendingSubscribe.PopFront(); + } + while (!bucket.PendingUpdate.Empty()) { + bucket.PendingUpdate.PopFront(); + } + + if (!AppData(ctx)->FeatureFlags.GetEnableGranularTimecast()) { + bucket.SubscriptionId = Max(); + return; + } + + bucket.SubscriptionId = NextWatchSubscriptionId++; + auto req = std::make_unique(bucketId, bucket.SubscriptionId); + req->Record.SetMinStep(bucket.LatestStep->Get()); + + for (ui64 tabletId : bucket.Tablets) { + auto& tabletInfo = Tablets.at(tabletId); + Y_ABORT_UNLESS(tabletInfo.Mediator == &mediator); + Y_ABORT_UNLESS(tabletInfo.BucketId == bucketId); + tabletInfo.SubscriptionId = bucket.SubscriptionId; + bucket.PendingSubscribe.PushBack(&tabletInfo); + tabletInfo.FrozenStep = Max(); // will be unfrozen unless reported as frozen + tabletInfo.MinStep = tabletInfo.Entry->Get(); // will not freeze below this step + tabletInfo.Synced = false; + req->Record.AddTablets(tabletId); + } + + const auto& client = MediatorPipe(mediator, ctx); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " SEND to Mediator# " << mediator.TabletId << " " << req->ToString()); + NTabletPipe::SendData(ctx, client, req.release()); + } + + void SendGranularWatchAdd(TMediator& mediator, ui32 bucketId, ui64 tabletId, const TActorContext& ctx) { + auto& bucket = mediator.Buckets[bucketId]; + + Y_ABORT_UNLESS(bucket.Tablets.contains(tabletId)); + auto& tabletInfo = Tablets.at(tabletId); + Y_ABORT_UNLESS(tabletInfo.Mediator == &mediator); + Y_ABORT_UNLESS(tabletInfo.BucketId == bucketId); + + if (bucket.SubscriptionId == Max()) { + tabletInfo.SubscriptionId = Max(); + return; + } + + tabletInfo.SubscriptionId = NextWatchSubscriptionId++; + bucket.PendingSubscribe.PushBack(&tabletInfo); + tabletInfo.FrozenStep = Max(); // will be unfrozen unless reported as frozen + tabletInfo.MinStep = tabletInfo.Entry->Get(); // will not freeze below this step + tabletInfo.Synced = false; + + auto req = std::make_unique(bucketId, tabletInfo.SubscriptionId); + req->Record.AddAddTablets(tabletId); + + const auto& client = MediatorPipe(mediator, ctx); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " SEND to Mediator# " << mediator.TabletId << " " << req->ToString()); + NTabletPipe::SendData(ctx, client, req.release()); + } + + void SendGranularWatchRemove(TMediator& mediator, ui32 bucketId, ui64 tabletId, const TActorContext& ctx) { + auto& bucket = mediator.Buckets[bucketId]; + + Y_ABORT_UNLESS(bucket.Tablets.contains(tabletId)); + auto& tabletInfo = Tablets.at(tabletId); + Y_ABORT_UNLESS(tabletInfo.Mediator == &mediator); + Y_ABORT_UNLESS(tabletInfo.BucketId == bucketId); + tabletInfo.SubscriptionId = Max(); + tabletInfo.Synced = false; + bucket.PendingSubscribe.Remove(&tabletInfo); + bucket.PendingUpdate.Remove(&tabletInfo); + bucket.Tablets.erase(tabletId); + + if (bucket.SubscriptionId == Max()) { + return; + } + + auto req = std::make_unique(bucketId, NextWatchSubscriptionId++); + req->Record.AddRemoveTablets(tabletId); + + const auto& client = MediatorPipe(mediator, ctx); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " SEND to Mediator# " << mediator.TabletId << " " << req->ToString()); + NTabletPipe::SendData(ctx, client, req.release()); + } + + void RegisterMediator(TMediator& mediator, const TActorContext& ctx) { + auto req = std::make_unique(); + for (ui32 bucketId = 0, e = mediator.BucketsSz; bucketId != e; ++bucketId) { - auto &x = mediator.Buckets[bucketId]; - if (!x.Entry) - continue; - if (x.Entry.RefCount() == 1) { - x.Entry.Drop(); - x.Waiters = { }; + auto& bucket = mediator.Buckets[bucketId]; + if (bucket.Tablets.empty()) { + // There are no tablets interested in this bucket, avoid unnecessary watches + bucket.WatchSent = false; + bucket.Waiters = { }; continue; } - ev->Record.AddBucket(bucketId); + SendGranularWatch(mediator, bucketId, ctx); + + req->Record.AddBucket(bucketId); + bucket.WatchSent = true; + bucket.WatchSynced = false; } - if (ev->Record.BucketSize()) { - mediator.PipeClient = ctx.RegisterWithSameMailbox(NTabletPipe::CreateClient(ctx.SelfID, mediatorTabletId)); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() - << " SEND to# " << mediatorTabletId << " Mediator " << ev->ToString()); - NTabletPipe::SendData(ctx, mediator.PipeClient, ev.Release()); + if (req->Record.BucketSize()) { + const auto& client = MediatorPipe(mediator, ctx); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " SEND to Mediator# " << mediator.TabletId << " " << req->ToString()); + NTabletPipe::SendData(ctx, client, req.release()); } } - void SubscribeBucket(ui64 mediatorTabletId, TMediator &mediator, ui32 bucketId, const TActorContext &ctx) { - if (mediator.PipeClient) { - TAutoPtr ev(new TEvMediatorTimecast::TEvWatch(bucketId)); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() - << " SEND to# " << mediatorTabletId << " Mediator " << ev->ToString()); - NTabletPipe::SendData(ctx, mediator.PipeClient, ev.Release()); + void SubscribeBucket(TMediator& mediator, ui32 bucketId, ui64 tabletId, const TActorContext& ctx) { + auto& bucket = mediator.Buckets[bucketId]; + Y_ABORT_UNLESS(bucket.Tablets.contains(tabletId)); + if (bucket.WatchSent) { + SendGranularWatchAdd(mediator, bucketId, tabletId, ctx); } else { - RegisterMediator(mediatorTabletId, mediator, ctx); + SendGranularWatch(mediator, bucketId, ctx); + + auto req = std::make_unique(bucketId); + + const auto& client = MediatorPipe(mediator, ctx); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " SEND to Mediator# " << mediator.TabletId << " " << req->ToString()); + NTabletPipe::SendData(ctx, client, req.release()); + + bucket.WatchSent = true; } } - void TryResync(const TActorId &pipeClient, ui64 tabletId, const TActorContext &ctx) { + void TryResync(const TActorId& pipeClient, ui64 tabletId, const TActorContext& ctx) { ResyncCoordinator(tabletId, pipeClient, ctx); auto it = Mediators.find(tabletId); @@ -181,34 +379,35 @@ class TMediatorTimecastProxy : public TActor { return; } - TMediator &mediator = it->second; + TMediator& mediator = it->second; if (mediator.PipeClient == pipeClient) { mediator.PipeClient = TActorId(); - RegisterMediator(tabletId, mediator, ctx); + RegisterMediator(mediator, ctx); } } - void SyncCoordinator(ui64 coordinatorId, TCoordinator &coordinator, const TActorContext &ctx); - void ResyncCoordinator(ui64 coordinatorId, const TActorId &pipeClient, const TActorContext &ctx); - void NotifyCoordinatorWaiters(ui64 coordinatorId, TCoordinator &coordinator, const TActorContext &ctx); + void SyncCoordinator(ui64 coordinatorId, TCoordinator& coordinator, const TActorContext& ctx); + void ResyncCoordinator(ui64 coordinatorId, const TActorId& pipeClient, const TActorContext& ctx); + void NotifyCoordinatorWaiters(ui64 coordinatorId, TCoordinator& coordinator, const TActorContext& ctx); - void Handle(TEvMediatorTimecast::TEvRegisterTablet::TPtr &ev, const TActorContext &ctx); - void Handle(TEvMediatorTimecast::TEvUnregisterTablet::TPtr &ev, const TActorContext &ctx); - void Handle(TEvMediatorTimecast::TEvWaitPlanStep::TPtr &ev, const TActorContext &ctx); - void Handle(TEvMediatorTimecast::TEvUpdate::TPtr &ev, const TActorContext &ctx); - void Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx); - void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActorContext &ctx); - void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr &ev, const TActorContext &ctx); - void Handle(TEvPrivate::TEvRetryCoordinator::TPtr &ev, const TActorContext &ctx); + void Handle(TEvMediatorTimecast::TEvRegisterTablet::TPtr& ev, const TActorContext& ctx); + void Handle(TEvMediatorTimecast::TEvUnregisterTablet::TPtr& ev, const TActorContext& ctx); + void Handle(TEvMediatorTimecast::TEvWaitPlanStep::TPtr& ev, const TActorContext& ctx); + void Handle(TEvMediatorTimecast::TEvUpdate::TPtr& ev, const TActorContext& ctx); + void Handle(TEvMediatorTimecast::TEvGranularUpdate::TPtr& ev, const TActorContext& ctx); + void Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx); + void Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPrivate::TEvRetryCoordinator::TPtr& ev, const TActorContext& ctx); // Client requests for readstep subscriptions - void Handle(TEvMediatorTimecast::TEvSubscribeReadStep::TPtr &ev, const TActorContext &ctx); - void Handle(TEvMediatorTimecast::TEvUnsubscribeReadStep::TPtr &ev, const TActorContext &ctx); - void Handle(TEvMediatorTimecast::TEvWaitReadStep::TPtr &ev, const TActorContext &ctx); + void Handle(TEvMediatorTimecast::TEvSubscribeReadStep::TPtr& ev, const TActorContext& ctx); + void Handle(TEvMediatorTimecast::TEvUnsubscribeReadStep::TPtr& ev, const TActorContext& ctx); + void Handle(TEvMediatorTimecast::TEvWaitReadStep::TPtr& ev, const TActorContext& ctx); // Coordinator replies for readstep subscriptions - void Handle(TEvTxProxy::TEvSubscribeReadStepResult::TPtr &ev, const TActorContext &ctx); - void Handle(TEvTxProxy::TEvSubscribeReadStepUpdate::TPtr &ev, const TActorContext &ctx); + void Handle(TEvTxProxy::TEvSubscribeReadStepResult::TPtr& ev, const TActorContext& ctx); + void Handle(TEvTxProxy::TEvSubscribeReadStepUpdate::TPtr& ev, const TActorContext& ctx); public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { @@ -225,6 +424,7 @@ class TMediatorTimecastProxy : public TActor { HFunc(TEvMediatorTimecast::TEvUnregisterTablet, Handle); HFunc(TEvMediatorTimecast::TEvWaitPlanStep, Handle); HFunc(TEvMediatorTimecast::TEvUpdate, Handle); + HFunc(TEvMediatorTimecast::TEvGranularUpdate, Handle); HFunc(TEvMediatorTimecast::TEvSubscribeReadStep, Handle); HFunc(TEvMediatorTimecast::TEvUnsubscribeReadStep, Handle); @@ -241,90 +441,123 @@ class TMediatorTimecastProxy : public TActor { } }; -void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvRegisterTablet::TPtr &ev, const TActorContext &ctx) { - const TEvMediatorTimecast::TEvRegisterTablet *msg = ev->Get(); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvRegisterTablet::TPtr& ev, const TActorContext& ctx) { + const TEvMediatorTimecast::TEvRegisterTablet* msg = ev->Get(); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE " << msg->ToString()); const ui64 tabletId = msg->TabletId; - const NKikimrSubDomains::TProcessingParams &processingParams = msg->ProcessingParams; + const NKikimrSubDomains::TProcessingParams& processingParams = msg->ProcessingParams; - auto& tabletInfo = Tablets[tabletId]; + auto it = Tablets.find(tabletId); + if (it == Tablets.end()) { + auto res = Tablets.emplace( + std::piecewise_construct, + std::forward_as_tuple(tabletId), + std::forward_as_tuple(tabletId)); + it = res.first; + } + auto& tabletInfo = it->second; TMediators mediators(processingParams); Y_ABORT_UNLESS(mediators.List().size()); const ui64 mediatorTabletId = mediators.Select(tabletId); - tabletInfo.MediatorTabletId = mediatorTabletId; TTimeCastBuckets buckets(processingParams); const ui32 bucketId = buckets.Select(tabletId); - tabletInfo.BucketId = bucketId; - TMediator &mediator = MediatorInfo(mediatorTabletId, processingParams); - tabletInfo.Mediator = &mediator; + TMediator& mediator = MediatorInfo(mediatorTabletId, processingParams); Y_ABORT_UNLESS(bucketId < mediator.BucketsSz); - auto &bucket = mediator.Buckets[bucketId]; - if (!bucket.Entry) - bucket.Entry = new TMediatorTimecastEntry(); + auto& bucket = mediator.Buckets[bucketId]; + + if (tabletInfo.Mediator && (tabletInfo.Mediator != &mediator || tabletInfo.BucketId != bucketId)) { + // Tablet unexpectedly changed their mediator/bucket + SendGranularWatchRemove(*tabletInfo.Mediator, tabletInfo.BucketId, tabletId, ctx); + tabletInfo.Mediator = nullptr; + tabletInfo.Entry.Reset(); + } + + tabletInfo.Mediator = &mediator; + tabletInfo.BucketId = bucketId; + if (!tabletInfo.Entry) { + tabletInfo.Entry = new TMediatorTimecastEntry(bucket.SafeStep, bucket.LatestStep); + } ++tabletInfo.RefCount; + if (bucket.AddNewTablet(tabletId)) { + // New tablet for this bucket, make sure we subscribe + SubscribeBucket(mediator, bucketId, tabletId, ctx); + } TAutoPtr result( - new TEvMediatorTimecast::TEvRegisterTabletResult(tabletId, bucket.Entry)); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() - << " SEND to# " << ev->Sender.ToString() << " Sender " << result->ToString()); + new TEvMediatorTimecast::TEvRegisterTabletResult(tabletId, tabletInfo.Entry)); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " SEND to Sender# " << ev->Sender << " " << result->ToString()); ctx.Send(ev->Sender, result.Release()); - - SubscribeBucket(mediatorTabletId, mediator, bucketId, ctx); } -void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUnregisterTablet::TPtr &ev, const TActorContext &ctx) { - const auto *msg = ev->Get(); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUnregisterTablet::TPtr& ev, const TActorContext& ctx) { + const auto* msg = ev->Get(); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE " << msg->ToString()); const ui64 tabletId = msg->TabletId; auto it = Tablets.find(tabletId); - if (it != Tablets.end()) { - Y_ABORT_UNLESS(it->second.RefCount > 0); - if (0 == --it->second.RefCount) { - // Note: buckets are unsubscribed lazily when entry refcount reaches zero on reconnect - Tablets.erase(it); + if (it == Tablets.end()) { + return; + } + + auto& tabletInfo = it->second; + + Y_ABORT_UNLESS(tabletInfo.RefCount > 0); + if (0 == --tabletInfo.RefCount) { + if (tabletInfo.Mediator) { + SendGranularWatchRemove(*tabletInfo.Mediator, tabletInfo.BucketId, tabletId, ctx); } + // Note: buckets are unsubscribed lazily when entry refcount reaches zero on reconnect + Tablets.erase(it); } } -void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvWaitPlanStep::TPtr &ev, const TActorContext &ctx) { - const auto *msg = ev->Get(); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvWaitPlanStep::TPtr& ev, const TActorContext& ctx) { + const auto* msg = ev->Get(); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE " << msg->ToString()); const ui64 tabletId = msg->TabletId; const ui64 planStep = msg->PlanStep; auto it = Tablets.find(tabletId); Y_ABORT_UNLESS(it != Tablets.end(), "TEvWaitPlanStep TabletId# %" PRIu64 " is not subscribed", tabletId); + auto& tabletInfo = it->second; - TMediator &mediator = *it->second.Mediator; - const ui32 bucketId = it->second.BucketId; - auto &bucket = mediator.Buckets[bucketId]; - Y_DEBUG_ABORT_UNLESS(bucket.Entry, "TEvWaitPlanStep TabletId# %" PRIu64 " has no timecast entry, possible race", tabletId); - if (!bucket.Entry) { + Y_DEBUG_ABORT_UNLESS(tabletInfo.Entry, "TEvWaitPlanStep TabletId# %" PRIu64 " has no timecast entry, possible race", tabletId); + if (!tabletInfo.Entry) { return; } - const ui64 currentStep = bucket.Entry->Get(tabletId); + Y_ABORT_UNLESS(tabletInfo.Mediator); + TMediator& mediator = *tabletInfo.Mediator; + auto& bucket = mediator.Buckets[tabletInfo.BucketId]; + + const ui64 currentStep = tabletInfo.Entry->Get(tabletId); if (currentStep < planStep) { - bucket.Waiters.emplace(ev->Sender, tabletId, planStep); - for (ui64 coordinatorId : mediator.Coordinators) { - TMediatorCoordinator &coordinator = MediatorCoordinators[coordinatorId]; - if (coordinator.WaitingSteps.insert(planStep).second && !coordinator.RetryPending) { - Send(MakePipePerNodeCacheID(false), - new TEvPipeCache::TEvForward( - new TEvTxProxy::TEvRequirePlanSteps(coordinatorId, planStep), - coordinatorId, - !coordinator.Subscribed)); - coordinator.Subscribed = true; + if (bucket.LatestStep->Get() < planStep) { + bucket.Waiters.emplace(ev->Sender, tabletId, planStep); + for (ui64 coordinatorId : mediator.Coordinators) { + TMediatorCoordinator& coordinator = MediatorCoordinators[coordinatorId]; + if (coordinator.WaitingSteps.insert(planStep).second && !coordinator.RetryPending) { + Send(MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward( + new TEvTxProxy::TEvRequirePlanSteps(coordinatorId, planStep), + coordinatorId, + !coordinator.Subscribed)); + coordinator.Subscribed = true; + } } + } else { + // Either frozen or not synchronized yet + tabletInfo.Waiters.emplace(ev->Sender, tabletId, planStep); + bucket.TabletWaiters.insert(TTabletWaiter(planStep, tabletId)); } return; } @@ -333,25 +566,25 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvWaitPlanStep::TPtr & ctx.Send(ev->Sender, new TEvMediatorTimecast::TEvNotifyPlanStep(tabletId, currentStep)); } -void TMediatorTimecastProxy::Handle(TEvTabletPipe::TEvClientConnected::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() +void TMediatorTimecastProxy::Handle(TEvTabletPipe::TEvClientConnected::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE EvClientConnected"); - TEvTabletPipe::TEvClientConnected *msg = ev->Get(); + TEvTabletPipe::TEvClientConnected* msg = ev->Get(); if (msg->Status != NKikimrProto::OK) { TryResync(msg->ClientId, msg->TabletId, ctx); } } -void TMediatorTimecastProxy::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() +void TMediatorTimecastProxy::Handle(TEvTabletPipe::TEvClientDestroyed::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE EvClientDestroyed"); - TEvTabletPipe::TEvClientDestroyed *msg = ev->Get(); + TEvTabletPipe::TEvClientDestroyed* msg = ev->Get(); TryResync(msg->ClientId, msg->TabletId, ctx); } -void TMediatorTimecastProxy::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr &ev, const TActorContext &ctx) { - auto *msg = ev->Get(); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() +void TMediatorTimecastProxy::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev, const TActorContext& ctx) { + auto* msg = ev->Get(); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE EvDeliveryProblem " << msg->TabletId); auto it = MediatorCoordinators.find(msg->TabletId); if (it != MediatorCoordinators.end()) { @@ -362,9 +595,9 @@ void TMediatorTimecastProxy::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr &ev, } } -void TMediatorTimecastProxy::Handle(TEvPrivate::TEvRetryCoordinator::TPtr &ev, const TActorContext &ctx) { - auto *msg = ev->Get(); - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() +void TMediatorTimecastProxy::Handle(TEvPrivate::TEvRetryCoordinator::TPtr& ev, const TActorContext& ctx) { + auto* msg = ev->Get(); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE EvRetryCoordinator " << msg->Coordinator); auto it = MediatorCoordinators.find(msg->Coordinator); if (it != MediatorCoordinators.end() && it->second.RetryPending) { @@ -380,28 +613,28 @@ void TMediatorTimecastProxy::Handle(TEvPrivate::TEvRetryCoordinator::TPtr &ev, c } } -void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUpdate::TPtr &ev, const TActorContext &ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID.ToString() - << " HANDLE "<< ev->Get()->ToString()); +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUpdate::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " HANDLE " << ev->Get()->ToString()); - const NKikimrTxMediatorTimecast::TEvUpdate &record = ev->Get()->Record; + const NKikimrTxMediatorTimecast::TEvUpdate& record = ev->Get()->Record; const ui64 mediatorTabletId = record.GetMediator(); auto it = Mediators.find(mediatorTabletId); if (it != Mediators.end()) { - auto &mediator = it->second; + auto& mediator = it->second; Y_ABORT_UNLESS(record.GetBucket() < mediator.BucketsSz); - auto &bucket = mediator.Buckets[record.GetBucket()]; + auto& bucket = mediator.Buckets[record.GetBucket()]; const ui64 step = record.GetTimeBarrier(); - switch (bucket.Entry.RefCount()) { + switch (bucket.SafeStep.RefCount()) { case 0: break; case 1: - bucket.Entry.Drop(); bucket.Waiters = { }; - break; + bucket.TabletWaiters = { }; + [[fallthrough]]; default: { - bucket.Entry->Update(step, nullptr, 0); + bucket.SafeStep->Set(step); THashSet> processed; // a set of processed tablets while (!bucket.Waiters.empty()) { const auto& top = bucket.Waiters.top(); @@ -413,6 +646,27 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUpdate::TPtr &ev, co } bucket.Waiters.pop(); } + while (!bucket.TabletWaiters.empty()) { + const auto& candidate = *bucket.TabletWaiters.begin(); + if (step < candidate.PlanStep) { + break; + } + auto it = Tablets.find(candidate.TabletId); + if (it != Tablets.end()) { + auto& tabletInfo = it->second; + while (!tabletInfo.Waiters.empty()) { + const auto& top = *tabletInfo.Waiters.begin(); + if (step < top.PlanStep) { + break; + } + if (processed.insert(std::make_pair(top.Sender, top.TabletId)).second) { + ctx.Send(top.Sender, new TEvMediatorTimecast::TEvNotifyPlanStep(top.TabletId, step)); + } + tabletInfo.Waiters.erase(tabletInfo.Waiters.begin()); + } + } + bucket.TabletWaiters.erase(bucket.TabletWaiters.begin()); + } break; } } @@ -427,7 +681,179 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUpdate::TPtr &ev, co } } -void TMediatorTimecastProxy::SyncCoordinator(ui64 coordinatorId, TCoordinator &coordinator, const TActorContext &ctx) { +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvGranularUpdate::TPtr& ev, const TActorContext& ctx) { + auto* msg = ev->Get(); + LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " HANDLE " << msg->ToString()); + + const ui64 mediatorTabletId = msg->Record.GetMediator(); + auto it = Mediators.find(mediatorTabletId); + if (it == Mediators.end()) { + // Ignore messages from unknown mediators + // Note: we don't clean up mediators, so this shouldn't happen + return; + } + + auto& mediator = it->second; + const ui32 bucketId = msg->Record.GetBucket(); + if (bucketId >= mediator.BucketsSz) { + LOG_CRIT_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " got update from Mediator# " << mediatorTabletId << " Bucket# " << bucketId + << " expecting only " << mediator.BucketsSz << " buckets"); + return; + } + + auto& bucket = mediator.Buckets[bucketId]; + if (msg->Record.GetSubscriptionId() < bucket.SubscriptionId) { + // Ignore messages from old subscriptions + return; + } + + if (msg->Record.FrozenTabletsSize() != msg->Record.FrozenStepsSize()) { + LOG_CRIT_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " got update from Mediator# " << mediatorTabletId << " Bucket# " << bucketId + << " with mismatched frozen records"); + return; + } + + size_t frozenSize = msg->Record.FrozenTabletsSize(); + size_t unfrozenSize = msg->Record.UnfrozenTabletsSize(); + + for (size_t i = 0; i < frozenSize; ++i) { + ui64 tabletId = msg->Record.GetFrozenTablets(i); + ui64 step = msg->Record.GetFrozenSteps(i); + if (!bucket.Tablets.contains(tabletId)) { + // We are not interested in this tablet + continue; + } + auto& tabletInfo = Tablets.at(tabletId); + if (msg->Record.GetSubscriptionId() < tabletInfo.SubscriptionId) { + // We have resubscribed to this tablet and this update must be ignored + continue; + } + // This tablet is now frozen at this step + tabletInfo.FrozenStep = step; + // Update frozen steps for synchronized tablets + if (tabletInfo.Synced && bucket.WatchSynced) { + bucket.PendingUpdate.PushBack(&tabletInfo); + } + } + + for (size_t i = 0; i < unfrozenSize; ++i) { + ui64 tabletId = msg->Record.GetUnfrozenTablets(i); + if (!bucket.Tablets.contains(tabletId)) { + // We are not interested in this tablet + continue; + } + auto& tabletInfo = Tablets.at(tabletId); + if (msg->Record.GetSubscriptionId() < tabletInfo.SubscriptionId) { + // We have resubscribed to this tablet and this update must be ignored + continue; + } + // This tablet is no longer frozen + tabletInfo.FrozenStep = Max(); + // Update frozen steps for synchronized tablets + if (tabletInfo.Synced && bucket.WatchSynced) { + bucket.PendingUpdate.PushBack(&tabletInfo); + } + } + + while (!bucket.PendingSubscribe.Empty()) { + auto* tabletInfo = bucket.PendingSubscribe.Front(); + if (msg->Record.GetSubscriptionId() < tabletInfo->SubscriptionId) { + // This tablet (and all others) is not subscribed yet + break; + } + // This tablet is now subscribed and needs to be updated + bucket.PendingUpdate.PushBack(tabletInfo); + tabletInfo->Synced = true; + } + + bucket.SubscriptionId = msg->Record.GetSubscriptionId(); + + const ui64 latestStep = msg->Record.GetLatestStep(); + + if (!bucket.WatchSynced) { + // We may have connected to a lagging mediator, make sure we don't sync too early + if (latestStep < bucket.LatestStep->Get()) { + return; + } + + // We have reached an expected state, now this bucket is synchronized + bucket.WatchSynced = true; + } else if (latestStep < bucket.LatestStep->Get()) { + // This should never happen, but since we need to protect against + // mediator time jumping backwards for running instances we will ignore + // this update. Note that the current state is already updated, it's + // just not published yet, and will be published later. + LOG_CRIT_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID + << " got update from Mediator# " << mediatorTabletId + << " with LatestStep# " << latestStep + << " previous LatestStep# " << bucket.LatestStep->Get()); + return; + } + + TIntrusiveList checkWaiters; + + // Process frozen step updates + while (!bucket.PendingUpdate.Empty()) { + auto* tabletInfo = bucket.PendingUpdate.PopFront(); + ui64 frozenStep = Max(tabletInfo->FrozenStep, tabletInfo->MinStep); + tabletInfo->Entry->SetFrozenStep(frozenStep); + // Note: all waiters in tabletInfo are always below LatestStep + if (!tabletInfo->Waiters.empty() && tabletInfo->Waiters.begin()->PlanStep <= frozenStep) { + checkWaiters.PushBack(tabletInfo); + } + } + + // Publish the new latest step + bucket.LatestStep->Set(latestStep); + + THashSet> processed; // a set of processed tablets + while (!bucket.Waiters.empty()) { + const auto& top = bucket.Waiters.top(); + if (latestStep < top.PlanStep) { + break; + } + auto it = Tablets.find(top.TabletId); + if (it != Tablets.end() && bucket.Tablets.contains(top.TabletId)) { + auto& tabletInfo = it->second; + ui64 frozenStep = Max(tabletInfo.FrozenStep, tabletInfo.MinStep); + if (tabletInfo.Synced && top.PlanStep <= frozenStep) { + // Send update immediately + if (processed.insert(std::make_pair(top.Sender, top.TabletId)).second) { + ctx.Send(top.Sender, new TEvMediatorTimecast::TEvNotifyPlanStep(top.TabletId, Min(latestStep, frozenStep))); + } + } else { + // Move this waiter to the per-tablet waiter + tabletInfo.Waiters.insert(top); + bucket.TabletWaiters.insert(TTabletWaiter(top.PlanStep, top.TabletId)); + } + } + bucket.Waiters.pop(); + } + + while (!checkWaiters.Empty()) { + auto* tabletInfo = checkWaiters.PopFront(); + ui64 frozenStep = Max(tabletInfo->FrozenStep, tabletInfo->MinStep); + ui64 step = Min(latestStep, frozenStep); + while (!tabletInfo->Waiters.empty()) { + const auto& top = *tabletInfo->Waiters.begin(); + if (step < top.PlanStep) { + break; + } + if (processed.insert(std::make_pair(top.Sender, top.TabletId)).second) { + ctx.Send(top.Sender, new TEvMediatorTimecast::TEvNotifyPlanStep(top.TabletId, step)); + } + // Current queue may have more waiters for the same plan step, but + // all of them will be processed and erased in this loop. + bucket.TabletWaiters.erase(TTabletWaiter(top.PlanStep, top.TabletId)); + tabletInfo->Waiters.erase(tabletInfo->Waiters.begin()); + } + } +} + +void TMediatorTimecastProxy::SyncCoordinator(ui64 coordinatorId, TCoordinator& coordinator, const TActorContext& ctx) { const ui64 seqNo = LastSeqNo; if (!coordinator.PipeClient) { @@ -445,13 +871,13 @@ void TMediatorTimecastProxy::SyncCoordinator(ui64 coordinatorId, TCoordinator &c NTabletPipe::SendData(ctx, coordinator.PipeClient, new TEvTxProxy::TEvSubscribeReadStep(coordinatorId, seqNo)); } -void TMediatorTimecastProxy::ResyncCoordinator(ui64 coordinatorId, const TActorId &pipeClient, const TActorContext &ctx) { +void TMediatorTimecastProxy::ResyncCoordinator(ui64 coordinatorId, const TActorId& pipeClient, const TActorContext& ctx) { auto itCoordinator = Coordinators.find(coordinatorId); if (itCoordinator == Coordinators.end()) { return; } - auto &coordinator = itCoordinator->second; + auto& coordinator = itCoordinator->second; if (coordinator.PipeClient != pipeClient) { return; } @@ -467,14 +893,14 @@ void TMediatorTimecastProxy::ResyncCoordinator(ui64 coordinatorId, const TActorI SyncCoordinator(coordinatorId, coordinator, ctx); } -void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvSubscribeReadStep::TPtr &ev, const TActorContext &ctx) { - const auto *msg = ev->Get(); +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvSubscribeReadStep::TPtr& ev, const TActorContext& ctx) { + const auto* msg = ev->Get(); LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE " << msg->ToString()); const ui64 coordinatorId = msg->CoordinatorId; - auto &subscriber = CoordinatorSubscribers[ev->Sender]; - auto &coordinator = Coordinators[coordinatorId]; + auto& subscriber = CoordinatorSubscribers[ev->Sender]; + auto& coordinator = Coordinators[coordinatorId]; subscriber.Coordinators.insert(coordinatorId); coordinator.Subscribers.insert(ev->Sender); ui64 seqNo = ++LastSeqNo; @@ -483,16 +909,16 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvSubscribeReadStep::T SyncCoordinator(coordinatorId, coordinator, ctx); } -void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUnsubscribeReadStep::TPtr &ev, const TActorContext &ctx) { - const auto *msg = ev->Get(); +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUnsubscribeReadStep::TPtr& ev, const TActorContext& ctx) { + const auto* msg = ev->Get(); LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE " << msg->ToString()); - auto &subscriber = CoordinatorSubscribers[ev->Sender]; + auto& subscriber = CoordinatorSubscribers[ev->Sender]; if (msg->CoordinatorId == 0) { // Unsubscribe from all coordinators for (ui64 coordinatorId : subscriber.Coordinators) { - auto &coordinator = Coordinators[coordinatorId]; + auto& coordinator = Coordinators[coordinatorId]; coordinator.Subscribers.erase(ev->Sender); if (coordinator.Subscribers.empty()) { coordinator.IdleStart = ctx.Monotonic(); @@ -501,7 +927,7 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUnsubscribeReadStep: subscriber.Coordinators.clear(); } else if (subscriber.Coordinators.contains(msg->CoordinatorId)) { // Unsubscribe from specific coordinator - auto &coordinator = Coordinators[msg->CoordinatorId]; + auto& coordinator = Coordinators[msg->CoordinatorId]; coordinator.Subscribers.erase(ev->Sender); if (coordinator.Subscribers.empty()) { coordinator.IdleStart = ctx.Monotonic(); @@ -515,8 +941,8 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvUnsubscribeReadStep: } } -void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvWaitReadStep::TPtr &ev, const TActorContext &ctx) { - const auto *msg = ev->Get(); +void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvWaitReadStep::TPtr& ev, const TActorContext& ctx) { + const auto* msg = ev->Get(); LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE " << msg->ToString()); @@ -525,7 +951,7 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvWaitReadStep::TPtr & if (itCoordinator == Coordinators.end()) { return; } - auto &coordinator = itCoordinator->second; + auto& coordinator = itCoordinator->second; if (!coordinator.Subscribers.contains(ev->Sender)) { return; @@ -544,8 +970,8 @@ void TMediatorTimecastProxy::Handle(TEvMediatorTimecast::TEvWaitReadStep::TPtr & coordinator.WaitRequests[key] = ev->Cookie; } -void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepResult::TPtr &ev, const TActorContext &ctx) { - const auto &record = ev->Get()->Record; +void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepResult::TPtr& ev, const TActorContext& ctx) { + const auto& record = ev->Get()->Record; LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE TEvSubscribeReadStepResult " << record.ShortDebugString()); @@ -554,7 +980,7 @@ void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepResult::TPtr if (itCoordinator == Coordinators.end()) { return; } - auto &coordinator = itCoordinator->second; + auto& coordinator = itCoordinator->second; bool updated = false; const ui64 nextReadStep = record.GetNextAcquireStep(); @@ -594,8 +1020,8 @@ void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepResult::TPtr } } -void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepUpdate::TPtr &ev, const TActorContext &ctx) { - const auto &record = ev->Get()->Record; +void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepUpdate::TPtr& ev, const TActorContext& ctx) { + const auto& record = ev->Get()->Record; LOG_DEBUG_S(ctx, NKikimrServices::TX_MEDIATOR_TIMECAST, "Actor# " << ctx.SelfID << " HANDLE TEvSubscribeReadStepUpdate " << record.ShortDebugString()); @@ -604,7 +1030,7 @@ void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepUpdate::TPtr if (itCoordinator == Coordinators.end()) { return; } - auto &coordinator = itCoordinator->second; + auto& coordinator = itCoordinator->second; const ui64 nextReadStep = record.GetNextAcquireStep(); if (coordinator.LastObservedReadStep < nextReadStep) { @@ -625,7 +1051,7 @@ void TMediatorTimecastProxy::Handle(TEvTxProxy::TEvSubscribeReadStepUpdate::TPtr } } -void TMediatorTimecastProxy::NotifyCoordinatorWaiters(ui64 coordinatorId, TCoordinator &coordinator, const TActorContext &ctx) { +void TMediatorTimecastProxy::NotifyCoordinatorWaiters(ui64 coordinatorId, TCoordinator& coordinator, const TActorContext& ctx) { const ui64 step = coordinator.LastObservedReadStep; for (auto it = coordinator.WaitRequests.begin(); it != coordinator.WaitRequests.end();) { const ui64 waitStep = it->first.first; diff --git a/ydb/core/tx/time_cast/time_cast.h b/ydb/core/tx/time_cast/time_cast.h index 4fa496bdb1d7..d3b459016312 100644 --- a/ydb/core/tx/time_cast/time_cast.h +++ b/ydb/core/tx/time_cast/time_cast.h @@ -10,15 +10,43 @@ namespace NKikimr { +class TMediatorTimecastSharedEntry : public TThrRefBase { +public: + using TPtr = TIntrusivePtr; + using TCPtr = TIntrusiveConstPtr; + + TMediatorTimecastSharedEntry() noexcept = default; + ~TMediatorTimecastSharedEntry() noexcept = default; + + ui64 Get() const noexcept; + void Set(ui64 step) noexcept; + +private: + std::atomic Step{ 0 }; +}; + class TMediatorTimecastEntry : public TThrRefBase { - TAtomic Step; public: - TMediatorTimecastEntry() - : Step(0) - {} + using TPtr = TIntrusivePtr; + using TCPtr = TIntrusiveConstPtr; + + TMediatorTimecastEntry( + const TMediatorTimecastSharedEntry::TPtr& safeStep, + const TMediatorTimecastSharedEntry::TPtr& latestStep) noexcept; + ~TMediatorTimecastEntry() noexcept; + + /** + * Note: tabletId argument is not used (for compatibility only) + */ + ui64 Get(ui64 tabletId = 0) const noexcept; - ui64 Get(ui64 tabletId) const; - void Update(ui64 step, ui64 *exemption, ui64 exsz); + ui64 GetFrozenStep() const noexcept; + void SetFrozenStep(ui64 step) noexcept; + +private: + const TMediatorTimecastSharedEntry::TCPtr SafeStep; + const TMediatorTimecastSharedEntry::TCPtr LatestStep; + std::atomic FrozenStep{ 0 }; }; class TMediatorTimecastReadStep : public TThrRefBase { @@ -75,7 +103,7 @@ struct TEvMediatorTimecast { NKikimrSubDomains::TProcessingParams ProcessingParams; - TEvRegisterTablet(ui64 tabletId, const NKikimrSubDomains::TProcessingParams &processing) + TEvRegisterTablet(ui64 tabletId, const NKikimrSubDomains::TProcessingParams& processing) : TabletId(tabletId) , ProcessingParams(processing) {} @@ -94,11 +122,11 @@ struct TEvMediatorTimecast { struct TEvRegisterTabletResult : public TEventLocal { const ui64 TabletId; - const TIntrusivePtr Entry; + const TMediatorTimecastEntry::TCPtr Entry; - TEvRegisterTabletResult(ui64 tabletId, TIntrusivePtr &entry) + TEvRegisterTabletResult(ui64 tabletId, TMediatorTimecastEntry::TCPtr entry) : TabletId(tabletId) - , Entry(entry) + , Entry(std::move(entry)) {} TString ToString() const { diff --git a/ydb/core/tx/time_cast/time_cast_ut.cpp b/ydb/core/tx/time_cast/time_cast_ut.cpp index 0acb2ee12898..85b2c037fcad 100644 --- a/ydb/core/tx/time_cast/time_cast_ut.cpp +++ b/ydb/core/tx/time_cast/time_cast_ut.cpp @@ -1,4 +1,8 @@ #include "time_cast.h" +#include +#include +#include +#include #include #include #include @@ -41,7 +45,7 @@ namespace NKikimr::NMediatorTimeCastTest { Tests::TServer::TPtr server = new TServer(serverSettings); - auto &runtime = *server->GetRuntime(); + auto& runtime = *server->GetRuntime(); runtime.SetLogPriority(NKikimrServices::TX_MEDIATOR_TIMECAST, NActors::NLog::PRI_DEBUG); auto sender = runtime.AllocateEdgeActor(); @@ -79,6 +83,344 @@ namespace NKikimr::NMediatorTimeCastTest { } } + class TTargetTablet + : public TActor + , public NTabletFlatExecutor::TTabletExecutedFlat + { + public: + TTargetTablet(const TActorId& tablet, TTabletStorageInfo* info) + : TActor(&TThis::StateInit) + , TTabletExecutedFlat(info, tablet, nullptr) + {} + + private: + STFUNC(StateInit) { + StateInitImpl(ev, SelfId()); + } + + STFUNC(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTxProcessing::TEvPlanStep, Handle); + default: + HandleDefaultEvents(ev, SelfId()); + } + } + + void Handle(TEvTxProcessing::TEvPlanStep::TPtr& ev) { + ui64 step = ev->Get()->Record.GetStep(); + THashMap> acks; + for (const auto& tx : ev->Get()->Record.GetTransactions()) { + ui64 txId = tx.GetTxId(); + TActorId owner = ActorIdFromProto(tx.GetAckTo()); + acks[owner].push_back(txId); + } + for (auto& pr : acks) { + auto owner = pr.first; + Send(owner, new TEvTxProcessing::TEvPlanStepAck(TabletID(), step, pr.second.begin(), pr.second.end())); + } + Send(ev->Sender, new TEvTxProcessing::TEvPlanStepAccepted(TabletID(), step)); + } + + private: + void DefaultSignalTabletActive(const TActorContext&) override { + // must be empty + } + + void OnActivateExecutor(const TActorContext& ctx) override { + Become(&TThis::StateWork); + SignalTabletActive(ctx); + } + + void OnDetach(const TActorContext&) override { + PassAway(); + } + + void OnTabletDead(TEvTablet::TEvTabletDead::TPtr&, const TActorContext&) override { + PassAway(); + } + }; + + TActorId BootTarget(TTestActorRuntime& runtime, ui64 tabletId) { + auto boot = CreateTestBootstrapper(runtime, + CreateTestTabletInfo(tabletId, TTabletTypes::Dummy), + [](const TActorId& tablet, TTabletStorageInfo* info) { + return new TTargetTablet(tablet, info); + }); + + { + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot, 1)); + runtime.DispatchEvents(options); + } + + return boot; + } + + ui64 PlanTx(const TServer::TPtr& server, ui64 txId, std::vector tablets) { + auto& runtime = *server->GetRuntime(); + ui64 coordinatorId = ChangeStateStorage(TTestTxConfig::Coordinator, server->GetSettings().Domain); + auto req = std::make_unique( + coordinatorId, txId, 0, Min(), Max()); + auto* affectedSet = req->Record.MutableTransaction()->MutableAffectedSet(); + for (ui64 tabletId : tablets) { + auto* x = affectedSet->Add(); + x->SetTabletId(tabletId); + x->SetFlags(TEvTxProxy::TEvProposeTransaction::AffectedWrite); + } + auto sender = runtime.AllocateEdgeActor(); + ForwardToTablet(runtime, coordinatorId, sender, req.release()); + for (;;) { + auto ev = runtime.GrabEdgeEventRethrow(sender); + auto* msg = ev->Get(); + switch (msg->GetStatus()) { + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusAccepted: + break; + case TEvTxProxy::TEvProposeTransactionStatus::EStatus::StatusPlanned: + return msg->Record.GetStepId(); + default: + UNIT_ASSERT_C(false, "Unexpected status " << int(msg->GetStatus())); + } + } + } + + TMediatorTimecastEntry::TCPtr RegisterTablet(const TServer::TPtr& server, ui64 tabletId) { + auto& runtime = *server->GetRuntime(); + auto sender = runtime.AllocateEdgeActor(); + auto params = ExtractProcessingParams(*runtime.GetAppData().DomainsInfo->GetDomain()); + auto req = std::make_unique(tabletId, params); + runtime.Send(new IEventHandle(MakeMediatorTimecastProxyID(), sender, req.release()), 0, true); + auto ev = runtime.GrabEdgeEventRethrow(sender); + return ev->Get()->Entry; + } + + Y_UNIT_TEST(GranularTimecast) { + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetDomainPlanResolution(500) + .SetDomainTimecastBuckets(1) + .SetEnableGranularTimecast(true) + .SetUseRealThreads(false); + + TServer::TPtr server = new TServer(serverSettings); + + auto& runtime = *server->GetRuntime(); + runtime.SetLogPriority(NKikimrServices::TX_MEDIATOR_TIMECAST, NActors::NLog::PRI_DEBUG); + + auto sender = runtime.AllocateEdgeActor(); + + ui64 tablet1 = ChangeStateStorage(TTestTxConfig::TxTablet0, server->GetSettings().Domain); + BootTarget(runtime, tablet1); + ui64 tablet2 = ChangeStateStorage(TTestTxConfig::TxTablet1, server->GetSettings().Domain); + BootTarget(runtime, tablet2); + ui64 tablet3 = ChangeStateStorage(TTestTxConfig::TxTablet3, server->GetSettings().Domain); + BootTarget(runtime, tablet3); + + auto entry1 = RegisterTablet(server, tablet1); + auto entry2 = RegisterTablet(server, tablet2); + auto entry3 = RegisterTablet(server, tablet3); + + ui64 testStep1 = entry1->Get(); + runtime.SimulateSleep(TDuration::Seconds(2)); + ui64 testStep2 = entry1->Get(); + Cerr << "... have step " << testStep1 << " and " << testStep2 << " after sleep" << Endl; + UNIT_ASSERT_C(testStep1 < testStep2, "Have step " << testStep1 << " and " << testStep2 << " after sleep"); + + TBlockEvents blockAcks(runtime); + TBlockEvents blockPlan1(runtime, + [&](auto& ev) { return ev->Get()->Record.GetTabletID() == tablet1; }); + TBlockEvents blockPlan2(runtime, + [&](auto& ev) { return ev->Get()->Record.GetTabletID() == tablet2; }); + + ui64 stepTx1 = PlanTx(server, 1, { tablet1, tablet2 }); + Cerr << "... tx1 planned at step " << stepTx1 << Endl; + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + UNIT_ASSERT_VALUES_EQUAL(blockPlan1.size(), 1u); + UNIT_ASSERT_VALUES_EQUAL(blockPlan2.size(), 1u); + + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + + UNIT_ASSERT_C(entry1->Get() == stepTx1-1, "tablet1 step " << entry1->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry2->Get() == stepTx1-1, "tablet2 step " << entry2->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry3->Get() == stepTx1, "tablet3 step " << entry3->Get() << " not at " << stepTx1); + + ui64 stepTx2 = PlanTx(server, 2, { tablet1, tablet2 }); + Cerr << "... tx2 planned at step " << stepTx2 << Endl; + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + UNIT_ASSERT_VALUES_EQUAL(blockPlan1.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(blockPlan2.size(), 2u); + + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + + UNIT_ASSERT_C(entry1->Get() == stepTx1-1, "tablet1 step " << entry1->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry2->Get() == stepTx1-1, "tablet2 step " << entry2->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry3->Get() == stepTx2, "tablet3 step " << entry3->Get() << " not at " << stepTx2); + + Cerr << "... unblocking tx1 at tablet2" << Endl; + blockPlan2.Unblock(1); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + + UNIT_ASSERT_C(entry1->Get() == stepTx1-1, "tablet1 step " << entry1->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry2->Get() == stepTx2-1, "tablet2 step " << entry2->Get() << " not at " << (stepTx2-1)); + UNIT_ASSERT_C(entry3->Get() == stepTx2, "tablet3 step " << entry3->Get() << " not at " << stepTx2); + + Cerr << "... unblocking tx2 at tablet2" << Endl; + blockPlan2.Unblock(1); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + + UNIT_ASSERT_C(entry1->Get() == stepTx1-1, "tablet1 step " << entry1->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry2->Get() == stepTx2, "tablet2 step " << entry2->Get() << " not at " << stepTx2); + UNIT_ASSERT_C(entry3->Get() == stepTx2, "tablet3 step " << entry3->Get() << " not at " << stepTx2); + + TBlockEvents blockNotify(runtime); + + // This tests notify when latest step is updated for a non-frozen tablet + runtime.Send(new IEventHandle(MakeMediatorTimecastProxyID(), sender, new TEvMediatorTimecast::TEvWaitPlanStep(tablet2, stepTx2+1))); + runtime.WaitFor("step notify", [&]{ return blockNotify.size() > 0; }); + UNIT_ASSERT_VALUES_EQUAL(blockNotify.size(), 1u); + ui64 notifiedStep = blockNotify[0]->Get()->PlanStep; + blockNotify.clear(); + Cerr << "... notified at step " << notifiedStep << Endl; + UNIT_ASSERT_C(notifiedStep >= stepTx2+1, "notified at " << notifiedStep << " expected at least " << (stepTx2+1)); + + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + UNIT_ASSERT_C(entry1->Get() == stepTx1-1, "tablet1 step " << entry1->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry2->Get() == notifiedStep, "tablet2 step " << entry2->Get() << " not at " << notifiedStep); + UNIT_ASSERT_C(entry3->Get() == notifiedStep, "tablet3 step " << entry3->Get() << " not at " << notifiedStep); + + TBlockEvents blockUpdate(runtime); + + // Plan a new tx and double check nothing has changed yet + ui64 stepTx3 = PlanTx(server, 3, { tablet2 }); + Cerr << "... tx3 planned at step " << stepTx3 << Endl; + UNIT_ASSERT_VALUES_EQUAL(entry1->Get(), stepTx1-1); + UNIT_ASSERT_VALUES_EQUAL(entry2->Get(), notifiedStep); + UNIT_ASSERT_VALUES_EQUAL(entry3->Get(), notifiedStep); + + // This tests notify after latest step is updated and then tablet's frozen step changes + runtime.Send(new IEventHandle(MakeMediatorTimecastProxyID(), sender, new TEvMediatorTimecast::TEvWaitPlanStep(tablet2, stepTx3))); + runtime.WaitFor("granular update", [&]{ return blockUpdate.size() > 0 && blockUpdate.back()->Get()->Record.GetLatestStep() >= stepTx3; }); + UNIT_ASSERT_VALUES_EQUAL(blockUpdate.size(), 1u); // new step + freeze + blockUpdate.Unblock(); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + UNIT_ASSERT_C(entry1->Get() == stepTx1-1, "tablet1 step " << entry1->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry2->Get() == stepTx3-1, "tablet2 step " << entry2->Get() << " not at " << (stepTx3-1)); + UNIT_ASSERT_C(entry3->Get() == stepTx3, "tablet3 step " << entry3->Get() << " not at " << stepTx3); + UNIT_ASSERT_VALUES_EQUAL(blockNotify.size(), 0u); // not notified yet! + + blockPlan2.Unblock(); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + UNIT_ASSERT_VALUES_EQUAL(blockUpdate.size(), 1u); // unfreeze + blockUpdate.Unblock(); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + UNIT_ASSERT_C(entry1->Get() == stepTx1-1, "tablet1 step " << entry1->Get() << " not at " << (stepTx1-1)); + UNIT_ASSERT_C(entry2->Get() == stepTx3, "tablet2 step " << entry2->Get() << " not at " << stepTx3); + UNIT_ASSERT_C(entry3->Get() == stepTx3, "tablet3 step " << entry3->Get() << " not at " << stepTx3); + + UNIT_ASSERT_VALUES_EQUAL(blockNotify.size(), 1u); // notified! + Cerr << "... notified at step " << blockNotify[0]->Get()->PlanStep << Endl; + UNIT_ASSERT_VALUES_EQUAL(blockNotify[0]->Get()->PlanStep, stepTx3); + blockNotify.clear(); + + blockAcks.clear(); + blockPlan1.clear(); + blockPlan2.clear(); + blockUpdate.clear(); + + // Test that attaching to a lagging mediator doesn't cause time to go backwards + Cerr << "... restarting mediator" << Endl; + ui64 mediatorId = ChangeStateStorage(TTestTxConfig::Mediator, server->GetSettings().Domain); + RebootTablet(runtime, mediatorId, sender); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + UNIT_ASSERT_VALUES_EQUAL(blockPlan1.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(blockPlan2.size(), 3u); + for (;;) { + while (blockUpdate.size() > 0) { + Cerr << "... unblocking update: " << blockUpdate[0]->Get()->Record.ShortDebugString() << Endl; + blockUpdate.Unblock(1); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + UNIT_ASSERT_VALUES_EQUAL(entry1->Get(), stepTx1-1); + UNIT_ASSERT_VALUES_EQUAL(entry2->Get(), stepTx3); + UNIT_ASSERT_VALUES_EQUAL(entry3->Get(), stepTx3); + } + if (blockPlan2.empty()) { + break; + } + Cerr << "... unblocking plan for tablet2" << Endl; + blockPlan2.Unblock(1); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + } + + blockAcks.clear(); + blockPlan1.clear(); + blockPlan2.clear(); + blockUpdate.clear(); + + // Test that attaching to a mediator without granular support allows time to go forward + Cerr << "... restarting mediator" << Endl; + RebootTablet(runtime, mediatorId, sender); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + UNIT_ASSERT_VALUES_EQUAL(blockPlan1.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(blockPlan2.size(), 3u); + UNIT_ASSERT_VALUES_EQUAL(entry1->Get(), stepTx1-1); + UNIT_ASSERT_VALUES_EQUAL(entry2->Get(), stepTx3); + UNIT_ASSERT_VALUES_EQUAL(entry3->Get(), stepTx3); + + Cerr << "... fully unblocking tx1" << Endl; + blockPlan1.Unblock(1); + blockPlan2.Unblock(1); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + UNIT_ASSERT_VALUES_EQUAL(entry1->Get(), stepTx1); + UNIT_ASSERT_VALUES_EQUAL(entry2->Get(), stepTx3); + UNIT_ASSERT_VALUES_EQUAL(entry3->Get(), stepTx3); + + Cerr << "... fully unblocking tx2" << Endl; + blockPlan1.Unblock(1); + blockPlan2.Unblock(1); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + UNIT_ASSERT_VALUES_EQUAL(entry1->Get(), stepTx2); + UNIT_ASSERT_VALUES_EQUAL(entry2->Get(), stepTx3); + UNIT_ASSERT_VALUES_EQUAL(entry3->Get(), stepTx3); + + Cerr << "... fully unblocking tx3" << Endl; + blockPlan2.Unblock(1); + runtime.SimulateSleep(TDuration::MilliSeconds(1)); + Cerr << "... tablet1 at " << entry1->Get() << Endl; + Cerr << "... tablet2 at " << entry2->Get() << Endl; + Cerr << "... tablet3 at " << entry3->Get() << Endl; + UNIT_ASSERT_VALUES_EQUAL(entry1->Get(), stepTx3); + UNIT_ASSERT_VALUES_EQUAL(entry2->Get(), stepTx3); + UNIT_ASSERT_VALUES_EQUAL(entry3->Get(), stepTx3); + } + } // Y_UNIT_TEST_SUITE(MediatorTimeCast) } // namespace NKikimr::NMediatorTimeCastTest From 18e9ed8d0bfd854ef6388086d7aded05829f123e Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Thu, 22 Aug 2024 13:18:35 +0300 Subject: [PATCH 042/261] Update postgis functions from tests (#8133) --- .../yql/parser/pg_catalog/postgis_procs.h | 71 ++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/ydb/library/yql/parser/pg_catalog/postgis_procs.h b/ydb/library/yql/parser/pg_catalog/postgis_procs.h index 21e2f4b792a7..8fbbc740d4c2 100644 --- a/ydb/library/yql/parser/pg_catalog/postgis_procs.h +++ b/ydb/library/yql/parser/pg_catalog/postgis_procs.h @@ -7,23 +7,83 @@ "box3d_in", "box3d_out", "brin_summarize_new_values", +"bytea", "geography", +"geography_cmp", +"geography_distance_knn", +"geography_eq", +"geography_ge", +"geography_gt", "geography_in", +"geography_le", +"geography_lt", "geography_out", +"geography_overlaps", "geography_recv", "geography_send", "geography_typmod_in", "geography_typmod_out", "geometry", +"geometry_above", +"geometry_below", +"geometry_cmp", +"geometry_contains", +"geometry_contains_nd", +"geometry_distance_box", +"geometry_distance_centroid", +"geometry_distance_centroid_nd", +"geometry_distance_cpa", +"geometry_eq", +"geometry_ge", +"geometry_gt", +"geometry_hash", "geometry_in", +"geometry_le", +"geometry_left", +"geometry_lt", "geometry_out", +"geometry_overabove", +"geometry_overbelow", +"geometry_overlaps", +"geometry_overlaps_nd", +"geometry_overleft", +"geometry_overright", "geometry_recv", +"geometry_right", +"geometry_same", +"geometry_same_nd", "geometry_send", +"geometry_sortsupport", "geometrytype", "geometry_typmod_in", "geometry_typmod_out", +"geometry_within", +"geometry_within_nd", "gidx_in", "gidx_out", +"json", +"jsonb", +"pgis_asflatgeobuf_finalfn", +"pgis_asflatgeobuf_transfn", +"pgis_asgeobuf_finalfn", +"pgis_asgeobuf_transfn", +"pgis_asmvt_combinefn", +"pgis_asmvt_deserialfn", +"pgis_asmvt_finalfn", +"pgis_asmvt_serialfn", +"pgis_asmvt_transfn", +"pgis_geometry_accum_transfn", +"pgis_geometry_clusterintersecting_finalfn", +"pgis_geometry_clusterwithin_finalfn", +"pgis_geometry_collect_finalfn", +"pgis_geometry_coverageunion_finalfn", +"pgis_geometry_makeline_finalfn", +"pgis_geometry_polygonize_finalfn", +"pgis_geometry_union_parallel_combinefn", +"pgis_geometry_union_parallel_deserialfn", +"pgis_geometry_union_parallel_finalfn", +"pgis_geometry_union_parallel_serialfn", +"pgis_geometry_union_parallel_transfn", "postgis_addbbox", "postgis_dropbbox", "_postgis_index_extent", @@ -63,16 +123,20 @@ "st_astext", "st_astwkb", "st_azimuth", +"_st_bestsrid", "st_boundary", +"st_boundingdiagonal", "st_box2dfromgeohash", "st_buffer", "st_buildarea", "st_centroid", "st_chaikinsmoothing", +"st_clipbybox2d", "st_closestpoint", "st_closestpointofapproach", "st_collect", "st_collectionextract", +"st_combinebbox", "st_contains", "st_containsproperly", "st_convexhull", @@ -135,6 +199,7 @@ "st_intersects", "st_isclosed", "st_iscollection", +"st_isempty", "st_ispolygonccw", "st_ispolygoncw", "st_isring", @@ -163,7 +228,6 @@ "st_m", "st_makebox2d", "st_makeenvelope", -"st_makeline", "st_makepoint", "st_makepointm", "st_makepolygon", @@ -196,7 +260,6 @@ "st_points", "st_pointz", "st_pointzm", -"st_polygonize", "st_project", "st_quantizecoordinates", "st_reduceprecision", @@ -237,5 +300,9 @@ "st_within", "st_wkbtosql", "st_x", +"st_xmax", +"st_xmin", "st_y", +"st_ymax", +"st_ymin", "st_z", From 85e615bd89fe1f72c006e206488b83dc4b8d112e Mon Sep 17 00:00:00 2001 From: kruall Date: Thu, 22 Aug 2024 13:44:42 +0300 Subject: [PATCH 043/261] Add UseRingQueue to ActorSystemConfig (#8122) --- ydb/core/driver_lib/run/config_helpers.cpp | 2 ++ ydb/core/protos/config.proto | 1 + ydb/library/actors/core/config.h | 2 ++ ydb/library/actors/core/executor_pool_basic.cpp | 2 +- ydb/library/actors/core/executor_pool_io.cpp | 7 ++++--- ydb/library/actors/core/executor_pool_io.h | 2 +- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ydb/core/driver_lib/run/config_helpers.cpp b/ydb/core/driver_lib/run/config_helpers.cpp index bb2753759121..2d0b834eb713 100644 --- a/ydb/core/driver_lib/run/config_helpers.cpp +++ b/ydb/core/driver_lib/run/config_helpers.cpp @@ -52,6 +52,7 @@ void AddExecutorPool(NActors::TCpuManagerConfig& cpuManager, const NKikimrConfig NActors::TBasicExecutorPoolConfig basic; basic.PoolId = poolId; basic.PoolName = poolConfig.GetName(); + basic.UseRingQueue = systemConfig.HasUseRingQueue() && systemConfig.GetUseRingQueue(); if (poolConfig.HasMaxAvgPingDeviation() && counters) { auto poolGroup = counters->GetSubgroup("execpool", basic.PoolName); auto &poolInfo = cpuManager.PingInfoByPool[poolId]; @@ -92,6 +93,7 @@ void AddExecutorPool(NActors::TCpuManagerConfig& cpuManager, const NKikimrConfig io.Threads = poolConfig.GetThreads(); io.Affinity = ParseAffinity(poolConfig.GetAffinity()); cpuManager.IO.emplace_back(std::move(io)); + io.UseRingQueue = systemConfig.HasUseRingQueue() && systemConfig.GetUseRingQueue(); break; } diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index fba99342ef41..a137cb1495df 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -118,6 +118,7 @@ message TActorSystemConfig { optional ENodeType NodeType = 14 [default = COMPUTE]; optional uint32 ForceIOPoolThreads = 17; optional bool UseSharedThreads = 18; + optional bool UseRingQueue = 19; optional bool MonitorStuckActors = 15; optional EActorSystemProfile ActorSystemProfile = 16; diff --git a/ydb/library/actors/core/config.h b/ydb/library/actors/core/config.h index 654e092a088a..038f5673c3c0 100644 --- a/ydb/library/actors/core/config.h +++ b/ydb/library/actors/core/config.h @@ -31,6 +31,7 @@ namespace NActors { i16 SoftProcessingDurationTs = 0; EASProfile ActorSystemProfile = EASProfile::Default; bool HasSharedThread = false; + bool UseRingQueue = false; }; struct TSharedExecutorPoolConfig { @@ -47,6 +48,7 @@ namespace NActors { TString PoolName; ui32 Threads = 1; TCpuMask Affinity; // Executor thread affinity + bool UseRingQueue = false; }; struct TSelfPingInfo { diff --git a/ydb/library/actors/core/executor_pool_basic.cpp b/ydb/library/actors/core/executor_pool_basic.cpp index 98b14b077c2f..284392c36961 100644 --- a/ydb/library/actors/core/executor_pool_basic.cpp +++ b/ydb/library/actors/core/executor_pool_basic.cpp @@ -82,7 +82,7 @@ namespace NActors { } TBasicExecutorPool::TBasicExecutorPool(const TBasicExecutorPoolConfig& cfg, IHarmonizer *harmonizer, TExecutorPoolJail *jail) - : TExecutorPoolBase(cfg.PoolId, cfg.Threads, new TAffinity(cfg.Affinity), false) + : TExecutorPoolBase(cfg.PoolId, cfg.Threads, new TAffinity(cfg.Affinity), cfg.UseRingQueue) , DefaultSpinThresholdCycles(cfg.SpinThreshold * NHPTimer::GetCyclesPerSecond() * 0.000001) // convert microseconds to cycles , SpinThresholdCycles(DefaultSpinThresholdCycles) , SpinThresholdCyclesPerThread(new NThreading::TPadded>[cfg.Threads]) diff --git a/ydb/library/actors/core/executor_pool_io.cpp b/ydb/library/actors/core/executor_pool_io.cpp index 2f9edb22b43c..31592f06a6ed 100644 --- a/ydb/library/actors/core/executor_pool_io.cpp +++ b/ydb/library/actors/core/executor_pool_io.cpp @@ -6,8 +6,8 @@ #include namespace NActors { - TIOExecutorPool::TIOExecutorPool(ui32 poolId, ui32 threads, const TString& poolName, TAffinity* affinity) - : TExecutorPoolBase(poolId, threads, affinity, false) + TIOExecutorPool::TIOExecutorPool(ui32 poolId, ui32 threads, const TString& poolName, TAffinity* affinity, bool useRingQueue) + : TExecutorPoolBase(poolId, threads, affinity, useRingQueue) , Threads(new TExecutorThreadCtx[threads]) , PoolName(poolName) {} @@ -17,7 +17,8 @@ namespace NActors { cfg.PoolId, cfg.Threads, cfg.PoolName, - new TAffinity(cfg.Affinity) + new TAffinity(cfg.Affinity), + cfg.UseRingQueue ) { Harmonizer = harmonizer; diff --git a/ydb/library/actors/core/executor_pool_io.h b/ydb/library/actors/core/executor_pool_io.h index c4fd95b4890b..67bbd66c32a1 100644 --- a/ydb/library/actors/core/executor_pool_io.h +++ b/ydb/library/actors/core/executor_pool_io.h @@ -26,7 +26,7 @@ namespace NActors { const TString PoolName; const ui32 ActorSystemIndex = NActors::TActorTypeOperator::GetActorSystemIndex(); public: - TIOExecutorPool(ui32 poolId, ui32 threads, const TString& poolName = "", TAffinity* affinity = nullptr); + TIOExecutorPool(ui32 poolId, ui32 threads, const TString& poolName = "", TAffinity* affinity = nullptr, bool useRingQueue = false); explicit TIOExecutorPool(const TIOExecutorPoolConfig& cfg, IHarmonizer *harmonizer = nullptr); ~TIOExecutorPool(); From 48b3de97e391cfb9d49e9138157755b60089d36a Mon Sep 17 00:00:00 2001 From: Semyon Danilov Date: Thu, 22 Aug 2024 14:54:48 +0400 Subject: [PATCH 044/261] CMS editor async load fix (#8140) Loading editor.js doesn't block the rest of the page's content loading anymore --- ydb/core/cms/ui/configs_dispatcher_main.js | 92 ++++----- ydb/core/cms/ui/main.js | 9 +- ydb/core/cms/ui/yaml_config.js | 212 +++++++++++---------- 3 files changed, 162 insertions(+), 151 deletions(-) diff --git a/ydb/core/cms/ui/configs_dispatcher_main.js b/ydb/core/cms/ui/configs_dispatcher_main.js index 2e9341bd168e..28e57583b53b 100644 --- a/ydb/core/cms/ui/configs_dispatcher_main.js +++ b/ydb/core/cms/ui/configs_dispatcher_main.js @@ -1,4 +1,4 @@ -var createEditor; +var createEditorPromise = new $.Deferred(); var codeMirror; var codeMirrorResolved; @@ -7,9 +7,12 @@ function replaceWithEditor(selector) { var container = $(selector); var value = container.text(); container.text(""); - var editor = createEditor(container.get(0), true, 1068) - editor.setValue(value); - return editor; + var editorPromise = createEditorPromise.then(function(createEditor) { + var editor = createEditor(container.get(0), true, 1068) + editor.setValue(value); + return editor; + }); + return editorPromise; } function main() { @@ -22,51 +25,53 @@ function main() { } }); - codeMirror = replaceWithEditor("#yaml-config-item"); - codeMirror.trigger('fold', 'editor.foldLevel2'); - - $("#fold-yaml-config").click(function() { + replaceWithEditor("#yaml-config-item").then(function(codeMirror) { codeMirror.trigger('fold', 'editor.foldLevel2'); + + $("#fold-yaml-config").click(function() { + codeMirror.trigger('fold', 'editor.foldLevel2'); + }); + + $("#unfold-yaml-config").click(function() { + codeMirror.trigger('fold', 'editor.unfoldAll'); + }); + + $("#copy-yaml-config").click(function() { + copyToClipboard(codeMirror.getValue()); + }); }); - $("#unfold-yaml-config").click(function() { - codeMirror.trigger('fold', 'editor.unfoldAll'); - }); - - $("#copy-yaml-config").click(function() { - copyToClipboard(codeMirror.getValue()); - }); - - codeMirrorResolved = replaceWithEditor("#resolved-yaml-config-item"); - codeMirrorResolved.trigger('fold', 'editor.foldLevel1'); - - $("#fold-resolved-yaml-config").click(function() { + replaceWithEditor("#resolved-yaml-config-item").then(function(codeMirrorResolved) { codeMirrorResolved.trigger('fold', 'editor.foldLevel1'); - }); - - $("#unfold-resolved-yaml-config").click(function() { - codeMirrorResolved.trigger('fold', 'editor.unfoldAll'); - }); - - $("#copy-resolved-yaml-config").click(function() { - copyToClipboard(codeMirrorResolved.getValue()); + + $("#fold-resolved-yaml-config").click(function() { + codeMirrorResolved.trigger('fold', 'editor.foldLevel1'); + }); + + $("#unfold-resolved-yaml-config").click(function() { + codeMirrorResolved.trigger('fold', 'editor.unfoldAll'); + }); + + $("#copy-resolved-yaml-config").click(function() { + copyToClipboard(codeMirrorResolved.getValue()); + }); }); $("#host-ref").text("YDB Developer UI - " + window.location.hostname); $(".yaml-config-item").each(function() { - let editor = replaceWithEditor(this); - - $(this).parent().find('.fold-yaml-config').click(function() { - editor.trigger('fold', 'editor.foldLevel2'); - }); - - $(this).parent().find('.unfold-yaml-config').click(function() { - editor.trigger('fold', 'editor.unfoldAll'); - }); - - $(this).parent().find('.copy-yaml-config').click(function() { - copyToClipboard(editor.getValue()); + replaceWithEditor(this).then(function(editor) { + $(this).parent().find('.fold-yaml-config').click(function() { + editor.trigger('fold', 'editor.foldLevel2'); + }); + + $(this).parent().find('.unfold-yaml-config').click(function() { + editor.trigger('fold', 'editor.unfoldAll'); + }); + + $(this).parent().find('.copy-yaml-config').click(function() { + copyToClipboard(editor.getValue()); + }); }); }); } @@ -74,11 +79,12 @@ function main() { let run = () => { require.config({ urlArgs: "v=0.27.0.2", - paths: { vs: "../cms/ext/monaco-editor/vs" } + paths: { vs: "../cms/ext/monaco-editor/vs" }, + waitSeconds: 60 // Since editor is quite large }); require(["vs/editor/editor.main"], function () { - createEditor = (container, readOnly, width) => { + createEditorPromise.resolve((container, readOnly, width) => { var editor; container.style.border = '1px solid #eee'; container.style.borderRadius = '8px'; @@ -112,7 +118,7 @@ let run = () => { editor.onDidContentSizeChange(updateHeight); updateHeight(); return editor; - } + }); $(document).ready(main); }); diff --git a/ydb/core/cms/ui/main.js b/ydb/core/cms/ui/main.js index c09d5019e5d3..073fe9c69fb4 100644 --- a/ydb/core/cms/ui/main.js +++ b/ydb/core/cms/ui/main.js @@ -2,10 +2,11 @@ requirejs.config({ baseUrl: 'js/lib', paths: { jquery: 'ext/jquery.min' - } + }, + waitSeconds: 60 // Since editor is quite large }); -var createEditor; +var createEditorPromise = new $.Deferred(); let run = () => { require.config({ @@ -14,7 +15,7 @@ let run = () => { }); require(["vs/editor/editor.main"], function () { - createEditor = (container, readOnly, width) => { + createEditorPromise.resolve((container, readOnly, width) => { var editor; container.style.border = '1px solid #eee'; container.style.borderRadius = '8px'; @@ -48,7 +49,7 @@ let run = () => { editor.onDidContentSizeChange(updateHeight); updateHeight(); return editor; - } + }); $(document).ready(main); }); diff --git a/ydb/core/cms/ui/yaml_config.js b/ydb/core/cms/ui/yaml_config.js index 0c2bb0d405b0..a2aa9de918f0 100644 --- a/ydb/core/cms/ui/yaml_config.js +++ b/ydb/core/cms/ui/yaml_config.js @@ -284,13 +284,15 @@ class YamlConfigState { configsRoot.append(remove); configsRoot.append(head); configsRoot.append(form); - let cm = createEditor(div.get(0), true, 1068); - cm.setValue(volatileConfig.config); - copy.click(function() { - copyToClipboard(cm.getValue()); + createEditorPromise.then((createEditor) => { + let cm = createEditor(div.get(0), true, 1068); + cm.setValue(volatileConfig.config); + copy.click(function() { + copyToClipboard(cm.getValue()); + }); + + this.codeMirrors.push(cm); }); - - this.codeMirrors.push(cm); } } } else { @@ -415,137 +417,139 @@ class YamlConfigState { initTab() { var self = this; - this.codeMirror = createEditor($("#main-editor-container").get(0), this.readOnly, 1068); - this.config = ""; - this.codeMirror.setValue(this.config); + createEditorPromise.then((createEditor) => { + this.codeMirror = createEditor($("#main-editor-container").get(0), this.readOnly, 1068); + this.config = ""; + this.codeMirror.setValue(this.config); - this.volatileCodeMirror = createEditor(document.getElementById("volatile-yaml-config-item"), false, 1068); + this.volatileCodeMirror = createEditor(document.getElementById("volatile-yaml-config-item"), false, 1068); - this.volatileConfig = ""; - this.volatileCodeMirror.setValue(this.volatileConfig); + this.volatileConfig = ""; + this.volatileCodeMirror.setValue(this.volatileConfig); - this.additionalResolveCodeMirror = createEditor($("#additional-yaml-config-resolve").get(0), false, 1034); - this.additionalResolveCodeMirror.setValue(""); + this.additionalResolveCodeMirror = createEditor($("#additional-yaml-config-resolve").get(0), false, 1034); + this.additionalResolveCodeMirror.setValue(""); - $("#copy-additional-yaml-config").click(function() { - copyToClipboard(self.additionalResolveCodeMirror.getValue()); - }); + $("#copy-additional-yaml-config").click(function() { + copyToClipboard(self.additionalResolveCodeMirror.getValue()); + }); - $("#link-additional-yaml-config").click(function() { - $(".yaml-btn").first().click(); - self.volatileCodeMirror.setValue(self.additionalResolveCodeMirror.getValue()); - }); + $("#link-additional-yaml-config").click(function() { + $(".yaml-btn").first().click(); + self.volatileCodeMirror.setValue(self.additionalResolveCodeMirror.getValue()); + }); - this.arbitraryResolveCodeMirror = createEditor($("#arbitrary-yaml-config-resolve").get(0), false, 1034); - this.arbitraryResolveCodeMirror.setValue(""); + this.arbitraryResolveCodeMirror = createEditor($("#arbitrary-yaml-config-resolve").get(0), false, 1034); + this.arbitraryResolveCodeMirror.setValue(""); - $("#copy-arbitrary-yaml-config").click(function() { - copyToClipboard(self.arbitraryResolveCodeMirror.getValue()); - }); + $("#copy-arbitrary-yaml-config").click(function() { + copyToClipboard(self.arbitraryResolveCodeMirror.getValue()); + }); - $("#fold-arbitrary-yaml-config").click(function() { - self.arbitraryResolveCodeMirror.trigger('fold', 'editor.foldLevel2'); - }); + $("#fold-arbitrary-yaml-config").click(function() { + self.arbitraryResolveCodeMirror.trigger('fold', 'editor.foldLevel2'); + }); - $("#unfold-arbitrary-yaml-config").click(function() { - self.arbitraryResolveCodeMirror.trigger('fold', 'editor.unfoldAll'); - }); + $("#unfold-arbitrary-yaml-config").click(function() { + self.arbitraryResolveCodeMirror.trigger('fold', 'editor.unfoldAll'); + }); - var elemResolved = $("#yaml-config-resolved"); + var elemResolved = $("#yaml-config-resolved"); - var copyResolved = $(` -
-
-
- `); + var copyResolved = $(` +
+
+
+ `); - elemResolved.parent().prepend(copyResolved); + elemResolved.parent().prepend(copyResolved); - this.resolvedCodeMirror = createEditor(elemResolved.get(0), true, 1050); - this.resolvedCodeMirror.setValue(""); + this.resolvedCodeMirror = createEditor(elemResolved.get(0), true, 1050); + this.resolvedCodeMirror.setValue(""); - copyResolved.click(function() { - copyToClipboard(this.resolvedCodeMirror.getValue()); - }); + copyResolved.click(function() { + copyToClipboard(this.resolvedCodeMirror.getValue()); + }); - $("#yaml-label-add").click(() => addLabel()); - $("#yaml-label-clear").click(() => clearLabels()); + $("#yaml-label-add").click(() => addLabel()); + $("#yaml-label-clear").click(() => clearLabels()); - $(".yaml-floater").children().click(function() { - var floater = $(this); - var submenu = $(floater.attr('data-target')); - if (!$(".yaml-floater").children().hasClass("active")) { - floater.addClass("active"); - submenu.addClass("active"); - setTimeout(() => hideOnClickOutside(".yaml-submenu"), 100); - } - }); + $(".yaml-floater").children().click(function() { + var floater = $(this); + var submenu = $(floater.attr('data-target')); + if (!$(".yaml-floater").children().hasClass("active")) { + floater.addClass("active"); + submenu.addClass("active"); + setTimeout(() => hideOnClickOutside(".yaml-submenu"), 100); + } + }); - $("#link-yaml-config").click(function() { - $(".yaml-btn").first().click(); - $("#yaml-arbitrary-config-tab").click(); - self.arbitraryResolveCodeMirror.setValue(self.codeMirror.getValue()); - }); + $("#link-yaml-config").click(function() { + $(".yaml-btn").first().click(); + $("#yaml-arbitrary-config-tab").click(); + self.arbitraryResolveCodeMirror.setValue(self.codeMirror.getValue()); + }); - $("#copy-yaml-config").click(function() { - copyToClipboard(self.codeMirror.getValue()); - }); + $("#copy-yaml-config").click(function() { + copyToClipboard(self.codeMirror.getValue()); + }); - $("#fold-yaml-config").click(function() { - self.codeMirror.trigger('fold', 'editor.foldLevel2'); - }); + $("#fold-yaml-config").click(function() { + self.codeMirror.trigger('fold', 'editor.foldLevel2'); + }); - $("#unfold-yaml-config").click(function() { - self.codeMirror.trigger('fold', 'editor.unfoldAll'); - }); + $("#unfold-yaml-config").click(function() { + self.codeMirror.trigger('fold', 'editor.unfoldAll'); + }); - $("#copy-volatile-yaml-config").click(function() { - copyToClipboard(self.volatileCodeMirror.getValue()); - }); + $("#copy-volatile-yaml-config").click(function() { + copyToClipboard(self.volatileCodeMirror.getValue()); + }); - $("#link-volatile-yaml-config").click(function() { - $(".yaml-btn").first().click(); - $("#yaml-current-config-tab").click(); - $("#yaml-resolve-include-volatile").prop('checked', true); - self.additionalResolveCodeMirror.setValue(self.volatileCodeMirror.getValue()); - }); + $("#link-volatile-yaml-config").click(function() { + $(".yaml-btn").first().click(); + $("#yaml-current-config-tab").click(); + $("#yaml-resolve-include-volatile").prop('checked', true); + self.additionalResolveCodeMirror.setValue(self.volatileCodeMirror.getValue()); + }); - $("#volatile-yaml-resolve-all").click(() => self.resolveAll()); - $("#volatile-yaml-resolve-for-labels").click(() => self.resolveForLabels()); + $("#volatile-yaml-resolve-all").click(() => self.resolveAll()); + $("#volatile-yaml-resolve-for-labels").click(() => self.resolveForLabels()); - $('#volatile-yaml-apply-button').on('click', function(event) { - event.preventDefault(); - showAck("Add volatile config?", " ", "Yes", "No", self.addVolatileConfig.bind(self)); - }); + $('#volatile-yaml-apply-button').on('click', function(event) { + event.preventDefault(); + showAck("Add volatile config?", " ", "Yes", "No", self.addVolatileConfig.bind(self)); + }); - $('#volatile-yaml-remove-all').on('click', function(event) { - event.preventDefault(); - showAck("Remove all volatile configs?", " ", "Yes", "No", self.removeAllVolatileConfigs.bind(self)); - }); + $('#volatile-yaml-remove-all').on('click', function(event) { + event.preventDefault(); + showAck("Remove all volatile configs?", " ", "Yes", "No", self.removeAllVolatileConfigs.bind(self)); + }); - $('.codeeditor-yaml').each(function () { - var self = $(this); - var copy = $(` -
-
-
- `); + $('.codeeditor-yaml').each(function () { + var self = $(this); + var copy = $(` +
+
+
+ `); - self.parent().prepend(copy); + self.parent().prepend(copy); - var value = self.text(); - self.text(""); + var value = self.text(); + self.text(""); - let cm = createEditor(self.get(0), true, 500); + let cm = createEditor(self.get(0), true, 500); - cm.setValue(value); + cm.setValue(value); - copy.click(function() { - copyToClipboard(cm.getValue()); + copy.click(function() { + copyToClipboard(cm.getValue()); + }); }); - }); - this.loadYaml(); + this.loadYaml(); + }); } } From 0cdb1fb5b9f602ff39532b52d923e26516f1f23c Mon Sep 17 00:00:00 2001 From: Artem Alekseev Date: Thu, 22 Aug 2024 13:56:25 +0300 Subject: [PATCH 045/261] Add scan/insert while alter test (#8094) --- ydb/tools/olap_workload/__main__.py | 177 +++++++++++++++++++++++----- 1 file changed, 146 insertions(+), 31 deletions(-) diff --git a/ydb/tools/olap_workload/__main__.py b/ydb/tools/olap_workload/__main__.py index f49232da9f43..02ee03f4f231 100644 --- a/ydb/tools/olap_workload/__main__.py +++ b/ydb/tools/olap_workload/__main__.py @@ -3,6 +3,8 @@ import ydb import time import os +import random +import string ydb.interceptor.monkey_patch_event_handler() @@ -15,12 +17,31 @@ def table_name_with_timestamp(): return os.path.join("column_table_" + str(timestamp())) +def random_string(length): + letters = string.ascii_lowercase + return bytes(''.join(random.choice(letters) for i in range(length)), encoding='utf8') + + +def random_type(): + return random.choice([ydb.PrimitiveType.Int64, ydb.PrimitiveType.String]) + + +def random_value(type): + if isinstance(type, ydb.OptionalType): + return random_value(type.item) + if type == ydb.PrimitiveType.Int64: + return random.randint(0, 1 << 31) + if type == ydb.PrimitiveType.String: + return random_string(random.randint(1, 32)) + + class Workload(object): - def __init__(self, endpoint, database, duration): + def __init__(self, endpoint, database, duration, batch_size): self.database = database self.driver = ydb.Driver(ydb.DriverConfig(endpoint, database)) self.pool = ydb.SessionPool(self.driver, size=200) self.duration = duration + self.batch_size = batch_size def __enter__(self): return self @@ -29,47 +50,140 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.pool.stop() self.driver.stop() + def run_query_ignore_errors(self, callee): + try: + self.pool.retry_operation_sync(callee) + except Exception as e: + print(type(e), e) + def create_table(self, table_name): - with self.pool.checkout() as s: - try: - s.execute_scheme( - """ - CREATE TABLE %s ( - id Int64 NOT NULL, - i64Val Int64, - PRIMARY KEY(id) - ) - PARTITION BY HASH(id) - WITH ( - STORE = COLUMN - ) - """ - % table_name + print(f"Create table {table_name}") + + def callee(session): + session.execute_scheme( + f""" + CREATE TABLE {table_name} ( + id Int64 NOT NULL, + i64Val Int64, + PRIMARY KEY(id) + ) + PARTITION BY HASH(id) + WITH ( + STORE = COLUMN ) + """ + ) - print("Table %s created" % table_name) - except ydb.SchemeError as e: - print(e) + self.run_query_ignore_errors(callee) def drop_table(self, table_name): - with self.pool.checkout() as s: - try: - s.drop_table(self.database + "/" + table_name) + print(f"Drop table {table_name}") + + def callee(session): + session.drop_table(self.database + "/" + table_name) + + self.run_query_ignore_errors(callee) + + def add_column(self, table_name, col_name, col_type): + print(f"Add column {table_name}.{col_name} {str(col_type)}") + + def callee(session): + session.execute_scheme(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {str(col_type)}") + + self.run_query_ignore_errors(callee) + + def drop_column(self, table_name, col_name): + print(f"Drop column {table_name}.{col_name}") + + def callee(session): + session.execute_scheme(f"ALTER TABLE {table_name} DROP COLUMN {col_name}") + + self.run_query_ignore_errors(callee) + + def generate_batch(self, schema): + data = [] + + for i in range(self.batch_size): + data.append({c.name: random_value(c.type) for c in schema}) + + return data + + def add_batch(self, table_name, schema): + print(f"Add batch {table_name}") - print("Table %s dropped" % table_name) - except ydb.SchemeError as e: - print(e) + column_types = ydb.BulkUpsertColumns() + + for c in schema: + column_types.add_column(c.name, c.type) + + batch = self.generate_batch(schema) + + self.driver.table_client.bulk_upsert(self.database + "/" + table_name, batch, column_types) + + def list_tables(self): + db = self.driver.scheme_client.list_directory(self.database) + return [t.name for t in db.children if t.type == ydb.SchemeEntryType.COLUMN_TABLE] + + def list_columns(self, table_name): + path = self.database + "/" + table_name + + def callee(session): + return session.describe_table(path).columns + + return self.pool.retry_operation_sync(callee) + + def rows_count(self, table_name): + return self.driver.table_client.scan_query(f"SELECT count(*) FROM {table_name}").next().result_set.rows[0][0] + + def select_n(self, table_name, limit): + print(f"Select {limit} from {table_name}") + self.driver.table_client.scan_query(f"SELECT * FROM {table_name} limit {limit}").next() + + def drop_all_tables(self): + for t in self.list_tables(): + if t.startswith("column_table_"): + self.drop_table(t) + + def drop_all_columns(self, table_name): + for c in self.list_columns(table_name): + if c.name != "id": + self.drop_column(table_name, c.name) + + def queries_while_alter(self): + table_name = "queries_while_alter" + + schema = self.list_columns(table_name) + + self.select_n(table_name, 1000) + self.add_batch(table_name, schema) + self.select_n(table_name, 100) + self.add_batch(table_name, schema) + self.select_n(table_name, 300) + + if len(schema) > 50: + self.drop_all_columns(table_name) + + if self.rows_count(table_name) > 100000: + self.drop_table(table_name) + + col = "col_" + str(timestamp()) + self.add_column(table_name, col, random_type()) def run(self): started_at = time.time() while time.time() - started_at < self.duration: - table_name = table_name_with_timestamp() - self.create_table(table_name) + try: + self.create_table("queries_while_alter") - time.sleep(5) + self.drop_all_tables() - self.drop_table(table_name) + self.queries_while_alter() + + table_name = table_name_with_timestamp() + self.create_table(table_name) + except Exception as e: + print(type(e), e) if __name__ == '__main__': @@ -78,7 +192,8 @@ def run(self): ) parser.add_argument('--endpoint', default='localhost:2135', help="An endpoint to be used") parser.add_argument('--database', default=None, required=True, help='A database to connect') - parser.add_argument('--duration', default=10**9, type=lambda x: int(x), help='A duration of workload in seconds.') + parser.add_argument('--duration', default=120, type=lambda x: int(x), help='A duration of workload in seconds.') + parser.add_argument('--batch_size', default=1000, help='Batch size for bulk insert') args = parser.parse_args() - with Workload(args.endpoint, args.database, args.duration) as workload: + with Workload(args.endpoint, args.database, args.duration, args.batch_size) as workload: workload.run() From 00acf7d36ffd5a3ba6a638c3365e7bca8cfd96a1 Mon Sep 17 00:00:00 2001 From: azevaykin <145343289+azevaykin@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:57:46 +0300 Subject: [PATCH 046/261] Statistics: SendBaseStatsToSA return delta for schedule next event (#8119) --- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 22 +++++++++++++------- ydb/core/tx/schemeshard/schemeshard_impl.h | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 06e5c4a404a2..ba38452bc01d 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -7321,10 +7321,8 @@ void TSchemeShard::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& } void TSchemeShard::Handle(TEvPrivate::TEvSendBaseStatsToSA::TPtr&, const TActorContext& ctx) { - SendBaseStatsToSA(); - auto seconds = SendStatsIntervalMaxSeconds - SendStatsIntervalMinSeconds; - ctx.Schedule(TDuration::Seconds(SendStatsIntervalMinSeconds + RandomNumber(seconds)), - new TEvPrivate::TEvSendBaseStatsToSA()); + TDuration delta = SendBaseStatsToSA(); + ctx.Schedule(delta, new TEvPrivate::TEvSendBaseStatsToSA()); } void TSchemeShard::InitializeStatistics(const TActorContext& ctx) { @@ -7371,15 +7369,15 @@ void TSchemeShard::ConnectToSA() { << ", at schemeshard: " << TabletID()); } -void TSchemeShard::SendBaseStatsToSA() { +TDuration TSchemeShard::SendBaseStatsToSA() { if (!EnableStatistics) { - return; + return TDuration::Max(); } if (!SAPipeClientId) { ResolveSA(); if (!StatisticsAggregatorId) { - return; + return TDuration::Seconds(30); } } @@ -7411,6 +7409,13 @@ void TSchemeShard::SendBaseStatsToSA() { ++count; } + if (!count) { + LOG_DEBUG_S(TlsActivationContext->AsActorContext(), NKikimrServices::STATISTICS, + "SendBaseStatsToSA() No tables to send" + << ", at schemeshard: " << TabletID()); + return TDuration::Seconds(30); + } + TString stats; stats.clear(); Y_PROTOBUF_SUPPRESS_NODISCARD record.SerializeToString(&stats); @@ -7425,6 +7430,9 @@ void TSchemeShard::SendBaseStatsToSA() { "SendBaseStatsToSA()" << ", path count: " << count << ", at schemeshard: " << TabletID()); + + return TDuration::Seconds(SendStatsIntervalMinSeconds + + RandomNumber(SendStatsIntervalMaxSeconds - SendStatsIntervalMinSeconds)); } } // namespace NSchemeShard diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index ab1cbbcd5e45..70d5a71b3265 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -1396,7 +1396,7 @@ class TSchemeShard void InitializeStatistics(const TActorContext& ctx); void ResolveSA(); void ConnectToSA(); - void SendBaseStatsToSA(); + TDuration SendBaseStatsToSA(); From 3c7c1d0f2c0a3d2e95de3c922863dd75a10a127a Mon Sep 17 00:00:00 2001 From: Semyon Date: Thu, 22 Aug 2024 13:58:29 +0300 Subject: [PATCH 047/261] Add column shard Controller parameters to AppData (#8000) --- ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp | 16 +-- ydb/core/kqp/ut/olap/indexes_ut.cpp | 6 +- ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp | 4 +- ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 4 +- ydb/core/kqp/ut/olap/sparsed_ut.cpp | 2 +- ydb/core/kqp/ut/olap/write_ut.cpp | 8 +- ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 4 +- ydb/core/protos/config.proto | 6 + .../blobs_action/bs/blob_manager.cpp | 2 +- .../blobs_action/bs/blob_manager.h | 3 - ydb/core/tx/columnshard/columnshard.cpp | 2 +- ydb/core/tx/columnshard/columnshard_impl.cpp | 12 +- .../engines/column_engine_logs.cpp | 2 +- .../reader/plain_reader/iterator/context.h | 10 +- .../engines/storage/granule/storage.cpp | 2 +- .../optimizer/lbuckets/planner/optimizer.cpp | 3 +- .../optimizer/lbuckets/planner/optimizer.h | 8 +- .../optimizer/sbuckets/common/optimizer.cpp | 5 +- .../sbuckets/constructor/constructor.cpp | 3 +- .../sbuckets/constructor/constructor.h | 5 +- .../optimizer/sbuckets/index/bucket.cpp | 2 +- .../columnshard/engines/ut/ut_logs_engine.cpp | 6 +- .../columnshard/hooks/abstract/abstract.cpp | 22 +++ .../tx/columnshard/hooks/abstract/abstract.h | 130 ++++++++++++------ .../tx/columnshard/hooks/testing/controller.h | 78 +++++------ .../columnshard/hooks/testing/ro_controller.h | 4 +- .../tx/columnshard/test_helper/controllers.h | 13 +- .../ut_rw/ut_columnshard_read_write.cpp | 6 +- .../ut_schema/ut_columnshard_schema.cpp | 4 +- ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 4 +- ydb/core/tx/tiering/ut/ut_tiers.cpp | 12 +- 31 files changed, 223 insertions(+), 165 deletions(-) diff --git a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp index b8cc467fae2f..c57897a87594 100644 --- a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp +++ b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp @@ -93,8 +93,8 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { , Controller(NYDBTest::TControllers::RegisterCSControllerGuard()) { Controller->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable); Controller->SetExpectedShardsCount(ShardsCount); - Controller->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - Controller->SetReadTimeoutClean(TDuration::Seconds(1)); + Controller->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1)); Tests::NCommon::TLoggerInit(Kikimr).SetComponents({ NKikimrServices::TX_COLUMNSHARD }, "CS").Initialize(); @@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { } void WaitNormalization() { - Controller->SetReadTimeoutClean(TDuration::Seconds(1)); + Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1)); Controller->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Force); const auto start = TInstant::Now(); while (!Controller->IsTrivialLinks() && TInstant::Now() - start < TDuration::Seconds(30)) { @@ -120,11 +120,11 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { } AFL_VERIFY(Controller->IsTrivialLinks()); Controller->CheckInvariants(); - Controller->SetReadTimeoutClean(TDuration::Minutes(5)); + Controller->SetOverrideReadTimeoutClean(TDuration::Minutes(5)); } void Execute(const ui64 destinationIdx, const std::vector& sourceIdxs, const bool move, const NOlap::TSnapshot& snapshot, const std::set& pathIdxs) { - Controller->SetReadTimeoutClean(TDuration::Seconds(1)); + Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1)); AFL_VERIFY(destinationIdx < ShardIds.size()); const ui64 destination = ShardIds[destinationIdx]; std::vector sources; @@ -192,7 +192,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { CSTransferStatus->Reset(); AFL_VERIFY(!Controller->IsTrivialLinks()); Controller->CheckInvariants(); - Controller->SetReadTimeoutClean(TDuration::Minutes(5)); + Controller->SetOverrideReadTimeoutClean(TDuration::Minutes(5)); } }; Y_UNIT_TEST(BlobsSharingSplit1_1) { @@ -318,8 +318,8 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { void Execute() { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); TLocalHelper(Kikimr).SetShardingMethod(ShardingType).CreateTestOlapTable("olapTable", "olapStore", 24, 4); diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp index feff93803fc5..7699cd8d8b6e 100644 --- a/ydb/core/kqp/ut/olap/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes_ut.cpp @@ -17,8 +17,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { TKikimrRunner kikimr(settings); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); TLocalHelper(kikimr).CreateTestOlapTable(); @@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { TKikimrRunner kikimr(settings); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); TLocalHelper(kikimr).CreateTestOlapTable(); auto tableClient = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp index 4f6c90056e89..c1fcab4be0fd 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp @@ -21,10 +21,10 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { class TOlapStatsController : public NYDBTest::NColumnShard::TController { public: - TDuration GetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override { + TDuration DoGetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override { return TDuration::MilliSeconds(10); } - TDuration GetStatsReportInterval(const TDuration /*defaultValue*/) const override { + TDuration DoGetStatsReportInterval(const TDuration /*defaultValue*/) const override { return TDuration::MilliSeconds(10); } }; diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 090f54891054..180f5a7ebb17 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -2695,8 +2695,8 @@ Y_UNIT_TEST_SUITE(KqpOlap) { TLocalHelper testHelper(kikimr); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); diff --git a/ydb/core/kqp/ut/olap/sparsed_ut.cpp b/ydb/core/kqp/ut/olap/sparsed_ut.cpp index 1b5011128ded..88bc081fa357 100644 --- a/ydb/core/kqp/ut/olap/sparsed_ut.cpp +++ b/ydb/core/kqp/ut/olap/sparsed_ut.cpp @@ -95,7 +95,7 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { void Execute() { CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - CSController->SetPeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); + CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); Tests::NCommon::TLoggerInit(Kikimr).Initialize(); TTypedLocalHelper helper("Utf8", Kikimr); diff --git a/ydb/core/kqp/ut/olap/write_ut.cpp b/ydb/core/kqp/ut/olap/write_ut.cpp index ac63da37bc72..6293899abcd8 100644 --- a/ydb/core/kqp/ut/olap/write_ut.cpp +++ b/ydb/core/kqp/ut/olap/write_ut.cpp @@ -15,7 +15,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { Y_UNIT_TEST(TierDraftsGC) { auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csController->SetIndexWriteControllerEnabled(false); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); Singleton()->ResetWriteCounters(); auto settings = TKikimrSettings() @@ -50,7 +50,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { Y_UNIT_TEST(TierDraftsGCWithRestart) { auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csController->SetIndexWriteControllerEnabled(false); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1000)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1000)); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC); Singleton()->ResetWriteCounters(); @@ -133,7 +133,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { Y_UNIT_TEST(WriteDeleteCleanGC) { auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC); Singleton()->ResetWriteCounters(); @@ -176,7 +176,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { )", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync(); UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); } - csController->SetReadTimeoutClean(TDuration::Zero()); + csController->SetOverrideReadTimeoutClean(TDuration::Zero()); csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC); { const TInstant start = TInstant::Now(); diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 557eaf8b0180..7bf3746fabe0 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -3265,8 +3265,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto session = Kikimr->GetTableClient().CreateSession().GetValueSync().GetSession(); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); const TString query = Sprintf(R"( diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index a137cb1495df..79dec737eb5a 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1553,6 +1553,12 @@ message TColumnShardConfig { optional uint32 MaxInFlightIntervalsOnRequest = 16; optional uint32 MaxReadStaleness_ms = 18 [default = 300000]; + optional uint32 GCIntervalMs = 19 [default = 30000]; + optional uint32 CompactionActualizationLagMs = 20 [default = 1000]; + optional uint32 ActualizationTasksLagMs = 21 [default = 1000]; + optional uint32 LagForCompactionBeforeTieringsMs = 22 [default = 3600000]; + optional uint32 OptimizerFreshnessCheckDurationMs = 23 [default = 300000]; + optional uint32 SmallPortionDetectSizeLimit = 24 [default = 1048576]; // 1 << 20 } message TSchemeShardConfig { diff --git a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp index b4fb522e3d06..5008e3e57c7f 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp @@ -309,7 +309,7 @@ std::shared_ptr TBlobManager::BuildGCTas return nullptr; } - if (AppData()->TimeProvider->Now() - PreviousGCTime < NYDBTest::TControllers::GetColumnShardController()->GetOverridenGCPeriod(TDuration::Seconds(GC_INTERVAL_SECONDS))) { + if (AppData()->TimeProvider->Now() - PreviousGCTime < NYDBTest::TControllers::GetColumnShardController()->GetOverridenGCPeriod()) { ACFL_DEBUG("event", "TBlobManager::BuildGCTask skip")("current_gen", CurrentGen)("current_step", CurrentStep)("reason", "too_often"); BlobsManagerCounters.GCCounters.SkipCollectionThrottling->Add(1); return nullptr; diff --git a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h index 90094e62b7a3..52e0f573eb60 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h @@ -133,9 +133,6 @@ struct TBlobManagerCounters { // The implementation of BlobManager that hides all GC-related details class TBlobManager : public IBlobManager, public TCommonBlobsTracker { -private: - static constexpr ui64 GC_INTERVAL_SECONDS = 30; - private: using TBlobAddress = NBlobOperations::NBlobStorage::TBlobAddress; class TGCContext; diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 0d94deef36c1..3a5c969860aa 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -178,7 +178,7 @@ void TColumnShard::Handle(TEvPrivate::TEvReadFinished::TPtr& ev, const TActorCon void TColumnShard::Handle(TEvPrivate::TEvPingSnapshotsUsage::TPtr& /*ev*/, const TActorContext& ctx) { if (auto writeTx = InFlightReadsTracker.Ping( - this, NYDBTest::TControllers::GetColumnShardController()->GetPingCheckPeriod(0.6 * GetMaxReadStaleness()), TInstant::Now())) { + this, NYDBTest::TControllers::GetColumnShardController()->GetPingCheckPeriod(), TInstant::Now())) { Execute(writeTx.release(), ctx); } ctx.Schedule(0.3 * GetMaxReadStaleness(), new TEvPrivate::TEvPingSnapshotsUsage()); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 1875a14d9763..207554291d75 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -71,9 +71,8 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet) , ProgressTxController(std::make_unique(*this)) , StoragesManager(std::make_shared(*this)) , DataLocksManager(std::make_shared()) - , PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod( - TSettings::DefaultPeriodicWakeupActivationPeriod)) - , StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval(TSettings::DefaultStatsReportInterval)) + , PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod()) + , StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval()) , InFlightReadsTracker(StoragesManager, Counters.GetRequestsTracingCounters()) , TablesManager(StoragesManager, info->TabletID) , Subscribers(std::make_shared(*this)) @@ -680,8 +679,8 @@ void TColumnShard::SetupIndexation() { if (InsertTable->GetPathPriorities().size() && InsertTable->GetPathPriorities().rbegin()->first.GetCategory() == NOlap::TPathInfoIndexPriority::EIndexationPriority::PreventOverload) { force = true; } - const ui64 bytesLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationStartBytesLimit(TSettings::GuaranteeIndexationStartBytesLimit); - const TDuration durationLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationInterval(TSettings::GuaranteeIndexationInterval); + const ui64 bytesLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationStartBytesLimit(); + const TDuration durationLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationInterval(); if (!force && InsertTable->GetCountersCommitted().Bytes < bytesLimit && TMonotonic::Now() < BackgroundController.GetLastIndexationInstant() + durationLimit) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_indexation")("reason", "not_enough_data_and_too_frequency") @@ -1145,8 +1144,7 @@ const NKikimr::NColumnShard::NTiers::TManager* TColumnShard::GetTierManagerPoint } TDuration TColumnShard::GetMaxReadStaleness() { - return NYDBTest::TControllers::GetColumnShardController()->GetReadTimeoutClean( - TDuration::MilliSeconds(AppDataVerified().ColumnShardConfig.GetMaxReadStaleness_ms())); + return NYDBTest::TControllers::GetColumnShardController()->GetReadTimeoutClean(); } } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 63c3289f9c9d..9051ab02a1c4 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -421,7 +421,7 @@ std::vector> TColumnEngineForLogs::Star TSaverContext saverContext(StoragesManager); NActualizer::TTieringProcessContext context(memoryUsageLimit, saverContext, dataLocksManager, SignalCounters, ActualizationController); - const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetActualizationTasksLag(TDuration::Seconds(1)); + const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetActualizationTasksLag(); for (auto&& i : pathEviction) { auto g = GetGranuleOptional(i.first); if (g) { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h index 211f837074a0..1ae41c039808 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.h @@ -41,13 +41,9 @@ class TSpecialReadContext { CacheFetchingScripts; public: - static const inline ui64 DefaultRejectMemoryIntervalLimit = TGlobalLimits::DefaultRejectMemoryIntervalLimit; - static const inline ui64 DefaultReduceMemoryIntervalLimit = TGlobalLimits::DefaultReduceMemoryIntervalLimit; - static const inline ui64 DefaultReadSequentiallyBufferSize = TGlobalLimits::DefaultReadSequentiallyBufferSize; - - const ui64 ReduceMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetReduceMemoryIntervalLimit(DefaultReduceMemoryIntervalLimit); - const ui64 RejectMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetRejectMemoryIntervalLimit(DefaultRejectMemoryIntervalLimit); - const ui64 ReadSequentiallyBufferSize = DefaultReadSequentiallyBufferSize; + const ui64 ReduceMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetReduceMemoryIntervalLimit(); + const ui64 RejectMemoryIntervalLimit = NYDBTest::TControllers::GetColumnShardController()->GetRejectMemoryIntervalLimit(); + const ui64 ReadSequentiallyBufferSize = TGlobalLimits::DefaultReadSequentiallyBufferSize; ui64 GetProcessMemoryControlId() const { AFL_VERIFY(ProcessMemoryGuard); diff --git a/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp b/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp index 385f9d818d23..b017464eefeb 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/storage.cpp @@ -8,7 +8,7 @@ std::shared_ptr TGranulesStorage::GetGranuleForCom std::map> granulesSorted; ui32 countChecker = 0; std::optional priorityChecker; - const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetCompactionActualizationLag(TDuration::Seconds(1)); + const TDuration actualizationLag = NYDBTest::TControllers::GetColumnShardController()->GetCompactionActualizationLag(); for (auto&& i : Tables) { NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("path_id", i.first); i.second->ActualizeOptimizer(now, actualizationLag); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp index a36e976ed351..36f467a03133 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.cpp @@ -3,8 +3,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets { TDuration GetCommonFreshnessCheckDuration() { - static const TDuration CommonFreshnessCheckDuration = TDuration::Seconds(300); - return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(CommonFreshnessCheckDuration); + return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(); } } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h index f83183c04e39..d686fc719112 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h @@ -19,8 +19,6 @@ namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets { -static const ui64 SmallPortionDetectSizeLimit = 1 << 20; - TDuration GetCommonFreshnessCheckDuration(); class TSimplePortionsGroupInfo { @@ -683,7 +681,7 @@ class TPortionsBucket: public TMoveOnly { return; } MainPortion->InitRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized, Others.IsEmpty() && currentInstant > MainPortion->RecordSnapshotMax().GetPlanInstant() + - NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings(TDuration::Minutes(60))); + NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings()); } public: TTaskDescription GetTaskDescription() const { @@ -1104,7 +1102,7 @@ class TPortionBuckets { } void RemovePortion(const std::shared_ptr& portion) { - if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector(SmallPortionDetectSizeLimit)) { + if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector()) { Counters->SmallPortions->RemovePortion(portion); } if (!RemoveBucket(portion)) { @@ -1146,7 +1144,7 @@ class TPortionBuckets { } void AddPortion(const std::shared_ptr& portion, const TInstant now) { - if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector(SmallPortionDetectSizeLimit)) { + if (portion->GetTotalBlobBytes() < NYDBTest::TControllers::GetColumnShardController()->GetSmallPortionSizeDetector()) { Counters->SmallPortions->AddPortion(portion); AddOther(portion, now); return; diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp index baf229b3d1fd..2fe68710d805 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/common/optimizer.cpp @@ -4,8 +4,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { TDuration GetCommonFreshnessCheckDuration() { - static const TDuration CommonFreshnessCheckDuration = TDuration::Seconds(300); - return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(CommonFreshnessCheckDuration); + return NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(); } -} +} // namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp index 9e8d21bb9357..bc007d0fff10 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.cpp @@ -2,12 +2,11 @@ #include #include #include -#include namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { std::shared_ptr TOptimizerPlannerConstructor::BuildLogic() const { - const TDuration freshnessCheckDuration = NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(FreshnessCheckDuration); + const TDuration freshnessCheckDuration = NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration(); std::shared_ptr logic; if (LogicName == "one_head") { logic = std::make_shared(freshnessCheckDuration); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h index 8b85a25c0877..cabe72ccc2a9 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/constructor/constructor.h @@ -1,13 +1,16 @@ #pragma once +#include #include #include +#include namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { class TOptimizerPlannerConstructor: public IOptimizerPlannerConstructor { private: YDB_READONLY_DEF(TString, LogicName); - YDB_READONLY(TDuration, FreshnessCheckDuration, TDuration::Seconds(300)); + YDB_READONLY(TDuration, FreshnessCheckDuration, NYDBTest::TControllers::GetColumnShardController()->GetOptimizerFreshnessCheckDuration()); + public: static TString GetClassNameStatic() { return "s-buckets"; diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp index 5c70d26a38a3..ec344a674fd7 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/sbuckets/index/bucket.cpp @@ -9,7 +9,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NSBuckets { void TPortionsBucket::RebuildOptimizedFeature(const TInstant currentInstant) const { for (auto&& [_, p] : Portions) { p.MutablePortionInfo().InitRuntimeFeature(TPortionInfo::ERuntimeFeature::Optimized, Portions.size() == 1 && currentInstant > p->RecordSnapshotMax().GetPlanInstant() + - NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings(TDuration::Minutes(60)) + NYDBTest::TControllers::GetColumnShardController()->GetLagForCompactionBeforeTierings() ); } } diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index 0a82451bb7d5..cbe12efbd5ed 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -433,6 +433,7 @@ std::shared_ptr CommonStoragesManager = Initia Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { void WriteLoadRead(const std::vector& ydbSchema, const std::vector& key) { + TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); @@ -528,6 +529,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { void ReadWithPredicates(const std::vector& ydbSchema, const std::vector& key) { + TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(ydbSchema, key); @@ -624,6 +626,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexWriteOverload) { + TTestBasicRuntime runtime; TTestDbWrapper db; auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); TIndexInfo tableInfo = NColumnShard::BuildTableInfo(testColumns, testKey);; @@ -696,10 +699,11 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } Y_UNIT_TEST(IndexTtl) { + TTestBasicRuntime runtime; TTestDbWrapper db; TIndexInfo tableInfo = NColumnShard::BuildTableInfo(testColumns, testKey); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); - csDefaultControllerGuard->SetTasksActualizationLag(TDuration::Zero()); + csDefaultControllerGuard->SetOverrideTasksActualizationLag(TDuration::Zero()); ui64 pathId = 1; ui32 step = 1000; diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp b/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp index a4d916545eac..b275e17f2fdb 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.cpp @@ -1,4 +1,26 @@ #include "abstract.h" +#include + namespace NKikimr::NYDBTest { + +TDuration ICSController::GetGuaranteeIndexationInterval() const { + const TDuration defaultValue = NColumnShard::TSettings::GuaranteeIndexationInterval; + return DoGetGuaranteeIndexationInterval(defaultValue); +} + +TDuration ICSController::GetPeriodicWakeupActivationPeriod() const { + const TDuration defaultValue = NColumnShard::TSettings::DefaultPeriodicWakeupActivationPeriod; + return DoGetPeriodicWakeupActivationPeriod(defaultValue); +} + +TDuration ICSController::GetStatsReportInterval() const { + const TDuration defaultValue = NColumnShard::TSettings::DefaultStatsReportInterval; + return DoGetStatsReportInterval(defaultValue); +} + +ui64 ICSController::GetGuaranteeIndexationStartBytesLimit() const { + const ui64 defaultValue = NColumnShard::TSettings::GuaranteeIndexationStartBytesLimit; + return DoGetGuaranteeIndexationStartBytesLimit(defaultValue); +} } diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index 68a25a123a57..c104590235d3 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -84,13 +85,70 @@ class ICSController { virtual void DoOnDataSharingStarted(const ui64 /*tabletId*/, const TString& /*sessionId*/) { } + virtual TDuration DoGetPingCheckPeriod(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetOverridenGCPeriod(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetCompactionActualizationLag(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetActualizationTasksLag(const TDuration defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetReduceMemoryIntervalLimit(const ui64 defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetRejectMemoryIntervalLimit(const ui64 defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetReadSequentiallyBufferSize(const ui64 defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetSmallPortionSizeDetector(const ui64 defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetReadTimeoutClean(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetGuaranteeIndexationInterval(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetStatsReportInterval(const TDuration defaultValue) const { + return defaultValue; + } + virtual ui64 DoGetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const { + return defaultValue; + } + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration defaultValue) const { + return defaultValue; + } + +private: + inline static const NKikimrConfig::TColumnShardConfig DefaultConfig = {}; + + static const NKikimrConfig::TColumnShardConfig& GetConfig() { + if (HasAppData()) { + return AppDataVerified().ColumnShardConfig; + } + return DefaultConfig; + } + public: virtual void OnRequestTracingChanges( const std::set& /*snapshotsToSave*/, const std::set& /*snapshotsToRemove*/) { } - virtual TDuration GetPingCheckPeriod(const TDuration def) const { - return def; + TDuration GetPingCheckPeriod() const { + const TDuration defaultValue = 0.6 * GetReadTimeoutClean(); + return DoGetPingCheckPeriod(defaultValue); } virtual bool IsBackgroundEnabled(const EBackground /*id*/) const { @@ -100,15 +158,17 @@ class ICSController { using TPtr = std::shared_ptr; virtual ~ICSController() = default; - virtual TDuration GetOverridenGCPeriod(const TDuration def) const { - return def; + TDuration GetOverridenGCPeriod() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetGCIntervalMs()); + return DoGetOverridenGCPeriod(defaultValue); } virtual void OnSelectShardingFilter() { } - virtual TDuration GetCompactionActualizationLag(const TDuration def) const { - return def; + TDuration GetCompactionActualizationLag() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetCompactionActualizationLagMs()); + return DoGetCompactionActualizationLag(defaultValue); } virtual NColumnShard::TBlobPutResult::TPtr OverrideBlobPutResultOnCompaction( @@ -116,25 +176,25 @@ class ICSController { return original; } - virtual TDuration GetRemovedPortionLivetime(const TDuration def) const { - return def; - } - - virtual TDuration GetActualizationTasksLag(const TDuration d) const { - return d; + TDuration GetActualizationTasksLag() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetActualizationTasksLagMs()); + return DoGetActualizationTasksLag(defaultValue); } - virtual ui64 GetReduceMemoryIntervalLimit(const ui64 def) const { - return def; + ui64 GetReduceMemoryIntervalLimit() const { + const ui64 defaultValue = NOlap::TGlobalLimits::DefaultReduceMemoryIntervalLimit; + return DoGetReduceMemoryIntervalLimit(defaultValue); } - virtual ui64 GetRejectMemoryIntervalLimit(const ui64 def) const { - return def; + ui64 GetRejectMemoryIntervalLimit() const { + const ui64 defaultValue = NOlap::TGlobalLimits::DefaultRejectMemoryIntervalLimit; + return DoGetRejectMemoryIntervalLimit(defaultValue); } virtual bool NeedForceCompactionBacketsConstruction() const { return false; } - virtual ui64 GetSmallPortionSizeDetector(const ui64 def) const { - return def; + ui64 GetSmallPortionSizeDetector() const { + const ui64 defaultValue = GetConfig().GetSmallPortionDetectSizeLimit(); + return DoGetSmallPortionSizeDetector(defaultValue); } virtual void OnExportFinished() { } @@ -158,8 +218,9 @@ class ICSController { virtual void OnMaxValueUsage() { } - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration def) const { - return def; + virtual TDuration GetLagForCompactionBeforeTierings() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetLagForCompactionBeforeTieringsMs()); + return DoGetLagForCompactionBeforeTierings(defaultValue); } void OnTabletInitCompleted(const NColumnShard::TColumnShard& shard) { @@ -188,29 +249,20 @@ class ICSController { } virtual void OnIndexSelectProcessed(const std::optional /*result*/) { } - virtual TDuration GetReadTimeoutClean(const TDuration def) { - return def; + TDuration GetReadTimeoutClean() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetMaxReadStaleness_ms()); + return DoGetReadTimeoutClean(defaultValue); } virtual EOptimizerCompactionWeightControl GetCompactionControl() const { return EOptimizerCompactionWeightControl::Force; } - virtual TDuration GetTTLDefaultWaitingDuration(const TDuration defaultValue) const { - return defaultValue; - } - virtual TDuration GetGuaranteeIndexationInterval(const TDuration defaultValue) const { - return defaultValue; - } - virtual TDuration GetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const { - return defaultValue; - } - virtual TDuration GetStatsReportInterval(const TDuration defaultValue) const { - return defaultValue; - } - virtual ui64 GetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const { - return defaultValue; - } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const { - return defaultValue; + TDuration GetGuaranteeIndexationInterval() const; + TDuration GetPeriodicWakeupActivationPeriod() const; + TDuration GetStatsReportInterval() const; + ui64 GetGuaranteeIndexationStartBytesLimit() const; + TDuration GetOptimizerFreshnessCheckDuration() const { + const TDuration defaultValue = TDuration::MilliSeconds(GetConfig().GetOptimizerFreshnessCheckDurationMs()); + return DoGetOptimizerFreshnessCheckDuration(defaultValue); } virtual void OnTieringModified(const std::shared_ptr& /*tiers*/) { diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.h b/ydb/core/tx/columnshard/hooks/testing/controller.h index 031a8f144a95..a8e259877fd0 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/controller.h @@ -12,21 +12,21 @@ namespace NKikimr::NYDBTest::NColumnShard { class TController: public TReadOnlyController { private: using TBase = TReadOnlyController; - YDB_ACCESSOR_DEF(std::optional, RequestsTracePingCheckPeriod); - YDB_ACCESSOR_DEF(std::optional, LagForCompactionBeforeTierings); - YDB_ACCESSOR(std::optional, GuaranteeIndexationInterval, TDuration::Zero()); - YDB_ACCESSOR(std::optional, PeriodicWakeupActivationPeriod, std::nullopt); - YDB_ACCESSOR(std::optional, StatsReportInterval, std::nullopt); - YDB_ACCESSOR(std::optional, GuaranteeIndexationStartBytesLimit, 0); - YDB_ACCESSOR(std::optional, OptimizerFreshnessCheckDuration, TDuration::Zero()); - YDB_ACCESSOR_DEF(std::optional, CompactionActualizationLag); - YDB_ACCESSOR_DEF(std::optional, TasksActualizationLag); + YDB_ACCESSOR_DEF(std::optional, OverrideRequestsTracePingCheckPeriod); + YDB_ACCESSOR_DEF(std::optional, OverrideLagForCompactionBeforeTierings); + YDB_ACCESSOR(std::optional, OverrideGuaranteeIndexationInterval, TDuration::Zero()); + YDB_ACCESSOR(std::optional, OverridePeriodicWakeupActivationPeriod, std::nullopt); + YDB_ACCESSOR(std::optional, OverrideStatsReportInterval, std::nullopt); + YDB_ACCESSOR(std::optional, OverrideGuaranteeIndexationStartBytesLimit, 0); + YDB_ACCESSOR(std::optional, OverrideOptimizerFreshnessCheckDuration, TDuration::Zero()); + YDB_ACCESSOR_DEF(std::optional, OverrideCompactionActualizationLag); + YDB_ACCESSOR_DEF(std::optional, OverrideTasksActualizationLag); + YDB_ACCESSOR_DEF(std::optional, OverrideReadTimeoutClean); EOptimizerCompactionWeightControl CompactionControl = EOptimizerCompactionWeightControl::Force; YDB_ACCESSOR(std::optional, OverrideReduceMemoryIntervalLimit, 1024); YDB_ACCESSOR_DEF(std::optional, OverrideRejectMemoryIntervalLimit); - std::optional ReadTimeoutClean; std::optional ExpectedShardsCount; THashMap ShardActuals; @@ -130,16 +130,16 @@ class TController: public TReadOnlyController { THashSet SharingIds; protected: virtual ::NKikimr::NColumnShard::TBlobPutResult::TPtr OverrideBlobPutResultOnCompaction(const ::NKikimr::NColumnShard::TBlobPutResult::TPtr original, const NOlap::TWriteActionsCollection& actions) const override; - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration def) const override { - return LagForCompactionBeforeTierings.value_or(def); + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration def) const override { + return OverrideLagForCompactionBeforeTierings.value_or(def); } - virtual TDuration GetPingCheckPeriod(const TDuration def) const override { - return RequestsTracePingCheckPeriod.value_or(def); + virtual TDuration DoGetPingCheckPeriod(const TDuration def) const override { + return OverrideRequestsTracePingCheckPeriod.value_or(def); } - virtual TDuration GetCompactionActualizationLag(const TDuration def) const override { - return CompactionActualizationLag.value_or(def); + virtual TDuration DoGetCompactionActualizationLag(const TDuration def) const override { + return OverrideCompactionActualizationLag.value_or(def); } @@ -148,8 +148,8 @@ class TController: public TReadOnlyController { return !DisabledBackgrounds.contains(id); } - virtual TDuration GetActualizationTasksLag(const TDuration d) const override { - return TasksActualizationLag.value_or(d); + virtual TDuration DoGetActualizationTasksLag(const TDuration d) const override { + return OverrideTasksActualizationLag.value_or(d); } virtual void DoOnTabletInitCompleted(const ::NKikimr::NColumnShard::TColumnShard& shard) override; @@ -157,23 +157,29 @@ class TController: public TReadOnlyController { virtual void DoOnAfterGCAction(const ::NKikimr::NColumnShard::TColumnShard& shard, const NOlap::IBlobsGCAction& action) override; virtual bool DoOnWriteIndexComplete(const NOlap::TColumnEngineChanges& changes, const ::NKikimr::NColumnShard::TColumnShard& shard) override; - virtual TDuration GetGuaranteeIndexationInterval(const TDuration defaultValue) const override { - return GuaranteeIndexationInterval.value_or(defaultValue); + virtual TDuration DoGetGuaranteeIndexationInterval(const TDuration defaultValue) const override { + return OverrideGuaranteeIndexationInterval.value_or(defaultValue); } - TDuration GetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const override { - return PeriodicWakeupActivationPeriod.value_or(defaultValue); + virtual TDuration DoGetPeriodicWakeupActivationPeriod(const TDuration defaultValue) const override { + return OverridePeriodicWakeupActivationPeriod.value_or(defaultValue); } - TDuration GetStatsReportInterval(const TDuration defaultValue) const override { - return StatsReportInterval.value_or(defaultValue); + virtual TDuration DoGetStatsReportInterval(const TDuration defaultValue) const override { + return OverrideStatsReportInterval.value_or(defaultValue); } - virtual ui64 GetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const override { - return GuaranteeIndexationStartBytesLimit.value_or(defaultValue); + virtual ui64 DoGetGuaranteeIndexationStartBytesLimit(const ui64 defaultValue) const override { + return OverrideGuaranteeIndexationStartBytesLimit.value_or(defaultValue); } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const override { - return OptimizerFreshnessCheckDuration.value_or(defaultValue); + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const override { + return OverrideOptimizerFreshnessCheckDuration.value_or(defaultValue); } - virtual TDuration GetReadTimeoutClean(const TDuration def) override { - return ReadTimeoutClean.value_or(def); + virtual TDuration DoGetReadTimeoutClean(const TDuration def) const override { + return OverrideReadTimeoutClean.value_or(def); + } + virtual ui64 DoGetReduceMemoryIntervalLimit(const ui64 def) const override { + return OverrideReduceMemoryIntervalLimit.value_or(def); + } + virtual ui64 DoGetRejectMemoryIntervalLimit(const ui64 def) const override { + return OverrideRejectMemoryIntervalLimit.value_or(def); } virtual EOptimizerCompactionWeightControl GetCompactionControl() const override { return CompactionControl; @@ -190,18 +196,9 @@ class TController: public TReadOnlyController { } public: - virtual TDuration GetRemovedPortionLivetime(const TDuration /*def*/) const override { - return TDuration::Zero(); - } const TAtomicCounter& GetIndexWriteControllerBrokeCount() const { return IndexWriteControllerBrokeCount; } - virtual ui64 GetReduceMemoryIntervalLimit(const ui64 def) const override { - return OverrideReduceMemoryIntervalLimit.value_or(def); - } - virtual ui64 GetRejectMemoryIntervalLimit(const ui64 def) const override { - return OverrideRejectMemoryIntervalLimit.value_or(def); - } bool IsTrivialLinks() const; TCheckContext CheckInvariants() const; @@ -237,9 +234,6 @@ class TController: public TReadOnlyController { void SetCompactionControl(const EOptimizerCompactionWeightControl value) { CompactionControl = value; } - void SetReadTimeoutClean(const TDuration d) { - ReadTimeoutClean = d; - } bool HasPKSortingOnly() const; diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index 9f9e90d02987..c55be9455204 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -71,11 +71,11 @@ class TReadOnlyController: public ICSController { return EOptimizerCompactionWeightControl::Force; } -public: - virtual TDuration GetOverridenGCPeriod(const TDuration /*def*/) const override { + virtual TDuration DoGetOverridenGCPeriod(const TDuration /*def*/) const override { return TDuration::Zero(); } +public: void WaitCompactions(const TDuration d) const { TInstant start = TInstant::Now(); ui32 compactionsStart = GetCompactionStartedCounter().Val(); diff --git a/ydb/core/tx/columnshard/test_helper/controllers.h b/ydb/core/tx/columnshard/test_helper/controllers.h index 5b48f204e2bc..68cd6a1dc4ed 100644 --- a/ydb/core/tx/columnshard/test_helper/controllers.h +++ b/ydb/core/tx/columnshard/test_helper/controllers.h @@ -21,24 +21,21 @@ class TWaitCompactionController: public NYDBTest::NColumnShard::TController { virtual bool NeedForceCompactionBacketsConstruction() const override { return true; } - virtual ui64 GetSmallPortionSizeDetector(const ui64 /*def*/) const override { + virtual ui64 DoGetSmallPortionSizeDetector(const ui64 /*def*/) const override { return SmallSizeDetector.value_or(0); } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { return TDuration::Zero(); } - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { return TDuration::Zero(); } - virtual TDuration GetCompactionActualizationLag(const TDuration /*def*/) const override { + virtual TDuration DoGetCompactionActualizationLag(const TDuration /*def*/) const override { return TDuration::Zero(); } - virtual TDuration GetTTLDefaultWaitingDuration(const TDuration /*defaultValue*/) const override { - return TDuration::Seconds(1); - } public: TWaitCompactionController() { - SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); } ui32 GetFinishedExportsCount() const { diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index a2d4f9e36ab1..31d13085a186 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -608,7 +608,7 @@ void TestWriteReadLongTxDup() { void TestWriteRead(bool reboots, const TestTableDescription& table = {}, TString codec = "") { auto csControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - csControllerGuard->SetReadTimeoutClean(TDuration::Max()); + csControllerGuard->SetOverrideReadTimeoutClean(TDuration::Max()); TTestBasicRuntime runtime; TTester::Setup(runtime); @@ -2592,7 +2592,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { TTestBasicRuntime runtime; auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csDefaultControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); - csDefaultControllerGuard->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csDefaultControllerGuard->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); TTester::Setup(runtime); runtime.SetLogPriority(NKikimrServices::BLOB_CACHE, NActors::NLog::PRI_INFO); @@ -2824,7 +2824,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { PlanCommit(runtime, sender, planStep, txId); } UNIT_ASSERT_EQUAL(cleanupsHappened, 0); - csDefaultControllerGuard->SetRequestsTracePingCheckPeriod(TDuration::Zero()); + csDefaultControllerGuard->SetOverrideRequestsTracePingCheckPeriod(TDuration::Zero()); { auto read = std::make_unique(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, read.release()); diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index 29da71e9a8a8..deb7be3d89e9 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -174,7 +174,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {}, { auto csControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - csControllerGuard->SetTasksActualizationLag(TDuration::Zero()); + csControllerGuard->SetOverrideTasksActualizationLag(TDuration::Zero()); std::vector ts = {1600000000, 1620000000}; ui32 ttlIncSeconds = 1; @@ -526,7 +526,7 @@ std::vector> TestTiers(bool reboots, const std::vector(); csControllerGuard->DisableBackground(NYDBTest::ICSController::EBackground::TTL); - csControllerGuard->SetTasksActualizationLag(TDuration::Zero()); + csControllerGuard->SetOverrideTasksActualizationLag(TDuration::Zero()); TTestBasicRuntime runtime; TTester::Setup(runtime); diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index cd6004c88e29..1681aab2ee75 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -642,8 +642,8 @@ Y_UNIT_TEST_SUITE(TOlap) { runtime.UpdateCurrentTime(TInstant::Now() - TDuration::Seconds(600)); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1)); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30); // disable stats batching diff --git a/ydb/core/tx/tiering/ut/ut_tiers.cpp b/ydb/core/tx/tiering/ut/ut_tiers.cpp index 55fd3a3437e5..d80d6aae1c75 100644 --- a/ydb/core/tx/tiering/ut/ut_tiers.cpp +++ b/ydb/core/tx/tiering/ut/ut_tiers.cpp @@ -32,21 +32,15 @@ class TFastTTLCompactionController: public NKikimr::NYDBTest::ICSController { virtual bool NeedForceCompactionBacketsConstruction() const override { return true; } - virtual TDuration GetRemovedPortionLivetime(const TDuration /*def*/) const override { - return TDuration::Zero(); - } - virtual ui64 GetSmallPortionSizeDetector(const ui64 /*def*/) const override { + virtual ui64 DoGetSmallPortionSizeDetector(const ui64 /*def*/) const override { return 0; } - virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { + virtual TDuration DoGetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override { return TDuration::Zero(); } - virtual TDuration GetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { + virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration /*def*/) const override { return TDuration::Zero(); } - virtual TDuration GetTTLDefaultWaitingDuration(const TDuration /*defaultValue*/) const override { - return TDuration::Seconds(1); - } }; From 625f1875ea9d57585f9787d9996233e954befae1 Mon Sep 17 00:00:00 2001 From: Andrei Rykov Date: Thu, 22 Aug 2024 13:17:12 +0200 Subject: [PATCH 048/261] add http-handle returning configs (#7830) --- ydb/core/viewer/json_handlers_viewer.cpp | 6 + ydb/core/viewer/json_pipe_req.cpp | 8 + ydb/core/viewer/json_pipe_req.h | 1 + ydb/core/viewer/protos/viewer.proto | 16 ++ ydb/core/viewer/viewer_feature_flags.h | 183 +++++++++++++++++++++++ ydb/core/viewer/viewer_ut.cpp | 31 ++++ ydb/core/viewer/ya.make | 1 + 7 files changed, 246 insertions(+) create mode 100644 ydb/core/viewer/viewer_feature_flags.h diff --git a/ydb/core/viewer/json_handlers_viewer.cpp b/ydb/core/viewer/json_handlers_viewer.cpp index a9ec2d7c2cb9..d827a1180df4 100644 --- a/ydb/core/viewer/json_handlers_viewer.cpp +++ b/ydb/core/viewer/json_handlers_viewer.cpp @@ -11,6 +11,7 @@ #include "viewer_describe_consumer.h" #include "viewer_describe.h" #include "viewer_describe_topic.h" +#include "viewer_feature_flags.h" #include "viewer_graph.h" #include "viewer_healthcheck.h" #include "viewer_hiveinfo.h" @@ -265,6 +266,10 @@ void InitViewerCheckAccessJsonHandler(TJsonHandlers& jsonHandlers) { jsonHandlers.AddHandler("/viewer/check_access", new TJsonHandler(TCheckAccess::GetSwagger())); } +void InitViewerFeatureFlagsJsonHandler(TJsonHandlers& handlers) { + handlers.AddHandler("/viewer/feature_flags", new TJsonHandler(TJsonFeatureFlags::GetSwagger())); +} + void InitViewerJsonHandlers(TJsonHandlers& jsonHandlers) { InitViewerCapabilitiesJsonHandler(jsonHandlers); InitViewerNodelistJsonHandler(jsonHandlers); @@ -303,6 +308,7 @@ void InitViewerJsonHandlers(TJsonHandlers& jsonHandlers) { InitViewerRenderJsonHandler(jsonHandlers); InitViewerAutocompleteJsonHandler(jsonHandlers); InitViewerCheckAccessJsonHandler(jsonHandlers); + InitViewerFeatureFlagsJsonHandler(jsonHandlers); } } diff --git a/ydb/core/viewer/json_pipe_req.cpp b/ydb/core/viewer/json_pipe_req.cpp index 91ea3405cb15..f2d2c8172298 100644 --- a/ydb/core/viewer/json_pipe_req.cpp +++ b/ydb/core/viewer/json_pipe_req.cpp @@ -199,6 +199,14 @@ TViewerPipeClient::TRequestResponse(pipeClient, request.Release()); } +TViewerPipeClient::TRequestResponse TViewerPipeClient::MakeRequestConsoleNodeConfigByTenant(TString tenant, ui64 cookie) { + TActorId pipeClient = ConnectTabletPipe(GetConsoleId()); + auto request = MakeHolder(); + request->Record.MutableNode()->SetTenant(tenant); + request->Record.AddItemKinds(static_cast(NKikimrConsole::TConfigItem::FeatureFlagsItem)); + return MakeRequestToPipe(pipeClient, request.Release(), cookie); +} + void TViewerPipeClient::RequestConsoleGetTenantStatus(const TString& path) { TActorId pipeClient = ConnectTabletPipe(GetConsoleId()); THolder request = MakeHolder(); diff --git a/ydb/core/viewer/json_pipe_req.h b/ydb/core/viewer/json_pipe_req.h index 6b63ae778a1e..9d661ed4f9f6 100644 --- a/ydb/core/viewer/json_pipe_req.h +++ b/ydb/core/viewer/json_pipe_req.h @@ -190,6 +190,7 @@ class TViewerPipeClient : public TActorBootstrapped { TRequestResponse MakeRequestHiveStorageStats(TTabletId hiveId); void RequestConsoleListTenants(); TRequestResponse MakeRequestConsoleListTenants(); + TRequestResponse MakeRequestConsoleNodeConfigByTenant(TString tenant, ui64 cookie = 0); void RequestConsoleGetTenantStatus(const TString& path); TRequestResponse MakeRequestConsoleGetTenantStatus(const TString& path); void RequestBSControllerConfig(); diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index 8d3f69d074ae..222ef7c2fb1f 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -719,3 +719,19 @@ message TPDiskInfo { TPDiskInfoWhiteboard Whiteboard = 1; TPDiskInfoBSC BSC = 2; } + +message TFeatureFlagsConfig { + message TFeatureFlag { + string Name = 1; + bool Enabled = 2; + bool IsDefault = 3; + } + + message TDatabase { + string Name = 1; + repeated TFeatureFlag FeatureFlags = 2; + } + + uint32 Version = 1; + repeated TDatabase Databases = 2; +} diff --git a/ydb/core/viewer/viewer_feature_flags.h b/ydb/core/viewer/viewer_feature_flags.h new file mode 100644 index 000000000000..66e0410febf8 --- /dev/null +++ b/ydb/core/viewer/viewer_feature_flags.h @@ -0,0 +1,183 @@ +#pragma once +#include "json_pipe_req.h" +#include "viewer.h" + +namespace NKikimr::NViewer { + +using namespace NActors; + +class TJsonFeatureFlags : public TViewerPipeClient { + using TThis = TJsonFeatureFlags; + using TBase = TViewerPipeClient; + TJsonSettings JsonSettings; + ui32 Timeout = 0; + + TString FilterDatabase; + THashSet FilterFeatures; + THashMap DatabaseByCookie; + ui64 Cookie = 0; + TString DomainPath; + bool Direct = false; + + TRequestResponse TenantsResponse; + THashMap> NodeConfigResponses; + +public: + TJsonFeatureFlags(IViewer* viewer, NMon::TEvHttpInfo::TPtr &ev) + : TViewerPipeClient(viewer, ev) + {} + + void MakeNodeConfigRequest(const TString& database) { + NodeConfigResponses[database] = MakeRequestConsoleNodeConfigByTenant(database, Cookie); + DatabaseByCookie[Cookie++] = database; + } + + void Bootstrap() override { + const auto& params(Event->Get()->Request.GetParams()); + + JsonSettings.EnumAsNumbers = !FromStringWithDefault(params.Get("enums"), true); + JsonSettings.UI64AsString = !FromStringWithDefault(params.Get("ui64"), false); + FilterDatabase = params.Get("database"); + StringSplitter(params.Get("features")).Split(',').SkipEmpty().Collect(&FilterFeatures); + Direct = FromStringWithDefault(params.Get("direct"), Direct); + Timeout = FromStringWithDefault(params.Get("timeout"), 10000); + + TIntrusivePtr domains = AppData()->DomainsInfo; + auto* domain = domains->GetDomain(); + DomainPath = "/" + domain->Name; + + Direct |= Event->Get()->Request.GetUri().StartsWith("/node/"); // we're already forwarding + Direct |= (FilterDatabase == AppData()->TenantName); // we're already on the right node + if (FilterDatabase && !Direct) { + RequestStateStorageEndpointsLookup(FilterDatabase); // to find some dynamic node and redirect there + } else if (!FilterDatabase) { + MakeNodeConfigRequest(DomainPath); + TenantsResponse = MakeRequestConsoleListTenants(); + } else { + MakeNodeConfigRequest(FilterDatabase); + } + + Become(&TThis::StateWork, TDuration::MilliSeconds(Timeout), new TEvents::TEvWakeup()); + } + + void HandleReply(TEvStateStorage::TEvBoardInfo::TPtr& ev) { + TBase::ReplyAndPassAway(MakeForward(GetNodesFromBoardReply(ev))); + } + + STATEFN(StateWork) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvStateStorage::TEvBoardInfo, HandleReply); + hFunc(NConsole::TEvConsole::TEvListTenantsResponse, Handle); + hFunc(NConsole::TEvConsole::TEvGetNodeConfigResponse, Handle); + hFunc(TEvTabletPipe::TEvClientConnected, TBase::Handle); + cFunc(TEvents::TSystem::Wakeup, HandleTimeout); + } + } + + void Handle(NConsole::TEvConsole::TEvListTenantsResponse::TPtr& ev) { + TenantsResponse.Set(std::move(ev)); + Ydb::Cms::ListDatabasesResult listDatabasesResult; + TenantsResponse->Record.GetResponse().operation().result().UnpackTo(&listDatabasesResult); + for (const TString& path : listDatabasesResult.paths()) { + MakeNodeConfigRequest(path); + } + RequestDone(); + } + + void Handle(NConsole::TEvConsole::TEvGetNodeConfigResponse::TPtr& ev) { + TString database = DatabaseByCookie[ev.Get()->Cookie]; + NodeConfigResponses[database].Set(std::move(ev)); + RequestDone(); + } + + THashMap ParseFeatureFlags(const NKikimrConfig::TFeatureFlags& featureFlags) { + THashMap features; + const google::protobuf::Reflection* reflection = featureFlags.GetReflection(); + const google::protobuf::Descriptor* descriptor = featureFlags.GetDescriptor(); + + for (int i = 0; i < descriptor->field_count(); ++i) { + const google::protobuf::FieldDescriptor* field = descriptor->field(i); + if (field->cpp_type() == google::protobuf::FieldDescriptor::CPPTYPE_BOOL) { + auto& feat = features[field->name()]; + feat.SetName(field->name()); + feat.SetEnabled(reflection->GetBool(featureFlags, field)); + feat.SetIsDefault(field->default_value_bool()); + } + } + return features; + } + + void ReplyAndPassAway() override { + THashMap> FeatureFlagsByDatabase; + for (const auto& [database, response] : NodeConfigResponses) { + NKikimrConsole::TGetNodeConfigResponse rec = response->Record; + FeatureFlagsByDatabase[database] = ParseFeatureFlags(rec.GetConfig().GetFeatureFlags()); + } + + auto domainFeaturesIt = FeatureFlagsByDatabase.find(DomainPath); + if (domainFeaturesIt == FeatureFlagsByDatabase.end()) { + return TBase::ReplyAndPassAway(GetHTTPINTERNALERROR("text/plain", "No domain info from Console")); + } + + // prepare response + NKikimrViewer::TFeatureFlagsConfig Result; + Result.SetVersion(1); + for (const auto& [database, features] : FeatureFlagsByDatabase) { + auto databaseProto = Result.AddDatabases(); + databaseProto->SetName(database); + for (const auto& [name, featProto] : features) { + if (FilterFeatures.empty() || FilterFeatures.find(name) != FilterFeatures.end()) { + auto flag = databaseProto->AddFeatureFlags(); + flag->CopyFrom(featProto); + } + } + } + + TStringStream json; + TProtoToJson::ProtoToJson(json, Result, JsonSettings); + TBase::ReplyAndPassAway(GetHTTPOKJSON(json.Str())); + } + + static YAML::Node GetSwagger() { + TSimpleYamlBuilder yaml({ + .Method = "get", + .Tag = "viewer", + .Summary = "Feature flags", + .Description = "Returns feature flags of each database" + }); + yaml.AddParameter({ + .Name = "database", + .Description = "database name", + .Type = "string", + }); + yaml.AddParameter({ + .Name = "features", + .Description = "comma separated list of features", + .Type = "string", + }); + yaml.AddParameter({ + .Name = "direct", + .Description = "direct request to the node", + .Type = "boolean", + }); + yaml.AddParameter({ + .Name = "timeout", + .Description = "timeout in ms", + .Type = "integer", + }); + yaml.AddParameter({ + .Name = "enums", + .Description = "convert enums to strings", + .Type = "boolean", + }); + yaml.AddParameter({ + .Name = "ui64", + .Description = "return ui64 as number", + .Type = "boolean", + }); + yaml.SetResponseSchema(TProtoToYaml::ProtoToYamlSchema()); + return yaml; + } +}; + +} diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index 241080e56463..0a0fd0e77c27 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -1784,4 +1784,35 @@ Y_UNIT_TEST_SUITE(Viewer) { UNIT_ASSERT_VALUES_EQUAL_C(ticketParser->AuthorizeTicketRequests, 1, response); UNIT_ASSERT_VALUES_EQUAL_C(ticketParser->AuthorizeTicketSuccesses, 1, response); } + + Y_UNIT_TEST(SimpleFeatureFlags) { + TPortManager tp; + ui16 port = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + ui16 monPort = tp.GetPort(8765); + auto settings = TServerSettings(port); + + settings.InitKikimrRunConfig() + .SetNodeCount(1) + .SetUseRealThreads(true) + .SetDomainName("Root") + .SetMonitoringPortOffset(monPort, true); + + TServer server(settings); + server.EnableGRpc(grpcPort); + TClient client(settings); + + TKeepAliveHttpClient httpClient("localhost", monPort); + TStringStream responseStream; + TKeepAliveHttpClient::THeaders headers; + headers["Content-Type"] = "application/json"; + const TKeepAliveHttpClient::THttpCode statusCode = httpClient.DoGet("/viewer/feature_flags?timeout=600000&base64=false", &responseStream, headers); + const TString response = responseStream.ReadAll(); + UNIT_ASSERT_EQUAL_C(statusCode, HTTP_OK, statusCode << ": " << response); + NJson::TJsonReaderConfig jsonCfg; + NJson::TJsonValue json; + NJson::ReadJsonTree(response, &jsonCfg, &json, /* throwOnError = */ true); + auto resultSets = json["Databases"].GetArray(); + UNIT_ASSERT_EQUAL_C(1, resultSets.size(), response); + } } diff --git a/ydb/core/viewer/ya.make b/ydb/core/viewer/ya.make index aa5e4860c2e9..996f3e8ea358 100644 --- a/ydb/core/viewer/ya.make +++ b/ydb/core/viewer/ya.make @@ -64,6 +64,7 @@ SRCS( viewer_describe_consumer.h viewer_describe.h viewer_describe_topic.h + viewer_feature_flags.h viewer_graph.h viewer_healthcheck.h viewer_helper.h From 5a1fe68fa70d29b3bd112c72c25c3db43835851f Mon Sep 17 00:00:00 2001 From: kungurtsev Date: Thu, 22 Aug 2024 13:18:40 +0200 Subject: [PATCH 049/261] Apply old Resource Broker config (#8087) --- .../run/kikimr_services_initializers.cpp | 25 +++++- .../memory_controller/memory_controller.cpp | 45 +++++------ .../memory_controller/memory_controller.h | 17 +++- .../memory_controller_config.h | 2 +- .../memory_controller_ut.cpp | 81 ++++++++++++++++++- .../harness/resources/default_yaml.yml | 2 - 6 files changed, 137 insertions(+), 35 deletions(-) diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 35bcb044da82..5bdfad33692c 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -2047,8 +2047,29 @@ void TMemoryControllerInitializer::InitializeServices( NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - auto config = appData->MemoryControllerConfig; - auto* actor = NMemory::CreateMemoryController(TDuration::Seconds(1), ProcessMemoryInfoProvider, config, appData->Counters); + NMemory::TResourceBrokerConfig resourceBrokerSelfConfig; // for backward compatibility + auto mergeResourceBrokerConfigs = [&](const NKikimrResourceBroker::TResourceBrokerConfig& resourceBrokerConfig) { + if (resourceBrokerConfig.HasResourceLimit() && resourceBrokerConfig.GetResourceLimit().HasMemory()) { + resourceBrokerSelfConfig.LimitBytes = resourceBrokerConfig.GetResourceLimit().GetMemory(); + } + for (const auto& queue : resourceBrokerConfig.GetQueues()) { + if (queue.GetName() == NLocalDb::KqpResourceManagerQueue) { + if (queue.HasLimit() && queue.GetLimit().HasMemory()) { + resourceBrokerSelfConfig.QueryExecutionLimitBytes = queue.GetLimit().GetMemory(); + } + } + } + }; + if (Config.HasBootstrapConfig() && Config.GetBootstrapConfig().HasResourceBroker()) { + mergeResourceBrokerConfigs(Config.GetBootstrapConfig().GetResourceBroker()); + } + if (Config.HasResourceBrokerConfig()) { + mergeResourceBrokerConfigs(Config.GetResourceBrokerConfig()); + } + + auto* actor = NMemory::CreateMemoryController(TDuration::Seconds(1), ProcessMemoryInfoProvider, + Config.GetMemoryControllerConfig(), resourceBrokerSelfConfig, + appData->Counters); setup->LocalServices.emplace_back( NMemory::MakeMemoryControllerId(0), TActorSetupCmd(actor, TMailboxType::HTSwap, appData->BatchPoolId) diff --git a/ydb/core/memory_controller/memory_controller.cpp b/ydb/core/memory_controller/memory_controller.cpp index 0434dabfd26a..5938d58bf799 100644 --- a/ydb/core/memory_controller/memory_controller.cpp +++ b/ydb/core/memory_controller/memory_controller.cpp @@ -33,20 +33,6 @@ using namespace NActors; using namespace NResourceBroker; using TCounterPtr = ::NMonitoring::TDynamicCounters::TCounterPtr; -struct TResourceBrokerLimits { - ui64 LimitBytes; - ui64 QueryExecutionLimitBytes; - - auto operator<=>(const TResourceBrokerLimits&) const = default; - - TString ToString() const noexcept { - TStringBuilder result; - result << "LimitBytes: " << LimitBytes; - result << " QueryExecutionLimitBytes: " << QueryExecutionLimitBytes; - return result; - } -}; - class TMemoryConsumer : public IMemoryConsumer { public: TMemoryConsumer(EMemoryConsumerKind kind, TActorId actorId) @@ -109,12 +95,14 @@ class TMemoryController : public TActorBootstrapped { TDuration interval, TIntrusiveConstPtr processMemoryInfoProvider, const NKikimrConfig::TMemoryControllerConfig& config, + const TResourceBrokerConfig& resourceBrokerConfig, TIntrusivePtr<::NMonitoring::TDynamicCounters> counters) : Interval(interval) , MemTables(std::make_shared(counters, Consumers.emplace(EMemoryConsumerKind::MemTable, MakeIntrusive(EMemoryConsumerKind::MemTable, TActorId{})).first->second)) , ProcessMemoryInfoProvider(std::move(processMemoryInfoProvider)) , Config(config) + , ResourceBrokerSelfConfig(resourceBrokerConfig) , Counters(counters) {} @@ -127,7 +115,7 @@ class TMemoryController : public TActorBootstrapped { HandleWakeup(ctx); - LOG_INFO_S(ctx, NKikimrServices::MEMORY_CONTROLLER, "Bootstrapped"); + LOG_INFO_S(ctx, NKikimrServices::MEMORY_CONTROLLER, "Bootstrapped with config " << Config.ShortDebugString()); } private: @@ -148,7 +136,7 @@ class TMemoryController : public TActorBootstrapped { void HandleConfig(NConsole::TEvConsole::TEvConfigNotificationRequest::TPtr& ev, const TActorContext& ctx) { Config.Swap(ev->Get()->Record.MutableConfig()->MutableMemoryControllerConfig()); - LOG_INFO_S(ctx, NKikimrServices::MEMORY_CONTROLLER, "Config updated " << Config.DebugString()); + LOG_INFO_S(ctx, NKikimrServices::MEMORY_CONTROLLER, "Config updated " << Config.ShortDebugString()); } void HandleWakeup(const TActorContext& ctx) noexcept { @@ -158,7 +146,9 @@ class TMemoryController : public TActorBootstrapped { ui64 hardLimitBytes = GetHardLimitBytes(Config, processMemoryInfo, hasMemTotalHardLimit); ui64 softLimitBytes = GetSoftLimitBytes(Config, hardLimitBytes); ui64 targetUtilizationBytes = GetTargetUtilizationBytes(Config, hardLimitBytes); - ui64 activitiesLimitBytes = GetActivitiesLimitBytes(Config, hardLimitBytes); + ui64 activitiesLimitBytes = ResourceBrokerSelfConfig.LimitBytes + ? ResourceBrokerSelfConfig.LimitBytes // for backward compatibility + : GetActivitiesLimitBytes(Config, hardLimitBytes); TVector consumers(::Reserve(Consumers.size())); ui64 consumersConsumption = 0; @@ -264,7 +254,9 @@ class TMemoryController : public TActorBootstrapped { memoryStats.SetConsumersLimit(consumersLimitBytes); ui64 queryExecutionConsumption = TAlignedPagePool::GetGlobalPagePoolSize(); - ui64 queryExecutionLimitBytes = GetQueryExecutionLimitBytes(Config, hardLimitBytes); + ui64 queryExecutionLimitBytes = ResourceBrokerSelfConfig.QueryExecutionLimitBytes + ? ResourceBrokerSelfConfig.QueryExecutionLimitBytes // for backward compatibility + : GetQueryExecutionLimitBytes(Config, hardLimitBytes); LOG_INFO_S(ctx, NKikimrServices::MEMORY_CONTROLLER, "Consumer QueryExecution state:" << " Consumption: " << queryExecutionConsumption << " Limit: " << queryExecutionLimitBytes); Counters->GetCounter("Consumer/QueryExecution/Consumption")->Set(queryExecutionConsumption); @@ -273,7 +265,7 @@ class TMemoryController : public TActorBootstrapped { memoryStats.SetQueryExecutionLimit(queryExecutionLimitBytes); // Note: for now ResourceBroker and its queues aren't MemoryController consumers and don't share limits with other caches - ApplyResourceBrokerLimits({ + ApplyResourceBrokerConfig({ activitiesLimitBytes, queryExecutionLimitBytes }); @@ -363,22 +355,22 @@ class TMemoryController : public TActorBootstrapped { } } - void ApplyResourceBrokerLimits(TResourceBrokerLimits limits) { - if (limits == CurrentResourceBrokerLimits) { + void ApplyResourceBrokerConfig(TResourceBrokerConfig config) { + if (config == CurrentResourceBrokerConfig) { return; } TAutoPtr configure = new TEvResourceBroker::TEvConfigure(); configure->Merge = true; - configure->Record.MutableResourceLimit()->SetMemory(limits.LimitBytes); + configure->Record.MutableResourceLimit()->SetMemory(config.LimitBytes); auto queue = configure->Record.AddQueues(); queue->SetName(NLocalDb::KqpResourceManagerQueue); - queue->MutableLimit()->SetMemory(limits.QueryExecutionLimitBytes); + queue->MutableLimit()->SetMemory(config.QueryExecutionLimitBytes); Send(MakeResourceBrokerID(), configure.Release()); - CurrentResourceBrokerLimits.emplace(std::move(limits)); + CurrentResourceBrokerConfig.emplace(std::move(config)); } TConsumerCounters& GetConsumerCounters(EMemoryConsumerKind consumer) { @@ -445,9 +437,10 @@ class TMemoryController : public TActorBootstrapped { std::shared_ptr MemTables; const TIntrusiveConstPtr ProcessMemoryInfoProvider; NKikimrConfig::TMemoryControllerConfig Config; + TResourceBrokerConfig ResourceBrokerSelfConfig; const TIntrusivePtr<::NMonitoring::TDynamicCounters> Counters; TMap ConsumerCounters; - std::optional CurrentResourceBrokerLimits; + std::optional CurrentResourceBrokerConfig; }; } @@ -456,11 +449,13 @@ IActor* CreateMemoryController( TDuration interval, TIntrusiveConstPtr processMemoryInfoProvider, const NKikimrConfig::TMemoryControllerConfig& config, + const TResourceBrokerConfig& resourceBrokerSelfConfig, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters) { return new TMemoryController( interval, std::move(processMemoryInfoProvider), config, + resourceBrokerSelfConfig, GetServiceCounters(counters, "utils")->GetSubgroup("component", "memory_controller")); } diff --git a/ydb/core/memory_controller/memory_controller.h b/ydb/core/memory_controller/memory_controller.h index ac626fd92f51..f3428d173997 100644 --- a/ydb/core/memory_controller/memory_controller.h +++ b/ydb/core/memory_controller/memory_controller.h @@ -10,10 +10,25 @@ namespace NKikimr::NMemory { +struct TResourceBrokerConfig { + ui64 LimitBytes = 0; + ui64 QueryExecutionLimitBytes = 0; + + auto operator<=>(const TResourceBrokerConfig&) const = default; + + TString ToString() const noexcept { + TStringBuilder result; + result << "LimitBytes: " << LimitBytes; + result << " QueryExecutionLimitBytes: " << QueryExecutionLimitBytes; + return result; + } +}; + NActors::IActor* CreateMemoryController( TDuration interval, TIntrusiveConstPtr processMemoryInfoProvider, - const NKikimrConfig::TMemoryControllerConfig& config, + const NKikimrConfig::TMemoryControllerConfig& config, + const TResourceBrokerConfig& resourceBrokerSelfConfig, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& counters); } \ No newline at end of file diff --git a/ydb/core/memory_controller/memory_controller_config.h b/ydb/core/memory_controller/memory_controller_config.h index 49ce5e211766..fa8d2e68ac31 100644 --- a/ydb/core/memory_controller/memory_controller_config.h +++ b/ydb/core/memory_controller/memory_controller_config.h @@ -62,7 +62,7 @@ inline ui64 GetHardLimitBytes(const NKikimrConfig::TMemoryControllerConfig& conf hasMemTotalHardLimit = true; return info.MemTotal.value(); } - return 512_MB; // fallback + return 2_GB; // fallback } GET_LIMIT(SoftLimit) diff --git a/ydb/core/memory_controller/memory_controller_ut.cpp b/ydb/core/memory_controller/memory_controller_ut.cpp index 00fab4b7e8e2..0832ca2803f8 100644 --- a/ydb/core/memory_controller/memory_controller_ut.cpp +++ b/ydb/core/memory_controller/memory_controller_ut.cpp @@ -51,11 +51,27 @@ class TWithMemoryControllerServer : public TServer { ProcessMemoryInfoProvider = MakeIntrusive(); ProcessMemoryInfo = &ProcessMemoryInfoProvider->ProcessMemoryInfo; + // copy-paste from TMemoryControllerInitializer::InitializeServices + NMemory::TResourceBrokerConfig resourceBrokerSelfConfig; + const auto& resourceBrokerConfig = Settings->AppConfig->GetResourceBrokerConfig(); + if (resourceBrokerConfig.HasResourceLimit() && resourceBrokerConfig.GetResourceLimit().HasMemory()) { + resourceBrokerSelfConfig.LimitBytes = resourceBrokerConfig.GetResourceLimit().GetMemory(); + } + for (const auto& queue : resourceBrokerConfig.GetQueues()) { + if (queue.GetName() == NLocalDb::KqpResourceManagerQueue) { + if (queue.HasLimit() && queue.GetLimit().HasMemory()) { + resourceBrokerSelfConfig.QueryExecutionLimitBytes = queue.GetLimit().GetMemory(); + } + } + } + Cerr << "ResourceBrokerSelfConfig: " << resourceBrokerSelfConfig.ToString() << Endl; + for (ui32 nodeIndex = 0; nodeIndex < Runtime->GetNodeCount(); ++nodeIndex) { Runtime->AddLocalService(MakeMemoryControllerId(nodeIndex), TActorSetupCmd( CreateMemoryController(TDuration::Seconds(1), (TIntrusivePtr)ProcessMemoryInfoProvider, - {}, Runtime->GetDynamicCounters()), + Settings->AppConfig->GetMemoryControllerConfig(), resourceBrokerSelfConfig, + Runtime->GetDynamicCounters()), TMailboxType::ReadAsFilled, 0), nodeIndex); @@ -167,9 +183,9 @@ Y_UNIT_TEST(Counters_NoHardLimit) { UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/CGroupLimit")->Val(), 0_MB); UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/MemTotal")->Val(), 0_MB); UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/AllocatedMemory")->Val(), 0_MB); - UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/HardLimit")->Val(), 512_MB); // default - UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/SoftLimit")->Val(), 384_MB); - UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/TargetUtilization")->Val(), 256_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/HardLimit")->Val(), 2_GB); // default + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/SoftLimit")->Val(), 1536_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/TargetUtilization")->Val(), 1_GB); server->ProcessMemoryInfo->CGroupLimit = 200_MB; runtime.SimulateSleep(TDuration::Seconds(2)); @@ -401,12 +417,16 @@ Y_UNIT_TEST(ResourceBroker) { runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest(NLocalDb::KqpResourceManagerQueue))); auto config = runtime.GrabEdgeEvent(handle); UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 150_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Consumer/QueryExecution/Limit")->Val(), 150_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/ActivitiesLimitBytes")->Val(), 300_MB); server->ProcessMemoryInfo->CGroupLimit = 500_MB; runtime.SimulateSleep(TDuration::Seconds(2)); runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest(NLocalDb::KqpResourceManagerQueue))); config = runtime.GrabEdgeEvent(handle); UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 75_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Consumer/QueryExecution/Limit")->Val(), 75_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/ActivitiesLimitBytes")->Val(), 150_MB); // ensure that other settings are not affected: runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_ttl"))); @@ -417,7 +437,60 @@ Y_UNIT_TEST(ResourceBroker) { config = runtime.GrabEdgeEvent(handle); UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetCpu(), 3); UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 3221225472); +} + +Y_UNIT_TEST(ResourceBroker_ConfigLimit) { + using namespace NResourceBroker; + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root") + .SetUseRealThreads(false); + + auto memoryControllerConfig = serverSettings.AppConfig->MutableMemoryControllerConfig(); + memoryControllerConfig->SetQueryExecutionLimitPercent(15); + + auto resourceBrokerConfig = serverSettings.AppConfig->MutableResourceBrokerConfig(); + resourceBrokerConfig->MutableResourceLimit()->SetMemory(1000_MB); + auto queue = resourceBrokerConfig->AddQueues(); + queue->SetName("queue_kqp_resource_manager"); + queue->MutableLimit()->SetMemory(999_MB); + queue = resourceBrokerConfig->AddQueues(); + queue->SetName("queue_cs_ttl"); + queue->MutableLimit()->SetMemory(13_MB); + + auto server = MakeIntrusive(serverSettings); + server->ProcessMemoryInfo->CGroupLimit = 500_MB; + auto& runtime = *server->GetRuntime(); + TAutoPtr handle; + auto sender = runtime.AllocateEdgeActor(); + + InitRoot(server, sender); + + runtime.SimulateSleep(TDuration::Seconds(2)); + runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest(NLocalDb::KqpResourceManagerQueue))); + auto config = runtime.GrabEdgeEvent(handle); + UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 999_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Consumer/QueryExecution/Limit")->Val(), 999_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/ActivitiesLimitBytes")->Val(), 1000_MB); + + server->ProcessMemoryInfo->CGroupLimit = 200_MB; + runtime.SimulateSleep(TDuration::Seconds(2)); + runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest(NLocalDb::KqpResourceManagerQueue))); + config = runtime.GrabEdgeEvent(handle); + UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 999_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Consumer/QueryExecution/Limit")->Val(), 999_MB); + UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/ActivitiesLimitBytes")->Val(), 1000_MB); + + // ensure that other settings are not affected: + runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_ttl"))); + config = runtime.GrabEdgeEvent(handle); + UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetCpu(), 3); + UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 13_MB); + runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_general"))); + config = runtime.GrabEdgeEvent(handle); + UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetCpu(), 3); + UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 3221225472); } } diff --git a/ydb/tests/library/harness/resources/default_yaml.yml b/ydb/tests/library/harness/resources/default_yaml.yml index eda433fc7d23..7090fad7dedc 100644 --- a/ydb/tests/library/harness/resources/default_yaml.yml +++ b/ydb/tests/library/harness/resources/default_yaml.yml @@ -256,5 +256,3 @@ federated_query_config: uri: "" pinger: ping_period: "30s" -memory_controller_config: - hard_limit_bytes: 4294967296 From 56b79f55c2148124feb6f704a50012a98edb7e4d Mon Sep 17 00:00:00 2001 From: qrort <31865255+qrort@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:39:17 +0300 Subject: [PATCH 050/261] pass db path to pg_tables instead of tenant name. Resolves #7616 (#8061) --- ydb/core/grpc_services/rpc_read_columns.cpp | 1 + .../compute_actor/kqp_pure_compute_actor.cpp | 2 +- ydb/core/sys_view/pg_tables/pg_tables.cpp | 28 ++++++++++---- ydb/core/sys_view/pg_tables/pg_tables.h | 14 ++++--- ydb/core/sys_view/scan.cpp | 37 +++++++++++++------ ydb/core/sys_view/scan.h | 4 +- 6 files changed, 57 insertions(+), 29 deletions(-) diff --git a/ydb/core/grpc_services/rpc_read_columns.cpp b/ydb/core/grpc_services/rpc_read_columns.cpp index a7ab4cc54ab1..d9902c1fc720 100644 --- a/ydb/core/grpc_services/rpc_read_columns.cpp +++ b/ydb/core/grpc_services/rpc_read_columns.cpp @@ -317,6 +317,7 @@ class TReadColumnsRPC : public TActorBootstrapped { TTableRange range(MinKey.GetCells(), MinKeyInclusive, MaxKey.GetCells(), MaxKeyInclusive); auto tableScanActor = NSysView::CreateSystemViewScan(ctx.SelfID, 0, ResolveNamesResult->ResultSet.front().TableId, + JoinPath(ResolveNamesResult->ResultSet.front().Path), range, columns); diff --git a/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp index ab43bc1e5ff1..b08ba3bc64c3 100644 --- a/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_pure_compute_actor.cpp @@ -108,7 +108,7 @@ void TKqpComputeActor::DoBootstrap() { ScanData->TaskId = GetTask().GetId(); ScanData->TableReader = CreateKqpTableReader(*ScanData); - auto scanActor = NSysView::CreateSystemViewScan(SelfId(), 0, ScanData->TableId, ranges, columns); + auto scanActor = NSysView::CreateSystemViewScan(SelfId(), 0, ScanData->TableId, ScanData->TablePath, ranges, columns); if (!scanActor) { InternalError(TIssuesIds::DEFAULT_ERROR, TStringBuilder() diff --git a/ydb/core/sys_view/pg_tables/pg_tables.cpp b/ydb/core/sys_view/pg_tables/pg_tables.cpp index 1965f700eb95..c42bb2c7cb24 100644 --- a/ydb/core/sys_view/pg_tables/pg_tables.cpp +++ b/ydb/core/sys_view/pg_tables/pg_tables.cpp @@ -80,9 +80,13 @@ constexpr auto TPgTablesScanBase::ActorActivityType() { } void TPgTablesScanBase::ProceedToScan() { - auto request = MakeHolder(AppData()->TenantName); + auto request = MakeHolder(); NKikimrSchemeOp::TDescribePath& record = request->Record; - record.SetPath(AppData()->TenantName); + auto pathVec = SplitPath(TablePath_); + auto sz = pathVec.size(); + Y_ENSURE(sz > 2 && pathVec[sz - 2] == ".sys"); + pathVec.pop_back(); pathVec.pop_back(); + record.SetPath(JoinPath(pathVec)); record.MutableOptions()->SetReturnPartitioningInfo(false); record.MutableOptions()->SetReturnPartitionConfig(false); record.MutableOptions()->SetReturnChildren(true); @@ -158,12 +162,14 @@ void TPgTablesScanBase::StateWork(TAutoPtr& ev) { TPgTablesScanBase::TPgTablesScanBase( const NActors::TActorId &ownerId, ui32 scanId, const TTableId &tableId, + const TString& tablePath, const TTableRange &tableRange, const TArrayRef &columns, const TVector& schemaColumns, THashMap&& fillers, THashMap&& staticFillers) : NKikimr::NSysView::TScanActorBase(ownerId, scanId, tableId, tableRange, columns), + TablePath_(tablePath), SchemaColumns_(schemaColumns), Fillers_(std::move(fillers)), StaticFillers_(std::move(staticFillers)) {} @@ -171,12 +177,14 @@ TPgTablesScanBase::TPgTablesScanBase( TPgTablesScan::TPgTablesScan( const NActors::TActorId &ownerId, ui32 scanId, const TTableId &tableId, + const TString& tablePath, const TTableRange &tableRange, const TArrayRef &columns) : TPgTablesScanBase( ownerId, scanId, tableId, + tablePath, tableRange, columns, Singleton()->GetColumns(PgTablesName), @@ -214,12 +222,14 @@ TPgTablesScan::TPgTablesScan( TInformationSchemaTablesScan::TInformationSchemaTablesScan( const NActors::TActorId &ownerId, ui32 scanId, const TTableId &tableId, + const TString& tablePath, const TTableRange &tableRange, const TArrayRef &columns) : TPgTablesScanBase( ownerId, scanId, tableId, + tablePath, tableRange, columns, Singleton()->GetColumns(InformationSchemaTablesName), @@ -242,12 +252,14 @@ TInformationSchemaTablesScan::TInformationSchemaTablesScan( TPgClassScan::TPgClassScan( const NActors::TActorId &ownerId, ui32 scanId, const TTableId &tableId, + const TString& tablePath, const TTableRange &tableRange, const TArrayRef &columns) : TPgTablesScanBase( ownerId, scanId, tableId, + tablePath, tableRange, columns, Singleton()->GetColumns(PgClassName), @@ -304,22 +316,22 @@ TPgClassScan::TPgClassScan( }); } -THolder CreatePgTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, +THolder CreatePgTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns) { - return MakeHolder(ownerId, scanId, tableId, tableRange, columns); + return MakeHolder(ownerId, scanId, tableId, tablePath, tableRange, columns); } -THolder CreateInformationSchemaTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, +THolder CreateInformationSchemaTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns) { - return MakeHolder(ownerId, scanId, tableId, tableRange, columns); + return MakeHolder(ownerId, scanId, tableId, tablePath, tableRange, columns); } -THolder CreatePgClassScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, +THolder CreatePgClassScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns) { - return MakeHolder(ownerId, scanId, tableId, tableRange, columns); + return MakeHolder(ownerId, scanId, tableId, tablePath, tableRange, columns); } } // NSysView diff --git a/ydb/core/sys_view/pg_tables/pg_tables.h b/ydb/core/sys_view/pg_tables/pg_tables.h index 408d43b7b627..e7587106c087 100644 --- a/ydb/core/sys_view/pg_tables/pg_tables.h +++ b/ydb/core/sys_view/pg_tables/pg_tables.h @@ -27,6 +27,7 @@ class TPgTablesScanBase : public NKikimr::NSysView::TScanActorBase& columns, const TVector& schemaColumns, @@ -39,6 +40,7 @@ class TPgTablesScanBase : public NKikimr::NSysView::TScanActorBase& ev); protected: TString ConvertError_; + TString TablePath_; const TVector& SchemaColumns_; const THashMap Fillers_; const THashMap StaticFillers_; @@ -46,29 +48,29 @@ class TPgTablesScanBase : public NKikimr::NSysView::TScanActorBase& columns); }; class TInformationSchemaTablesScan : public TPgTablesScanBase { public: - TInformationSchemaTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, + TInformationSchemaTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns); }; class TPgClassScan : public TPgTablesScanBase { public: - TPgClassScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, + TPgClassScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns); private: THashMap namespaces; ui32 btreeAmOid; }; -THolder CreatePgTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, +THolder CreatePgTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns); -THolder CreateInformationSchemaTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, +THolder CreateInformationSchemaTablesScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns); -THolder CreatePgClassScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, +THolder CreatePgClassScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns); diff --git a/ydb/core/sys_view/scan.cpp b/ydb/core/sys_view/scan.cpp index 5ac381b9c3ee..87fb15cd966e 100644 --- a/ydb/core/sys_view/scan.cpp +++ b/ydb/core/sys_view/scan.cpp @@ -33,12 +33,14 @@ class TSysViewRangesReader : public TActor { const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, + const TString& tablePath, TVector ranges, const TArrayRef& columns) : TBase(&TSysViewRangesReader::ScanState) , OwnerId(ownerId) , ScanId(scanId) , TableId(tableId) + , TablePath(tablePath) , Ranges(std::move(ranges)) , Columns(columns.begin(), columns.end()) { @@ -74,7 +76,7 @@ class TSysViewRangesReader : public TActor { if (!ScanActorId) { if (CurrentRange < Ranges.size()) { auto actor = CreateSystemViewScan( - SelfId(), ScanId, TableId, Ranges[CurrentRange].ToTableRange(), + SelfId(), ScanId, TableId, TablePath, Ranges[CurrentRange].ToTableRange(), Columns); ScanActorId = Register(actor.Release()); CurrentRange += 1; @@ -135,6 +137,7 @@ class TSysViewRangesReader : public TActor { TActorId OwnerId; ui32 ScanId; TTableId TableId; + TString TablePath; TVector Ranges; TVector Columns; @@ -142,19 +145,29 @@ class TSysViewRangesReader : public TActor { TMaybe ScanActorId; }; -THolder CreateSystemViewScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, - TVector ranges, const TArrayRef& columns) -{ +THolder CreateSystemViewScan( + const NActors::TActorId& ownerId, + ui32 scanId, + const TTableId& tableId, + const TString& tablePath, + TVector ranges, + const TArrayRef& columns +) { if (ranges.size() == 1) { - return CreateSystemViewScan(ownerId, scanId, tableId, ranges[0].ToTableRange(), columns); + return CreateSystemViewScan(ownerId, scanId, tableId, tablePath, ranges[0].ToTableRange(), columns); } else { - return MakeHolder(ownerId, scanId, tableId, ranges, columns); + return MakeHolder(ownerId, scanId, tableId, tablePath, ranges, columns); } } -THolder CreateSystemViewScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef& columns) -{ +THolder CreateSystemViewScan( + const NActors::TActorId& ownerId, + ui32 scanId, + const TTableId& tableId, + const TString& tablePath, + const TTableRange& tableRange, + const TArrayRef& columns +) { if (tableId.SysViewInfo == PartitionStatsName) { return CreatePartitionStatsScan(ownerId, scanId, tableId, tableRange, columns); } @@ -214,14 +227,14 @@ THolder CreateSystemViewScan(const NActors::TActorId& ownerId, } if (tableId.SysViewInfo == PgTablesName) { - return CreatePgTablesScan(ownerId, scanId, tableId, tableRange, columns); + return CreatePgTablesScan(ownerId, scanId, tableId, tablePath, tableRange, columns); } if (tableId.SysViewInfo == InformationSchemaTablesName) { - return CreateInformationSchemaTablesScan(ownerId, scanId, tableId, tableRange, columns); + return CreateInformationSchemaTablesScan(ownerId, scanId, tableId, tablePath, tableRange, columns); } if (tableId.SysViewInfo == PgClassName) { - return CreatePgClassScan(ownerId, scanId, tableId, tableRange, columns); + return CreatePgClassScan(ownerId, scanId, tableId, tablePath, tableRange, columns); } return {}; diff --git a/ydb/core/sys_view/scan.h b/ydb/core/sys_view/scan.h index e0ecde4606c7..91a9c214e2b8 100644 --- a/ydb/core/sys_view/scan.h +++ b/ydb/core/sys_view/scan.h @@ -8,10 +8,10 @@ namespace NKikimr { namespace NSysView { THolder CreateSystemViewScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, - TVector ranges, const TArrayRef& columns); + const TString& tablePath, TVector ranges, const TArrayRef& columns); THolder CreateSystemViewScan(const NActors::TActorId& ownerId, ui32 scanId, const TTableId& tableId, - const TTableRange& tableRange, const TArrayRef& columns); + const TString& tablePath, const TTableRange& tableRange, const TArrayRef& columns); } // NSysView } // NKikimr From 5af399da4f90cc05a04e6f450757fa11c12f14f8 Mon Sep 17 00:00:00 2001 From: Pavel Velikhov Date: Thu, 22 Aug 2024 15:16:42 +0300 Subject: [PATCH 051/261] Lowered threshold for MapJoin (#8147) --- ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp | 2 +- ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json | 4 ++-- .../ut/join/data/join_order/tpcds64_1000s_column_store.json | 4 ++-- ydb/core/kqp/ut/join/data/join_order/tpch2_1000s.json | 6 +++--- .../ut/join/data/join_order/tpch2_1000s_column_store.json | 6 +++--- ydb/core/kqp/ut/join/data/join_order/tpch9_1000s.json | 4 ++-- .../ut/join/data/join_order/tpch9_1000s_column_store.json | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp b/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp index 218a7ee5d829..48537b8576c3 100644 --- a/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp +++ b/ydb/core/kqp/opt/logical/kqp_opt_cbo.cpp @@ -163,7 +163,7 @@ bool TKqpProviderContext::IsJoinApplicable(const std::shared_ptrStats->ByteSize < 5e8; + return joinKind != EJoinKind::OuterJoin && joinKind != EJoinKind::Exclusion && right->Stats->ByteSize < 1e8; case EJoinAlgoType::GraceJoin: return true; default: diff --git a/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json b/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json index b2c1692cdb65..b7353d9dd64f 100644 --- a/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json +++ b/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s.json @@ -89,7 +89,7 @@ "table":"test\/ds\/customer_address" }, { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { @@ -273,7 +273,7 @@ "table":"test\/ds\/customer_address" }, { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { diff --git a/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s_column_store.json b/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s_column_store.json index 3f01f39083a4..3a761c2bc9db 100644 --- a/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s_column_store.json +++ b/ydb/core/kqp/ut/join/data/join_order/tpcds64_1000s_column_store.json @@ -89,7 +89,7 @@ "table":"test\/ds\/customer_address" }, { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { @@ -273,7 +273,7 @@ "table":"test\/ds\/customer_address" }, { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { diff --git a/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s.json b/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s.json index 5fe7f772318f..f15b2c012f26 100644 --- a/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s.json +++ b/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s.json @@ -1,9 +1,9 @@ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { @@ -41,7 +41,7 @@ "args": [ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { diff --git a/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s_column_store.json b/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s_column_store.json index a969a23d9cf9..827ca629c612 100644 --- a/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s_column_store.json +++ b/ydb/core/kqp/ut/join/data/join_order/tpch2_1000s_column_store.json @@ -1,9 +1,9 @@ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { @@ -41,7 +41,7 @@ "args": [ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { diff --git a/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s.json b/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s.json index af2f896684ca..606c72699f4f 100644 --- a/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s.json +++ b/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s.json @@ -1,5 +1,5 @@ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { @@ -19,7 +19,7 @@ "table":"lineitem" }, { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { diff --git a/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s_column_store.json b/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s_column_store.json index af2f896684ca..606c72699f4f 100644 --- a/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s_column_store.json +++ b/ydb/core/kqp/ut/join/data/join_order/tpch9_1000s_column_store.json @@ -1,5 +1,5 @@ { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { @@ -19,7 +19,7 @@ "table":"lineitem" }, { - "op_name":"InnerJoin (MapJoin)", + "op_name":"InnerJoin (Grace)", "args": [ { From 6d6e3c366fee8e71247474feb2a87289c9a1b98e Mon Sep 17 00:00:00 2001 From: Marina Pereskokova Date: Thu, 22 Aug 2024 16:40:09 +0300 Subject: [PATCH 052/261] Fix skiff usage in dq (#7864) --- .../dq/provider/exec/yql_dq_exectransformer.cpp | 10 +++++----- .../dq/provider/exec/yql_dq_exectransformer.h | 6 +++++- ydb/library/yql/yt/native/plugin.cpp | 9 +++++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 6576342ff0e2..1caaa862d2b1 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -421,7 +421,7 @@ struct TDqsFinalPipelineConfigurator : public IPipelineConfigurator { class TSimpleSkiffConverter : public ISkiffConverter { public: - TString ConvertNodeToSkiff(const TDqStatePtr /*state*/, const IDataProvider::TFillSettings& /*fillSettings*/, const NYT::TNode& /*rowSpec*/, const NYT::TNode& /*item*/) override { + TString ConvertNodeToSkiff(const TDqStatePtr /*state*/, const IDataProvider::TFillSettings& /*fillSettings*/, const NYT::TNode& /*rowSpec*/, const NYT::TNode& /*item*/, const TVector& /*columns*/) override { Y_ABORT("not implemented"); } @@ -1137,7 +1137,7 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters break; } case IDataProvider::EResultFormat::Skiff: { - writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item)); + writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item, columns)); break; } default: { @@ -1179,7 +1179,7 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters break; } case IDataProvider::EResultFormat::Skiff: { - writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item)); + writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item, columns)); break; } default: { @@ -1588,7 +1588,7 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters break; } case IDataProvider::EResultFormat::Skiff: { - writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item)); + writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item, columns)); break; } default: { @@ -1615,7 +1615,7 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters break; } case IDataProvider::EResultFormat::Skiff: { - writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item)); + writer.OnStringScalar(skiffConverter->ConvertNodeToSkiff(state, fillSettings, rowSpec, item, columns)); break; } default: { diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h index 07bb1c1e70b3..9072ca47fc95 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h @@ -17,7 +17,11 @@ namespace NYql { NYT::TNode RowSpec; }; - virtual TString ConvertNodeToSkiff(const TDqStatePtr state, const IDataProvider::TFillSettings& fillSettings, const NYT::TNode& rowSpec, const NYT::TNode& item) = 0; + virtual TString ConvertNodeToSkiff( + const TDqStatePtr state, + const IDataProvider::TFillSettings& fillSettings, + const NYT::TNode& rowSpec, const NYT::TNode& item, + const TVector& columns) = 0; virtual TYtType ParseYTType(const TExprNode& node, TExprContext& ctx, const TMaybe& columns) = 0; }; using ISkiffConverterPtr = TIntrusivePtr; diff --git a/ydb/library/yql/yt/native/plugin.cpp b/ydb/library/yql/yt/native/plugin.cpp index 20bead9fb53a..fcc5268f5329 100644 --- a/ydb/library/yql/yt/native/plugin.cpp +++ b/ydb/library/yql/yt/native/plugin.cpp @@ -146,7 +146,12 @@ class TSkiffConverter : public ISkiffConverter { public: - TString ConvertNodeToSkiff(const TDqStatePtr state, const IDataProvider::TFillSettings& fillSettings, const NYT::TNode& rowSpec, const NYT::TNode& item) override + TString ConvertNodeToSkiff( + const TDqStatePtr state, + const IDataProvider::TFillSettings& fillSettings, + const NYT::TNode& rowSpec, + const NYT::TNode& item, + const TVector& columns) override { TMemoryUsageInfo memInfo("DqResOrPull"); TScopedAlloc alloc(__LOCATION__, NKikimr::TAlignedPagePoolCounters(), state->FunctionRegistry->SupportsSizedAllocators()); @@ -154,7 +159,7 @@ class TSkiffConverter TTypeEnvironment env(alloc); NYql::NCommon::TCodecContext codecCtx(env, *state->FunctionRegistry, &holderFactory); - auto skiffBuilder = MakeHolder(fillSettings.RowsLimitPerWrite, fillSettings.AllResultsBytesLimit, codecCtx, holderFactory, rowSpec, state->TypeCtx->OptLLVM.GetOrElse("OFF")); + auto skiffBuilder = MakeHolder(fillSettings.RowsLimitPerWrite, fillSettings.AllResultsBytesLimit, codecCtx, holderFactory, rowSpec, state->TypeCtx->OptLLVM.GetOrElse("OFF"), columns); if (item.IsList()) { skiffBuilder->SetListResult(); for (auto& node : item.AsList()) { From 447b895df1d6be9dc8a1515ad43b5e8b315be111 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Thu, 22 Aug 2024 16:47:30 +0300 Subject: [PATCH 053/261] Fixed validation of lambda before apply (#8153) --- ydb/library/yql/core/type_ann/type_ann_core.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp index ec9a0840e5a6..7509dbb37671 100644 --- a/ydb/library/yql/core/type_ann/type_ann_core.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp @@ -9098,6 +9098,11 @@ template if (extractKeyLambda->IsAtom()) { TExprNode::TPtr applied; if (udf->IsLambda()) { + if (udf->Head().ChildrenSize() != 1) { + ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(pos), TStringBuilder() << "Expected lambda with one argument, but got: " << udf->Head().ChildrenSize())); + return IGraphTransformer::TStatus::Error; + } + applied = ctx.Expr.Builder(pos) .Apply(udf) .With(0, udfInput) From bca70132a0ae4986145ddf20c2a13bd5f0f9fb4f Mon Sep 17 00:00:00 2001 From: vporyadke Date: Thu, 22 Aug 2024 16:56:54 +0300 Subject: [PATCH 054/261] actually tie per-dc-followers to dc (#8069) Co-authored-by: Aleksandr Zalialov --- ydb/core/mind/hive/data_center_info.h | 21 +++ ydb/core/mind/hive/follower_group.h | 11 ++ ydb/core/mind/hive/hive.cpp | 11 +- ydb/core/mind/hive/hive.h | 4 +- ydb/core/mind/hive/hive_events.h | 7 + ydb/core/mind/hive/hive_impl.cpp | 149 ++++++++++-------- ydb/core/mind/hive/hive_impl.h | 12 +- ydb/core/mind/hive/hive_schema.h | 3 +- ydb/core/mind/hive/hive_ut.cpp | 25 ++- ydb/core/mind/hive/leader_tablet_info.cpp | 1 + ydb/core/mind/hive/leader_tablet_info.h | 2 - ydb/core/mind/hive/node_info.cpp | 16 -- ydb/core/mind/hive/tablet_info.cpp | 7 +- ydb/core/mind/hive/tablet_info.h | 1 + ydb/core/mind/hive/tx__create_tablet.cpp | 10 +- ydb/core/mind/hive/tx__load_everything.cpp | 58 ++++++- .../mind/hive/tx__update_dc_followers.cpp | 79 ++++++++++ .../mind/hive/tx__update_tablet_status.cpp | 2 +- ydb/core/mind/hive/ya.make | 2 + ydb/core/protos/counters_hive.proto | 1 + .../flat_hive.schema | 8 +- 21 files changed, 321 insertions(+), 109 deletions(-) create mode 100644 ydb/core/mind/hive/data_center_info.h create mode 100644 ydb/core/mind/hive/tx__update_dc_followers.cpp diff --git a/ydb/core/mind/hive/data_center_info.h b/ydb/core/mind/hive/data_center_info.h new file mode 100644 index 000000000000..b7fd73faebd4 --- /dev/null +++ b/ydb/core/mind/hive/data_center_info.h @@ -0,0 +1,21 @@ +#include "hive.h" +#include "hive_events.h" + +namespace NKikimr { +namespace NHive { + +struct TDataCenterInfo { + using TFullFollowerGroupId = std::pair; + using TFollowerIter = TList::iterator; // list iterators are not invalidated + + std::unordered_set RegisteredNodes; + bool UpdateScheduled = false; + std::unordered_map> Followers; + + bool IsRegistered() const { + return !RegisteredNodes.empty(); + } +}; + +} // NHive +} // NKikimr diff --git a/ydb/core/mind/hive/follower_group.h b/ydb/core/mind/hive/follower_group.h index 1e6c4980190d..6d047cfb8cb2 100644 --- a/ydb/core/mind/hive/follower_group.h +++ b/ydb/core/mind/hive/follower_group.h @@ -67,6 +67,17 @@ struct TFollowerGroup { } } + ui32 GetFollowerCountForDataCenter(const TDataCenterId& dc) const { + if (!FollowerCountPerDataCenter) { + return 0; + } + if (NodeFilter.IsAllowedDataCenter(dc)) { + return FollowerCount; + } else { + return 0; + } + } + void SetFollowerCount(ui32 followerCount) { FollowerCount = followerCount; } diff --git a/ydb/core/mind/hive/hive.cpp b/ydb/core/mind/hive/hive.cpp index dff573a208ba..2368cd4f5619 100644 --- a/ydb/core/mind/hive/hive.cpp +++ b/ydb/core/mind/hive/hive.cpp @@ -82,11 +82,11 @@ NMetrics::EResource GetDominantResourceType(const TResourceNormalizedValues& nor } TNodeFilter::TNodeFilter(const THive& hive) - : Hive(hive) + : Hive(&hive) {} TArrayRef TNodeFilter::GetEffectiveAllowedDomains() const { - const auto* objectDomainInfo = Hive.FindDomain(ObjectDomain); + const auto* objectDomainInfo = Hive->FindDomain(ObjectDomain); if (!objectDomainInfo) { return {AllowedDomains.begin(), AllowedDomains.end()}; @@ -100,6 +100,13 @@ TArrayRef TNodeFilter::GetEffectiveAllowedDomains() const { } } +bool TNodeFilter::IsAllowedDataCenter(TDataCenterId dc) const { + if (AllowedDataCenters.empty()) { + return true; + } + return std::find(AllowedDataCenters.begin(), AllowedDataCenters.end(), dc) != AllowedDataCenters.end(); +} + template std::unordered_map MakeReverseMap(const std::unordered_map& map) { std::unordered_map result; diff --git a/ydb/core/mind/hive/hive.h b/ydb/core/mind/hive/hive.h index 8bc18522b7e7..cd2528e7e7e4 100644 --- a/ydb/core/mind/hive/hive.h +++ b/ydb/core/mind/hive/hive.h @@ -323,11 +323,13 @@ struct TNodeFilter { TSubDomainKey ObjectDomain; TTabletTypes::EType TabletType = TTabletTypes::TypeInvalid; - const THive& Hive; + const THive* Hive; explicit TNodeFilter(const THive& hive); TArrayRef GetEffectiveAllowedDomains() const; + + bool IsAllowedDataCenter(TDataCenterId dc) const; }; } // NHive diff --git a/ydb/core/mind/hive/hive_events.h b/ydb/core/mind/hive/hive_events.h index f42d64510656..60031c7cce9c 100644 --- a/ydb/core/mind/hive/hive_events.h +++ b/ydb/core/mind/hive/hive_events.h @@ -33,6 +33,7 @@ struct TEvPrivate { EvStorageBalancerOut, EvDeleteNode, EvCanMoveTablets, + EvUpdateDataCenterFollowers, EvEnd }; @@ -120,6 +121,12 @@ struct TEvPrivate { }; struct TEvCanMoveTablets : TEventLocal {}; + + struct TEvUpdateDataCenterFollowers : TEventLocal { + TDataCenterId DataCenter; + + TEvUpdateDataCenterFollowers(TDataCenterId dataCenter) : DataCenter(dataCenter) {}; + }; }; } // NHive diff --git a/ydb/core/mind/hive/hive_impl.cpp b/ydb/core/mind/hive/hive_impl.cpp index 0d8b59939eb4..56fc274e3848 100644 --- a/ydb/core/mind/hive/hive_impl.cpp +++ b/ydb/core/mind/hive/hive_impl.cpp @@ -206,7 +206,7 @@ TInstant THive::GetAllowedBootingTime() { return result; } -void THive::ExecuteProcessBootQueue(NIceDb::TNiceDb& db, TSideEffects& sideEffects) { +void THive::ExecuteProcessBootQueue(NIceDb::TNiceDb&, TSideEffects& sideEffects) { TInstant now = TActivationContext::Now(); if (WarmUp) { TInstant allowed = GetAllowedBootingTime(); @@ -257,10 +257,6 @@ void THive::ExecuteProcessBootQueue(NIceDb::TNiceDb& db, TSideEffects& sideEffec sideEffects.Send(actorToNotify, new TEvPrivate::TEvRestartComplete(tablet->GetFullTabletId(), "boot delay")); } tablet->ActorsToNotifyOnRestart.clear(); - if (tablet->IsFollower()) { - TLeaderTabletInfo& leader = tablet->GetLeader(); - UpdateTabletFollowersNumber(leader, db, sideEffects); - } BootQueue.AddToWaitQueue(record); // waiting for new node continue; } @@ -786,16 +782,11 @@ void THive::Handle(TEvInterconnect::TEvNodeInfo::TPtr &ev) { } void THive::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) { - THashSet dataCenters; for (const TEvInterconnect::TNodeInfo& node : ev->Get()->Nodes) { NodesInfo[node.NodeId] = node; - dataCenters.insert(node.Location.GetDataCenterId()); - } - dataCenters.erase(0); // remove default data center id if exists - if (!dataCenters.empty()) { - if (DataCenters != dataCenters.size()) { - DataCenters = dataCenters.size(); - BLOG_D("TEvInterconnect::TEvNodesInfo DataCenters=" << DataCenters << " RegisteredDataCenters=" << RegisteredDataCenters); + auto dataCenterId = node.Location.GetDataCenterId(); + if (dataCenterId != 0) { + DataCenters[dataCenterId]; // just create entry in hash map } } Execute(CreateLoadEverything()); @@ -2737,81 +2728,97 @@ void THive::SendReconnect(const TActorId& local) { } ui32 THive::GetDataCenters() { - return DataCenters ? DataCenters : 1; -} - -ui32 THive::GetRegisteredDataCenters() { - return RegisteredDataCenters ? RegisteredDataCenters : 1; -} - -void THive::UpdateRegisteredDataCenters() { - if (RegisteredDataCenters != RegisteredDataCenterNodes.size()) { - BLOG_D("THive (UpdateRegisteredDC) DataCenters=" << DataCenters << " RegisteredDataCenters=" << RegisteredDataCenters << "->" << RegisteredDataCenterNodes.size()); - RegisteredDataCenters = RegisteredDataCenterNodes.size(); - } + return DataCenters.size() ? DataCenters.size() : 1; } void THive::AddRegisteredDataCentersNode(TDataCenterId dataCenterId, TNodeId nodeId) { + BLOG_D("AddRegisteredDataCentersNode(" << dataCenterId << ", " << nodeId << ")"); if (dataCenterId != 0) { // ignore default data center id if exists - if (RegisteredDataCenterNodes[dataCenterId].insert(nodeId).second) { - if (RegisteredDataCenters != RegisteredDataCenterNodes.size()) { - UpdateRegisteredDataCenters(); - } + auto& dataCenter = DataCenters[dataCenterId]; + bool wasRegistered = dataCenter.IsRegistered(); + dataCenter.RegisteredNodes.insert(nodeId); + if (!wasRegistered && !dataCenter.UpdateScheduled) { + dataCenter.UpdateScheduled = true; + Schedule(TDuration::Seconds(1), new TEvPrivate::TEvUpdateDataCenterFollowers(dataCenterId)); } } } void THive::RemoveRegisteredDataCentersNode(TDataCenterId dataCenterId, TNodeId nodeId) { + BLOG_D("RemoveRegisteredDataCentersNode(" << dataCenterId << ", " << nodeId << ")"); if (dataCenterId != 0) { // ignore default data center id if exists - RegisteredDataCenterNodes[dataCenterId].erase(nodeId); - if (RegisteredDataCenterNodes[dataCenterId].size() == 0) { - RegisteredDataCenterNodes.erase(dataCenterId); - } - if (RegisteredDataCenters != RegisteredDataCenterNodes.size()) { - UpdateRegisteredDataCenters(); + auto& dataCenter = DataCenters[dataCenterId]; + bool wasRegistered = dataCenter.IsRegistered(); + dataCenter.RegisteredNodes.erase(nodeId); + if (wasRegistered && !dataCenter.IsRegistered() && !dataCenter.UpdateScheduled) { + dataCenter.UpdateScheduled = true; + Schedule(TDuration::Seconds(1), new TEvPrivate::TEvUpdateDataCenterFollowers(dataCenterId)); } } } -void THive::UpdateTabletFollowersNumber(TLeaderTabletInfo& tablet, NIceDb::TNiceDb& db, TSideEffects& sideEffects) { - BLOG_D("UpdateTabletFollowersNumber Tablet " << tablet.ToString() << " RegisteredDataCenters=" << GetRegisteredDataCenters()); - for (TFollowerGroup& group : tablet.FollowerGroups) { - ui32 followerCount = tablet.GetActualFollowerCount(group.Id); - ui32 requiredFollowerCount = group.GetComputedFollowerCount(GetRegisteredDataCenters()); - - while (followerCount < requiredFollowerCount) { - BLOG_D("UpdateTabletFollowersNumber Tablet " << tablet.ToString() << " is increasing number of followers (" << followerCount << "<" << requiredFollowerCount << ")"); - - TFollowerTabletInfo& follower = tablet.AddFollower(group); - follower.Statistics.SetLastAliveTimestamp(TlsActivationContext->Now().MilliSeconds()); - db.Table().Key(tablet.Id, follower.Id).Update( - NIceDb::TUpdate(follower.FollowerGroup.Id), - NIceDb::TUpdate(0), - NIceDb::TUpdate(follower.Statistics)); - follower.InitTabletMetrics(); - follower.BecomeStopped(); - ++followerCount; - } +void THive::CreateTabletFollowers(TLeaderTabletInfo& tablet, NIceDb::TNiceDb& db, TSideEffects& sideEffects) { + BLOG_D("CreateTabletFollowers Tablet " << tablet.ToString()); - while (followerCount > requiredFollowerCount) { - BLOG_D("UpdateTabletFollowersNumber Tablet " << tablet.ToString() << " is decreasing number of followers (" << followerCount << ">" << requiredFollowerCount << ")"); + // In case tablet already has followers (happens if tablet is modified through CreateTablet), delete them + // But create new ones before deleting old ones, to avoid issues with reusing ids + decltype(tablet.Followers)::iterator oldFollowersIt; + if (tablet.Followers.empty()) { + oldFollowersIt = tablet.Followers.end(); + } else { + oldFollowersIt = std::prev(tablet.Followers.end()); + } - auto itFollower = tablet.Followers.rbegin(); - while (itFollower != tablet.Followers.rend() && itFollower->FollowerGroup.Id != group.Id) { - ++itFollower; + for (TFollowerGroup& group : tablet.FollowerGroups) { + if (group.FollowerCountPerDataCenter) { + for (auto& [dataCenterId, dataCenter] : DataCenters) { + if (!dataCenter.IsRegistered()) { + continue; + } + for (ui32 i = 0; i < group.GetFollowerCountForDataCenter(dataCenterId); ++i) { + TFollowerTabletInfo& follower = tablet.AddFollower(group); + follower.NodeFilter.AllowedDataCenters = {dataCenterId}; + follower.Statistics.SetLastAliveTimestamp(TlsActivationContext->Now().MilliSeconds()); + db.Table().Key(tablet.Id, follower.Id).Update( + NIceDb::TUpdate(follower.FollowerGroup.Id), + NIceDb::TUpdate(0), + NIceDb::TUpdate(follower.Statistics), + NIceDb::TUpdate(dataCenterId)); + follower.InitTabletMetrics(); + follower.BecomeStopped(); + dataCenter.Followers[{tablet.Id, group.Id}].push_back(std::prev(tablet.Followers.end())); + BLOG_D("Created follower " << follower.GetFullTabletId() << " for dc " << dataCenterId); + } } - if (itFollower == tablet.Followers.rend()) { - break; + } else { + for (ui32 i = 0; i < group.GetRawFollowerCount(); ++i) { + TFollowerTabletInfo& follower = tablet.AddFollower(group); + follower.Statistics.SetLastAliveTimestamp(TlsActivationContext->Now().MilliSeconds()); + db.Table().Key(tablet.Id, follower.Id).Update( + NIceDb::TUpdate(follower.FollowerGroup.Id), + NIceDb::TUpdate(0), + NIceDb::TUpdate(follower.Statistics)); + follower.InitTabletMetrics(); + follower.BecomeStopped(); + BLOG_D("Created follower " << follower.GetFullTabletId()); } - TFollowerTabletInfo& follower = *itFollower; - db.Table().Key(tablet.Id, follower.Id).Delete(); - db.Table().Key(tablet.Id, follower.Id).Delete(); - follower.InitiateStop(sideEffects); - tablet.Followers.erase(std::prev(itFollower.base())); - UpdateCounterTabletsTotal(-1); - --followerCount; + } } + + if (oldFollowersIt == tablet.Followers.end()) { + return; + } + auto endIt = std::next(oldFollowersIt); + for (auto followerIt = tablet.Followers.begin(); followerIt != endIt; ++followerIt) { + TFollowerTabletInfo& follower = *followerIt; + BLOG_D("Deleting follower " << follower.GetFullTabletId()); + db.Table().Key(tablet.Id, follower.Id).Delete(); + db.Table().Key(tablet.Id, follower.Id).Delete(); + follower.InitiateStop(sideEffects); + UpdateCounterTabletsTotal(-1); + } + tablet.Followers.erase(tablet.Followers.begin(), endIt); } TDuration THive::GetBalancerCooldown(EBalancerType balancerType) const { @@ -3036,6 +3043,7 @@ void THive::ProcessEvent(std::unique_ptr event) { hFunc(TEvHive::TEvUpdateDomain, Handle); hFunc(TEvPrivate::TEvDeleteNode, Handle); hFunc(TEvHive::TEvRequestTabletDistribution, Handle); + hFunc(TEvPrivate::TEvUpdateDataCenterFollowers, Handle); } } @@ -3138,6 +3146,7 @@ STFUNC(THive::StateWork) { fFunc(TEvPrivate::TEvProcessStorageBalancer::EventType, EnqueueIncomingEvent); fFunc(TEvPrivate::TEvDeleteNode::EventType, EnqueueIncomingEvent); fFunc(TEvHive::TEvRequestTabletDistribution::EventType, EnqueueIncomingEvent); + fFunc(TEvPrivate::TEvUpdateDataCenterFollowers::EventType, EnqueueIncomingEvent); hFunc(TEvPrivate::TEvProcessIncomingEvent, Handle); default: if (!HandleDefaultEvents(ev, SelfId())) { @@ -3435,6 +3444,10 @@ void THive::Handle(TEvHive::TEvRequestTabletDistribution::TPtr& ev) { Send(ev->Sender, response.release()); } +void THive::Handle(TEvPrivate::TEvUpdateDataCenterFollowers::TPtr& ev) { + Execute(CreateUpdateDcFollowers(ev->Get()->DataCenter)); +} + TVector THive::GetNodesForWhiteboardBroadcast(size_t maxNodesToReturn) { TVector nodes; TNodeId selfNodeId = SelfId().NodeId(); diff --git a/ydb/core/mind/hive/hive_impl.h b/ydb/core/mind/hive/hive_impl.h index 4a8b8003f5bd..30724c7ade00 100644 --- a/ydb/core/mind/hive/hive_impl.h +++ b/ydb/core/mind/hive/hive_impl.h @@ -54,6 +54,7 @@ #include "sequencer.h" #include "boot_queue.h" #include "object_distribution.h" +#include "data_center_info.h" #define DEPRECATED_CTX (ActorContext()) #define DEPRECATED_NOW (TActivationContext::Now()) @@ -239,6 +240,7 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar friend class TTxUpdateTabletGroups; friend class TTxMonEvent_TabletAvailability; friend class TLoggedMonTransaction; + friend class TTxUpdateDcFollowers; friend class TDeleteTabletActor; @@ -301,6 +303,7 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar ITransaction* CreateRequestTabletOwners(TEvHive::TEvRequestTabletOwners::TPtr event); ITransaction* CreateUpdateTabletsObject(TEvHive::TEvUpdateTabletsObject::TPtr event); ITransaction* CreateUpdateDomain(TSubDomainKey subdomainKey, TEvHive::TEvUpdateDomain::TPtr event = {}); + ITransaction* CreateUpdateDcFollowers(const TDataCenterId& dc); public: TDomainsView DomainsView; @@ -329,8 +332,6 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar ui32 ConfigurationGeneration = 0; ui64 TabletsTotal = 0; ui64 TabletsAlive = 0; - ui32 DataCenters = 1; - ui32 RegisteredDataCenters = 1; TObjectDistributions ObjectDistributions; double StorageScatter = 0; std::set SeenTabletTypes; @@ -448,7 +449,7 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar TDuration NodeBrokerEpoch; std::unordered_map TabletLimit; // built from CurrentConfig std::unordered_map DefaultDataCentersPreference; - std::unordered_map> RegisteredDataCenterNodes; + std::unordered_map DataCenters; std::unordered_set ConnectedNodes; // normalized to be sorted list of unique values @@ -576,6 +577,7 @@ class THive : public TActor, public TTabletExecutedFlat, public THiveShar void Handle(TEvHive::TEvUpdateDomain::TPtr& ev); void Handle(TEvPrivate::TEvDeleteNode::TPtr& ev); void Handle(TEvHive::TEvRequestTabletDistribution::TPtr& ev); + void Handle(TEvPrivate::TEvUpdateDataCenterFollowers::TPtr& ev); protected: void RestartPipeTx(ui64 tabletId); @@ -680,8 +682,6 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId void FillTabletInfo(NKikimrHive::TEvResponseHiveInfo& response, ui64 tabletId, const TLeaderTabletInfo* info, const NKikimrHive::TEvRequestHiveInfo& req); void ExecuteStartTablet(TFullTabletId tabletId, const TActorId& local, ui64 cookie, bool external); ui32 GetDataCenters(); - ui32 GetRegisteredDataCenters(); - void UpdateRegisteredDataCenters(); void AddRegisteredDataCentersNode(TDataCenterId dataCenterId, TNodeId nodeId); void RemoveRegisteredDataCentersNode(TDataCenterId dataCenterId, TNodeId nodeId); void QueuePing(const TActorId& local); @@ -694,7 +694,7 @@ TTabletInfo* FindTabletEvenInDeleting(TTabletId tabletId, TFollowerId followerId void StopTablet(const TActorId& local, const TTabletInfo& tablet); void StopTablet(const TActorId& local, TFullTabletId tabletId); void ExecuteProcessBootQueue(NIceDb::TNiceDb& db, TSideEffects& sideEffects); - void UpdateTabletFollowersNumber(TLeaderTabletInfo& tablet, NIceDb::TNiceDb& db, TSideEffects& sideEffects); + void CreateTabletFollowers(TLeaderTabletInfo& tablet, NIceDb::TNiceDb& db, TSideEffects& sideEffects); TDuration GetBalancerCooldown(EBalancerType balancerType) const; void UpdateObjectCount(const TLeaderTabletInfo& tablet, const TNodeInfo& node, i64 diff); ui64 GetObjectImbalance(TFullObjectId object); diff --git a/ydb/core/mind/hive/hive_schema.h b/ydb/core/mind/hive/hive_schema.h index bd7ffe7a7205..21e325fc1f81 100644 --- a/ydb/core/mind/hive/hive_schema.h +++ b/ydb/core/mind/hive/hive_schema.h @@ -151,9 +151,10 @@ struct Schema : NIceDb::Schema { struct GroupID : Column<3, Schema::TabletFollowerGroup::GroupID::ColumnType> {}; struct FollowerNode : Column<4, NScheme::NTypeIds::Uint32> {}; struct Statistics : Column<5, NScheme::NTypeIds::String> { using Type = NKikimrHive::TTabletStatistics; }; + struct DataCenter : Column<6, NScheme::NTypeIds::String> {}; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; struct TabletChannel : Table<2> { diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index fd84a130f5f6..e3c2a6adbea7 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -3603,6 +3603,13 @@ Y_UNIT_TEST_SUITE(THiveTest) { const TActorId hiveActor = CreateTestBootstrapper(runtime, CreateTestTabletInfo(hiveTablet, TTabletTypes::Hive), &CreateDefaultHive); runtime.EnableScheduleForActor(hiveActor); + // wait for creation of nodes + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvLocal::EvStatus, 2); + runtime.DispatchEvents(options); + } + // creating NUM_TABLETS tablets TTabletTypes::EType tabletType = TTabletTypes::Dummy; @@ -3640,7 +3647,13 @@ Y_UNIT_TEST_SUITE(THiveTest) { CreateLocal(runtime, 2); CreateLocal(runtime, 3); - // kill all tablets + // no need to kill all tablets, hive must update followers on its own + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvLocal::EvTabletStatus); + runtime.DispatchEvents(options); + } + /* for (ui64 tabletId : tablets) { runtime.Register(CreateTabletKiller(tabletId)); @@ -3649,7 +3662,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { // leader (death, start) + new extra follower options.FinalEvents.emplace_back(TDispatchOptions::TFinalEventCondition(TEvLocal::EvTabletStatus, 3)); runtime.DispatchEvents(options); - } + }*/ { int leaders = 0; @@ -3675,7 +3688,7 @@ Y_UNIT_TEST_SUITE(THiveTest) { CreateLocal(runtime, 4); CreateLocal(runtime, 5); - // kill all tablets + /* for (ui64 tabletId : tablets) { runtime.Register(CreateTabletKiller(tabletId)); @@ -3685,6 +3698,12 @@ Y_UNIT_TEST_SUITE(THiveTest) { options.FinalEvents.emplace_back(TDispatchOptions::TFinalEventCondition(TEvLocal::EvTabletStatus, 3)); runtime.DispatchEvents(options); } + */ + { + TDispatchOptions options; + options.FinalEvents.emplace_back(TEvLocal::EvTabletStatus); + runtime.DispatchEvents(options); + } { int leaders = 0; diff --git a/ydb/core/mind/hive/leader_tablet_info.cpp b/ydb/core/mind/hive/leader_tablet_info.cpp index 5b30189103fb..9c23c2a153b1 100644 --- a/ydb/core/mind/hive/leader_tablet_info.cpp +++ b/ydb/core/mind/hive/leader_tablet_info.cpp @@ -144,6 +144,7 @@ TFollowerTabletInfo& TLeaderTabletInfo::AddFollower(TFollowerGroup& followerGrou } else { follower.Id = followerId; } + follower.NodeFilter = followerGroup.NodeFilter; Hive.UpdateCounterTabletsTotal(+1); Hive.UpdateDomainTabletsTotal(ObjectDomain, +1); return follower; diff --git a/ydb/core/mind/hive/leader_tablet_info.h b/ydb/core/mind/hive/leader_tablet_info.h index d86808090521..30f584608984 100644 --- a/ydb/core/mind/hive/leader_tablet_info.h +++ b/ydb/core/mind/hive/leader_tablet_info.h @@ -68,7 +68,6 @@ struct TLeaderTabletInfo : TTabletInfo { TTabletTypes::EType Type; TFullObjectId ObjectId; TSubDomainKey ObjectDomain; - TNodeFilter NodeFilter; NKikimrHive::TDataCentersPreference DataCentersPreference; TIntrusivePtr TabletStorageInfo; TChannelsBindings BoundChannels; @@ -96,7 +95,6 @@ struct TLeaderTabletInfo : TTabletInfo { , State(ETabletState::Unknown) , Type(TTabletTypes::TypeInvalid) , ObjectId(0, 0) - , NodeFilter(hive) , ChannelProfileReassignReason(NKikimrHive::TEvReassignTablet::HIVE_REASSIGN_REASON_NO) , KnownGeneration(0) , Category(nullptr) diff --git a/ydb/core/mind/hive/node_info.cpp b/ydb/core/mind/hive/node_info.cpp index 6acc1853abae..7d46f8717411 100644 --- a/ydb/core/mind/hive/node_info.cpp +++ b/ydb/core/mind/hive/node_info.cpp @@ -238,22 +238,6 @@ bool TNodeInfo::IsAbleToRunTablet(const TTabletInfo& tablet, TTabletDebugState* const TFollowerTabletInfo& follower = tablet.AsFollower(); const TFollowerGroup& followerGroup = follower.FollowerGroup; const TLeaderTabletInfo& leader = follower.LeaderTablet; - if (followerGroup.RequireAllDataCenters) { - auto dataCenters = Hive.GetRegisteredDataCenters(); - ui32 maxFollowersPerDataCenter = (followerGroup.GetComputedFollowerCount(Hive.GetDataCenters()) + dataCenters - 1) / dataCenters; // ceil - ui32 existingFollowers; - if (tablet.IsAlive()) { - existingFollowers = leader.GetFollowersAliveOnDataCenterExcludingFollower(Location.GetDataCenterId(), tablet); - } else { - existingFollowers = leader.GetFollowersAliveOnDataCenter(Location.GetDataCenterId()); - } - if (maxFollowersPerDataCenter <= existingFollowers) { - if (debugState) { - debugState->NodesFilledWithDatacenterFollowers++; - } - return false; - } - } if (followerGroup.RequireDifferentNodes) { if (leader.IsSomeoneAliveOnNode(Id)) { if (debugState) { diff --git a/ydb/core/mind/hive/tablet_info.cpp b/ydb/core/mind/hive/tablet_info.cpp index 3b04e443020d..b0047634e440 100644 --- a/ydb/core/mind/hive/tablet_info.cpp +++ b/ydb/core/mind/hive/tablet_info.cpp @@ -22,6 +22,7 @@ TTabletInfo::TTabletInfo(ETabletRole role, THive& hive) , ResourceMetricsAggregates(Hive.GetDefaultResourceMetricsAggregates()) , Weight(0) , BalancerPolicy(EBalancerPolicy::POLICY_BALANCE) + , NodeFilter(hive) {} const TLeaderTabletInfo& TTabletInfo::GetLeader() const { @@ -492,11 +493,7 @@ void TTabletInfo::ActualizeCounter() { } const TNodeFilter& TTabletInfo::GetNodeFilter() const { - if (IsLeader()) { - return AsLeader().NodeFilter; - } else { - return AsFollower().FollowerGroup.NodeFilter; - } + return NodeFilter; } bool TTabletInfo::InitiateStart(TNodeInfo* node) { diff --git a/ydb/core/mind/hive/tablet_info.h b/ydb/core/mind/hive/tablet_info.h index adbc141fe18c..217a699fbf0d 100644 --- a/ydb/core/mind/hive/tablet_info.h +++ b/ydb/core/mind/hive/tablet_info.h @@ -163,6 +163,7 @@ struct TTabletInfo { EBalancerPolicy BalancerPolicy; TNodeId FailedNodeId = 0; // last time we tried to start the tablet, we failed on this node TInstant BootTime; + TNodeFilter NodeFilter; TTabletInfo(ETabletRole role, THive& hive); TTabletInfo(const TTabletInfo&) = delete; diff --git a/ydb/core/mind/hive/tx__create_tablet.cpp b/ydb/core/mind/hive/tx__create_tablet.cpp index 3a0f3f9bfeca..887a0bf92515 100644 --- a/ydb/core/mind/hive/tx__create_tablet.cpp +++ b/ydb/core/mind/hive/tx__create_tablet.cpp @@ -304,7 +304,13 @@ class TTxCreateTablet : public TTransactionBase { NIceDb::TUpdate(followerGroup->RequireDifferentNodes)); } - Self->UpdateTabletFollowersNumber(*tablet, db, SideEffects); + auto followerGroupsEnd = itFollowerGroup; + for (; itFollowerGroup != tablet->FollowerGroups.end(); ++itFollowerGroup) { + db.Table().Key(TabletId, itFollowerGroup->Id).Delete(); + } + tablet->FollowerGroups.erase(followerGroupsEnd, tablet->FollowerGroups.end()); + + Self->CreateTabletFollowers(*tablet, db, SideEffects); ProcessTablet(*tablet); BLOG_D("THive::TTxCreateTablet::Execute Existing tablet " << tablet->ToString() << " has been successfully updated"); @@ -461,7 +467,7 @@ class TTxCreateTablet : public TTransactionBase { NIceDb::TUpdate(followerGroup.FollowerCountPerDataCenter)); } - Self->UpdateTabletFollowersNumber(tablet, db, SideEffects); + Self->CreateTabletFollowers(tablet, db, SideEffects); Self->OwnerToTablet.emplace(ownerIdx, TabletId); Self->ObjectToTabletMetrics[tablet.ObjectId].IncreaseCount(); Self->TabletTypeToTabletMetrics[tablet.Type].IncreaseCount(); diff --git a/ydb/core/mind/hive/tx__load_everything.cpp b/ydb/core/mind/hive/tx__load_everything.cpp index 8017ae554b61..fb97348010df 100644 --- a/ydb/core/mind/hive/tx__load_everything.cpp +++ b/ydb/core/mind/hive/tx__load_everything.cpp @@ -34,7 +34,6 @@ class TTxLoadEverything : public TTransactionBase { Self->Keeper.Clear(); Self->Domains.clear(); Self->BlockedOwners.clear(); - Self->RegisteredDataCenterNodes.clear(); Self->Domains[Self->RootDomainKey].Path = Self->RootDomainName; Self->Domains[Self->RootDomainKey].HiveId = rootHiveId; @@ -615,6 +614,11 @@ class TTxLoadEverything : public TTransactionBase { TFollowerGroup& followerGroup = tablet->GetFollowerGroup(followerGroupId); TFollowerTabletInfo& follower = tablet->AddFollower(followerGroup, followerId); follower.Statistics = tabletFollowerRowset.GetValueOrDefault(); + if (tabletFollowerRowset.HaveValue()) { + auto dc = tabletFollowerRowset.GetValue(); + follower.NodeFilter.AllowedDataCenters = {dc}; + Self->DataCenters[dc].Followers[{tabletId, followerGroup.Id}].push_back(std::prev(tablet->Followers.end())); + } if (nodeId == 0) { follower.BecomeStopped(); } else { @@ -636,6 +640,58 @@ class TTxLoadEverything : public TTransactionBase { << numMissingTablets << " for missing tablets)"); } + // Compatability: some per-dc followers do not have their datacenter set - try to set it now + for (auto& [tabletId, tablet] : Self->Tablets) { + for (auto& group : tablet.FollowerGroups) { + if (!group.FollowerCountPerDataCenter) { + continue; + } + std::map dataCentersToCover; // dc -> need x more followers in dc + for (const auto& [dc, _] : Self->DataCenters) { + dataCentersToCover[dc] = group.GetFollowerCountForDataCenter(dc); + } + auto groupId = group.Id; + auto filterGroup = [groupId](auto&& follower) { return follower->FollowerGroup.Id == groupId;}; + auto groupFollowersIters = std::views::iota(tablet.Followers.begin(), tablet.Followers.end()) | std::views::filter(filterGroup); + std::vector followersWithoutDc; + for (auto followerIt : groupFollowersIters) { + auto& allowedDc = followerIt->NodeFilter.AllowedDataCenters; + if (allowedDc.size() == 1) { + --dataCentersToCover[allowedDc.front()]; + continue; + } + bool ok = false; + if (followerIt->Node) { + auto dc = followerIt->Node->Location.GetDataCenterId(); + auto& cnt = dataCentersToCover[dc]; + if (cnt > 0) { + --cnt; + allowedDc = {dc}; + Self->DataCenters[dc].Followers[{tabletId, groupId}].push_back(followerIt); + db.Table().Key(tabletId, followerIt->Id).Update(dc); + ok = true; + } + } + if (!ok) { + followersWithoutDc.push_back(followerIt); + } + } + auto dcIt = dataCentersToCover.begin(); + for (auto follower : followersWithoutDc) { + while (dcIt != dataCentersToCover.end() && dcIt->second <= 0) { + ++dcIt; + } + if (dcIt == dataCentersToCover.end()) { + break; + } + follower->NodeFilter.AllowedDataCenters = {dcIt->first}; + Self->DataCenters[dcIt->first].Followers[{tabletId, groupId}].push_back(follower); + db.Table().Key(follower->GetFullTabletId()).Update(dcIt->first); + --dcIt->second; + } + } + } + { size_t numMetrics = 0; size_t numMissingTablets = 0; diff --git a/ydb/core/mind/hive/tx__update_dc_followers.cpp b/ydb/core/mind/hive/tx__update_dc_followers.cpp new file mode 100644 index 000000000000..9d6c26591bc4 --- /dev/null +++ b/ydb/core/mind/hive/tx__update_dc_followers.cpp @@ -0,0 +1,79 @@ +#include "hive_impl.h" +#include "hive_log.h" + +namespace NKikimr { +namespace NHive { + +class TTxUpdateDcFollowers : public TTransactionBase { + TDataCenterId DataCenterId; + TSideEffects SideEffects; +public: + TTxUpdateDcFollowers(const TDataCenterId& dataCenter, THive* hive) + : TBase(hive) + , DataCenterId(dataCenter) + {} + + TTxType GetTxType() const override { return NHive::TXTYPE_UPDATE_DC_FOLLOWERS; } + + bool Execute(TTransactionContext& txc, const TActorContext&) override { + BLOG_D("THive::TTxUpdateDcFollowers::Execute(" << DataCenterId << ")"); + SideEffects.Reset(Self->SelfId()); + NIceDb::TNiceDb db(txc.DB); + auto& dataCenter = Self->DataCenters[DataCenterId]; + if (!dataCenter.UpdateScheduled) { + return true; + } + dataCenter.UpdateScheduled = false; + if (dataCenter.IsRegistered()) { + for (auto& [tabletId, tablet] : Self->Tablets) { + for (auto& group : tablet.FollowerGroups) { + auto& followers = dataCenter.Followers[{tabletId, group.Id}]; + auto neededCount = group.GetFollowerCountForDataCenter(DataCenterId); + while (followers.size() < neededCount) { + TFollowerTabletInfo& follower = tablet.AddFollower(group); + follower.NodeFilter.AllowedDataCenters = {DataCenterId}; + follower.Statistics.SetLastAliveTimestamp(TlsActivationContext->Now().MilliSeconds()); + db.Table().Key(tabletId, follower.Id).Update( + NIceDb::TUpdate(follower.FollowerGroup.Id), + NIceDb::TUpdate(0), + NIceDb::TUpdate(follower.Statistics), + NIceDb::TUpdate(DataCenterId)); + follower.InitTabletMetrics(); + follower.BecomeStopped(); + follower.InitiateBoot(); + followers.push_back(std::prev(tablet.Followers.end())); + BLOG_D("THive::TTxUpdateDcFollowers::Execute(" << DataCenterId << "): created follower " << follower.GetFullTabletId()); + } + } + } + } else { + // deleting followers + i64 deletedFollowers = 0; + for (auto& [_, followers] : dataCenter.Followers) { + for (auto follower : followers) { + db.Table().Key(follower->GetFullTabletId()).Delete(); + db.Table().Key(follower->GetFullTabletId()).Delete(); + follower->InitiateStop(SideEffects); + auto& leader = follower->GetLeader(); + leader.Followers.erase(follower); + ++deletedFollowers; + } + } + BLOG_D("THive::TTxUpdateDcFollowers::Execute(" << DataCenterId << "): deleted " << deletedFollowers << " followers"); + Self->UpdateCounterTabletsTotal(-deletedFollowers); + dataCenter.Followers.clear(); + } + return true; + } + + void Complete(const TActorContext& ctx) override { + SideEffects.Complete(ctx); + } +}; + +ITransaction* THive::CreateUpdateDcFollowers(const TDataCenterId& dc) { + return new TTxUpdateDcFollowers(dc, this); +} + +} // NHive +} // NKikimr diff --git a/ydb/core/mind/hive/tx__update_tablet_status.cpp b/ydb/core/mind/hive/tx__update_tablet_status.cpp index 2cd239fe6d52..ab3513ef1577 100644 --- a/ydb/core/mind/hive/tx__update_tablet_status.cpp +++ b/ydb/core/mind/hive/tx__update_tablet_status.cpp @@ -123,7 +123,7 @@ class TTxUpdateTabletStatus : public TTransactionBase { db.Table().Key(TabletId).Update(NIceDb::TUpdate(tablet->NodeId), NIceDb::TUpdate(Generation), NIceDb::TUpdate(tablet->Statistics)); - Self->UpdateTabletFollowersNumber(leader, db, SideEffects); + // tablet booted successfully, we may actually cut history now while (!leader.DeletedHistory.empty() && leader.DeletedHistory.front().DeletedAtGeneration < leader.KnownGeneration) { diff --git a/ydb/core/mind/hive/ya.make b/ydb/core/mind/hive/ya.make index 5d4132950cd9..ec05c7fd9a06 100644 --- a/ydb/core/mind/hive/ya.make +++ b/ydb/core/mind/hive/ya.make @@ -5,6 +5,7 @@ SRCS( balancer.h boot_queue.cpp boot_queue.h + data_center_info.h domain_info.cpp domain_info.h drain.cpp @@ -75,6 +76,7 @@ SRCS( tx__sync_tablets.cpp tx__tablet_owners_reply.cpp tx__unlock_tablet.cpp + tx__update_dc_followers.cpp tx__update_domain.cpp tx__update_tablet_groups.cpp tx__update_tablet_metrics.cpp diff --git a/ydb/core/protos/counters_hive.proto b/ydb/core/protos/counters_hive.proto index 9f58674505ee..19aa4928885a 100644 --- a/ydb/core/protos/counters_hive.proto +++ b/ydb/core/protos/counters_hive.proto @@ -163,4 +163,5 @@ enum ETxTypes { TXTYPE_MON_OBJECT_STATS = 63 [(TxTypeOpts) = {Name: "TxMonObjectStats"}]; TXTYPE_MON_SUBACTORS = 64 [(TxTypeOpts) = {Name: "TxMonSubactors"}]; TXTYPE_MON_TABLET_AVAILABILITY = 65 [(TxTypeOpts) = {Name: "TxMonTabletAvailability"}]; + TXTYPE_UPDATE_DC_FOLLOWERS = 66 [(TxTypeOpts) = {Name: "TxUpdateDcFollowers"}]; } diff --git a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema index 861980fd77d7..9b2f219b7896 100644 --- a/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema +++ b/ydb/tests/functional/scheme_tests/canondata/tablet_scheme_tests.TestTabletSchemes.test_tablet_schemes_flat_hive_/flat_hive.schema @@ -884,6 +884,11 @@ "ColumnId": 5, "ColumnName": "Statistics", "ColumnType": "String" + }, + { + "ColumnId": 6, + "ColumnName": "DataCenter", + "ColumnType": "String" } ], "ColumnsDropped": [], @@ -894,7 +899,8 @@ 2, 3, 4, - 5 + 5, + 6 ], "RoomID": 0, "Codec": 0, From d6e1db345b89ef777ac45f37572648d610f6eeee Mon Sep 17 00:00:00 2001 From: vporyadke Date: Thu, 22 Aug 2024 16:57:07 +0300 Subject: [PATCH 055/261] treat initial vdisk status as ok (#8150) --- ydb/core/health_check/health_check.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/health_check/health_check.cpp b/ydb/core/health_check/health_check.cpp index 87f5ea9542df..680336fd686b 100644 --- a/ydb/core/health_check/health_check.cpp +++ b/ydb/core/health_check/health_check.cpp @@ -1984,9 +1984,9 @@ class TSelfCheckRequest : public TActorBootstrapped { switch (vDiskInfo.GetVDiskState()) { case NKikimrWhiteboard::EVDiskState::OK: + case NKikimrWhiteboard::EVDiskState::Initial: context.ReportStatus(Ydb::Monitoring::StatusFlag::GREEN); break; - case NKikimrWhiteboard::EVDiskState::Initial: case NKikimrWhiteboard::EVDiskState::SyncGuidRecovery: context.IssueRecords.clear(); context.ReportStatus(Ydb::Monitoring::StatusFlag::YELLOW, From ee5cd28d601443b05bf843f323b0c3485ab5ba05 Mon Sep 17 00:00:00 2001 From: Igor Munkin Date: Thu, 22 Aug 2024 19:20:41 +0500 Subject: [PATCH 056/261] Refactor tests for BlockMapJoinCore computation node (#8129) --- .../comp_nodes/ut/mkql_block_map_join_ut.cpp | 375 ++++++++++-------- 1 file changed, 204 insertions(+), 171 deletions(-) diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp index 37032b901cb0..920f6e664051 100644 --- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp +++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_map_join_ut.cpp @@ -13,195 +13,228 @@ namespace NKikimr { namespace NMiniKQL { namespace { - TMap NameToIndex(const TStructType* structType) { - TMap map; - for (size_t i = 0; i < structType->GetMembersCount(); i++) { - const TString name(structType->GetMemberName(i)); - map[name] = i; - } - return map; - } - TVector GeneratePayload(size_t level) { - constexpr size_t alphaSize = 'Z' - 'A' + 1; - if (level == 1) { - TVector alphabet(alphaSize); - std::iota(alphabet.begin(), alphabet.end(), 'A'); - return alphabet; - } - const auto subPayload = GeneratePayload(level - 1); - TVector payload; - payload.reserve(alphaSize * subPayload.size()); - for (char ch = 'A'; ch <= 'Z'; ch++) { - for (const auto& tail : subPayload) { - payload.emplace_back(ch + tail); - } - } - return payload; - } +using TKSV = std::tuple; +using TArrays = std::array, std::tuple_size_v>; - constexpr size_t payloadSize = 2; - static const TVector twoLetterPayloads = GeneratePayload(payloadSize); - - template - const TRuntimeNode MakeSimpleKey( - TProgramBuilder& pgmBuilder, - T value, - bool isEmpty = false - ) { - if constexpr (!isOptional) { - return pgmBuilder.NewDataLiteral(value); - } - const auto keyType = pgmBuilder.NewDataType(NUdf::TDataType::Id, true); - if (isEmpty) { - return pgmBuilder.NewEmptyOptional(keyType); +TVector GenerateValues(size_t level) { + constexpr size_t alphaSize = 'Z' - 'A' + 1; + if (level == 1) { + TVector alphabet(alphaSize); + std::iota(alphabet.begin(), alphabet.end(), 'A'); + return alphabet; + } + const auto subValues = GenerateValues(level - 1); + TVector values; + values.reserve(alphaSize * subValues.size()); + for (char ch = 'A'; ch <= 'Z'; ch++) { + for (const auto& tail : subValues) { + values.emplace_back(ch + tail); } - return pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral(value)); } + return values; +} - template - const TRuntimeNode MakeSet( - TProgramBuilder& pgmBuilder, - const TVector& keyValues - ) { - const auto keyType = pgmBuilder.NewDataType(NUdf::TDataType::Id); - - TRuntimeNode::TList keyListItems; - std::transform(keyValues.cbegin(), keyValues.cend(), - std::back_inserter(keyListItems), [&pgmBuilder](const auto key) { - return pgmBuilder.NewDataLiteral(key); - }); - - const auto keyList = pgmBuilder.NewList(keyType, keyListItems); - return pgmBuilder.ToHashedDict(keyList, false, - [&](TRuntimeNode item) { - return item; - }, [&](TRuntimeNode) { - return pgmBuilder.NewVoid(); - }); +template +const TRuntimeNode MakeSimpleKey(TProgramBuilder& pgmBuilder, T value, bool isEmpty = false) { + if constexpr (!isOptional) { + return pgmBuilder.NewDataLiteral(value); + } + const auto keyType = pgmBuilder.NewDataType(NUdf::TDataType::Id, true); + if (isEmpty) { + return pgmBuilder.NewEmptyOptional(keyType); } + return pgmBuilder.NewOptional(pgmBuilder.NewDataLiteral(value)); +} + +template +const TRuntimeNode MakeSet(TProgramBuilder& pgmBuilder, const TSet& keyValues) { + const auto keyType = pgmBuilder.NewDataType(NUdf::TDataType::Id); - void DoTestBlockJoinOnUint64(EJoinKind joinKind, size_t blockSize, size_t testSize) { - TSetup setup; - TProgramBuilder& pb = *setup.PgmBuilder; - - const TVector dictKeys = {1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144}; - const auto dict = MakeSet(pb, dictKeys); - - const auto ui64Type = pb.NewDataType(NUdf::TDataType::Id); - const auto strType = pb.NewDataType(NUdf::TDataType::Id); - const auto ui64BlockType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many); - const auto strBlockType = pb.NewBlockType(strType, TBlockType::EShape::Many); - const auto blockLenType = pb.NewBlockType(ui64Type, TBlockType::EShape::Scalar); - const auto structType = pb.NewStructType({ - {"key", ui64BlockType}, - {"subkey", ui64BlockType}, - {"payload", strBlockType}, - {"_yql_block_length", blockLenType} + TRuntimeNode::TList keyListItems; + std::transform(keyValues.cbegin(), keyValues.cend(), + std::back_inserter(keyListItems), [&pgmBuilder](const auto key) { + return pgmBuilder.NewDataLiteral(key); + }); + + const auto keyList = pgmBuilder.NewList(keyType, keyListItems); + return pgmBuilder.ToHashedDict(keyList, false, + [&](TRuntimeNode item) { + return item; + }, [&](TRuntimeNode) { + return pgmBuilder.NewVoid(); + }); +} + +TArrays KSVToArrays(const TVector& ksvVector, size_t current, + size_t blockSize, arrow::MemoryPool* memoryPool +) { + TArrays arrays; + arrow::UInt64Builder keysBuilder(memoryPool); + arrow::UInt64Builder subkeysBuilder(memoryPool); + arrow::BinaryBuilder valuesBuilder(memoryPool); + ARROW_OK(keysBuilder.Reserve(blockSize)); + ARROW_OK(subkeysBuilder.Reserve(blockSize)); + ARROW_OK(valuesBuilder.Reserve(blockSize)); + for (size_t i = 0; i < blockSize; i++) { + keysBuilder.UnsafeAppend(std::get<0>(ksvVector[current + i])); + subkeysBuilder.UnsafeAppend(std::get<1>(ksvVector[current + i])); + const TStringBuf string(std::get<2>(ksvVector[current + i])); + ARROW_OK(valuesBuilder.Append(string.data(), string.size())); + } + ARROW_OK(keysBuilder.FinishInternal(&arrays[0])); + ARROW_OK(subkeysBuilder.FinishInternal(&arrays[1])); + ARROW_OK(valuesBuilder.FinishInternal(&arrays[2])); + return arrays; +} + +TVector ArraysToKSV(const TArrays& arrays, const int64_t blockSize) { + TVector ksvVector; + for (size_t i = 0; i < std::tuple_size_v; i++) { + Y_ENSURE(arrays[i]->length == blockSize, + "Array size differs from the given block size"); + Y_ENSURE(arrays[i]->GetNullCount() == 0, + "Null values conversion is not supported"); + Y_ENSURE(arrays[i]->buffers.size() == 2 + (i > 1), + "Array layout doesn't respect the schema"); + } + const ui64* keyBuffer = arrays[0]->GetValuesSafe(1); + const ui64* subkeyBuffer = arrays[1]->GetValuesSafe(1); + const int32_t* offsets = arrays[2]->GetValuesSafe(1); + const char* valuesBuffer = arrays[2]->GetValuesSafe(2, 0); + for (auto i = 0; i < blockSize; i++) { + const TStringBuf value(valuesBuffer + offsets[i], offsets[i + 1] - offsets[i]); + ksvVector.push_back(std::make_tuple(keyBuffer[i], subkeyBuffer[i], value)); + } + return ksvVector; +} + +const TRuntimeNode BuildBlockJoin(TProgramBuilder& pgmBuilder, EJoinKind joinKind, + TVector keyColumns, TRuntimeNode& leftArg, TType* leftTuple, + const TRuntimeNode& dictNode +) { + const auto tupleType = AS_TYPE(TTupleType, leftTuple); + const auto listTupleType = pgmBuilder.NewListType(leftTuple); + leftArg = pgmBuilder.Arg(listTupleType); + + const auto leftWideFlow = pgmBuilder.ExpandMap(pgmBuilder.ToFlow(leftArg), + [&](TRuntimeNode tupleNode) -> TRuntimeNode::TList { + TRuntimeNode::TList wide; + wide.reserve(tupleType->GetElementsCount()); + for (size_t i = 0; i < tupleType->GetElementsCount(); i++) { + wide.emplace_back(pgmBuilder.Nth(tupleNode, i)); + } + return wide; }); - const auto fields = NameToIndex(AS_TYPE(TStructType, structType)); - const auto listStructType = pb.NewListType(structType); - - const auto leftArg = pb.Arg(listStructType); - - const auto leftWideFlow = pb.ExpandMap(pb.ToFlow(leftArg), - [&](TRuntimeNode item) -> TRuntimeNode::TList { - return { - pb.Member(item, "key"), - pb.Member(item, "subkey"), - pb.Member(item, "payload"), - pb.Member(item, "_yql_block_length") - }; - }); - - const auto joinNode = pb.BlockMapJoinCore(leftWideFlow, dict, joinKind, {0}); - - const auto rootNode = pb.Collect(pb.NarrowMap(joinNode, - [&](TRuntimeNode::TList items) -> TRuntimeNode { - return pb.NewStruct(structType, { - {"key", items[0]}, - {"subkey", items[1]}, - {"payload", items[2]}, - {"_yql_block_length", items[3]} - }); - })); - - const auto graph = setup.BuildGraph(rootNode, {leftArg.GetNode()}); - const auto& leftBlocks = graph->GetEntryPoint(0, true); - const auto& holderFactory = graph->GetHolderFactory(); - auto& ctx = graph->GetContext(); - - TVector keys(testSize); - TVector subkeys; - std::iota(keys.begin(), keys.end(), 1); - std::transform(keys.cbegin(), keys.cend(), std::back_inserter(subkeys), - [](const auto& value) { return value * 1001; }); - - TVector payloads; - std::transform(keys.cbegin(), keys.cend(), std::back_inserter(payloads), - [](const auto& value) { return twoLetterPayloads[value].c_str(); }); - - size_t current = 0; - TDefaultListRepresentation leftListValues; - while (current < testSize) { - arrow::UInt64Builder keysBuilder(&ctx.ArrowMemoryPool); - arrow::UInt64Builder subkeysBuilder(&ctx.ArrowMemoryPool); - arrow::BinaryBuilder payloadsBuilder(&ctx.ArrowMemoryPool); - ARROW_OK(keysBuilder.Reserve(blockSize)); - ARROW_OK(subkeysBuilder.Reserve(blockSize)); - ARROW_OK(payloadsBuilder.Reserve(blockSize)); - for (size_t i = 0; i < blockSize; i++, current++) { - keysBuilder.UnsafeAppend(keys[current]); - subkeysBuilder.UnsafeAppend(subkeys[current]); - ARROW_OK(payloadsBuilder.Append(payloads[current], payloadSize)); + + const auto joinNode = pgmBuilder.BlockMapJoinCore(leftWideFlow, dictNode, joinKind, keyColumns); + + const auto rootNode = pgmBuilder.Collect(pgmBuilder.NarrowMap(joinNode, + [&](TRuntimeNode::TList items) -> TRuntimeNode { + TVector tupleElements; + tupleElements.reserve(tupleType->GetElementsCount()); + for (size_t i = 0; i < tupleType->GetElementsCount(); i++) { + tupleElements.emplace_back(items[i]); } - std::shared_ptr keysData; - ARROW_OK(keysBuilder.FinishInternal(&keysData)); - std::shared_ptr subkeysData; - ARROW_OK(subkeysBuilder.FinishInternal(&subkeysData)); - std::shared_ptr payloadsData; - ARROW_OK(payloadsBuilder.FinishInternal(&payloadsData)); - - NUdf::TUnboxedValue* items = nullptr; - const auto structObj = holderFactory.CreateDirectArrayHolder(fields.size(), items); - items[fields.at("key")] = holderFactory.CreateArrowBlock(keysData); - items[fields.at("subkey")] = holderFactory.CreateArrowBlock(subkeysData); - items[fields.at("payload")] = holderFactory.CreateArrowBlock(payloadsData); - items[fields.at("_yql_block_length")] = MakeBlockCount(holderFactory, blockSize); - leftListValues = leftListValues.Append(std::move(structObj)); + return pgmBuilder.NewTuple(tupleElements); + })); + + return rootNode; +} + +TVector DoTestBlockJoinOnUint64(EJoinKind joinKind, TVector values, + TSet set, size_t blockSize +) { + TSetup setup; + TProgramBuilder& pb = *setup.PgmBuilder; + + const auto dict = MakeSet(pb, set); + + const auto ui64Type = pb.NewDataType(NUdf::TDataType::Id); + const auto strType = pb.NewDataType(NUdf::EDataSlot::String); + const auto ui64BlockType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many); + const auto strBlockType = pb.NewBlockType(strType, TBlockType::EShape::Many); + const auto blockLenType = pb.NewBlockType(ui64Type, TBlockType::EShape::Scalar); + const auto ksvType = pb.NewTupleType({ + ui64BlockType, ui64BlockType, strBlockType, blockLenType + }); + // Mind the last block length column. + const auto ksvWidth = AS_TYPE(TTupleType, ksvType)->GetElementsCount() - 1; + + TRuntimeNode leftArg; + const auto rootNode = BuildBlockJoin(pb, joinKind, {0}, leftArg, ksvType, dict); + + const auto graph = setup.BuildGraph(rootNode, {leftArg.GetNode()}); + const auto& leftBlocks = graph->GetEntryPoint(0, true); + const auto& holderFactory = graph->GetHolderFactory(); + auto& ctx = graph->GetContext(); + + const size_t testSize = values.size(); + size_t current = 0; + TDefaultListRepresentation leftListValues; + while (current < testSize) { + const auto arrays = KSVToArrays(values, current, blockSize, &ctx.ArrowMemoryPool); + current += blockSize; + + NUdf::TUnboxedValue* items = nullptr; + const auto tuple = holderFactory.CreateDirectArrayHolder(ksvWidth + 1, items); + for (size_t i = 0; i < ksvWidth; i++) { + items[i] = holderFactory.CreateArrowBlock(arrays[i]); } - leftBlocks->SetValue(ctx, holderFactory.CreateDirectListHolder(std::move(leftListValues))); - const auto joinIterator = graph->GetValue().GetListIterator(); - - NUdf::TUnboxedValue item; - TVector joinResult; - while (joinIterator.Next(item)) { - joinResult.push_back(item); + items[ksvWidth] = MakeBlockCount(holderFactory, blockSize); + leftListValues = leftListValues.Append(std::move(tuple)); + } + leftBlocks->SetValue(ctx, holderFactory.CreateDirectListHolder(std::move(leftListValues))); + const auto joinIterator = graph->GetValue().GetListIterator(); + + TVector resultKSV; + TArrays arrays; + NUdf::TUnboxedValue value; + while (joinIterator.Next(value)) { + for (size_t i = 0; i < ksvWidth; i++) { + const auto arrayValue = value.GetElement(i); + const auto arrayDatum = TArrowBlock::From(arrayValue).GetDatum(); + UNIT_ASSERT(arrayDatum.is_array()); + arrays[i] = arrayDatum.array(); } - - UNIT_ASSERT_VALUES_EQUAL(joinResult.size(), 1); - const auto blocks = joinResult.front(); - const auto blockLengthValue = blocks.GetElement(fields.at("_yql_block_length")); + const auto blockLengthValue = value.GetElement(ksvWidth); const auto blockLengthDatum = TArrowBlock::From(blockLengthValue).GetDatum(); - UNIT_ASSERT(blockLengthDatum.is_scalar()); + Y_ENSURE(blockLengthDatum.is_scalar()); const auto blockLength = blockLengthDatum.scalar_as().value; - const auto dictSize = std::count_if(dictKeys.cbegin(), dictKeys.cend(), - [testSize](ui64 key) { return key < testSize; }); - const auto expectedLength = joinKind == EJoinKind::LeftSemi ? dictSize - : joinKind == EJoinKind::LeftOnly ? testSize - dictSize - : -1; - UNIT_ASSERT_VALUES_EQUAL(expectedLength, blockLength); + const auto blockKSV = ArraysToKSV(arrays, blockLength); + resultKSV.insert(resultKSV.end(), blockKSV.cbegin(), blockKSV.cend()); + } + std::sort(resultKSV.begin(), resultKSV.end()); + return resultKSV; +} + +void TestBlockJoinOnUint64(EJoinKind joinKind) { + constexpr size_t testSize = 1 << 14; + constexpr size_t valueSize = 3; + static const TVector threeLetterValues = GenerateValues(valueSize); + static const TSet fib = {1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, + 233, 377, 610, 987, 1597, 2584, 4181, 6765, 10946, 17711}; + + TVector testKSV; + for (size_t k = 0; k < testSize; k++) { + testKSV.push_back(std::make_tuple(k, k * 1001, threeLetterValues[k])); } + TVector expectedKSV; + std::copy_if(testKSV.cbegin(), testKSV.cend(), std::back_inserter(expectedKSV), + [&joinKind](const auto& ksv) { + const auto contains = fib.contains(std::get<0>(ksv)); + return joinKind == EJoinKind::LeftSemi ? contains : !contains; + }); - void TestBlockJoinOnUint64(EJoinKind joinKind) { - const size_t testSize = 512; - for (size_t blockSize = 8; blockSize <= testSize; blockSize <<= 2) { - DoTestBlockJoinOnUint64(joinKind, blockSize, testSize); - } + for (size_t blockSize = 8; blockSize <= testSize; blockSize <<= 1) { + const auto gotKSV = DoTestBlockJoinOnUint64(joinKind, testKSV, fib, blockSize); + UNIT_ASSERT_EQUAL(expectedKSV, gotKSV); } +} + } // namespace -Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinTest) { +Y_UNIT_TEST_SUITE(TMiniKQLBlockMapJoinBasicTest) { Y_UNIT_TEST(TestLeftSemiOnUint64) { TestBlockJoinOnUint64(EJoinKind::LeftSemi); } From b6202d757022b72f9df9b6aa51de98c61edc36d1 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 22 Aug 2024 17:36:56 +0300 Subject: [PATCH 057/261] Fix config resolver in ui (#8157) --- ydb/core/cms/ui/yaml_config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ydb/core/cms/ui/yaml_config.js b/ydb/core/cms/ui/yaml_config.js index a2aa9de918f0..1352da8446ed 100644 --- a/ydb/core/cms/ui/yaml_config.js +++ b/ydb/core/cms/ui/yaml_config.js @@ -398,8 +398,8 @@ class YamlConfigState { var rawLabels = yamlCollectLabels(); for (let label in rawLabels) { labels.push({ - Label: label, - Value: rawLabels[label], + label: label, + value: rawLabels[label], }); } var cmd = { From 66486183ef3ed77cadd6999bd796df338bb05ab9 Mon Sep 17 00:00:00 2001 From: yumkam Date: Thu, 22 Aug 2024 17:40:33 +0300 Subject: [PATCH 058/261] tpc ds: manually optimize join order to workaround #7565 (#8163) --- ydb/library/benchmarks/queries/tpcds/yql/q18.sql | 5 +++-- ydb/library/benchmarks/queries/tpcds/yql/q30.sql | 3 ++- ydb/library/benchmarks/queries/tpcds/yql/q64.sql | 9 +++++---- ydb/library/benchmarks/queries/tpcds/yql/q65.sql | 2 ++ ydb/library/benchmarks/queries/tpcds/yql/q81.sql | 3 ++- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ydb/library/benchmarks/queries/tpcds/yql/q18.sql b/ydb/library/benchmarks/queries/tpcds/yql/q18.sql index f4115828ea79..84ee7247d3e7 100644 --- a/ydb/library/benchmarks/queries/tpcds/yql/q18.sql +++ b/ydb/library/benchmarks/queries/tpcds/yql/q18.sql @@ -1,5 +1,6 @@ {% include 'header.sql.jinja' %} +-- TODO this commit should be reverted upon proper fix for https://github.com/ydb-platform/ydb/issues/7565 -- NB: Subquerys -- start query 1 in stream 0 using template query18.tpl and seed 1978355063 select item.i_item_id, @@ -15,10 +16,10 @@ select item.i_item_id, avg( cast(cd1.cd_dep_count as float)) agg7 from {{catalog_sales}} as catalog_sales cross join {{customer_demographics}} cd1 - cross join {{customer_demographics}} cd2 + cross join {{date_dim}} as date_dim cross join {{customer}} as customer + cross join {{customer_demographics}} cd2 cross join {{customer_address}} as customer_address - cross join {{date_dim}} as date_dim cross join {{item}} as item where cs_sold_date_sk = d_date_sk and cs_item_sk = i_item_sk and diff --git a/ydb/library/benchmarks/queries/tpcds/yql/q30.sql b/ydb/library/benchmarks/queries/tpcds/yql/q30.sql index e2ca5074bd21..63832cbed0b5 100644 --- a/ydb/library/benchmarks/queries/tpcds/yql/q30.sql +++ b/ydb/library/benchmarks/queries/tpcds/yql/q30.sql @@ -1,5 +1,6 @@ {% include 'header.sql.jinja' %} +-- TODO this commit should be reverted upon proper fix for https://github.com/ydb-platform/ydb/issues/7565 -- NB: Subquerys $customer_total_return = (select web_returns.wr_returning_customer_sk as ctr_customer_sk @@ -25,8 +26,8 @@ $avg_total_return_by_state = ,c_last_review_date,ctr_total_return from $customer_total_return ctr1 join $avg_total_return_by_state ctr2 on ctr1.ctr_state = ctr2.ctr_state - cross join {{customer_address}} as customer_address cross join {{customer}} as customer + cross join {{customer_address}} as customer_address where ctr1.ctr_total_return > $z1_2_35*ctr2.avg_return and ca_address_sk = c_current_addr_sk and ca_state = 'GA' diff --git a/ydb/library/benchmarks/queries/tpcds/yql/q64.sql b/ydb/library/benchmarks/queries/tpcds/yql/q64.sql index 926a59f81195..3e31c40466c0 100644 --- a/ydb/library/benchmarks/queries/tpcds/yql/q64.sql +++ b/ydb/library/benchmarks/queries/tpcds/yql/q64.sql @@ -1,5 +1,6 @@ {% include 'header.sql.jinja' %} +-- TODO this commit should be reverted upon proper fix for https://github.com/ydb-platform/ydb/issues/7565 -- NB: Subquerys $cs_ui = (select catalog_sales.cs_item_sk cs_item_sk @@ -31,17 +32,17 @@ $cross_sales = ,sum(ss_list_price) s2 ,sum(ss_coupon_amt) s3 FROM {{store_sales}} as store_sales + cross join {{customer_demographics}} cd1 + cross join {{household_demographics}} hd1 cross join {{store_returns}} as store_returns cross join $cs_ui cs_ui cross join {{date_dim}} d1 - cross join {{date_dim}} d2 - cross join {{date_dim}} d3 cross join {{store}} as store cross join {{customer}} as customer - cross join {{customer_demographics}} cd1 + cross join {{date_dim}} d2 + cross join {{date_dim}} d3 cross join {{customer_demographics}} cd2 cross join {{promotion}} as promotion - cross join {{household_demographics}} hd1 cross join {{household_demographics}} hd2 cross join {{customer_address}} ad1 cross join {{customer_address}} ad2 diff --git a/ydb/library/benchmarks/queries/tpcds/yql/q65.sql b/ydb/library/benchmarks/queries/tpcds/yql/q65.sql index 28dac50f5629..8ccfac75f7cb 100644 --- a/ydb/library/benchmarks/queries/tpcds/yql/q65.sql +++ b/ydb/library/benchmarks/queries/tpcds/yql/q65.sql @@ -1,5 +1,6 @@ {% include 'header.sql.jinja' %} +-- TODO this commit should be reverted upon proper fix for https://github.com/ydb-platform/ydb/issues/7565 -- NB: Subquerys -- start query 1 in stream 0 using template query65.tpl and seed 1819994127 select @@ -28,6 +29,7 @@ select where sb.ss_store_sk = sc.ss_store_sk and sc.revenue <= $z0_1_35 * sb.ave and s_store_sk = sc.ss_store_sk and + s_store_sk = sb.ss_store_sk and i_item_sk = sc.ss_item_sk order by s_store_name, i_item_desc , i_wholesale_cost, sc.revenue, i_current_price diff --git a/ydb/library/benchmarks/queries/tpcds/yql/q81.sql b/ydb/library/benchmarks/queries/tpcds/yql/q81.sql index 011f25d5907c..204cab8f67ee 100644 --- a/ydb/library/benchmarks/queries/tpcds/yql/q81.sql +++ b/ydb/library/benchmarks/queries/tpcds/yql/q81.sql @@ -1,5 +1,6 @@ {% include 'header.sql.jinja' %} +-- TODO this commit should be reverted upon proper fix for https://github.com/ydb-platform/ydb/issues/7565 -- NB: Subquerys $customer_total_return = (select catalog_returns.cr_returning_customer_sk as ctr_customer_sk @@ -20,8 +21,8 @@ $avg_ctr_total_return = (select ctr_state, avg(ctr_total_return) as ctr_total_re ,ca_location_type,ctr1.ctr_total_return from $customer_total_return ctr1 join $avg_ctr_total_return ctr2 on (ctr1.ctr_state = ctr2.ctr_state) - cross join {{customer_address}} as customer_address cross join {{customer}} as customer + cross join {{customer_address}} as customer_address where ctr1.ctr_total_return > ctr2.ctr_total_return*$z1_2_35 and ca_address_sk = c_current_addr_sk and ca_state = 'TX' From fe47a19f58a3aa71ea94611d9e3b5d30f243197c Mon Sep 17 00:00:00 2001 From: Andrei Rykov Date: Thu, 22 Aug 2024 16:50:50 +0200 Subject: [PATCH 059/261] query float test asan (#8161) --- ydb/core/viewer/viewer_ut.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index 0a0fd0e77c27..3427f3baa6a5 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -1709,7 +1709,7 @@ Y_UNIT_TEST_SUITE(Viewer) { "database": "/Root", "action": "execute-script", "syntax": "yql_v1", - "stats": "profile" + "stats": "none" })json"; const TKeepAliveHttpClient::THttpCode statusCode = httpClient.DoPost("/viewer/query?timeout=600000&base64=false&schema=modern", requestBody, &responseStream, headers); const TString response = responseStream.ReadAll(); From 6293fc66dcb75590939cb3c8350bbb121e1f3ee1 Mon Sep 17 00:00:00 2001 From: nikita kozlovsky Date: Thu, 22 Aug 2024 18:03:19 +0300 Subject: [PATCH 060/261] ci: fix logic in check-is-mergeable check (#8152) --- .github/workflows/pr_check.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index 6b8cc8f57dda..d414682b3726 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -139,7 +139,7 @@ jobs: let i = 0; - while (pr.mergeable == null || i >= 60) { + while (pr.mergeable == null && i < 60) { console.log("get pull-request status"); let result = await github.rest.pulls.get({ @@ -158,6 +158,11 @@ jobs: console.log("pr.mergeable=%o", pr.mergeable); + if (pr.mergeable === null) { + core.setFailed("Unable to check if the PR is mergeable, please re-run the check."); + return false; + } + const { data: comments } = await github.rest.issues.listComments({ issue_number: context.issue.number, owner: context.repo.owner, @@ -166,7 +171,7 @@ jobs: const commentToUpdate = comments.find(comment => comment.body.startsWith(header)); - if (pr.mergeable === false) { + if (!pr.mergeable) { let commentParams = { ...context.repo, issue_number: context.issue.number, From 8a5aa434558cf8a949a1f15de9f97f0640cbd90c Mon Sep 17 00:00:00 2001 From: nikita kozlovsky Date: Thu, 22 Aug 2024 18:03:32 +0300 Subject: [PATCH 061/261] ci: upgrade actions/checkout@v4, actions/github-script@v7, add test_retry_count input for build_and_test_ya action (#8164) --- .github/actions/build_and_test_ya/action.yml | 4 ++++ .github/workflows/acceptance_run.yml | 2 +- .github/workflows/allowed_dirs.yml | 2 +- .github/workflows/build_analytics.yml | 2 +- .github/workflows/build_and_test_provisioned.yml | 2 +- .github/workflows/build_and_test_ya.yml | 2 +- .github/workflows/collect_analytics.yml | 2 +- .github/workflows/docs_build.yaml | 2 +- .github/workflows/docs_release.yaml | 2 +- .github/workflows/postcommit_asan.yml | 2 +- .github/workflows/postcommit_relwithdebinfo.yml | 2 +- .github/workflows/pr_check.yml | 10 +++++----- .github/workflows/pr_labels.yaml | 2 +- .github/workflows/prewarm-ccache.yml | 2 +- 14 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml index 6b921e4c5cfe..1b2b41262a91 100644 --- a/.github/actions/build_and_test_ya/action.yml +++ b/.github/actions/build_and_test_ya/action.yml @@ -42,6 +42,9 @@ inputs: additional_ya_make_args: type: string default: "" + test_retry_count: + default: "" + description: "how many times to retry failed tests" secs: type: string default: "" @@ -124,6 +127,7 @@ runs: bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }} bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }} put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }} + test_retry_count: ${{ inputs.test_retry_count }} - name: build_stats shell: bash diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index 02fb4edbc122..d0f9d131c08d 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -29,7 +29,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.commit_sha }} diff --git a/.github/workflows/allowed_dirs.yml b/.github/workflows/allowed_dirs.yml index 98167cb10a04..bcb4a59c0167 100644 --- a/.github/workflows/allowed_dirs.yml +++ b/.github/workflows/allowed_dirs.yml @@ -11,7 +11,7 @@ jobs: cancel-in-progress: true runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check dirs run: ${{github.workspace}}/.github/check_dirs.sh ${{github.workspace}} diff --git a/.github/workflows/build_analytics.yml b/.github/workflows/build_analytics.yml index 5f97b6e2ce53..d0614dfc6637 100644 --- a/.github/workflows/build_analytics.yml +++ b/.github/workflows/build_analytics.yml @@ -33,7 +33,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.commit_sha }} diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml index 9a9b7a8c9b93..28958c015d2d 100644 --- a/.github/workflows/build_and_test_provisioned.yml +++ b/.github/workflows/build_and_test_provisioned.yml @@ -48,7 +48,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label }}" ] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.checkout_ref }} - name: Build diff --git a/.github/workflows/build_and_test_ya.yml b/.github/workflows/build_and_test_ya.yml index 10d9723330c5..877f0d2667d7 100644 --- a/.github/workflows/build_and_test_ya.yml +++ b/.github/workflows/build_and_test_ya.yml @@ -65,7 +65,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label }}", "${{ inputs.runner_additional_label || inputs.runner_label }}"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.commit_sha }} diff --git a/.github/workflows/collect_analytics.yml b/.github/workflows/collect_analytics.yml index 5473986512ec..eda7e83df403 100644 --- a/.github/workflows/collect_analytics.yml +++ b/.github/workflows/collect_analytics.yml @@ -17,7 +17,7 @@ jobs: runs-on: [ self-hosted ] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.commit_sha }} - name: Setup ydb access diff --git a/.github/workflows/docs_build.yaml b/.github/workflows/docs_build.yaml index 1a2ef9a9d77d..4b632912b589 100644 --- a/.github/workflows/docs_build.yaml +++ b/.github/workflows/docs_build.yaml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - name: Build diff --git a/.github/workflows/docs_release.yaml b/.github/workflows/docs_release.yaml index 0c7ae9da9f82..1560e6d8dd61 100644 --- a/.github/workflows/docs_release.yaml +++ b/.github/workflows/docs_release.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Extract version shell: bash run: echo "version=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | sed -e 's|stable-|v|g' -e 's|-|.|g' >> $GITHUB_OUTPUT diff --git a/.github/workflows/postcommit_asan.yml b/.github/workflows/postcommit_asan.yml index 3465ec17fd5c..b49551e327d8 100644 --- a/.github/workflows/postcommit_asan.yml +++ b/.github/workflows/postcommit_asan.yml @@ -15,7 +15,7 @@ jobs: name: Build and test release-asan steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 2 - name: Setup ydb access diff --git a/.github/workflows/postcommit_relwithdebinfo.yml b/.github/workflows/postcommit_relwithdebinfo.yml index a5dccc8df82a..aca0428c2b03 100644 --- a/.github/workflows/postcommit_relwithdebinfo.yml +++ b/.github/workflows/postcommit_relwithdebinfo.yml @@ -14,7 +14,7 @@ jobs: name: Build and test relwithdebinfo steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 2 - name: Setup ydb access diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index d414682b3726..95a7515b338e 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -30,7 +30,7 @@ jobs: - name: Check if running tests is allowed id: check-ownership-membership - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} script: | @@ -81,7 +81,7 @@ jobs: - name: comment-if-waiting-on-ok if: steps.check-ownership-membership.outputs.result == 'false' && github.event.action == 'opened' - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | let externalContributorLabel = 'external'; @@ -100,7 +100,7 @@ jobs: }); - name: cleanup-test-label - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: script: | let labelsToRemove = ['ok-to-test', 'rebase-and-check']; @@ -126,7 +126,7 @@ jobs: - name: check is mergeable id: check-is-mergeable if: steps.check-ownership-membership.outputs.result == 'true' - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: result-encoding: string script: | @@ -210,7 +210,7 @@ jobs: name: Build and test ${{ matrix.build_preset }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.check-running-allowed.outputs.commit_sha }} fetch-depth: 2 diff --git a/.github/workflows/pr_labels.yaml b/.github/workflows/pr_labels.yaml index 032994cecb07..efcf96c9dc9b 100644 --- a/.github/workflows/pr_labels.yaml +++ b/.github/workflows/pr_labels.yaml @@ -15,7 +15,7 @@ jobs: steps: - name: Update PR labels id: update-pr-labels - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/prewarm-ccache.yml b/.github/workflows/prewarm-ccache.yml index dc86bb83f49d..080849d97ab9 100644 --- a/.github/workflows/prewarm-ccache.yml +++ b/.github/workflows/prewarm-ccache.yml @@ -13,7 +13,7 @@ jobs: version: ["ubuntu-2204", "ubuntu-2004", "ubuntu-1804"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: build shell: bash run: | From 1620fa52a2269e9f0ff74163d0aea9663a2aed4c Mon Sep 17 00:00:00 2001 From: azevaykin <145343289+azevaykin@users.noreply.github.com> Date: Thu, 22 Aug 2024 18:17:37 +0300 Subject: [PATCH 062/261] Statistics: disable some internal events without feature flag (#8159) --- ydb/core/statistics/aggregator/tx_init.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index de0bc44a0bca..d97521eda571 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -270,8 +270,13 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { Self->SubscribeForConfigChanges(ctx); Self->Schedule(Self->PropagateInterval, new TEvPrivate::TEvPropagate()); - Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); - Self->Schedule(Self->SendAnalyzePeriod, new TEvPrivate::TEvSendAnalyze()); + + if (Self->EnableColumnStatistics) { + Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); + Self->Schedule(Self->SendAnalyzePeriod, new TEvPrivate::TEvSendAnalyze()); + } else { + SA_LOG_W("[" << Self->TabletID() << "] TTxInit::Complete. EnableColumnStatistics=false"); + } Self->InitializeStatisticsTable(); From 12eb1d7e47b95a9bb9c1dedd2bf93726d24ee155 Mon Sep 17 00:00:00 2001 From: Marina Pereskokova Date: Thu, 22 Aug 2024 18:26:00 +0300 Subject: [PATCH 063/261] YQLAgent, free progress memory (#7585) --- ydb/library/yql/yt/bridge/interface.h | 14 +++++++++ ydb/library/yql/yt/dynamic/impl.cpp | 42 ++++++++++++--------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/ydb/library/yql/yt/bridge/interface.h b/ydb/library/yql/yt/bridge/interface.h index 7279867eeefc..f6d07ba5f182 100644 --- a/ydb/library/yql/yt/bridge/interface.h +++ b/ydb/library/yql/yt/bridge/interface.h @@ -73,6 +73,14 @@ struct TBridgeQueryResult ssize_t YsonErrorLength = 0; }; +#define FOR_EACH_QUERY_RESULT_STRING_FIELD(XX) \ + XX(YsonResult) \ + XX(Plan) \ + XX(Statistics) \ + XX(Progress) \ + XX(TaskInfo) \ + XX(YsonError) + struct TBridgeClustersResult { const char** Clusters = nullptr; @@ -82,6 +90,9 @@ struct TBridgeClustersResult ssize_t YsonErrorLength = 0; }; +#define FOR_EACH_BRIDGE_RESULT_STRING_FIELD(XX) \ + XX(YsonError) + enum EQueryFileContentType { RawInlineData, @@ -105,6 +116,9 @@ struct TBridgeAbortResult ssize_t YsonErrorLength = 0; }; +#define FOR_EACH_ABORT_RESULT_STRING_FIELD(XX) \ + XX(YsonError) + using TFuncBridgeFreeQueryResult = void(TBridgeQueryResult* result); using TFuncBridgeFreeClustersResult = void(TBridgeClustersResult* result); using TFuncBridgeRun = TBridgeQueryResult*( diff --git a/ydb/library/yql/yt/dynamic/impl.cpp b/ydb/library/yql/yt/dynamic/impl.cpp index 94b0a914e5f0..247b7fd40b1c 100644 --- a/ydb/library/yql/yt/dynamic/impl.cpp +++ b/ydb/library/yql/yt/dynamic/impl.cpp @@ -8,6 +8,9 @@ using namespace NYT::NYson; extern "C" { +#define FREE_STRING_FIELD(StringField) delete[] result->StringField; +#define FILL_STRING_FIELD(StringField) FillString(bridgeResult->StringField, bridgeResult->StringField##Length, result.StringField); + //////////////////////////////////////////////////////////////////////////////// ssize_t BridgeGetAbiVersion() @@ -59,17 +62,14 @@ void BridgeFreeClustersResult(TBridgeClustersResult* result) delete[] result->Clusters[i]; } delete[] result->Clusters; - delete[] result->YsonError; + + FOR_EACH_BRIDGE_RESULT_STRING_FIELD(FREE_STRING_FIELD); delete result; } void BridgeFreeQueryResult(TBridgeQueryResult* result) { - delete[] result->TaskInfo; - delete[] result->Statistics; - delete[] result->Plan; - delete[] result->YsonResult; - delete[] result->YsonError; + FOR_EACH_QUERY_RESULT_STRING_FIELD(FREE_STRING_FIELD); delete result; } @@ -97,7 +97,7 @@ TBridgeClustersResult* BridgeGetUsedClusters( static const auto EmptyMap = TYsonString(TString("{}")); auto* nativePlugin = reinterpret_cast(plugin); - auto* bridgeClustersResult = new TBridgeClustersResult; + auto* bridgeResult = new TBridgeClustersResult; std::vector files(bridgeFileCount); for (int index = 0; index < bridgeFileCount; index++) { @@ -109,21 +109,21 @@ TBridgeClustersResult* BridgeGetUsedClusters( }); } - auto clustersResult = nativePlugin->GetUsedClusters( + auto result = nativePlugin->GetUsedClusters( TString(queryText), settings ? TYsonString(TString(settings, settingsLength)) : EmptyMap, files); - bridgeClustersResult->Clusters = new const char*[clustersResult.Clusters.size()]; - for (size_t i = 0; i < clustersResult.Clusters.size(); i++) { + bridgeResult->Clusters = new const char*[result.Clusters.size()]; + for (size_t i = 0; i < result.Clusters.size(); i++) { ssize_t clusterLength; - FillString(bridgeClustersResult->Clusters[i], clusterLength, clustersResult.Clusters[i]); + FillString(bridgeResult->Clusters[i], clusterLength, result.Clusters[i]); } - bridgeClustersResult->ClusterCount = clustersResult.Clusters.size(); + bridgeResult->ClusterCount = result.Clusters.size(); - FillString(bridgeClustersResult->YsonError, bridgeClustersResult->YsonErrorLength, clustersResult.YsonError); + FOR_EACH_BRIDGE_RESULT_STRING_FIELD(FILL_STRING_FIELD); - return bridgeClustersResult; + return bridgeResult; } TBridgeQueryResult* BridgeRun( @@ -161,12 +161,7 @@ TBridgeQueryResult* BridgeRun( settings ? TYsonString(TString(settings, settingsLength)) : EmptyMap, files, executeMode); - FillString(bridgeResult->YsonResult, bridgeResult->YsonResultLength, result.YsonResult); - FillString(bridgeResult->Plan, bridgeResult->PlanLength, result.Plan); - FillString(bridgeResult->Statistics, bridgeResult->StatisticsLength, result.Statistics); - FillString(bridgeResult->Progress, bridgeResult->ProgressLength, result.Progress); - FillString(bridgeResult->TaskInfo, bridgeResult->TaskInfoLength, result.TaskInfo); - FillString(bridgeResult->YsonError, bridgeResult->YsonErrorLength, result.YsonError); + FOR_EACH_QUERY_RESULT_STRING_FIELD(FILL_STRING_FIELD); return bridgeResult; } @@ -177,8 +172,7 @@ TBridgeQueryResult* BridgeGetProgress(TBridgeYqlPlugin* plugin, const char* quer auto* bridgeResult = new TBridgeQueryResult; auto result = nativePlugin->GetProgress(NYT::TGuid::FromString(queryId)); - FillString(bridgeResult->Plan, bridgeResult->PlanLength, result.Plan); - FillString(bridgeResult->Progress, bridgeResult->ProgressLength, result.Progress); + FOR_EACH_QUERY_RESULT_STRING_FIELD(FILL_STRING_FIELD); return bridgeResult; } @@ -189,14 +183,14 @@ TBridgeAbortResult* BridgeAbort(TBridgeYqlPlugin* plugin, const char* queryId) auto* bridgeResult = new TBridgeAbortResult; auto result = nativePlugin->Abort(NYT::TGuid::FromString(queryId)); - FillString(bridgeResult->YsonError, bridgeResult->YsonErrorLength, result.YsonError); + FOR_EACH_ABORT_RESULT_STRING_FIELD(FILL_STRING_FIELD); return bridgeResult; } void BridgeFreeAbortResult(TBridgeAbortResult* result) { - delete[] result->YsonError; + FOR_EACH_ABORT_RESULT_STRING_FIELD(FREE_STRING_FIELD); delete result; } From 81fa5f2aa7161c1f9e27a2d8693fa37937b8698d Mon Sep 17 00:00:00 2001 From: Grigory Reznikov Date: Thu, 22 Aug 2024 17:26:10 +0200 Subject: [PATCH 064/261] Do not use YQL_LOG during YQL plugin initialization (#6798) --- ydb/library/yql/yt/native/plugin.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ydb/library/yql/yt/native/plugin.cpp b/ydb/library/yql/yt/native/plugin.cpp index fcc5268f5329..024d84ea7018 100644 --- a/ydb/library/yql/yt/native/plugin.cpp +++ b/ydb/library/yql/yt/native/plugin.cpp @@ -356,7 +356,9 @@ class TYqlPlugin ProgramFactory_->SetFileStorage(FileStorage_); ProgramFactory_->SetUrlPreprocessing(MakeIntrusive(GatewaysConfig_)); } catch (const std::exception& ex) { - YQL_LOG(FATAL) << "Unexpected exception while initializing YQL plugin: " << ex.what(); + // NB: YQL_LOG may be not initialized yet (for example, during singletons config parse), + // so we use std::cerr instead of it. + std::cerr << "Unexpected exception while initializing YQL plugin: " << ex.what() << std::endl; exit(1); } YQL_LOG(INFO) << "YQL plugin initialized"; From 7f83ce588d56b70a9a8af48af4ccfd96b0a490a0 Mon Sep 17 00:00:00 2001 From: Maxim Kovalev Date: Thu, 22 Aug 2024 18:35:56 +0300 Subject: [PATCH 065/261] YQL: Fuse reduces with presort settings (#8117) --- .../provider/phy_opt/yql_yt_phy_opt_fuse.cpp | 16 ++++- .../sql/dq_file/part15/canondata/result.json | 22 ++++++ .../hybrid_file/part8/canondata/result.json | 14 ++++ .../tests/sql/sql2yql/canondata/result.json | 14 ++++ .../produce/fuse_reduces_with_presort.cfg | 1 + .../produce/fuse_reduces_with_presort.sql | 69 +++++++++++++++++++ .../tests/sql/suites/produce/input_sorted.txt | 4 ++ .../sql/suites/produce/input_sorted.txt.attr | 15 ++++ .../part15/canondata/result.json | 21 ++++++ 9 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.cfg create mode 100644 ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.sql create mode 100644 ydb/library/yql/tests/sql/suites/produce/input_sorted.txt create mode 100644 ydb/library/yql/tests/sql/suites/produce/input_sorted.txt.attr diff --git a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp index ef849b752cd5..84a455f6f960 100644 --- a/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp +++ b/ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp @@ -120,9 +120,15 @@ TMaybeNode TYtPhysicalOptProposalTransformer::FuseReduce(TExprBase no !NYql::HasSetting(innerReduce.Settings().Ref(), EYtSettingType::ReduceBy)) { return node; } - if (NYql::HasSetting(outerReduce.Settings().Ref(), EYtSettingType::SortBy)) { - return node; + auto innerSortBy = NYql::GetSettingAsColumnList(innerReduce.Settings().Ref(), EYtSettingType::SortBy); + auto outerSortBy = NYql::GetSettingAsColumnList(outerReduce.Settings().Ref(), EYtSettingType::SortBy); + if (outerSortBy.size() > innerSortBy.size()) { + return node; + } + if (!std::equal(outerSortBy.cbegin(), outerSortBy.cend(), innerSortBy.cbegin())) { + return node; + } } if (NYql::HasSettingsExcept(innerReduce.Settings().Ref(), EYtSettingType::ReduceBy | @@ -130,6 +136,7 @@ TMaybeNode TYtPhysicalOptProposalTransformer::FuseReduce(TExprBase no EYtSettingType::Flow | EYtSettingType::FirstAsPrimary | EYtSettingType::SortBy | + EYtSettingType::KeepSorted | EYtSettingType::NoDq)) { return node; } @@ -227,6 +234,11 @@ TMaybeNode TYtPhysicalOptProposalTransformer::FuseReduce(TExprBase no newSettings = NYql::AddSetting(*newSettings, EYtSettingType::NoDq, {}, ctx); } + if (NYql::HasSetting(outerReduce.Settings().Ref(), EYtSettingType::KeepSorted) && + !NYql::HasSetting(innerReduce.Settings().Ref(), EYtSettingType::KeepSorted)) { + newSettings = NYql::AddSetting(*newSettings, EYtSettingType::KeepSorted, {}, ctx); + } + return Build(ctx, node.Pos()) .InitFrom(outerReduce) .World() diff --git a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json index bebc52376d31..1094588c94f5 100644 --- a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json @@ -2410,6 +2410,28 @@ } ], "test.test[pragma-yson_auto_convert--Results]": [], + "test.test[produce-fuse_reduces_with_presort--Analyze]": [ + { + "checksum": "325a59b196f61e264182ef061c4cc672", + "size": 13222, + "uri": "https://{canondata_backend}/1937367/c8f509a79779b30b722211cfeb063fe74251b5ea/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Analyze_/plan.txt" + } + ], + "test.test[produce-fuse_reduces_with_presort--Debug]": [ + { + "checksum": "161aa9d1d7f46024786b2745eefd0bfc", + "size": 4976, + "uri": "https://{canondata_backend}/1937367/c8f509a79779b30b722211cfeb063fe74251b5ea/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Debug_/opt.yql_patched" + } + ], + "test.test[produce-fuse_reduces_with_presort--Plan]": [ + { + "checksum": "325a59b196f61e264182ef061c4cc672", + "size": 13222, + "uri": "https://{canondata_backend}/1937367/c8f509a79779b30b722211cfeb063fe74251b5ea/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Plan_/plan.txt" + } + ], + "test.test[produce-fuse_reduces_with_presort--Results]": [], "test.test[produce-yql-10297-default.txt-Analyze]": [ { "checksum": "a3b64a2cf9903b3868a2dd88a18fc46e", diff --git a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json index ad702320778c..fc7243a720a7 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json @@ -2589,6 +2589,20 @@ "uri": "https://{canondata_backend}/1923547/94f377eaa1d93890e1345ac4940cc6fa07bddd4f/resource.tar.gz#test.test_pg_catalog-pg_timezone_abbrevs-default.txt-Plan_/plan.txt" } ], + "test.test[produce-fuse_reduces_with_presort--Debug]": [ + { + "checksum": "a0be01e2962b39871f4097f1d598a1ca", + "size": 7199, + "uri": "https://{canondata_backend}/1942525/70eec467682e96dfaac0f3c809005b8a22cce54f/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Debug_/opt.yql_patched" + } + ], + "test.test[produce-fuse_reduces_with_presort--Plan]": [ + { + "checksum": "b7d4ddc5c9f07fedb74a72b0789bd050", + "size": 7633, + "uri": "https://{canondata_backend}/1942525/70eec467682e96dfaac0f3c809005b8a22cce54f/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Plan_/plan.txt" + } + ], "test.test[produce-reduce_lambda_list_mem-default.txt-Debug]": [ { "checksum": "940dd13b9a62a90b5c95a4a12a765fe7", diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index 637835b9785b..9cc6e50ffe64 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -15336,6 +15336,13 @@ "uri": "https://{canondata_backend}/1784117/d56ae82ad9d30397a41490647be1bd2124718f98/resource.tar.gz#test_sql2yql.test_produce-discard_reduce_lambda_/sql.yql" } ], + "test_sql2yql.test[produce-fuse_reduces_with_presort]": [ + { + "checksum": "e379da7ed6a051092b9b7125ec129f09", + "size": 11861, + "uri": "https://{canondata_backend}/1923547/a000f225caa0fe7476a3e0c0740662cd5bbe1d5e/resource.tar.gz#test_sql2yql.test_produce-fuse_reduces_with_presort_/sql.yql" + } + ], "test_sql2yql.test[produce-native_desc_reduce_with_presort]": [ { "checksum": "78eedc889a54e4499e62fb3752015a4a", @@ -31688,6 +31695,13 @@ "uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_produce-discard_reduce_lambda_/formatted.sql" } ], + "test_sql_format.test[produce-fuse_reduces_with_presort]": [ + { + "checksum": "8f8335d2d11bff52392da6cc36645c55", + "size": 1636, + "uri": "https://{canondata_backend}/1923547/a000f225caa0fe7476a3e0c0740662cd5bbe1d5e/resource.tar.gz#test_sql_format.test_produce-fuse_reduces_with_presort_/formatted.sql" + } + ], "test_sql_format.test[produce-native_desc_reduce_with_presort]": [ { "checksum": "6af6ad9ee36a41b35cec0e63c725eae8", diff --git a/ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.cfg b/ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.cfg new file mode 100644 index 000000000000..612a5060aaa9 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.cfg @@ -0,0 +1 @@ +in Input input_sorted.txt \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.sql b/ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.sql new file mode 100644 index 000000000000..b4f5afd8c22b --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.sql @@ -0,0 +1,69 @@ +USE plato; + +$reduce = ($_, $TableRows) -> { + return Yql::Condense1( + $TableRows, + ($item) -> ($item), + ($_, $_) -> (false), + ($item, $_) -> ($item) + ); +}; + +$stream = +select * from Input; + +-- + +$stream1 = Reduce $stream +presort value1 +on key, subkey +using $reduce(TableRow()) +assume order by key, subkey, value1; + +$stream1 = Reduce $stream1 +presort value1 +on key, subkey +using $reduce(TableRow()) +assume order by key, subkey, value1; + +-- + +$stream2 = Reduce $stream +presort value1, value2 +on key, subkey +using $reduce(TableRow()) +assume order by key, subkey, value1, value2; + +$stream2 = Reduce $stream2 +presort value1 +on key, subkey +using $reduce(TableRow()) +assume order by key, subkey, value1; + +-- + +$stream3 = Reduce $stream +presort value1, value2, value3 +on key, subkey +using $reduce(TableRow()) +assume order by key, subkey, value1, value2, value3; + +$stream3 = Reduce $stream3 +on key, subkey +using $reduce(TableRow()) +assume order by key, subkey; + +select + * +from $stream1 +ASSUME ORDER BY `key`, `subkey`; + +select + * +from $stream2 +ASSUME ORDER BY `key`, `subkey`; + +select + * +from $stream3 +ASSUME ORDER BY `key`, `subkey`; diff --git a/ydb/library/yql/tests/sql/suites/produce/input_sorted.txt b/ydb/library/yql/tests/sql/suites/produce/input_sorted.txt new file mode 100644 index 000000000000..070ba2f98786 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/produce/input_sorted.txt @@ -0,0 +1,4 @@ +{"key"="020";"subkey"="1";"value1"="abc";"value2"="aabbcc";"value3"="aa"}; +{"key"="075";"subkey"="2";"value1"="ddd";"value2"="dddddd";"value3"="bb"}; +{"key"="150";"subkey"="3";"value1"="q";"value2"="qq";"value3"="cc"}; +{"key"="800";"subkey"="4";"value1"="qzz";"value2"="qqzzzz";"value3"="dd"}; \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/suites/produce/input_sorted.txt.attr b/ydb/library/yql/tests/sql/suites/produce/input_sorted.txt.attr new file mode 100644 index 000000000000..f091ca59dde5 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/produce/input_sorted.txt.attr @@ -0,0 +1,15 @@ +{ + "_yql_row_spec"={ + "Type"=["StructType";[ + ["key";["DataType";"String"]]; + ["subkey";["DataType";"String"]]; + ["value1";["DataType";"String"]]; + ["value2";["DataType";"String"]]; + ["value3";["DataType";"String"]] + ]]; + "SortDirections"=[1;1;1;1;1]; + "SortedBy"=["key";"subkey";"value1";"value2";"value3"]; + "SortedByTypes"=[["DataType";"String"];["DataType";"String"];["DataType";"String"];["DataType";"String"];["DataType";"String"]]; + "SortMembers"=["key";"subkey";"value1";"value2";"value3"]; + } +} diff --git a/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json index d69fb9c3debb..4c2116d640ae 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json @@ -2261,6 +2261,27 @@ "uri": "https://{canondata_backend}/1903280/4c77300cd3fef018d27d7f75b6ff956e63258b21/resource.tar.gz#test.test_pragma-yson_auto_convert--Results_/results.txt" } ], + "test.test[produce-fuse_reduces_with_presort--Debug]": [ + { + "checksum": "7eec75c62c340d46b9fd0d652345e21a", + "size": 3577, + "uri": "https://{canondata_backend}/1889210/f2ee730cb1f26eb197a3bfb0493ec60757c5a916/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Debug_/opt.yql" + } + ], + "test.test[produce-fuse_reduces_with_presort--Plan]": [ + { + "checksum": "9e8381980dc1e802f465fa49ee5dfd14", + "size": 11259, + "uri": "https://{canondata_backend}/1889210/f2ee730cb1f26eb197a3bfb0493ec60757c5a916/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Plan_/plan.txt" + } + ], + "test.test[produce-fuse_reduces_with_presort--Results]": [ + { + "checksum": "487e3e6ee23dd6103b213486901febc4", + "size": 7301, + "uri": "https://{canondata_backend}/1889210/f2ee730cb1f26eb197a3bfb0493ec60757c5a916/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Results_/results.txt" + } + ], "test.test[produce-yql-10297-default.txt-Debug]": [ { "checksum": "6ee64112721e725a4bd2da387df401e6", From ebbf9543a1eea1f2d52ffbaa444a399af901ef8f Mon Sep 17 00:00:00 2001 From: Alexander Rutkovsky Date: Thu, 22 Aug 2024 18:39:14 +0300 Subject: [PATCH 066/261] Report UNKNOWN status to SysView for disks that have not checked in yet (#8144) --- ydb/core/mind/bscontroller/bsc.cpp | 4 +-- ydb/core/mind/bscontroller/cmds_box.cpp | 2 +- .../mind/bscontroller/cmds_storage_pool.cpp | 6 ++--- ydb/core/mind/bscontroller/config.cpp | 6 ++--- .../mind/bscontroller/config_fit_groups.cpp | 2 +- ydb/core/mind/bscontroller/impl.h | 26 ++++++++++++------- ydb/core/mind/bscontroller/monitoring.cpp | 2 +- ydb/core/mind/bscontroller/register_node.cpp | 2 +- ydb/core/mind/bscontroller/self_heal.cpp | 10 +++---- ydb/core/mind/bscontroller/sys_view.cpp | 16 ++++++++---- ydb/core/sys_view/ut_kqp.cpp | 2 +- 11 files changed, 46 insertions(+), 32 deletions(-) diff --git a/ydb/core/mind/bscontroller/bsc.cpp b/ydb/core/mind/bscontroller/bsc.cpp index 9ce2d1ca2700..d9ae407390b6 100644 --- a/ydb/core/mind/bscontroller/bsc.cpp +++ b/ydb/core/mind/bscontroller/bsc.cpp @@ -337,7 +337,7 @@ void TBlobStorageController::ValidateInternalState() { Y_ABORT_UNLESS(donor->GetShortVDiskId() == vslot->GetShortVDiskId()); } if (vslot->Group) { - if (vslot->Status == NKikimrBlobStorage::EVDiskStatus::READY) { + if (vslot->GetStatus() == NKikimrBlobStorage::EVDiskStatus::READY) { Y_DEBUG_ABORT_UNLESS(vslot->IsReady || vslot->IsInVSlotReadyTimestampQ()); } else { Y_DEBUG_ABORT_UNLESS(!vslot->IsReady && !vslot->IsInVSlotReadyTimestampQ()); @@ -401,7 +401,7 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) { const auto& record = msg->Record; for (const auto& item : record.GetVDiskStatus()) { const TVSlotId vslotId(item.GetNodeId(), item.GetPDiskId(), item.GetVSlotId()); - if (TVSlotInfo *slot = FindVSlot(vslotId); slot && slot->Status > item.GetStatus()) { + if (TVSlotInfo *slot = FindVSlot(vslotId); slot && slot->GetStatus() > item.GetStatus()) { return 1; } else if (const auto it = StaticVSlots.find(vslotId); it != StaticVSlots.end() && it->second.VDiskStatus > item.GetStatus()) { return 1; diff --git a/ydb/core/mind/bscontroller/cmds_box.cpp b/ydb/core/mind/bscontroller/cmds_box.cpp index e1e821c5c643..4b7f53c6beb0 100644 --- a/ydb/core/mind/bscontroller/cmds_box.cpp +++ b/ydb/core/mind/bscontroller/cmds_box.cpp @@ -213,7 +213,7 @@ namespace NKikimr::NBsController { for (const auto& [id, slot] : pdisk->VSlotsOnPDisk) { if (slot->Group) { auto *m = VSlots.FindForUpdate(slot->VSlotId); - m->Status = NKikimrBlobStorage::EVDiskStatus::ERROR; + m->VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR; m->IsReady = false; TGroupInfo *group = Groups.FindForUpdate(slot->Group->ID); GroupFailureModelChanged.insert(slot->Group->ID); diff --git a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp index 516ab802d6ad..01a09938030b 100644 --- a/ydb/core/mind/bscontroller/cmds_storage_pool.cpp +++ b/ydb/core/mind/bscontroller/cmds_storage_pool.cpp @@ -557,7 +557,7 @@ namespace NKikimr::NBsController { x->MutableVDiskMetrics()->CopyFrom(*vslot.VDiskMetrics); x->MutableVDiskMetrics()->ClearVDiskId(); } - x->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.VDiskStatus)); + x->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.VDiskStatus.value_or(NKikimrBlobStorage::EVDiskStatus::ERROR))); x->SetReady(vslot.ReadySince <= mono); } if (const auto& s = Self.StorageConfig; s.HasBlobStorageConfig()) { @@ -698,7 +698,7 @@ namespace NKikimr::NBsController { TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId); vslot->Mood = TMood::Wipe; - vslot->Status = NKikimrBlobStorage::EVDiskStatus::ERROR; + vslot->VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR; vslot->IsReady = false; GroupFailureModelChanged.insert(group->ID); group->CalculateGroupStatus(); @@ -744,7 +744,7 @@ namespace NKikimr::NBsController { TGroupInfo *group = Groups.FindForUpdate(vslot->GroupId); vslot->Mood = targetMood; - vslot->Status = NKikimrBlobStorage::EVDiskStatus::ERROR; + vslot->VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR; vslot->IsReady = false; GroupFailureModelChanged.insert(group->ID); group->CalculateGroupStatus(); diff --git a/ydb/core/mind/bscontroller/config.cpp b/ydb/core/mind/bscontroller/config.cpp index 4ed5a2cffc5b..b63680720525 100644 --- a/ydb/core/mind/bscontroller/config.cpp +++ b/ydb/core/mind/bscontroller/config.cpp @@ -486,9 +486,9 @@ namespace NKikimr::NBsController { if (!overlay->second || !overlay->second->Group) { // deleted one (overlay->second ? overlay->second : base->second)->DropFromVSlotReadyTimestampQ(); NotReadyVSlotIds.erase(overlay->first); - } else if (overlay->second->Status != NKikimrBlobStorage::EVDiskStatus::READY) { + } else if (overlay->second->GetStatus() != NKikimrBlobStorage::EVDiskStatus::READY) { overlay->second->DropFromVSlotReadyTimestampQ(); - } else if (!base || base->second->Status != NKikimrBlobStorage::EVDiskStatus::READY) { + } else if (!base || base->second->GetStatus() != NKikimrBlobStorage::EVDiskStatus::READY) { overlay->second->PutInVSlotReadyTimestampQ(now); } else { Y_DEBUG_ABORT_UNLESS(overlay->second->IsReady || overlay->second->IsInVSlotReadyTimestampQ()); @@ -998,7 +998,7 @@ namespace NKikimr::NBsController { pb->SetAllocatedSize(vslot.Metrics.GetAllocatedSize()); pb->MutableVDiskMetrics()->CopyFrom(vslot.Metrics); pb->MutableVDiskMetrics()->ClearVDiskId(); - pb->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.Status)); + pb->SetStatus(NKikimrBlobStorage::EVDiskStatus_Name(vslot.GetStatus())); for (const TVSlotId& vslotId : vslot.Donors) { auto *item = pb->AddDonors(); Serialize(item->MutableVSlotId(), vslotId); diff --git a/ydb/core/mind/bscontroller/config_fit_groups.cpp b/ydb/core/mind/bscontroller/config_fit_groups.cpp index e72184045d26..0953ad03f97b 100644 --- a/ydb/core/mind/bscontroller/config_fit_groups.cpp +++ b/ydb/core/mind/bscontroller/config_fit_groups.cpp @@ -524,7 +524,7 @@ namespace NKikimr { // also we have to find replicating VSlots on this PDisk and assume they consume up to // max(vslotSize for every slot in group), not their actual AllocatedSize for (const auto& [id, slot] : info.VSlotsOnPDisk) { - if (slot->Group && slot->Status != NKikimrBlobStorage::EVDiskStatus::READY) { + if (slot->Group && slot->GetStatus() != NKikimrBlobStorage::EVDiskStatus::READY) { ui64 maxGroupSlotSize = 0; for (const TVSlotInfo *peer : slot->Group->VDisksInGroup) { maxGroupSlotSize = Max(maxGroupSlotSize, peer->Metrics.GetAllocatedSize()); diff --git a/ydb/core/mind/bscontroller/impl.h b/ydb/core/mind/bscontroller/impl.h index d55e57b34d63..f8a3c9a36703 100644 --- a/ydb/core/mind/bscontroller/impl.h +++ b/ydb/core/mind/bscontroller/impl.h @@ -124,16 +124,17 @@ class TBlobStorageController : public TActor, public TTa TVSlotReadyTimestampQ::iterator VSlotReadyTimestampIter; public: - NKikimrBlobStorage::EVDiskStatus Status = NKikimrBlobStorage::EVDiskStatus::ERROR; + std::optional VDiskStatus; + NHPTimer::STime VDiskStatusTimestamp = GetCycleCountFast(); bool IsReady = false; bool OnlyPhantomsRemain = false; public: void SetStatus(NKikimrBlobStorage::EVDiskStatus status, TMonotonic now, TInstant instant, bool onlyPhantomsRemain) { - if (status != Status) { + if (status != VDiskStatus) { if (status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // became "replicating" LastGotReplicating = instant; - } else if (Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // was "replicating" + } else if (VDiskStatus == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { // was "replicating" Y_DEBUG_ABORT_UNLESS(LastGotReplicating != TInstant::Zero()); ReplicationTime += instant - LastGotReplicating; LastGotReplicating = {}; @@ -145,7 +146,7 @@ class TBlobStorageController : public TActor, public TTa LastSeenReady = instant; } - Status = status; + VDiskStatus = status; IsReady = false; if (status == NKikimrBlobStorage::EVDiskStatus::READY) { PutInVSlotReadyTimestampQ(now); @@ -159,6 +160,10 @@ class TBlobStorageController : public TActor, public TTa } } + NKikimrBlobStorage::EVDiskStatus GetStatus() const { + return VDiskStatus.value_or(NKikimrBlobStorage::EVDiskStatus::ERROR); + } + void PutInVSlotReadyTimestampQ(TMonotonic now) { const TMonotonic readyAfter = now + ReadyStablePeriod; // vdisk will be treated as READY one shortly, but not now Y_ABORT_UNLESS(VSlotReadyTimestampIter == TVSlotReadyTimestampQ::iterator()); @@ -291,15 +296,16 @@ class TBlobStorageController : public TActor, public TTa TString GetStatusString() const { TStringStream s; - s << NKikimrBlobStorage::EVDiskStatus_Name(Status); - if (Status == NKikimrBlobStorage::REPLICATING && OnlyPhantomsRemain) { + const auto status = GetStatus(); + s << NKikimrBlobStorage::EVDiskStatus_Name(status); + if (status == NKikimrBlobStorage::REPLICATING && OnlyPhantomsRemain) { s << "/p"; } return s.Str(); } bool IsOperational() const { - return Status >= NKikimrBlobStorage::REPLICATING; + return GetStatus() >= NKikimrBlobStorage::REPLICATING; } void OnCommit(); @@ -2276,7 +2282,7 @@ class TBlobStorageController : public TActor, public TTa histo.IncrementFor(passed.Seconds()); TDuration timeBeingReplicating = slot->ReplicationTime; - if (slot->Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { + if (slot->GetStatus() == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { timeBeingReplicating += now - slot->LastGotReplicating; } @@ -2301,7 +2307,8 @@ class TBlobStorageController : public TActor, public TTa const NKikimrBlobStorage::TVDiskKind::EVDiskKind VDiskKind; std::optional VDiskMetrics; - NKikimrBlobStorage::EVDiskStatus VDiskStatus = NKikimrBlobStorage::EVDiskStatus::ERROR; + std::optional VDiskStatus; + NHPTimer::STime VDiskStatusTimestamp = GetCycleCountFast(); TMonotonic ReadySince = TMonotonic::Max(); // when IsReady becomes true for this disk; Max() in non-READY state TStaticVSlotInfo(const NKikimrBlobStorage::TNodeWardenServiceSet::TVDisk& vdisk, @@ -2315,6 +2322,7 @@ class TBlobStorageController : public TActor, public TTa TStaticVSlotInfo& item = it->second; VDiskMetrics = std::move(item.VDiskMetrics); VDiskStatus = item.VDiskStatus; + VDiskStatusTimestamp = item.VDiskStatusTimestamp; ReadySince = item.ReadySince; } } diff --git a/ydb/core/mind/bscontroller/monitoring.cpp b/ydb/core/mind/bscontroller/monitoring.cpp index 9f6fa7597e6d..374955bc70f7 100644 --- a/ydb/core/mind/bscontroller/monitoring.cpp +++ b/ydb/core/mind/bscontroller/monitoring.cpp @@ -1296,7 +1296,7 @@ void TBlobStorageController::RenderVSlotRow(IOutputStream& out, const TVSlotInfo } TABLED() { TDuration time = vslot.ReplicationTime; - if (vslot.Status == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { + if (vslot.GetStatus() == NKikimrBlobStorage::EVDiskStatus::REPLICATING) { time += TActivationContext::Now() - vslot.LastGotReplicating; } out << time; diff --git a/ydb/core/mind/bscontroller/register_node.cpp b/ydb/core/mind/bscontroller/register_node.cpp index 69d05dcdfd8d..8dd84e535250 100644 --- a/ydb/core/mind/bscontroller/register_node.cpp +++ b/ydb/core/mind/bscontroller/register_node.cpp @@ -547,7 +547,7 @@ void TBlobStorageController::OnWardenDisconnected(TNodeId nodeId, TActorId serve updates.push_back({ .VDiskId = it->second->GetVDiskId(), .IsReady = it->second->IsReady, - .VDiskStatus = it->second->Status, + .VDiskStatus = it->second->GetStatus(), }); ScrubState.UpdateVDiskState(&*it->second); SysViewChangedVSlots.insert(it->second->VSlotId); diff --git a/ydb/core/mind/bscontroller/self_heal.cpp b/ydb/core/mind/bscontroller/self_heal.cpp index 2d13a1d86cfd..e75c4ef7333e 100644 --- a/ydb/core/mind/bscontroller/self_heal.cpp +++ b/ydb/core/mind/bscontroller/self_heal.cpp @@ -913,7 +913,7 @@ namespace NKikimr::NBsController { slot->OnlyPhantomsRemain, slot->IsReady, TMonotonic::Zero(), - slot->Status, + slot->GetStatus(), }; } } @@ -960,7 +960,7 @@ namespace NKikimr::NBsController { false, /* OnlyPhantomsRemain */ true, /* IsReady; decision is based on ReadySince */ info.ReadySince, - info.VDiskStatus, + info.VDiskStatus.value_or(NKikimrBlobStorage::EVDiskStatus::ERROR), }; } } @@ -987,7 +987,7 @@ namespace NKikimr::NBsController { const bool was = slot->IsOperational(); if (const TGroupInfo *group = slot->Group) { const bool wasReady = slot->IsReady; - if (slot->Status != m.GetStatus() || slot->OnlyPhantomsRemain != m.GetOnlyPhantomsRemain()) { + if (slot->GetStatus() != m.GetStatus() || slot->OnlyPhantomsRemain != m.GetOnlyPhantomsRemain()) { slot->SetStatus(m.GetStatus(), mono, now, m.GetOnlyPhantomsRemain()); if (slot->IsReady != wasReady) { ScrubState.UpdateVDiskState(slot); @@ -1001,14 +1001,14 @@ namespace NKikimr::NBsController { .VDiskId = vdiskId, .OnlyPhantomsRemain = slot->OnlyPhantomsRemain, .IsReady = slot->IsReady, - .VDiskStatus = slot->Status, + .VDiskStatus = slot->GetStatus(), }); if (!was && slot->IsOperational() && !group->SeenOperational) { groups.insert(const_cast(group)); } SysViewChangedVSlots.insert(vslotId); } - if (slot->Status == NKikimrBlobStorage::EVDiskStatus::READY) { + if (slot->GetStatus() == NKikimrBlobStorage::EVDiskStatus::READY) { // we can release donor slots without further notice then the VDisk is completely replicated; we // intentionally use GetStatus() here instead of IsReady() to prevent waiting for (const TVSlotId& donorVSlotId : slot->Donors) { diff --git a/ydb/core/mind/bscontroller/sys_view.cpp b/ydb/core/mind/bscontroller/sys_view.cpp index 6526402a825c..c2f9f737de45 100644 --- a/ydb/core/mind/bscontroller/sys_view.cpp +++ b/ydb/core/mind/bscontroller/sys_view.cpp @@ -325,7 +325,8 @@ void CopyInfo(NKikimrSysView::TPDiskInfo* info, const THolder status, NHPTimer::STime statusTimestamp, + NKikimrBlobStorage::TVDiskKind::EVDiskKind kind, bool isBeingDeleted) { pb->SetGroupId(vdiskId.GroupID.GetRawId()); pb->SetGroupGeneration(vdiskId.GroupGeneration); pb->SetFailRealm(vdiskId.FailRealm); @@ -337,7 +338,12 @@ void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId, if (m.HasAvailableSize()) { pb->SetAvailableSize(m.GetAvailableSize()); } - pb->SetStatusV2(NKikimrBlobStorage::EVDiskStatus_Name(status)); + if (!status && CyclesToDuration(GetCycleCountFast() - statusTimestamp) > TDuration::Seconds(15)) { + status = NKikimrBlobStorage::EVDiskStatus::ERROR; + } + if (status) { + pb->SetStatusV2(NKikimrBlobStorage::EVDiskStatus_Name(*status)); + } pb->SetKind(NKikimrBlobStorage::TVDiskKind::EVDiskKind_Name(kind)); if (isBeingDeleted) { pb->SetIsBeingDeleted(true); @@ -345,8 +351,8 @@ void SerializeVSlotInfo(NKikimrSysView::TVSlotInfo *pb, const TVDiskID& vdiskId, } void CopyInfo(NKikimrSysView::TVSlotInfo* info, const THolder& vSlotInfo) { - SerializeVSlotInfo(info, vSlotInfo->GetVDiskId(), vSlotInfo->Metrics, vSlotInfo->Status, vSlotInfo->Kind, - vSlotInfo->IsBeingDeleted()); + SerializeVSlotInfo(info, vSlotInfo->GetVDiskId(), vSlotInfo->Metrics, vSlotInfo->VDiskStatus, + vSlotInfo->VDiskStatusTimestamp, vSlotInfo->Kind, vSlotInfo->IsBeingDeleted()); } void CopyInfo(NKikimrSysView::TGroupInfo* info, const THolder& groupInfo) { @@ -462,7 +468,7 @@ void TBlobStorageController::UpdateSystemViews() { if (SysViewChangedVSlots.count(vslotId)) { static const NKikimrBlobStorage::TVDiskMetrics zero; SerializeVSlotInfo(&state.VSlots[vslotId], vslot.VDiskId, vslot.VDiskMetrics ? *vslot.VDiskMetrics : zero, - vslot.VDiskStatus, vslot.VDiskKind, false); + vslot.VDiskStatus, vslot.VDiskStatusTimestamp, vslot.VDiskKind, false); } } if (StorageConfig.HasBlobStorageConfig()) { diff --git a/ydb/core/sys_view/ut_kqp.cpp b/ydb/core/sys_view/ut_kqp.cpp index cf8f7f95974d..8191e9b1ff34 100644 --- a/ydb/core/sys_view/ut_kqp.cpp +++ b/ydb/core/sys_view/ut_kqp.cpp @@ -997,7 +997,7 @@ Y_UNIT_TEST_SUITE(SystemView) { check.String("Default"); // Kind check.Uint64(env.GetServer().GetRuntime()->GetNodeId(0)); // NodeId check.Uint64(1u); // PDiskId - check.String("ERROR"); // Status + check.Null(); // Status check.Uint64(0u); // VDisk check.Uint64(1000u); // VSlotId } From 205f1c33702d66106792533cb4c8fc5332cb559c Mon Sep 17 00:00:00 2001 From: Evgeniy Ivanov Date: Thu, 22 Aug 2024 18:17:45 +0200 Subject: [PATCH 067/261] use new avx macros (#7494) --- ydb/core/blobstorage/crypto/crypto.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/blobstorage/crypto/crypto.h b/ydb/core/blobstorage/crypto/crypto.h index 9255aaf3d470..1d0a84175175 100644 --- a/ydb/core/blobstorage/crypto/crypto.h +++ b/ydb/core/blobstorage/crypto/crypto.h @@ -10,7 +10,7 @@ #define ChaChaVec ChaCha #define Poly1305Vec Poly1305 #define CHACHA_BPI 1 -#elif __AVX512__ +#elif __AVX512F__ #include #include #include From db45919892f3be6444f4076d6d1c71f884e80ffa Mon Sep 17 00:00:00 2001 From: Vlad Kuznetsov Date: Thu, 22 Aug 2024 18:40:49 +0200 Subject: [PATCH 068/261] PDisk LWTracing improvements (#8160) Co-authored-by: Vlad Kuznecov --- .../lwtrace_probes/blobstorage_probes.h | 4 ++-- .../blobstorage_pdisk_blockdevice_async.cpp | 6 +++++ .../blobstorage_pdisk_completion_impl.cpp | 3 ++- .../pdisk/blobstorage_pdisk_completion_impl.h | 1 + .../pdisk/blobstorage_pdisk_impl.cpp | 22 ++++++++++--------- .../pdisk/blobstorage_pdisk_impl.h | 2 +- .../pdisk/blobstorage_pdisk_impl_log.cpp | 20 ++++++++++++----- 7 files changed, 38 insertions(+), 20 deletions(-) diff --git a/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h b/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h index 35db647385ce..bf12934e97ad 100644 --- a/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h +++ b/ydb/core/blobstorage/lwtrace_probes/blobstorage_probes.h @@ -214,10 +214,10 @@ struct TEventTypeField { PROBE(PDiskChunkReadPieceComplete, GROUPS("PDisk", "PDiskRequest"), \ TYPES(TPDiskIdField, ui64, ui64, double), \ NAMES("pdisk", "size", "relativeOffset", "deviceTimeMs")) \ - PROBE(PDiskAddWritePieceToScheduler, GROUPS("PDisk", "PDiskRequest"), \ + PROBE(PDiskChunkWriteAddToScheduler, GROUPS("PDisk", "PDiskRequest"), \ TYPES(TPDiskIdField, ui64, double, ui64, bool, ui64, ui64), \ NAMES("pdisk", "reqId", "creationTimeSec", "owner", "isFast", "priorityClass", "size")) \ - PROBE(PDiskChunkWritePieceSendToDevice, GROUPS("PDisk", "PDiskRequest"), \ + PROBE(PDiskChunkWriteLastPieceSendToDevice, GROUPS("PDisk", "PDiskRequest"), \ TYPES(TPDiskIdField, ui64, ui64, ui64, ui64), \ NAMES("pdisk", "owner", "chunkIdx", "pieceOffset", "pieceSize")) \ PROBE(PDiskLogWriteComplete, GROUPS("PDisk", "PDiskRequest"), \ diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_blockdevice_async.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_blockdevice_async.cpp index 02b75eee4eec..4844e5b2db91 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_blockdevice_async.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_blockdevice_async.cpp @@ -242,6 +242,9 @@ class TRealBlockDevice : public IBlockDevice { EIoResult ret = EIoResult::TryAgain; while (ret == EIoResult::TryAgain) { action->SubmitTime = HPNow(); + if (action->FlushAction) { + action->FlushAction->SubmitTime = action->SubmitTime; + } if (op->GetType() == IAsyncIoOperation::EType::PWrite) { PDISK_FAIL_INJECTION(1); @@ -562,6 +565,9 @@ class TRealBlockDevice : public IBlockDevice { EIoResult ret = EIoResult::TryAgain; while (ret == EIoResult::TryAgain) { action->SubmitTime = HPNow(); + if (action->FlushAction) { + action->FlushAction->SubmitTime = action->SubmitTime; + } ret = Device.IoContext->Submit(op, Device.SharedCallback.Get()); if (ret == EIoResult::Ok) { return true; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.cpp index b274561f5867..5caaf8ad124d 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.cpp @@ -276,7 +276,8 @@ void TCompletionChunkReadPart::Exec(TActorSystem *actorSystem) { } double deviceTimeMs = HPMilliSecondsFloat(GetTime - SubmitTime); - LWTRACK(PDiskChunkReadPieceComplete, Read->Orbit, PDisk->PDiskId, RawReadSize, CommonBufferOffset, deviceTimeMs); + LWTRACK(PDiskChunkReadPieceComplete, Orbit, PDisk->PDiskId, RawReadSize, CommonBufferOffset, deviceTimeMs); + Read->Orbit.Join(Orbit); CumulativeCompletion->PartReadComplete(actorSystem); CumulativeCompletion = nullptr; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h index e79e3ae81fc2..c3be2b61d913 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_completion_impl.h @@ -111,6 +111,7 @@ class TCompletionChunkWrite : public TCompletionAction { Mon->IncrementResponseTime(PriorityClass, responseTimeMs, SizeBytes); } LWTRACK(PDiskChunkResponseTime, Orbit, PDiskId, ReqId.Id, PriorityClass, responseTimeMs, SizeBytes); + Event->Orbit = std::move(Orbit); actorSystem->Send(Recipient, Event.Release()); if (Mon) { Mon->GetWriteCounter(PriorityClass)->CountResponse(); diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index c5b648477b23..d4a387c2aa2c 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -848,9 +848,6 @@ bool TPDisk::ChunkWritePiece(TChunkWrite *evChunkWrite, ui32 pieceShift, ui32 pi guard.Release(); - LWTRACK(PDiskChunkWritePieceSendToDevice, evChunkWrite->Orbit, PDiskId, evChunkWrite->Owner, chunkIdx, - pieceShift, pieceSize); - ui32 bytesAvailable = pieceSize; Y_ABORT_UNLESS(evChunkWrite->BytesWritten == pieceShift); const ui32 count = evChunkWrite->PartsPtr->Size(); @@ -909,6 +906,9 @@ bool TPDisk::ChunkWritePiece(TChunkWrite *evChunkWrite, ui32 pieceShift, ui32 pi LOG_INFO(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# %" PRIu32 " chunkIdx# %" PRIu32 " was zero-padded after writing", (ui32)PDiskId, (ui32)chunkIdx); } + LWTRACK(PDiskChunkWriteLastPieceSendToDevice, evChunkWrite->Orbit, PDiskId, evChunkWrite->Owner, chunkIdx, + pieceShift, pieceSize); + auto traceId = evChunkWrite->SpanStack.GetTraceId(); evChunkWrite->Completion->Orbit = std::move(evChunkWrite->Orbit); writer.Flush(evChunkWrite->ReqId, &traceId, evChunkWrite->Completion.Release()); @@ -951,7 +951,7 @@ void TPDisk::SendChunkReadError(const TIntrusivePtr& read, TStringSt } TPDisk::EChunkReadPieceResult TPDisk::ChunkReadPiece(TIntrusivePtr &read, ui64 pieceCurrentSector, - ui64 pieceSizeLimit, ui64 *reallyReadDiskBytes, NWilson::TTraceId traceId) { + ui64 pieceSizeLimit, ui64 *reallyReadDiskBytes, NWilson::TTraceId traceId, NLWTrace::TOrbit&& orbit) { if (read->IsReplied) { return ReadPieceResultOk; } @@ -1020,6 +1020,8 @@ TPDisk::EChunkReadPieceResult TPDisk::ChunkReadPiece(TIntrusivePtr & THolder completion(new TCompletionChunkReadPart(this, read, bytesToRead, payloadBytesToRead, payloadOffset, read->FinalCompletion, isTheLastPart, std::move(span))); completion->CostNs = DriveModel.TimeForSizeNs(bytesToRead, read->ChunkIdx, TDriveModel::OP_TYPE_READ); + LWTRACK(PDiskChunkReadPiecesSendToDevice, orbit, PDiskId); + completion->Orbit = std::move(orbit); Y_ABORT_UNLESS(bytesToRead <= completion->GetBuffer()->Size()); ui8 *data = completion->GetBuffer()->Data(); BlockDevice->PreadAsync(data, bytesToRead, readOffset, completion.Release(), @@ -2289,7 +2291,6 @@ void TPDisk::ProcessChunkReadQueue() { bool isComplete = false; ui8 priorityClass = read->PriorityClass; NHPTimer::STime creationTime = read->CreationTime; - LWTRACK(PDiskChunkReadPiecesSendToDevice, read->Orbit, PDiskId); if (!read->IsReplied) { LOG_DEBUG_S(*ActorSystem, NKikimrServices::BS_PDISK, "PDiskId# " << (ui32)PDiskId << " ReqId# " << reqId @@ -2301,7 +2302,7 @@ void TPDisk::ProcessChunkReadQueue() { ui64 currentLimit = Min(bufferSize, piece->PieceSizeLimit - size); ui64 reallyReadDiskBytes; EChunkReadPieceResult result = ChunkReadPiece(read, piece->PieceCurrentSector + size / Format.SectorSize, - currentLimit, &reallyReadDiskBytes, piece->SpanStack.GetTraceId()); + currentLimit, &reallyReadDiskBytes, piece->SpanStack.GetTraceId(), std::move(piece->Orbit)); isComplete = (result != ReadPieceResultInProgress); // Read pieces is sliced previously and it is expected that ChunkReadPiece will read exactly // currentLimit bytes @@ -2977,7 +2978,6 @@ bool TPDisk::PreprocessRequest(TRequestBase *request) { auto result = std::make_unique(NKikimrProto::OK, ev.ChunkIdx, ev.Cookie, GetStatusFlags(ev.Owner, ev.OwnerGroupType), TString()); - result->Orbit = std::move(ev.Orbit); ++state.OperationsInProgress; ++ownerData.InFlight->ChunkWrites; @@ -3219,7 +3219,7 @@ void TPDisk::PushRequestToForseti(TRequestBase *request) { TChunkWritePiece *piece = new TChunkWritePiece(whole, smallJobCount * smallJobSize, largeJobSize, std::move(span)); piece->EstimateCost(DriveModel); AddJobToForseti(cbs, piece, request->JobKind); - LWTRACK(PDiskAddWritePieceToScheduler, request->Orbit, PDiskId, request->ReqId.Id, + LWTRACK(PDiskChunkWriteAddToScheduler, request->Orbit, PDiskId, request->ReqId.Id, HPSecondsFloat(HPNow() - request->CreationTime), request->Owner, request->IsFast, request->PriorityClass, whole->TotalSize); } else if (request->GetType() == ERequestType::RequestChunkRead) { @@ -3240,7 +3240,8 @@ void TPDisk::PushRequestToForseti(TRequestBase *request) { // Schedule small job. auto piece = new TChunkReadPiece(read, idx * smallJobSize, smallJobSize * Format.SectorSize, false, std::move(span)); - LWTRACK(PDiskChunkReadPieceAddToScheduler, read->Orbit, PDiskId, idx, idx * smallJobSize * Format.SectorSize, + read->Orbit.Fork(piece->Orbit); + LWTRACK(PDiskChunkReadPieceAddToScheduler, piece->Orbit, PDiskId, idx, idx * smallJobSize * Format.SectorSize, smallJobSize * Format.SectorSize); piece->EstimateCost(DriveModel); piece->SelfPointer = piece; @@ -3251,7 +3252,8 @@ void TPDisk::PushRequestToForseti(TRequestBase *request) { span.Attribute("is_last_piece", true); auto piece = new TChunkReadPiece(read, smallJobCount * smallJobSize, largeJobSize * Format.SectorSize, true, std::move(span)); - LWTRACK(PDiskChunkReadPieceAddToScheduler, read->Orbit, PDiskId, smallJobCount, + read->Orbit.Fork(piece->Orbit); + LWTRACK(PDiskChunkReadPieceAddToScheduler, piece->Orbit, PDiskId, smallJobCount, smallJobCount * smallJobSize * Format.SectorSize, largeJobSize * Format.SectorSize); piece->EstimateCost(DriveModel); piece->SelfPointer = piece; diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h index 590479df68f4..036c817fb36e 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.h @@ -291,7 +291,7 @@ class TPDisk : public IPDisk { void SendChunkReadError(const TIntrusivePtr& read, TStringStream& errorReason, NKikimrProto::EReplyStatus status); EChunkReadPieceResult ChunkReadPiece(TIntrusivePtr &read, ui64 pieceCurrentSector, ui64 pieceSizeLimit, - ui64 *reallyReadBytes, NWilson::TTraceId traceId); + ui64 *reallyReadBytes, NWilson::TTraceId traceId, NLWTrace::TOrbit&& orbit); void SplitChunkJobSize(ui32 totalSize, ui32 *outSmallJobSize, ui32 *outLargeJObSize, ui32 *outSmallJobCount); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Chunk locking diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp index fab461ce2d27..d24e16751b89 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl_log.cpp @@ -19,23 +19,31 @@ class TLogFlushCompletionAction : public TCompletionAction { : EndChunkIdx(endChunkIdx) , EndSectorIdx(endSectorIdx) , CommonLogger(commonLogger) - , CompletionLogWrite(completionLogWrite) { } + , CompletionLogWrite(completionLogWrite) + { + Orbit = std::move(completionLogWrite->Orbit); + } + + void SetUpCompletionLogWrite() { + CompletionLogWrite->SubmitTime = SubmitTime; + CompletionLogWrite->GetTime = GetTime; + CompletionLogWrite->SetResult(Result); + CompletionLogWrite->SetErrorReason(ErrorReason); + CompletionLogWrite->Orbit = std::move(Orbit); + } void Exec(TActorSystem *actorSystem) override { CommonLogger->FirstUncommitted = TFirstUncommitted(EndChunkIdx, EndSectorIdx); - CompletionLogWrite->SetResult(Result); - CompletionLogWrite->SetErrorReason(ErrorReason); + SetUpCompletionLogWrite(); CompletionLogWrite->Exec(actorSystem); delete this; } void Release(TActorSystem *actorSystem) override { - CompletionLogWrite->SetResult(Result); - CompletionLogWrite->SetErrorReason(ErrorReason); + SetUpCompletionLogWrite(); CompletionLogWrite->Release(actorSystem); - delete this; } }; From ab6f78d00ab0c6e323d7782531d07e60fda43fa9 Mon Sep 17 00:00:00 2001 From: Alek5andr-Kotov Date: Thu, 22 Aug 2024 19:51:00 +0300 Subject: [PATCH 069/261] The value of the `WriteInflightSize` in the main partition (#8116) (#8142) --- ydb/core/persqueue/partition.cpp | 1 + ydb/core/persqueue/partition_write.cpp | 8 +++++++ .../client/ydb_topic/ut/topic_to_table_ut.cpp | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 1de8209a5607..5d8123b0f982 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -2229,6 +2229,7 @@ void TPartition::CommitWriteOperations(TTransaction& t) }, std::nullopt}; msg.Internal = true; + WriteInflightSize += msg.Msg.Data.size(); ExecRequest(msg, *Parameters, PersistRequest.Get()); auto& info = TxSourceIdForPostPersist[blob.SourceId]; diff --git a/ydb/core/persqueue/partition_write.cpp b/ydb/core/persqueue/partition_write.cpp index 0afb95c2fc42..4d85b65a813a 100644 --- a/ydb/core/persqueue/partition_write.cpp +++ b/ydb/core/persqueue/partition_write.cpp @@ -1122,6 +1122,10 @@ bool TPartition::ExecRequest(TWriteMsg& p, ProcessParameters& parameters, TEvKey auto& sourceIdBatch = parameters.SourceIdBatch; auto sourceId = sourceIdBatch.GetSource(p.Msg.SourceId); + Y_DEBUG_ABORT_UNLESS(WriteInflightSize >= p.Msg.Data.size(), + "PQ %" PRIu64 ", Partition {%" PRIu32 ", %" PRIu32 "}, WriteInflightSize=%" PRIu64 ", p.Msg.Data.size=%" PRISZT, + TabletID, Partition.OriginalPartitionId, Partition.InternalPartitionId, + WriteInflightSize, p.Msg.Data.size()); WriteInflightSize -= p.Msg.Data.size(); TabletCounters.Percentile()[COUNTER_LATENCY_PQ_RECEIVE_QUEUE].IncrementFor(ctx.Now().MilliSeconds() - p.Msg.ReceiveTimestamp); @@ -1540,6 +1544,10 @@ void TPartition::FilterDeadlinedWrites(const TActorContext& ctx, TMessageQueue& TabletCounters.Cumulative()[COUNTER_PQ_WRITE_ERROR].Increment(1); TabletCounters.Cumulative()[COUNTER_PQ_WRITE_BYTES_ERROR].Increment(msg.Data.size() + msg.SourceId.size()); + Y_DEBUG_ABORT_UNLESS(WriteInflightSize >= msg.Data.size(), + "PQ %" PRIu64 ", Partition {%" PRIu32 ", %" PRIu32 "}, WriteInflightSize=%" PRIu64 ", msg.Data.size=%" PRISZT, + TabletID, Partition.OriginalPartitionId, Partition.InternalPartitionId, + WriteInflightSize, msg.Data.size()); WriteInflightSize -= msg.Data.size(); } diff --git a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp b/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp index 010c1d6e5849..224870d2edf1 100644 --- a/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp +++ b/ydb/public/sdk/cpp/client/ydb_topic/ut/topic_to_table_ut.cpp @@ -1898,6 +1898,29 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_27, TFixture) } } +Y_UNIT_TEST_F(WriteToTopic_Demo_28, TFixture) +{ + // The test verifies that the `WriteInflightSize` is correctly considered for the main partition. + // Writing to the service partition does not change the `WriteInflightSize` of the main one. + CreateTopic("topic_A", TEST_CONSUMER); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + TString message(16'000, 'a'); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, TString(16'000, 'a'), &tx, 0); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_1); + + CommitTx(tx, EStatus::SUCCESS); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, TString(20'000, 'b'), nullptr, 0); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_2); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), nullptr, 0); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); +} + } } From f6611d65e951b3c43ee64c87403846c8b91c25c2 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Thu, 22 Aug 2024 22:25:53 +0300 Subject: [PATCH 070/261] Do docker-compose tests in parallel (#8165) --- ydb/core/external_sources/hive_metastore/ut/ya.make | 1 + ydb/core/external_sources/s3/ut/ya.make | 1 + .../generic/connector/tests/datasource/clickhouse/ya.make | 2 +- .../generic/connector/tests/datasource/ms_sql_server/ya.make | 2 +- .../providers/generic/connector/tests/datasource/mysql/ya.make | 2 +- .../providers/generic/connector/tests/datasource/oracle/ya.make | 2 +- .../generic/connector/tests/datasource/postgresql/ya.make | 2 +- .../providers/generic/connector/tests/datasource/ydb/ya.make | 2 +- ydb/library/yql/providers/generic/connector/tests/join/ya.make | 2 +- ydb/tests/fq/generic/ya.make | 2 +- 10 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ydb/core/external_sources/hive_metastore/ut/ya.make b/ydb/core/external_sources/hive_metastore/ut/ya.make index 6fd18fcb47f2..49cb3844651a 100644 --- a/ydb/core/external_sources/hive_metastore/ut/ya.make +++ b/ydb/core/external_sources/hive_metastore/ut/ya.make @@ -29,6 +29,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) diff --git a/ydb/core/external_sources/s3/ut/ya.make b/ydb/core/external_sources/s3/ut/ya.make index 12301fcfe8f5..b9e006d13ec9 100644 --- a/ydb/core/external_sources/s3/ut/ya.make +++ b/ydb/core/external_sources/s3/ut/ya.make @@ -28,6 +28,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make index c7153bf98c25..2efb90299631 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/ya.make @@ -34,6 +34,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -52,7 +53,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() TEST_SRCS( diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make index 42fc2aac49af..9caaafdae3f3 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/ya.make @@ -34,6 +34,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -52,7 +53,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() TEST_SRCS( diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make index 8d5275732ae7..2e6af03f6088 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/ya.make @@ -34,6 +34,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -52,7 +53,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() TEST_SRCS( diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make index 184890bb9740..034e0f2e4d41 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/ya.make @@ -34,6 +34,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -52,7 +53,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() TEST_SRCS( diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make index 950432aec24d..8f9c84aeeb78 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/ya.make @@ -34,6 +34,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -52,7 +53,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() TEST_SRCS( diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make index cab225427e95..1f6397637f33 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/ya.make @@ -34,6 +34,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -52,7 +53,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() TEST_SRCS( diff --git a/ydb/library/yql/providers/generic/connector/tests/join/ya.make b/ydb/library/yql/providers/generic/connector/tests/join/ya.make index d8aa54bc8173..413c782ab3ac 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/ya.make +++ b/ydb/library/yql/providers/generic/connector/tests/join/ya.make @@ -34,6 +34,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -52,7 +53,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() TEST_SRCS( diff --git a/ydb/tests/fq/generic/ya.make b/ydb/tests/fq/generic/ya.make index 09e62fd560e7..a4b89c247483 100644 --- a/ydb/tests/fq/generic/ya.make +++ b/ydb/tests/fq/generic/ya.make @@ -30,6 +30,7 @@ IF (AUTOCHECK) ) ENDIF() +ENV(COMPOSE_HTTP_TIMEOUT=1200) # during parallel tests execution there could be huge disk io, which triggers timeouts in docker-compose INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) IF (OPENSOURCE) @@ -48,7 +49,6 @@ IF (OPENSOURCE) # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. # See DEVTOOLSSUPPORT-44103, YA-1759 for details. TAG(ya:not_autocheck) - REQUIREMENTS(cpu:all) ENDIF() DEPENDS( From 5f7177dc524e6a4c7324f7408de384da1f7f67ac Mon Sep 17 00:00:00 2001 From: Vitaly Isaev Date: Thu, 22 Aug 2024 23:21:36 +0300 Subject: [PATCH 071/261] Update github.com/ydb-platform/fq-connector-go to 0.5.5-rc.2 (#8183) --- .../connector/tests/datasource/clickhouse/docker-compose.yml | 2 +- .../connector/tests/datasource/ms_sql_server/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/mysql/docker-compose.yml | 2 +- .../connector/tests/datasource/oracle/docker-compose.yml | 2 +- .../connector/tests/datasource/postgresql/docker-compose.yml | 2 +- .../generic/connector/tests/datasource/ydb/docker-compose.yml | 2 +- .../providers/generic/connector/tests/join/docker-compose.yml | 2 +- ydb/tests/fq/generic/docker-compose.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml index f5c85822772f..1d56c724e1c4 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/clickhouse/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-ch-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml index f3f0d8528f42..cdc3e53f678f 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ms_sql_server/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mssql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml index d828f70ea5cf..d5d8d6e44fcd 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/mysql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-mysql-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml index ed29ad70d07c..8ddbc979cf0c 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/oracle/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-oracle-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml index 74c98a3ad1a5..747a58cb4ce3 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/postgresql/docker-compose.yml @@ -1,7 +1,7 @@ services: fq-connector-go: container_name: fq-tests-pg-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml index a3c3303a74e0..2b475de150a7 100644 --- a/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/datasource/ydb/docker-compose.yml @@ -5,7 +5,7 @@ services: echo \"$$(dig fq-tests-ydb-ydb +short) fq-tests-ydb-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: fq-tests-ydb-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - 2130 volumes: diff --git a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml index b824f638c024..db0a5589ef1d 100644 --- a/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/join/docker-compose.yml @@ -12,7 +12,7 @@ services: - 8123 fq-connector-go: container_name: fq-tests-join-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - 2130 volumes: diff --git a/ydb/tests/fq/generic/docker-compose.yml b/ydb/tests/fq/generic/docker-compose.yml index 915a7ae0fdec..042d924109cd 100644 --- a/ydb/tests/fq/generic/docker-compose.yml +++ b/ydb/tests/fq/generic/docker-compose.yml @@ -15,7 +15,7 @@ services: echo \"$$(dig tests-fq-generic-ydb +short) tests-fq-generic-ydb\" >> /etc/hosts; cat /etc/hosts; /opt/ydb/bin/fq-connector-go server -c /opt/ydb/cfg/fq-connector-go.yaml" container_name: tests-fq-generic-fq-connector-go - image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.1@sha256:06a5685dd13a4263e25043300af09140c8227dee7d69f46954cd4b7dee401d05 + image: ghcr.io/ydb-platform/fq-connector-go:v0.5.5-rc.2@sha256:eac7a3107f4357d515573dd0907c76227726ffe54e612e580d52c33488b06ef5 ports: - "2130" postgresql: From 97f5b3b14f6701e23a76db5df949c7d2da865263 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Fri, 23 Aug 2024 02:16:49 +0300 Subject: [PATCH 072/261] Revert "ci: upgrade actions/checkout@v4, actions/github-script@v7, add test_retry_count input for build_and_test_ya action" (#8185) --- .github/actions/build_and_test_ya/action.yml | 4 ---- .github/workflows/acceptance_run.yml | 2 +- .github/workflows/allowed_dirs.yml | 2 +- .github/workflows/build_analytics.yml | 2 +- .github/workflows/build_and_test_provisioned.yml | 2 +- .github/workflows/build_and_test_ya.yml | 2 +- .github/workflows/collect_analytics.yml | 2 +- .github/workflows/docs_build.yaml | 2 +- .github/workflows/docs_release.yaml | 2 +- .github/workflows/postcommit_asan.yml | 2 +- .github/workflows/postcommit_relwithdebinfo.yml | 2 +- .github/workflows/pr_check.yml | 10 +++++----- .github/workflows/pr_labels.yaml | 2 +- .github/workflows/prewarm-ccache.yml | 2 +- 14 files changed, 17 insertions(+), 21 deletions(-) diff --git a/.github/actions/build_and_test_ya/action.yml b/.github/actions/build_and_test_ya/action.yml index 1b2b41262a91..6b921e4c5cfe 100644 --- a/.github/actions/build_and_test_ya/action.yml +++ b/.github/actions/build_and_test_ya/action.yml @@ -42,9 +42,6 @@ inputs: additional_ya_make_args: type: string default: "" - test_retry_count: - default: "" - description: "how many times to retry failed tests" secs: type: string default: "" @@ -127,7 +124,6 @@ runs: bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }} bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }} put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }} - test_retry_count: ${{ inputs.test_retry_count }} - name: build_stats shell: bash diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index d0f9d131c08d..02fb4edbc122 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -29,7 +29,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: ref: ${{ inputs.commit_sha }} diff --git a/.github/workflows/allowed_dirs.yml b/.github/workflows/allowed_dirs.yml index bcb4a59c0167..98167cb10a04 100644 --- a/.github/workflows/allowed_dirs.yml +++ b/.github/workflows/allowed_dirs.yml @@ -11,7 +11,7 @@ jobs: cancel-in-progress: true runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - name: Check dirs run: ${{github.workspace}}/.github/check_dirs.sh ${{github.workspace}} diff --git a/.github/workflows/build_analytics.yml b/.github/workflows/build_analytics.yml index d0614dfc6637..5f97b6e2ce53 100644 --- a/.github/workflows/build_analytics.yml +++ b/.github/workflows/build_analytics.yml @@ -33,7 +33,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: ref: ${{ inputs.commit_sha }} diff --git a/.github/workflows/build_and_test_provisioned.yml b/.github/workflows/build_and_test_provisioned.yml index 28958c015d2d..9a9b7a8c9b93 100644 --- a/.github/workflows/build_and_test_provisioned.yml +++ b/.github/workflows/build_and_test_provisioned.yml @@ -48,7 +48,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label }}" ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: ref: ${{ inputs.checkout_ref }} - name: Build diff --git a/.github/workflows/build_and_test_ya.yml b/.github/workflows/build_and_test_ya.yml index 877f0d2667d7..10d9723330c5 100644 --- a/.github/workflows/build_and_test_ya.yml +++ b/.github/workflows/build_and_test_ya.yml @@ -65,7 +65,7 @@ jobs: runs-on: [ self-hosted, "${{ inputs.runner_label }}", "${{ inputs.runner_additional_label || inputs.runner_label }}"] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: ref: ${{ inputs.commit_sha }} diff --git a/.github/workflows/collect_analytics.yml b/.github/workflows/collect_analytics.yml index eda7e83df403..5473986512ec 100644 --- a/.github/workflows/collect_analytics.yml +++ b/.github/workflows/collect_analytics.yml @@ -17,7 +17,7 @@ jobs: runs-on: [ self-hosted ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: ref: ${{ inputs.commit_sha }} - name: Setup ydb access diff --git a/.github/workflows/docs_build.yaml b/.github/workflows/docs_build.yaml index 4b632912b589..1a2ef9a9d77d 100644 --- a/.github/workflows/docs_build.yaml +++ b/.github/workflows/docs_build.yaml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} - name: Build diff --git a/.github/workflows/docs_release.yaml b/.github/workflows/docs_release.yaml index 1560e6d8dd61..0c7ae9da9f82 100644 --- a/.github/workflows/docs_release.yaml +++ b/.github/workflows/docs_release.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: Extract version shell: bash run: echo "version=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" | sed -e 's|stable-|v|g' -e 's|-|.|g' >> $GITHUB_OUTPUT diff --git a/.github/workflows/postcommit_asan.yml b/.github/workflows/postcommit_asan.yml index b49551e327d8..3465ec17fd5c 100644 --- a/.github/workflows/postcommit_asan.yml +++ b/.github/workflows/postcommit_asan.yml @@ -15,7 +15,7 @@ jobs: name: Build and test release-asan steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 2 - name: Setup ydb access diff --git a/.github/workflows/postcommit_relwithdebinfo.yml b/.github/workflows/postcommit_relwithdebinfo.yml index aca0428c2b03..a5dccc8df82a 100644 --- a/.github/workflows/postcommit_relwithdebinfo.yml +++ b/.github/workflows/postcommit_relwithdebinfo.yml @@ -14,7 +14,7 @@ jobs: name: Build and test relwithdebinfo steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: fetch-depth: 2 - name: Setup ydb access diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index 95a7515b338e..d414682b3726 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -30,7 +30,7 @@ jobs: - name: Check if running tests is allowed id: check-ownership-membership - uses: actions/github-script@v7 + uses: actions/github-script@v6 with: github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} script: | @@ -81,7 +81,7 @@ jobs: - name: comment-if-waiting-on-ok if: steps.check-ownership-membership.outputs.result == 'false' && github.event.action == 'opened' - uses: actions/github-script@v7 + uses: actions/github-script@v6 with: script: | let externalContributorLabel = 'external'; @@ -100,7 +100,7 @@ jobs: }); - name: cleanup-test-label - uses: actions/github-script@v7 + uses: actions/github-script@v6 with: script: | let labelsToRemove = ['ok-to-test', 'rebase-and-check']; @@ -126,7 +126,7 @@ jobs: - name: check is mergeable id: check-is-mergeable if: steps.check-ownership-membership.outputs.result == 'true' - uses: actions/github-script@v7 + uses: actions/github-script@v6 with: result-encoding: string script: | @@ -210,7 +210,7 @@ jobs: name: Build and test ${{ matrix.build_preset }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 with: ref: ${{ needs.check-running-allowed.outputs.commit_sha }} fetch-depth: 2 diff --git a/.github/workflows/pr_labels.yaml b/.github/workflows/pr_labels.yaml index efcf96c9dc9b..032994cecb07 100644 --- a/.github/workflows/pr_labels.yaml +++ b/.github/workflows/pr_labels.yaml @@ -15,7 +15,7 @@ jobs: steps: - name: Update PR labels id: update-pr-labels - uses: actions/github-script@v7 + uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/prewarm-ccache.yml b/.github/workflows/prewarm-ccache.yml index 080849d97ab9..dc86bb83f49d 100644 --- a/.github/workflows/prewarm-ccache.yml +++ b/.github/workflows/prewarm-ccache.yml @@ -13,7 +13,7 @@ jobs: version: ["ubuntu-2204", "ubuntu-2004", "ubuntu-1804"] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v3 - name: build shell: bash run: | From dabebe011f23472b1e3aa1e94bfbb8ca0e8c0731 Mon Sep 17 00:00:00 2001 From: pilik Date: Fri, 23 Aug 2024 09:55:43 +0300 Subject: [PATCH 073/261] [KQP] ANALYZE retries has been added. (#8115) --- ydb/core/kqp/gateway/actors/analyze_actor.cpp | 112 ++++++++---------- ydb/core/kqp/gateway/actors/analyze_actor.h | 27 +++-- 2 files changed, 70 insertions(+), 69 deletions(-) diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.cpp b/ydb/core/kqp/gateway/actors/analyze_actor.cpp index 2ababa6954bc..e9384a78e8ba 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.cpp +++ b/ydb/core/kqp/gateway/actors/analyze_actor.cpp @@ -1,7 +1,6 @@ #include "analyze_actor.h" #include -#include #include #include #include @@ -42,19 +41,6 @@ void TAnalyzeActor::Bootstrap() { Become(&TAnalyzeActor::StateWork); } -void TAnalyzeActor::SendAnalyzeStatus() { - Y_ABORT_UNLESS(StatisticsAggregatorId.has_value()); - - auto getStatus = std::make_unique(); - auto& record = getStatus->Record; - record.SetOperationId(OperationId); - - Send( - MakePipePerNodeCacheID(false), - new TEvPipeCache::TEvForward(getStatus.release(), StatisticsAggregatorId.value(), true) - ); -} - void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ctx); @@ -67,50 +53,10 @@ void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, c << " , but expected " << OperationId); } - - // TODO Don't send EvAnalyzeStatus, EvAnalyzeResponse is already here - SendAnalyzeStatus(); -} - -void TAnalyzeActor::Handle(TEvAnalyzePrivate::TEvAnalyzeStatusCheck::TPtr& ev, const TActorContext& ctx) { - Y_UNUSED(ev); - Y_UNUSED(ctx); - - SendAnalyzeStatus(); -} - -void TAnalyzeActor::Handle(NStat::TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev, const TActorContext& ctx) { - auto& record = ev->Get()->Record; - switch (record.GetStatus()) { - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_UNSPECIFIED: { - Promise.SetValue( - NYql::NCommon::ResultFromError( - YqlIssue( - {}, NYql::TIssuesIds::UNEXPECTED, - TStringBuilder() << "Statistics Aggregator unspecified error" - ) - ) - ); - this->Die(ctx); - return; - } - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_NO_OPERATION: { - NYql::IKikimrGateway::TGenericResult result; - result.SetSuccess(); - Promise.SetValue(std::move(result)); - - this->Die(ctx); - return; - } - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_ENQUEUED: { - Schedule(TDuration::Seconds(10), new TEvAnalyzePrivate::TEvAnalyzeStatusCheck()); - return; - } - case NKikimrStat::TEvAnalyzeStatusResponse::STATUS_IN_PROGRESS: { - Schedule(TDuration::Seconds(5), new TEvAnalyzePrivate::TEvAnalyzeStatusCheck()); - return; - } - } + NYql::IKikimrGateway::TGenericResult result; + result.SetSuccess(); + Promise.SetValue(std::move(result)); + this->Die(ctx); } void TAnalyzeActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx) { @@ -188,13 +134,56 @@ void TAnalyzeActor::Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& } } +TDuration TAnalyzeActor::CalcBackoffTime() { + ui32 backoffSlots = 1 << RetryCount; + TDuration maxDuration = RetryInterval * backoffSlots; + + double uncertaintyRatio = std::max(std::min(UncertainRatio, 1.0), 0.0); + double uncertaintyMultiplier = RandomNumber() * uncertaintyRatio - uncertaintyRatio + 1.0; + + double durationMs = round(maxDuration.MilliSeconds() * uncertaintyMultiplier); + durationMs = std::max(std::min(durationMs, MaxBackoffDurationMs), 0.0); + return TDuration::MilliSeconds(durationMs); +} + +void TAnalyzeActor::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev, ctx); + + if (RetryCount >= MaxRetryCount) { + Promise.SetValue( + NYql::NCommon::ResultFromError( + YqlIssue( + {}, NYql::TIssuesIds::UNEXPECTED, + TStringBuilder() << "Can't establish connection with the Statistics Aggregator!" + ) + ) + ); + this->Die(ctx); + return; + } + + ++RetryCount; + Schedule(CalcBackoffTime(), new TEvAnalyzePrivate::TEvAnalyzeRetry()); +} + +void TAnalyzeActor::Handle(TEvAnalyzePrivate::TEvAnalyzeRetry::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ev, ctx); + + auto analyzeRequest = std::make_unique(); + analyzeRequest->Record = Request.Record; + Send( + MakePipePerNodeCacheID(false), + new TEvPipeCache::TEvForward(analyzeRequest.release(), StatisticsAggregatorId.value(), true), + IEventHandle::FlagTrackDelivery + ); +} + void TAnalyzeActor::SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeCacheNavigate::TEntry& entry, const TActorContext& ctx) { Y_ABORT_UNLESS(entry.DomainInfo->Params.HasStatisticsAggregator()); StatisticsAggregatorId = entry.DomainInfo->Params.GetStatisticsAggregator(); - auto analyzeRequest = std::make_unique(); - auto& record = analyzeRequest->Record; + auto& record = Request.Record; record.SetOperationId(OperationId); auto table = record.AddTables(); @@ -223,7 +212,8 @@ void TAnalyzeActor::SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeC *table->MutableColumnTags()->Add() = tagByColumnName[columnName]; } - // TODO This request should be retried if StatisticsAggregator fails + auto analyzeRequest = std::make_unique(); + analyzeRequest->Record = Request.Record; Send( MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(analyzeRequest.release(), entry.DomainInfo->Params.GetStatisticsAggregator(), true), diff --git a/ydb/core/kqp/gateway/actors/analyze_actor.h b/ydb/core/kqp/gateway/actors/analyze_actor.h index 8ea018c63d94..f59fba90c2b1 100644 --- a/ydb/core/kqp/gateway/actors/analyze_actor.h +++ b/ydb/core/kqp/gateway/actors/analyze_actor.h @@ -3,6 +3,7 @@ #include #include +#include #include @@ -11,11 +12,11 @@ namespace NKikimr::NKqp { struct TEvAnalyzePrivate { enum EEv { - EvAnalyzeStatusCheck = EventSpaceBegin(TEvents::ES_PRIVATE), + EvAnalyzeRetry = EventSpaceBegin(TEvents::ES_PRIVATE), EvEnd }; - struct TEvAnalyzeStatusCheck : public TEventLocal {}; + struct TEvAnalyzeRetry : public TEventLocal {}; }; class TAnalyzeActor : public NActors::TActorBootstrapped { @@ -28,8 +29,8 @@ class TAnalyzeActor : public NActors::TActorBootstrapped { switch(ev->GetTypeRewrite()) { HFunc(TEvTxProxySchemeCache::TEvNavigateKeySetResult, Handle); HFunc(NStat::TEvStatistics::TEvAnalyzeResponse, Handle); - HFunc(NStat::TEvStatistics::TEvAnalyzeStatusResponse, Handle); - HFunc(TEvAnalyzePrivate::TEvAnalyzeStatusCheck, Handle); + HFunc(TEvPipeCache::TEvDeliveryProblem, Handle); + HFunc(TEvAnalyzePrivate::TEvAnalyzeRetry, Handle); default: HandleUnexpectedEvent(ev->GetTypeRewrite()); } @@ -38,17 +39,18 @@ class TAnalyzeActor : public NActors::TActorBootstrapped { private: void Handle(NStat::TEvStatistics::TEvAnalyzeResponse::TPtr& ev, const TActorContext& ctx); - void Handle(NStat::TEvStatistics::TEvAnalyzeStatusResponse::TPtr& ev, const TActorContext& ctx); - void Handle(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev, const TActorContext& ctx); - void Handle(TEvAnalyzePrivate::TEvAnalyzeStatusCheck::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev, const TActorContext& ctx); + + void Handle(TEvAnalyzePrivate::TEvAnalyzeRetry::TPtr& ev, const TActorContext& ctx); void HandleUnexpectedEvent(ui32 typeRewrite); +private: void SendStatisticsAggregatorAnalyze(const NSchemeCache::TSchemeCacheNavigate::TEntry&, const TActorContext&); - void SendAnalyzeStatus(); + TDuration CalcBackoffTime(); private: TString TablePath; @@ -58,6 +60,15 @@ class TAnalyzeActor : public NActors::TActorBootstrapped { std::optional StatisticsAggregatorId; TPathId PathId; TString OperationId; + + // for retries + NStat::TEvStatistics::TEvAnalyze Request; + TDuration RetryInterval = TDuration::MilliSeconds(5); + size_t RetryCount = 0; + + constexpr static size_t MaxRetryCount = 10; + constexpr static double UncertainRatio = 0.5; + constexpr static double MaxBackoffDurationMs = TDuration::Seconds(15).MilliSeconds(); }; } // end of NKikimr::NKqp From a1bf3f37c32e1710f93ec4622b51701d8704067b Mon Sep 17 00:00:00 2001 From: Vlad Kuznetsov Date: Fri, 23 Aug 2024 10:10:04 +0200 Subject: [PATCH 074/261] Fix counter names in group_write load actor (#8058) --- ydb/core/load_test/group_write.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ydb/core/load_test/group_write.cpp b/ydb/core/load_test/group_write.cpp index 11d1d3b21baf..71467399d0ee 100644 --- a/ydb/core/load_test/group_write.cpp +++ b/ydb/core/load_test/group_write.cpp @@ -500,10 +500,10 @@ class TLogWriterLoadTestActor : public TActorBootstrappedGetCounter("tabletId") = tabletId; - const auto& percCounters = Counters->GetSubgroup("sensor", "microseconds"); - MaxInFlightLatency = percCounters->GetCounter("MaxInFlightLatency"); - ResponseQT->Initialize(percCounters->GetSubgroup("metric", "writeResponse"), Percentiles); - ReadResponseQT->Initialize(percCounters->GetSubgroup("metric", "readResponse"), Percentiles); + const auto& percCounters = Counters->GetSubgroup("subsystem", "latency"); + MaxInFlightLatency = percCounters->GetCounter("MaxInFlightLatencyUs"); + ResponseQT->Initialize(percCounters->GetSubgroup("sensor", "writeResponseUs"), Percentiles); + ReadResponseQT->Initialize(percCounters->GetSubgroup("sensor", "readResponseUs"), Percentiles); } TString PrintMe() { From 528bde451e4bf4e1cc2546fc98bd4c15b6511cf5 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Fri, 23 Aug 2024 11:51:02 +0300 Subject: [PATCH 075/261] Added sparsed columns tests for standalone tables (#8167) --- ydb/core/kqp/ut/olap/helpers/typed_local.h | 2 +- ydb/core/kqp/ut/olap/sparsed_ut.cpp | 56 +++++++++++++++++----- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/ydb/core/kqp/ut/olap/helpers/typed_local.h b/ydb/core/kqp/ut/olap/helpers/typed_local.h index a72cef64e33e..0ae2c1a544ad 100644 --- a/ydb/core/kqp/ut/olap/helpers/typed_local.h +++ b/ydb/core/kqp/ut/olap/helpers/typed_local.h @@ -29,7 +29,7 @@ class TTypedLocalHelper: public Tests::NCS::THelper { : TBase(kikimrRunner.GetTestServer()) , TypeName(typeName) , KikimrRunner(kikimrRunner) - , TablePath("/Root/" + storeName + "/" + tableName) + , TablePath(storeName.empty() ? "/Root/" + tableName : "/Root/" + storeName + "/" + tableName) , TableName(tableName) , StoreName(storeName) { SetShardingMethod("HASH_FUNCTION_CONSISTENCY_64"); diff --git a/ydb/core/kqp/ut/olap/sparsed_ut.cpp b/ydb/core/kqp/ut/olap/sparsed_ut.cpp index 88bc081fa357..cc40397496ee 100644 --- a/ydb/core/kqp/ut/olap/sparsed_ut.cpp +++ b/ydb/core/kqp/ut/olap/sparsed_ut.cpp @@ -18,10 +18,12 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { const TKikimrSettings Settings = TKikimrSettings().SetWithSampleTables(false); TKikimrRunner Kikimr; NKikimr::NYDBTest::TControllers::TGuard CSController; + const TString StoreName; public: - TSparsedDataTest() + TSparsedDataTest(const char* storeName) : Kikimr(Settings) , CSController(NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard()) + , StoreName(storeName) { } @@ -30,8 +32,7 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { auto selectQuery = TString(R"( SELECT count(*) as count, - FROM `/Root/olapStore/olapTable` - )"); + FROM `/Root/)") + (StoreName.empty() ? "" : StoreName + "/") + "olapTable`"; auto tableClient = Kikimr.GetTableClient(); auto rows = ExecuteScanQuery(tableClient, selectQuery); @@ -42,9 +43,9 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { auto selectQuery = TString(R"( SELECT count(*) as count, - FROM `/Root/olapStore/olapTable` + FROM `/Root/)") + (StoreName.empty() ? "" : StoreName + "/") + R"(olapTable` WHERE field == 'abcde' - )"); + )"; auto tableClient = Kikimr.GetTableClient(); auto rows = ExecuteScanQuery(tableClient, selectQuery); @@ -52,7 +53,7 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { } void FillCircle(const double shiftKff, const ui32 countExpectation) { - TTypedLocalHelper helper("Utf8", Kikimr); + TTypedLocalHelper helper("Utf8", Kikimr, "olapTable", StoreName); const double frq = 0.9; { NArrow::NConstruction::TStringPoolFiller sPool(1000, 52, "abcde", frq); @@ -98,23 +99,52 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); Tests::NCommon::TLoggerInit(Kikimr).Initialize(); - TTypedLocalHelper helper("Utf8", Kikimr); - helper.CreateTestOlapTable(); + TTypedLocalHelper helper("Utf8", Kikimr, "olapTable", StoreName); + if (!StoreName.empty()) { + helper.CreateTestOlapTable(); + } else { + auto tableClient = Kikimr.GetTableClient(); + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + + auto query = TStringBuilder() << R"( + --!syntax_v1 + CREATE TABLE `/Root/olapTable` + ( + pk_int int64 NOT NULL, + field )" << "Utf8" << R"(, + ts TimeStamp, + PRIMARY KEY (pk_int) + ) + PARTITION BY HASH(pk_int) + WITH ( + STORE = COLUMN + ))"; + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + } + + TString type = StoreName.empty() ? "TABLE" : "TABLESTORE"; + TString name = StoreName.empty() ? "olapTable" : "olapStore"; FillCircle(0, 10000); - helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`SPARSED`, `DEFAULT_VALUE`=`abcde`);"); + helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/" + name + "`(TYPE " + type + ") SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`SPARSED`, `DEFAULT_VALUE`=`abcde`);"); FillCircle(0.1, 11000); - helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`PLAIN`);"); + helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/" + name + "`(TYPE " + type + ") SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`PLAIN`);"); FillCircle(0.2, 12000); - helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`SPARSED`);"); + helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/" + name + "`(TYPE " + type + ") SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`SPARSED`);"); FillCircle(0.3, 13000); - helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`PLAIN`);"); + helper.ExecuteSchemeQuery("ALTER OBJECT `/Root/" + name + "`(TYPE " + type + ") SET (ACTION=ALTER_COLUMN, NAME=field, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`PLAIN`);"); FillCircle(0.4, 14000); } }; Y_UNIT_TEST(Switching) { - TSparsedDataTest test; + TSparsedDataTest test("olapStore"); + test.Execute(); + } + + Y_UNIT_TEST(SwitchingStandalone) { + TSparsedDataTest test(""); test.Execute(); } } From 9dc1b51219594239f6c4b86e1b94d9d186013cc6 Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Fri, 23 Aug 2024 12:02:13 +0300 Subject: [PATCH 076/261] Removed weird restriction for optional pg (#8175) --- ydb/library/yql/ast/yql_type_string.cpp | 7 ------- ydb/library/yql/ast/yql_type_string_ut.cpp | 5 +++++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ydb/library/yql/ast/yql_type_string.cpp b/ydb/library/yql/ast/yql_type_string.cpp index d693299f39aa..9bbfb12bf6f3 100644 --- a/ydb/library/yql/ast/yql_type_string.cpp +++ b/ydb/library/yql/ast/yql_type_string.cpp @@ -186,7 +186,6 @@ class TTypeParser private: TAstNode* ParseType() { - bool isPgType = false; TAstNode* type = nullptr; switch (Token) { @@ -324,13 +323,11 @@ class TTypeParser if (id.SkipPrefix("pg")) { if (NPg::HasType(TString(id))) { type = MakePgType(id); - isPgType = true; GetNextToken(); } } else if (id.SkipPrefix("_pg")) { if (NPg::HasType(TString(id)) && !id.StartsWith('_')) { type = MakePgType(TString("_") + id); - isPgType = true; GetNextToken(); } } @@ -342,10 +339,6 @@ class TTypeParser if (type) { while (Token == '?') { - if (isPgType) { - return AddError(TString("PG type can't be wrapped into Optional type")); - } - type = MakeOptionalType(type); GetNextToken(); } diff --git a/ydb/library/yql/ast/yql_type_string_ut.cpp b/ydb/library/yql/ast/yql_type_string_ut.cpp index 655d7a60910e..9b6db5f11e30 100644 --- a/ydb/library/yql/ast/yql_type_string_ut.cpp +++ b/ydb/library/yql/ast/yql_type_string_ut.cpp @@ -678,4 +678,9 @@ Y_UNIT_TEST_SUITE(TTypeString) TestFormat("((Pg int4))", "pgint4"); TestFormat("((Pg _int4))", "_pgint4"); } + + Y_UNIT_TEST(FormatOptionalPg) { + TestFormat("((Optional (Pg int4)))", "Optional"); + TestFormat("((Optional (Pg _int4)))", "Optional<_pgint4>"); + } } From e0bf421630e750725f31171eeab9cab1320d8865 Mon Sep 17 00:00:00 2001 From: azevaykin <145343289+azevaykin@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:12:28 +0300 Subject: [PATCH 077/261] Statistics: retry AnalyzeTable to ColumnShard (#8190) --- .../counters_statistics_aggregator.proto | 1 + .../statistics/aggregator/aggregator_impl.cpp | 13 +++++ .../statistics/aggregator/aggregator_impl.h | 7 +++ .../tx_analyze_table_delivery_problem.cpp | 48 +++++++++++++++++++ ydb/core/statistics/aggregator/tx_init.cpp | 1 + .../aggregator/ut/ut_analyze_columnshard.cpp | 19 +++++++- ydb/core/statistics/aggregator/ya.make | 1 + 7 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp diff --git a/ydb/core/protos/counters_statistics_aggregator.proto b/ydb/core/protos/counters_statistics_aggregator.proto index eb263bab214c..0947ca795180 100644 --- a/ydb/core/protos/counters_statistics_aggregator.proto +++ b/ydb/core/protos/counters_statistics_aggregator.proto @@ -22,4 +22,5 @@ enum ETxTypes { TXTYPE_ACK_TIMEOUT = 12 [(TxTypeOpts) = {Name: "TxAckTimeout"}]; TXTYPE_ANALYZE_TABLE_REQUEST = 13 [(TxTypeOpts) = {Name: "TxAnalyzeTableRequest"}]; TXTYPE_ANALYZE_TABLE_RESPONSE = 14 [(TxTypeOpts) = {Name: "TxAnalyzeTableResponse"}]; + TXTYPE_ANALYZE_TABLE_DELIVERY_PROBLEM = 15 [(TxTypeOpts) = {Name: "TTxAnalyzeTableDeliveryProblem"}]; } diff --git a/ydb/core/statistics/aggregator/aggregator_impl.cpp b/ydb/core/statistics/aggregator/aggregator_impl.cpp index 98dde3778cc3..baedc73b0c0b 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.cpp +++ b/ydb/core/statistics/aggregator/aggregator_impl.cpp @@ -404,11 +404,24 @@ void TStatisticsAggregator::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { auto tabletId = ev->Get()->TabletId; if (TraversalIsColumnTable) { if (tabletId == HiveId) { + SA_LOG_E("[" << TabletID() << "] TEvDeliveryProblem with HiveId=" << tabletId); Schedule(HiveRetryInterval, new TEvPrivate::TEvRequestDistribution); } else { + for (TForceTraversalOperation& operation : ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + for (TAnalyzedShard& shard : operationTable.AnalyzedShards) { + if (shard.ShardTabletId == tabletId) { + SA_LOG_E("[" << TabletID() << "] TEvDeliveryProblem with ColumnShard=" << tabletId); + shard.Status = TAnalyzedShard::EStatus::DeliveryProblem; + return; + } + } + } + } SA_LOG_CRIT("[" << TabletID() << "] TEvDeliveryProblem with unexpected tablet " << tabletId); } } else { + SA_LOG_E("[" << TabletID() << "] TEvDeliveryProblem with DataShard=" << tabletId); if (DatashardRanges.empty()) { return; } diff --git a/ydb/core/statistics/aggregator/aggregator_impl.h b/ydb/core/statistics/aggregator/aggregator_impl.h index e6b63b79e37f..f0419f7f2e0a 100644 --- a/ydb/core/statistics/aggregator/aggregator_impl.h +++ b/ydb/core/statistics/aggregator/aggregator_impl.h @@ -48,6 +48,7 @@ class TStatisticsAggregator : public TActor, public NTabl struct TTxAnalyze; struct TTxAnalyzeTableRequest; struct TTxAnalyzeTableResponse; + struct TTxAnalyzeTableDeliveryProblem; struct TTxNavigate; struct TTxResolve; struct TTxDatashardScanResponse; @@ -68,6 +69,7 @@ class TStatisticsAggregator : public TActor, public NTabl EvResolve, EvAckTimeout, EvSendAnalyze, + EvAnalyzeDeliveryProblem, EvEnd }; @@ -80,6 +82,7 @@ class TStatisticsAggregator : public TActor, public NTabl struct TEvRequestDistribution : public TEventLocal {}; struct TEvResolve : public TEventLocal {}; struct TEvSendAnalyze : public TEventLocal {}; + struct TEvAnalyzeDeliveryProblem : public TEventLocal {}; struct TEvAckTimeout : public TEventLocal { size_t SeqNo = 0; @@ -142,6 +145,7 @@ class TStatisticsAggregator : public TActor, public NTabl void Handle(TEvStatistics::TEvAggregateKeepAlive::TPtr& ev); void Handle(TEvPrivate::TEvAckTimeout::TPtr& ev); void Handle(TEvPrivate::TEvSendAnalyze::TPtr& ev); + void Handle(TEvPrivate::TEvAnalyzeDeliveryProblem::TPtr& ev); void InitializeStatisticsTable(); void Navigate(); @@ -204,6 +208,7 @@ class TStatisticsAggregator : public TActor, public NTabl hFunc(TEvStatistics::TEvAggregateKeepAlive, Handle); hFunc(TEvPrivate::TEvAckTimeout, Handle); hFunc(TEvPrivate::TEvSendAnalyze, Handle); + hFunc(TEvPrivate::TEvAnalyzeDeliveryProblem, Handle); default: if (!HandleDefaultEvents(ev, SelfId())) { @@ -311,6 +316,7 @@ class TStatisticsAggregator : public TActor, public NTabl static constexpr size_t SendAnalyzeCount = 100; static constexpr TDuration SendAnalyzePeriod = TDuration::Seconds(1); + static constexpr TDuration AnalyzeDeliveryProblemPeriod = TDuration::Seconds(1); enum ENavigateType { Analyze, @@ -348,6 +354,7 @@ class TStatisticsAggregator : public TActor, public NTabl enum class EStatus : ui8 { None, + DeliveryProblem, AnalyzeStarted, AnalyzeFinished, }; diff --git a/ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp b/ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp new file mode 100644 index 000000000000..b975069797e6 --- /dev/null +++ b/ydb/core/statistics/aggregator/tx_analyze_table_delivery_problem.cpp @@ -0,0 +1,48 @@ +#include "aggregator_impl.h" + +#include +#include + +#include + +namespace NKikimr::NStat { + +struct TStatisticsAggregator::TTxAnalyzeTableDeliveryProblem : public TTxBase { + std::vector> Events; + + TTxAnalyzeTableDeliveryProblem(TSelf* self) + : TTxBase(self) + {} + + TTxType GetTxType() const override { return TXTYPE_ANALYZE_TABLE_DELIVERY_PROBLEM; } + + bool Execute(TTransactionContext&, const TActorContext&) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeTableDeliveryProblem::Execute"); + + for (TForceTraversalOperation& operation : Self->ForceTraversals) { + for (TForceTraversalTable& operationTable : operation.Tables) { + for(TAnalyzedShard& analyzedShard : operationTable.AnalyzedShards) { + if (analyzedShard.Status == TAnalyzedShard::EStatus::DeliveryProblem) { + SA_LOG_D("[" << Self->TabletID() << "] Reset DeliveryProblem to ColumnShard=" << analyzedShard.ShardTabletId); + analyzedShard.Status = TAnalyzedShard::EStatus::None; + } + } + } + } + + return true; + } + + void Complete(const TActorContext& ctx) override { + SA_LOG_T("[" << Self->TabletID() << "] TTxAnalyzeTableDeliveryProblem::Complete"); + + ctx.Schedule(AnalyzeDeliveryProblemPeriod, new TEvPrivate::TEvAnalyzeDeliveryProblem()); + } +}; + +void TStatisticsAggregator::Handle(TEvPrivate::TEvAnalyzeDeliveryProblem::TPtr&) { + Execute(new TTxAnalyzeTableDeliveryProblem(this), + TActivationContext::AsActorContext()); +} + +} // NKikimr::NStat diff --git a/ydb/core/statistics/aggregator/tx_init.cpp b/ydb/core/statistics/aggregator/tx_init.cpp index d97521eda571..2a774d413206 100644 --- a/ydb/core/statistics/aggregator/tx_init.cpp +++ b/ydb/core/statistics/aggregator/tx_init.cpp @@ -274,6 +274,7 @@ struct TStatisticsAggregator::TTxInit : public TTxBase { if (Self->EnableColumnStatistics) { Self->Schedule(Self->TraversalPeriod, new TEvPrivate::TEvScheduleTraversal()); Self->Schedule(Self->SendAnalyzePeriod, new TEvPrivate::TEvSendAnalyze()); + Self->Schedule(Self->AnalyzeDeliveryProblemPeriod, new TEvPrivate::TEvAnalyzeDeliveryProblem()); } else { SA_LOG_W("[" << Self->TabletID() << "] TTxInit::Complete. EnableColumnStatistics=false"); } diff --git a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp index 6b5beea5ace7..6edffe5e31dc 100644 --- a/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp +++ b/ydb/core/statistics/aggregator/ut/ut_analyze_columnshard.cpp @@ -4,7 +4,7 @@ #include -#include +#include namespace NKikimr { namespace NStat { @@ -226,6 +226,23 @@ Y_UNIT_TEST_SUITE(AnalyzeColumnshard) { runtime.GrabEdgeEventRethrow(sender); } + Y_UNIT_TEST(AnalyzeRebootColumnShard) { + TTestEnv env(1, 1); + auto& runtime = *env.GetServer().GetRuntime(); + auto tableInfo = CreateDatabaseColumnTables(env, 1, 1)[0]; + auto sender = runtime.AllocateEdgeActor(); + + TBlockEvents block(runtime); + + auto analyzeRequest = MakeAnalyzeRequest({tableInfo.PathId}); + runtime.SendToPipe(tableInfo.SaTabletId, sender, analyzeRequest.release()); + + runtime.WaitFor("TEvAnalyzeTableResponse", [&]{ return block.size(); }); + block.Stop(); + RebootTablet(runtime, tableInfo.ShardIds[0], sender); + + runtime.GrabEdgeEventRethrow(sender); + } } } // NStat diff --git a/ydb/core/statistics/aggregator/ya.make b/ydb/core/statistics/aggregator/ya.make index 8898d92bfc59..e63230eebcc1 100644 --- a/ydb/core/statistics/aggregator/ya.make +++ b/ydb/core/statistics/aggregator/ya.make @@ -10,6 +10,7 @@ SRCS( tx_ack_timeout.cpp tx_aggr_stat_response.cpp tx_analyze.cpp + tx_analyze_table_delivery_problem.cpp tx_analyze_table_request.cpp tx_analyze_table_response.cpp tx_configure.cpp From 05ca170955ba25b70a43f2eb138fdc008d328635 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:15:46 +0300 Subject: [PATCH 078/261] Wait for all CAs inside Executer before shutdown (#7829) --- .gitignore | 3 + .../kqp/executer_actor/kqp_data_executer.cpp | 78 +++++++++++- .../kqp/executer_actor/kqp_executer_impl.h | 57 +++++---- ydb/core/kqp/executer_actor/kqp_planner.cpp | 14 ++- ydb/core/kqp/executer_actor/kqp_planner.h | 1 + .../kqp/executer_actor/kqp_scan_executer.cpp | 5 +- ydb/core/kqp/ut/query/kqp_limits_ut.cpp | 117 ++++++++++++++++++ .../dq/actors/compute/dq_compute_actor_impl.h | 30 +++-- 8 files changed, 257 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index ce49d73b454b..87b7b54d294d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ __pycache__/ *.pb.h *.pb.cc +# Other generated +*.fbs.h + # MacOS specific .DS_Store diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index fd16b7b441a6..1d18d034ac9d 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -205,6 +205,8 @@ class TKqpDataExecuter : public TKqpExecuterBaseResultsSize()); - Send(Target, ResponseEv.release()); + AlreadyReplied = true; PassAway(); } @@ -319,6 +320,8 @@ class TKqpDataExecuter : public TKqpExecuterBaseGetPendingComputeTasks().empty() && Planner->GetPendingComputeActors().empty()) { + LOG_I("Shutdown immediately - nothing to wait"); + PassAway(); + } else { + this->Become(&TThis::WaitShutdownState); + LOG_I("Waiting for shutdown of " << Planner->GetPendingComputeTasks().size() << " tasks and " + << Planner->GetPendingComputeActors().size() << " compute actors"); + TActivationContext::Schedule(TDuration::Seconds(10), new IEventHandle(SelfId(), SelfId(), new TEvents::TEvPoison)); + } + } else { + PassAway(); + } + } + void PassAway() override { auto totalTime = TInstant::Now() - StartTime; Counters->Counters->DataTxTotalTimeHistogram->Collect(totalTime.MilliSeconds()); @@ -2612,6 +2631,61 @@ class TKqpDataExecuter : public TKqpExecuterBaseGetTypeRewrite()) { + hFunc(TEvDqCompute::TEvState, HandleShutdown); + hFunc(TEvInterconnect::TEvNodeDisconnected, HandleShutdown); + hFunc(TEvents::TEvPoison, HandleShutdown); + default: + LOG_E("Unexpected event: " << ev->GetTypeName()); // ignore all other events + } + } + + void HandleShutdown(TEvDqCompute::TEvState::TPtr& ev) { + if (ev->Get()->Record.GetState() == NDqProto::COMPUTE_STATE_FAILURE) { + YQL_ENSURE(Planner); + + TActorId actor = ev->Sender; + ui64 taskId = ev->Get()->Record.GetTaskId(); + + Planner->CompletedCA(taskId, actor); + + if (Planner->GetPendingComputeTasks().empty() && Planner->GetPendingComputeActors().empty()) { + PassAway(); + } + } + } + + void HandleShutdown(TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { + const auto nodeId = ev->Get()->NodeId; + LOG_N("Node has disconnected while shutdown: " << nodeId); + + YQL_ENSURE(Planner); + + for (const auto& task : TasksGraph.GetTasks()) { + if (task.Meta.NodeId == nodeId && !task.Meta.Completed) { + if (task.ComputeActorId) { + Planner->CompletedCA(task.Id, task.ComputeActorId); + } else { + Planner->TaskNotStarted(task.Id); + } + } + } + + if (Planner->GetPendingComputeTasks().empty() && Planner->GetPendingComputeActors().empty()) { + PassAway(); + } + } + + void HandleShutdown(TEvents::TEvPoison::TPtr& ev) { + // Self-poison means timeout - don't wait anymore. + LOG_I("Timed out on waiting for Compute Actors to finish - forcing shutdown"); + + if (ev->Sender == SelfId()) { + PassAway(); + } + } + private: void ReplyTxStateUnknown(ui64 shardId) { auto message = TStringBuilder() << "Tx state unknown for shard " << shardId << ", txid " << TxId; diff --git a/ydb/core/kqp/executer_actor/kqp_executer_impl.h b/ydb/core/kqp/executer_actor/kqp_executer_impl.h index 37f8bec82831..2915775ea3fc 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_impl.h +++ b/ydb/core/kqp/executer_actor/kqp_executer_impl.h @@ -667,7 +667,7 @@ class TKqpExecuterBase : public TActorBootstrapped { if (statusCode == Ydb::StatusIds::INTERNAL_ERROR) { InternalError(issues); } else if (statusCode == Ydb::StatusIds::TIMEOUT) { - AbortExecutionAndDie(ev->Sender, NYql::NDqProto::StatusIds::TIMEOUT, "Request timeout exceeded"); + TimeoutError(ev->Sender); } else { RuntimeError(NYql::NDq::DqStatusToYdbStatus(msg.GetStatusCode()), issues); } @@ -1624,14 +1624,14 @@ class TKqpExecuterBase : public TActorBootstrapped { protected: void TerminateComputeActors(Ydb::StatusIds::StatusCode code, const NYql::TIssues& issues) { for (const auto& task : this->TasksGraph.GetTasks()) { - if (task.ComputeActorId) { + if (task.ComputeActorId && !task.Meta.Completed) { LOG_I("aborting compute actor execution, message: " << issues.ToOneLineString() << ", compute actor: " << task.ComputeActorId << ", task: " << task.Id); auto ev = MakeHolder(NYql::NDq::YdbStatusToDqStatus(code), issues); this->Send(task.ComputeActorId, ev.Release()); } else { - LOG_I("task: " << task.Id << ", does not have Compute ActorId yet"); + LOG_I("task: " << task.Id << ", does not have the CA id yet or is already complete"); } } } @@ -1649,7 +1649,6 @@ class TKqpExecuterBase : public TActorBootstrapped { void InternalError(const NYql::TIssues& issues) { LOG_E(issues.ToOneLineString()); - TerminateComputeActors(Ydb::StatusIds::INTERNAL_ERROR, issues); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::UNEXPECTED, "Internal error while executing transaction."); for (const NYql::TIssue& i : issues) { issue.AddSubIssue(MakeIntrusive(i)); @@ -1663,7 +1662,6 @@ class TKqpExecuterBase : public TActorBootstrapped { void ReplyUnavailable(const TString& message) { LOG_E("UNAVAILABLE: " << message); - TerminateComputeActors(Ydb::StatusIds::UNAVAILABLE, message); auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE); issue.AddSubIssue(new NYql::TIssue(message)); ReplyErrorAndDie(Ydb::StatusIds::UNAVAILABLE, issue); @@ -1671,7 +1669,6 @@ class TKqpExecuterBase : public TActorBootstrapped { void RuntimeError(Ydb::StatusIds::StatusCode code, const NYql::TIssues& issues) { LOG_E(Ydb::StatusIds_StatusCode_Name(code) << ": " << issues.ToOneLineString()); - TerminateComputeActors(code, issues); ReplyErrorAndDie(code, issues); } @@ -1687,11 +1684,19 @@ class TKqpExecuterBase : public TActorBootstrapped { ReplyErrorAndDie(status, &issues); } - void AbortExecutionAndDie(TActorId abortSender, NYql::NDqProto::StatusIds::StatusCode status, const TString& message) { + void TimeoutError(TActorId abortSender) { if (AlreadyReplied) { + LOG_E("Timeout when we already replied - not good" << Endl << TBackTrace().PrintToString() << Endl); return; } + const auto status = NYql::NDqProto::StatusIds::TIMEOUT; + const TString message = "Request timeout exceeded"; + + TerminateComputeActors(Ydb::StatusIds::TIMEOUT, message); + + AlreadyReplied = true; + LOG_E("Abort execution: " << NYql::NDqProto::StatusIds_StatusCode_Name(status) << "," << message); if (ExecuterSpan) { ExecuterSpan.EndError(TStringBuilder() << NYql::NDqProto::StatusIds_StatusCode_Name(status)); @@ -1701,17 +1706,14 @@ class TKqpExecuterBase : public TActorBootstrapped { // TEvAbortExecution can come from either ComputeActor or SessionActor (== Target). if (abortSender != Target) { - auto abortEv = MakeHolder(status, "Request timeout exceeded"); + auto abortEv = MakeHolder(status, message); this->Send(Target, abortEv.Release()); } - AlreadyReplied = true; LOG_E("Sending timeout response to: " << Target); - this->Send(Target, ResponseEv.release()); Request.Transactions.crop(0); - TerminateComputeActors(Ydb::StatusIds::TIMEOUT, message); - this->PassAway(); + this->Shutdown(); } void FillResponseStats(Ydb::StatusIds::StatusCode status) { @@ -1746,17 +1748,11 @@ class TKqpExecuterBase : public TActorBootstrapped { google::protobuf::RepeatedPtrField* issues) { if (AlreadyReplied) { + LOG_E("Error when we already replied - not good" << Endl << TBackTrace().PrintToString() << Endl); return; } - if (Planner) { - for (auto computeActor : Planner->GetPendingComputeActors()) { - LOG_D("terminate compute actor " << computeActor.first); - - auto ev = MakeHolder(NYql::NDq::YdbStatusToDqStatus(status), "Terminate execution"); - this->Send(computeActor.first, ev.Release()); - } - } + TerminateComputeActors(status, "Terminate execution"); AlreadyReplied = true; auto& response = *ResponseEv->Record.MutableResponse(); @@ -1782,8 +1778,7 @@ class TKqpExecuterBase : public TActorBootstrapped { ExecuterStateSpan.EndError(response.DebugString()); Request.Transactions.crop(0); - this->Send(Target, ResponseEv.release()); - this->PassAway(); + this->Shutdown(); } protected: @@ -1851,7 +1846,16 @@ class TKqpExecuterBase : public TActorBootstrapped { } protected: + // Introduced separate method from `PassAway()` - to not get confused with expectations from other actors, + // that `PassAway()` should kill actor immediately. + virtual void Shutdown() { + PassAway(); + } + void PassAway() override { + YQL_ENSURE(AlreadyReplied && ResponseEv); + this->Send(Target, ResponseEv.release()); + for (auto channelPair: ResultChannelProxies) { LOG_D("terminate result channel " << channelPair.first << " proxy at " << channelPair.second->SelfId()); @@ -1872,12 +1876,11 @@ class TKqpExecuterBase : public TActorBootstrapped { if (KqpTableResolverId) { this->Send(KqpTableResolverId, new TEvents::TEvPoison); - this->Send(this->SelfId(), new TEvents::TEvPoison); - LOG_T("Terminate, become ZombieState"); - this->Become(&TKqpExecuterBase::ZombieState); - } else { - IActor::PassAway(); } + + this->Send(this->SelfId(), new TEvents::TEvPoison); + LOG_T("Terminate, become ZombieState"); + this->Become(&TKqpExecuterBase::ZombieState); } STATEFN(ZombieState) { diff --git a/ydb/core/kqp/executer_actor/kqp_planner.cpp b/ydb/core/kqp/executer_actor/kqp_planner.cpp index 3a13c9fb7f6e..926dda2700ef 100644 --- a/ydb/core/kqp/executer_actor/kqp_planner.cpp +++ b/ydb/core/kqp/executer_actor/kqp_planner.cpp @@ -554,7 +554,19 @@ void TKqpPlanner::CompletedCA(ui64 taskId, TActorId computeActor) { YQL_ENSURE(it != PendingComputeActors.end()); LastStats.emplace_back(std::move(it->second)); PendingComputeActors.erase(it); - return; + + LOG_I("Compute actor has finished execution: " << computeActor.ToString()); +} + +void TKqpPlanner::TaskNotStarted(ui64 taskId) { + // NOTE: should be invoked only while shutting down - when node is disconnected. + + auto& task = TasksGraph.GetTask(taskId); + + YQL_ENSURE(!task.ComputeActorId); + YQL_ENSURE(!task.Meta.Completed); + + PendingComputeTasks.erase(taskId); } TProgressStat::TEntry TKqpPlanner::CalculateConsumptionUpdate() { diff --git a/ydb/core/kqp/executer_actor/kqp_planner.h b/ydb/core/kqp/executer_actor/kqp_planner.h index 639c53737055..eed887d6e9bc 100644 --- a/ydb/core/kqp/executer_actor/kqp_planner.h +++ b/ydb/core/kqp/executer_actor/kqp_planner.h @@ -74,6 +74,7 @@ class TKqpPlanner { std::unique_ptr AssignTasksToNodes(); bool AcknowledgeCA(ui64 taskId, TActorId computeActor, const NYql::NDqProto::TEvComputeActorState* state); void CompletedCA(ui64 taskId, TActorId computeActor); + void TaskNotStarted(ui64 taskId); TProgressStat::TEntry CalculateConsumptionUpdate(); void ShiftConsumption(); void Submit(); diff --git a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp index 277e77da71e0..c04d5e7573f4 100644 --- a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp @@ -273,6 +273,9 @@ class TKqpScanExecuter : public TKqpExecuterBaseOrbit, TxId, LastTaskId, LastComputeActorId, ResponseEv->ResultsSize()); @@ -281,8 +284,6 @@ class TKqpScanExecuter : public TKqpExecuterBase +#include #include +#include #include +#include #include #include @@ -832,6 +835,120 @@ Y_UNIT_TEST_SUITE(KqpLimits) { UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::TIMEOUT); } + /* Scenario: + - prepare and run query + - observe first EvState event from CA to Executer and replace it with EvAbortExecution + - count all EvState events from all CAs + - wait for final event EvTxResponse from Executer + - expect it to happen strictly after all EvState events + */ + Y_UNIT_TEST(WaitCAsStateOnAbort) { + TKikimrRunner kikimr(TKikimrSettings().SetUseRealThreads(false)); + auto db = kikimr.RunCall([&] { return kikimr.GetTableClient(); } ); + auto session = kikimr.RunCall([&] { return db.CreateSession().GetValueSync().GetSession(); } ); + + auto prepareResult = kikimr.RunCall([&] { return session.PrepareDataQuery(Q_(R"( + SELECT COUNT(*) FROM `/Root/TwoShard`; + )")).GetValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(prepareResult.GetStatus(), EStatus::SUCCESS, prepareResult.GetIssues().ToString()); + auto dataQuery = prepareResult.GetQuery(); + + bool firstEvState = false; + ui32 totalEvState = 0; + TActorId executerId; + ui32 actorCount = 3; // TODO: get number of actors properly. + + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + runtime.SetObserverFunc([&](TAutoPtr& ev) { + if (ev->GetTypeRewrite() == NYql::NDq::TEvDqCompute::TEvState::EventType) { + ++totalEvState; + if (!firstEvState) { + executerId = ev->Recipient; + ev = new IEventHandle(ev->Recipient, ev->Sender, + new NKikimr::NKqp::TEvKqp::TEvAbortExecution(NYql::NDqProto::StatusIds::UNSPECIFIED, NYql::TIssues())); + firstEvState = true; + } + } else if (ev->GetTypeRewrite() == NKikimr::NKqp::TEvKqpExecuter::TEvTxResponse::EventType && ev->Sender == executerId) { + UNIT_ASSERT_C(totalEvState == actorCount*2, "Executer sent response before waiting for CAs"); + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + auto settings = TExecDataQuerySettings().OperationTimeout(TDuration::MilliSeconds(500)); + kikimr.RunInThreadPool([&] { return dataQuery.Execute(TTxControl::BeginTx().CommitTx(), settings).GetValueSync(); }); + + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&](IEventHandle& ev) { + return ev.GetTypeRewrite() == NKikimr::NKqp::TEvKqpExecuter::TEvTxResponse::EventType + && ev.Sender == executerId && totalEvState == actorCount*2; + }); + + UNIT_ASSERT(runtime.DispatchEvents(opts)); + } + + /* Scenario: + - prepare and run query + - observe first EvState event from CA to Executer and replace it with EvAbortExecution + - count all EvState events from all CAs + - drop final EvState event from last CA + - wait for final event EvTxResponse from Executer after timeout poison + - expect it to happen strictly after all EvState events + */ + Y_UNIT_TEST(WaitCAsTimeout) { + TKikimrRunner kikimr(TKikimrSettings().SetUseRealThreads(false)); + auto db = kikimr.RunCall([&] { return kikimr.GetTableClient(); } ); + auto session = kikimr.RunCall([&] { return db.CreateSession().GetValueSync().GetSession(); } ); + + auto prepareResult = kikimr.RunCall([&] { return session.PrepareDataQuery(Q_(R"( + SELECT COUNT(*) FROM `/Root/TwoShard`; + )")).GetValueSync(); + }); + UNIT_ASSERT_VALUES_EQUAL_C(prepareResult.GetStatus(), EStatus::SUCCESS, prepareResult.GetIssues().ToString()); + auto dataQuery = prepareResult.GetQuery(); + + bool firstEvState = false; + bool timeoutPoison = false; + ui32 totalEvState = 0; + TActorId executerId; + ui32 actorCount = 3; // TODO: get number of actors properly. + + auto& runtime = *kikimr.GetTestServer().GetRuntime(); + runtime.SetObserverFunc([&](TAutoPtr& ev) { + if (ev->GetTypeRewrite() == NYql::NDq::TEvDqCompute::TEvState::EventType) { + ++totalEvState; + if (!firstEvState) { + executerId = ev->Recipient; + ev = new IEventHandle(ev->Recipient, ev->Sender, + new NKikimr::NKqp::TEvKqp::TEvAbortExecution(NYql::NDqProto::StatusIds::UNSPECIFIED, NYql::TIssues())); + firstEvState = true; + } else { + return TTestActorRuntime::EEventAction::DROP; + } + } else if (ev->GetTypeRewrite() == TEvents::TEvPoison::EventType && totalEvState == actorCount*2 && + ev->Sender == executerId && ev->Recipient == executerId) + { + timeoutPoison = true; + } else if (ev->GetTypeRewrite() == NKikimr::NKqp::TEvKqpExecuter::TEvTxResponse::EventType && ev->Sender == executerId) { + UNIT_ASSERT_C(timeoutPoison, "Executer sent response before waiting for CAs"); + } + + return TTestActorRuntime::EEventAction::PROCESS; + }); + + auto settings = TExecDataQuerySettings().OperationTimeout(TDuration::MilliSeconds(500)); + kikimr.RunInThreadPool([&] { return dataQuery.Execute(TTxControl::BeginTx().CommitTx(), settings).GetValueSync(); }); + + TDispatchOptions opts; + opts.FinalEvents.emplace_back([&](IEventHandle& ev) { + return ev.GetTypeRewrite() == NKikimr::NKqp::TEvKqpExecuter::TEvTxResponse::EventType + && ev.Sender == executerId && totalEvState == actorCount*2 && timeoutPoison; + }); + + UNIT_ASSERT(runtime.DispatchEvents(opts)); + } + Y_UNIT_TEST(ReplySizeExceeded) { auto kikimr = DefaultKikimrRunner(); auto db = kikimr.GetTableClient(); diff --git a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h index be7db0bca529..666d33b343e7 100644 --- a/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h +++ b/ydb/library/yql/dq/actors/compute/dq_compute_actor_impl.h @@ -554,7 +554,7 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped } } - void ReportStateAndMaybeDie(NYql::NDqProto::StatusIds::StatusCode statusCode, const TIssues& issues) + void ReportStateAndMaybeDie(NYql::NDqProto::StatusIds::StatusCode statusCode, const TIssues& issues, bool forceTerminate = false) { auto execEv = MakeHolder(); auto& record = execEv->Record; @@ -575,7 +575,7 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped this->Send(ExecuterId, execEv.Release()); - if (Checkpoints && State == NDqProto::COMPUTE_STATE_FINISHED) { + if (!forceTerminate && Checkpoints && State == NDqProto::COMPUTE_STATE_FINISHED) { // checkpointed CAs must not self-destroy return; } @@ -1032,10 +1032,6 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped auto tag = (EEvWakeupTag) ev->Get()->Tag; switch (tag) { case EEvWakeupTag::TimeoutTag: { - auto abortEv = MakeHolder(NYql::NDqProto::StatusIds::TIMEOUT, TStringBuilder() - << "Timeout event from compute actor " << this->SelfId() - << ", TxId: " << TxId << ", task: " << Task.GetId()); - if (ComputeActorSpan) { ComputeActorSpan.EndError( TStringBuilder() @@ -1044,10 +1040,8 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped ); } - this->Send(ExecuterId, abortEv.Release()); - - TerminateSources("timeout exceeded", false); - Terminate(false, "timeout exceeded"); + State = NDqProto::COMPUTE_STATE_FAILURE; + ReportStateAndMaybeDie(NYql::NDqProto::StatusIds::TIMEOUT, {TIssue("timeout exceeded")}, true); break; } case EEvWakeupTag::PeriodicStatsTag: { @@ -1071,8 +1065,9 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped switch (lostEventType) { case TEvDqCompute::TEvState::EventType: { CA_LOG_E("Handle undelivered TEvState event, abort execution"); - this->TerminateSources("executer lost", false); - Terminate(false, "executer lost"); + + TerminateSources("executer lost", false); + Terminate(false, "executer lost"); // Executer lost - no need to report state break; } default: { @@ -1118,14 +1113,17 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped InternalError(NYql::NDqProto::StatusIds::INTERNAL_ERROR, *ev->Get()->GetIssues().begin()); return; } + TIssues issues = ev->Get()->GetIssues(); CA_LOG_E("Handle abort execution event from: " << ev->Sender << ", status: " << NYql::NDqProto::StatusIds_StatusCode_Name(ev->Get()->Record.GetStatusCode()) << ", reason: " << issues.ToOneLineString()); - bool success = ev->Get()->Record.GetStatusCode() == NYql::NDqProto::StatusIds::SUCCESS; - - this->TerminateSources(issues, success); + if (ev->Get()->Record.GetStatusCode() == NYql::NDqProto::StatusIds::SUCCESS) { + State = NDqProto::COMPUTE_STATE_FINISHED; + } else { + State = NDqProto::COMPUTE_STATE_FAILURE; + } if (ev->Sender != ExecuterId) { if (ComputeActorSpan) { @@ -1135,7 +1133,7 @@ class TDqComputeActorBase : public NActors::TActorBootstrapped NActors::TActivationContext::Send(ev->Forward(ExecuterId)); } - Terminate(success, issues); + ReportStateAndMaybeDie(ev->Get()->Record.GetStatusCode(), issues, true); } void HandleExecuteBase(NActors::TEvInterconnect::TEvNodeDisconnected::TPtr& ev) { From 40a150290d8a3edf983a78525d9f4947cfd6fd86 Mon Sep 17 00:00:00 2001 From: Ivan Sukhov Date: Fri, 23 Aug 2024 13:20:21 +0300 Subject: [PATCH 079/261] inference json support (#8102) --- .../inference/arrow_fetcher.cpp | 79 ++++++++++++++----- .../inference/arrow_inferencinator.cpp | 67 ++++++++++++++-- .../inference/arrow_inferencinator.h | 6 ++ ydb/tests/fq/s3/test_format_data/test.json | 17 ---- .../fq/s3/test_format_data/test_each_row.json | 3 + .../fq/s3/test_format_data/test_list.json | 5 ++ ydb/tests/fq/s3/test_formats.py | 6 +- ydb/tests/fq/s3/test_s3_0.py | 2 +- 8 files changed, 136 insertions(+), 49 deletions(-) delete mode 100644 ydb/tests/fq/s3/test_format_data/test.json create mode 100644 ydb/tests/fq/s3/test_format_data/test_each_row.json create mode 100644 ydb/tests/fq/s3/test_format_data/test_list.json diff --git a/ydb/core/external_sources/object_storage/inference/arrow_fetcher.cpp b/ydb/core/external_sources/object_storage/inference/arrow_fetcher.cpp index 3f186f18f9b7..6119adeb3411 100644 --- a/ydb/core/external_sources/object_storage/inference/arrow_fetcher.cpp +++ b/ydb/core/external_sources/object_storage/inference/arrow_fetcher.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include @@ -58,7 +60,9 @@ class TArrowFileFetcher : public NActors::TActorBootstrapped switch (Format_) { case EFileFormat::CsvWithNames: - case EFileFormat::TsvWithNames: { + case EFileFormat::TsvWithNames: + case EFileFormat::JsonEachRow: + case EFileFormat::JsonList: { RequestPartialFile(std::move(localRequest), ctx, 0, 10_MB); break; } @@ -114,6 +118,12 @@ class TArrowFileFetcher : public NActors::TActorBootstrapped ctx.Send(request.Requester, new TEvArrowFile(std::move(file), request.Path)); break; } + case EFileFormat::JsonEachRow: + case EFileFormat::JsonList: { + file = CleanupJsonFile(data, request, arrow::json::ParseOptions::Defaults(), ctx); + ctx.Send(request.Requester, new TEvArrowFile(std::move(file), request.Path)); + break; + } case EFileFormat::Undefined: default: Y_ABORT("Invalid format should be unreachable"); @@ -220,23 +230,7 @@ class TArrowFileFetcher : public NActors::TActorBootstrapped std::shared_ptr CleanupCsvFile(const TString& data, const TRequest& request, const arrow::csv::ParseOptions& options, const NActors::TActorContext& ctx) { auto chunker = arrow::csv::MakeChunker(options); std::shared_ptr whole, partial; - auto arrowData = std::make_shared(nullptr, 0); - { - arrow::BufferBuilder builder; - auto buildRes = builder.Append(data.data(), data.size()); - if (buildRes.ok()) { - buildRes = builder.Finish(&arrowData); - } - if (!buildRes.ok()) { - auto error = MakeError( - request.Path, - NFq::TIssuesIds::INTERNAL_ERROR, - TStringBuilder{} << "couldn't consume buffer from S3Fetcher: " << buildRes.ToString() - ); - SendError(ctx, error); - return nullptr; - } - } + auto arrowData = BuildBufferFromData(data, request, ctx); auto status = chunker->Process(arrowData, &whole, &partial); if (!status.ok()) { @@ -277,7 +271,50 @@ class TArrowFileFetcher : public NActors::TActorBootstrapped } std::shared_ptr BuildParquetFileFromMetadata(const TString& data, const TRequest& request, const NActors::TActorContext& ctx) { - auto arrowData = std::make_shared(nullptr, 0); + auto arrowData = BuildBufferFromData(data, request, ctx); + return std::make_shared(std::move(arrowData)); + } + + std::shared_ptr CleanupJsonFile(const TString& data, const TRequest& request, const arrow::json::ParseOptions& options, const NActors::TActorContext& ctx) { + auto chunker = arrow::json::MakeChunker(options); + std::shared_ptr whole, partial; + auto arrowData = BuildBufferFromData(data, request, ctx); + + if (Format_ == EFileFormat::JsonList) { + auto empty = std::make_shared(nullptr, 0); + int64_t count = 1; + auto status = chunker->ProcessSkip(empty, arrowData, false, &count, &whole); + + if (!status.ok()) { + auto error = MakeError( + request.Path, + NFq::TIssuesIds::INTERNAL_ERROR, + TStringBuilder{} << "couldn't run arrow json chunker for " << request.Path << ": " << status.ToString() + ); + SendError(ctx, error); + return nullptr; + } + + arrowData = std::move(whole); + } + + auto status = chunker->Process(arrowData, &whole, &partial); + + if (!status.ok()) { + auto error = MakeError( + request.Path, + NFq::TIssuesIds::INTERNAL_ERROR, + TStringBuilder{} << "couldn't run arrow json chunker for " << request.Path << ": " << status.ToString() + ); + SendError(ctx, error); + return nullptr; + } + + return std::make_shared(std::move(whole)); + } + + std::shared_ptr BuildBufferFromData(const TString& data, const TRequest& request, const NActors::TActorContext& ctx) { + auto dataBuffer = std::make_shared(nullptr, 0); arrow::BufferBuilder builder; auto buildRes = builder.Append(data.data(), data.size()); if (!buildRes.ok()) { @@ -290,7 +327,7 @@ class TArrowFileFetcher : public NActors::TActorBootstrapped return nullptr; } - buildRes = builder.Finish(&arrowData); + buildRes = builder.Finish(&dataBuffer); if (!buildRes.ok()) { auto error = MakeError( request.Path, @@ -301,7 +338,7 @@ class TArrowFileFetcher : public NActors::TActorBootstrapped return nullptr; } - return std::make_shared(std::move(arrowData)); + return dataBuffer; } // Utility diff --git a/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.cpp b/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.cpp index d1757c66969d..36defbd72e68 100644 --- a/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.cpp +++ b/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -182,6 +184,10 @@ struct CsvConfig : public FormatConfig { arrow::csv::ConvertOptions ConvOpts = arrow::csv::ConvertOptions::Defaults(); }; +struct JsonConfig : public FormatConfig { + arrow::json::ParseOptions ParseOpts = arrow::json::ParseOptions::Defaults(); +}; + using TsvConfig = CsvConfig; namespace { @@ -190,23 +196,30 @@ using ArrowField = std::shared_ptr; using ArrowFields = std::vector; std::variant InferCsvTypes(std::shared_ptr file, const CsvConfig& config) { + int64_t fileSize; + if (auto sizeStatus = file->GetSize().Value(&fileSize); !sizeStatus.ok()) { + return TStringBuilder{} << "coudn't get file size: " << sizeStatus.ToString(); + } + std::shared_ptr reader; - auto fileSize = static_cast(file->GetSize().ValueOr(1 << 20)); - fileSize = std::min(fileSize, 1 << 20); auto readerStatus = arrow::csv::TableReader::Make( - arrow::io::default_io_context(), std::move(file), arrow::csv::ReadOptions{.use_threads = false, .block_size = fileSize}, config.ParseOpts, config.ConvOpts + arrow::io::default_io_context(), + std::move(file), + arrow::csv::ReadOptions{.use_threads = false, .block_size = static_cast(fileSize)}, + config.ParseOpts, + config.ConvOpts ) .Value(&reader); if (!readerStatus.ok()) { - return TString{TStringBuilder{} << "couldn't parse csv/tsv file, check format and compression params: " << readerStatus.ToString()}; + return TString{TStringBuilder{} << "couldn't open csv/tsv file, check format and compression params: " << readerStatus.ToString()}; } std::shared_ptr table; auto tableRes = reader->Read().Value(&table); if (!tableRes.ok()) { - return TStringBuilder{} << "couldn't parse csv/tsv file, check format and compression params: " << readerStatus.ToString(); + return TStringBuilder{} << "couldn't parse csv/tsv file, check format and compression params: " << tableRes.ToString(); } return table->fields(); @@ -217,24 +230,52 @@ std::variant InferParquetTypes(std::shared_ptr reader; auto readerStatus = builder.Build(&reader); if (!readerStatus.ok()) { - return TStringBuilder{} << "couldn't parse parquet file, check format params: " << openStatus.ToString(); + return TStringBuilder{} << "couldn't read parquet file, check format params: " << readerStatus.ToString(); } std::shared_ptr schema; auto schemaRes = reader->GetSchema(&schema); if (!schemaRes.ok()) { - return TStringBuilder{} << "couldn't parse parquet file, check format params: " << openStatus.ToString(); + return TStringBuilder{} << "couldn't parse parquet file, check format params: " << schemaRes.ToString(); } return schema->fields(); } +std::variant InferJsonTypes(std::shared_ptr file, const JsonConfig& config) { + int64_t fileSize; + if (auto sizeStatus = file->GetSize().Value(&fileSize); !sizeStatus.ok()) { + return TStringBuilder{} << "coudn't get file size: " << sizeStatus.ToString(); + } + + std::shared_ptr reader; + auto readerStatus = arrow::json::TableReader::Make( + arrow::default_memory_pool(), + std::move(file), + arrow::json::ReadOptions{.use_threads = false, .block_size = static_cast(fileSize)}, + config.ParseOpts + ).Value(&reader); + + if (!readerStatus.ok()) { + return TString{TStringBuilder{} << "couldn't open json file, check format and compression params: " << readerStatus.ToString()}; + } + + std::shared_ptr table; + auto tableRes = reader->Read().Value(&table); + + if (!tableRes.ok()) { + return TString{TStringBuilder{} << "couldn't parse json file, check format and compression params: " << tableRes.ToString()}; + } + + return table->fields(); +} + std::variant InferType(EFileFormat format, std::shared_ptr file, const FormatConfig& config) { switch (format) { case EFileFormat::CsvWithNames: @@ -243,6 +284,9 @@ std::variant InferType(EFileFormat format, std::shared_ptr return InferCsvTypes(std::move(file), static_cast(config)); case EFileFormat::Parquet: return InferParquetTypes(std::move(file)); + case EFileFormat::JsonEachRow: + case EFileFormat::JsonList: + return InferJsonTypes(std::move(file), static_cast(config)); case EFileFormat::Undefined: default: return std::variant{std::in_place_type_t{}, TStringBuilder{} << "unexpected format: " << ConvertFileFormat(format)}; @@ -259,12 +303,19 @@ std::unique_ptr MakeTsvConfig(const THashMap& param return config; } +std::unique_ptr MakeJsonConfig(const THashMap&) { + return std::make_unique(); +} + std::unique_ptr MakeFormatConfig(EFileFormat format, const THashMap& params) { switch (format) { case EFileFormat::CsvWithNames: return MakeCsvConfig(params); case EFileFormat::TsvWithNames: return MakeTsvConfig(params); + case EFileFormat::JsonEachRow: + case EFileFormat::JsonList: + return MakeJsonConfig(params); case EFileFormat::Undefined: default: return nullptr; diff --git a/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.h b/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.h index a6c15dda509a..e2b6d2dd5f0d 100644 --- a/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.h +++ b/ydb/core/external_sources/object_storage/inference/arrow_inferencinator.h @@ -9,6 +9,7 @@ enum class EFileFormat { CsvWithNames, TsvWithNames, JsonEachRow, + JsonList, Parquet, }; @@ -22,6 +23,9 @@ constexpr EFileFormat ConvertFileFormat(TStringBuf format) { if (format == "json_each_row") { return EFileFormat::JsonEachRow; } + if (format == "json_list") { + return EFileFormat::JsonList; + } if (format == "parquet") { return EFileFormat::Parquet; } @@ -37,6 +41,8 @@ constexpr TStringBuf ConvertFileFormat(EFileFormat format) { return "tsv_with_names"; case EFileFormat::JsonEachRow: return "json_each_row"; + case EFileFormat::JsonList: + return "json_list"; case EFileFormat::Parquet: return "parquet"; case EFileFormat::Undefined: diff --git a/ydb/tests/fq/s3/test_format_data/test.json b/ydb/tests/fq/s3/test_format_data/test.json deleted file mode 100644 index 488c60902e35..000000000000 --- a/ydb/tests/fq/s3/test_format_data/test.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "Fruit": "Banana", - "Price": 3, - "Weight": 100 - }, - { - "Fruit": "Apple", - "Price": 2, - "Weight": 22 - }, - { - "Fruit": "Pear", - "Price": 15, - "Weight": 33 - } -] \ No newline at end of file diff --git a/ydb/tests/fq/s3/test_format_data/test_each_row.json b/ydb/tests/fq/s3/test_format_data/test_each_row.json new file mode 100644 index 000000000000..2c32d057e153 --- /dev/null +++ b/ydb/tests/fq/s3/test_format_data/test_each_row.json @@ -0,0 +1,3 @@ +{ "Fruit": "Banana", "Price": 3, "Weight": 100 } +{ "Fruit": "Apple", "Price": 2, "Weight": 22 } +{ "Fruit": "Pear", "Price": 15, "Weight": 33 } \ No newline at end of file diff --git a/ydb/tests/fq/s3/test_format_data/test_list.json b/ydb/tests/fq/s3/test_format_data/test_list.json new file mode 100644 index 000000000000..d32cb7b04451 --- /dev/null +++ b/ydb/tests/fq/s3/test_format_data/test_list.json @@ -0,0 +1,5 @@ +[ + { "Fruit": "Banana", "Price": 3, "Weight": 100 }, + { "Fruit": "Apple", "Price": 2, "Weight": 22 }, + { "Fruit": "Pear", "Price": 15, "Weight": 33 } +] \ No newline at end of file diff --git a/ydb/tests/fq/s3/test_formats.py b/ydb/tests/fq/s3/test_formats.py index 5d4c6489edfd..ac41ca914993 100644 --- a/ydb/tests/fq/s3/test_formats.py +++ b/ydb/tests/fq/s3/test_formats.py @@ -86,8 +86,8 @@ def validate_pg_result(self, result_set): [ ("test.csv", "csv_with_names"), ("test.tsv", "tsv_with_names"), - ("test.json", "json_each_row"), - ("test.json", "json_list"), + ("test_each_row.json", "json_each_row"), + ("test_list.json", "json_list"), ("test.parquet", "parquet"), ], ) @@ -130,6 +130,8 @@ def test_format(self, kikimr, s3, client, filename, type_format, yq_version, uni [ ("test.csv", "csv_with_names"), ("test.tsv", "tsv_with_names"), + ("test_each_row.json", "json_each_row"), + ("test_list.json", "json_list"), ("test.parquet", "parquet"), ], ) diff --git a/ydb/tests/fq/s3/test_s3_0.py b/ydb/tests/fq/s3/test_s3_0.py index 3f8c758808fe..9be3a12baf20 100644 --- a/ydb/tests/fq/s3/test_s3_0.py +++ b/ydb/tests/fq/s3/test_s3_0.py @@ -339,7 +339,7 @@ def test_inference_file_error(self, kikimr, s3, client, unique_prefix): query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id client.wait_query_status(query_id, fq.QueryMeta.FAILED) - assert "couldn\\'t parse csv/tsv file, check format and compression params:" in str( + assert "couldn\\'t open csv/tsv file, check format and compression params:" in str( client.describe_query(query_id).result ) From 11b1438a1fa9ce6dbbf9f3fb5b73dda747a1aeb9 Mon Sep 17 00:00:00 2001 From: AlexSm Date: Fri, 23 Aug 2024 12:31:37 +0200 Subject: [PATCH 080/261] build: refresh Embedded UI (v6.16.0) (#8143) Co-authored-by: astandrik --- .../viewer/monitoring/asset-manifest.json | 112 ++++++------- ydb/core/viewer/monitoring/css.worker.js | 2 + ....LICENSE.txt => css.worker.js.LICENSE.txt} | 0 ydb/core/viewer/monitoring/editor.worker.js | 1 + ydb/core/viewer/monitoring/html.worker.js | 2 + ...LICENSE.txt => html.worker.js.LICENSE.txt} | 0 ydb/core/viewer/monitoring/index.html | 2 +- ydb/core/viewer/monitoring/json.worker.js | 2 + ...LICENSE.txt => json.worker.js.LICENSE.txt} | 0 .../static/css/124.0621cff7.chunk.css | 1 + .../static/css/1551.d5e5efc2.chunk.css | 3 - .../static/css/163.1284ddc4.chunk.css | 1 - .../static/css/321.e12415cc.chunk.css | 1 - .../static/css/4142.20a17b90.chunk.css | 1 + .../static/css/445.faba3b31.chunk.css | 1 + .../static/css/5246.49d67ade.chunk.css | 1 - .../static/css/5246.94947d4e.chunk.css | 1 + .../static/css/5715.07ca45c9.chunk.css | 1 - .../static/css/5784.92ae005b.chunk.css | 1 + .../static/css/6080.697a0a70.chunk.css | 3 + .../static/css/7045.ebbcb0f7.chunk.css | 1 - .../static/css/7542.9a1fbaca.chunk.css | 1 - .../static/css/8076.5d26c70c.chunk.css | 1 - .../static/css/8424.308a04db.chunk.css | 1 - .../static/css/8429.ef9c97b7.chunk.css | 1 - .../static/css/8579.fa282830.chunk.css | 1 + .../static/css/8667.0fa1f184.chunk.css | 1 + ...d329.chunk.css => 9399.a726d329.chunk.css} | 0 .../static/css/9818.3ebe673f.chunk.css | 1 - .../static/css/9818.82d95e45.chunk.css | 1 + .../{main.efd6a3a5.css => main.bb5c1fe0.css} | 4 +- .../static/js/124.9a7bec12.chunk.js | 1 + .../static/js/1343.b5e020af.chunk.js | 1 - ...44be66.chunk.js => 1478.631a9793.chunk.js} | 4 +- ...txt => 1478.631a9793.chunk.js.LICENSE.txt} | 0 .../static/js/1551.2e8e3e50.chunk.js | 2 - .../js/1551.2e8e3e50.chunk.js.LICENSE.txt | 1 - .../static/js/163.2451f8a1.chunk.js | 1 - .../static/js/163.eea01641.chunk.js | 1 + .../static/js/1745.9f25b511.chunk.js | 1 + .../static/js/1962.2fc0a4aa.chunk.js | 1 + ...169874.chunk.js => 2118.48c265c8.chunk.js} | 4 +- ...txt => 2118.48c265c8.chunk.js.LICENSE.txt} | 0 ...9a17949.chunk.js => 214.88d2c59f.chunk.js} | 4 +- ....txt => 214.88d2c59f.chunk.js.LICENSE.txt} | 0 .../static/js/2180.adcde51c.chunk.js | 1 - .../static/js/2335.ea670fe7.chunk.js | 1 + .../static/js/2335.eb54f5e5.chunk.js | 1 - ...cd0025.chunk.js => 2403.5de440f9.chunk.js} | 4 +- ...txt => 2403.5de440f9.chunk.js.LICENSE.txt} | 0 ...bfcc5a.chunk.js => 2418.4f82eefe.chunk.js} | 2 +- .../static/js/2535.4df017df.chunk.js | 1 + .../static/js/2937.09da0354.chunk.js | 1 + .../static/js/321.9a9868e1.chunk.js | 2 - .../js/321.9a9868e1.chunk.js.LICENSE.txt | 3 - .../static/js/328.4a94f418.chunk.js | 1 - .../static/js/3757.7c534899.chunk.js | 1 - .../static/js/4073.4c05eb94.chunk.js | 2 + .../js/4073.4c05eb94.chunk.js.LICENSE.txt | 10 ++ .../static/js/4142.0cece594.chunk.js | 1 + .../static/js/445.a5414de4.chunk.js | 1 + ...a9b6b7.chunk.js => 4635.a36f9ad7.chunk.js} | 4 +- ...txt => 4635.a36f9ad7.chunk.js.LICENSE.txt} | 0 ...2069de.chunk.js => 4789.4cbcac01.chunk.js} | 2 +- .../static/js/4842.57182d38.chunk.js | 1 - ...964f8aa.chunk.js => 496.0cdc7358.chunk.js} | 2 +- .../static/js/5246.224ba018.chunk.js | 1 - .../static/js/5246.f2a6e571.chunk.js | 1 + .../static/js/530.582a0d34.chunk.js | 1 - .../static/js/5715.0941d934.chunk.js | 1 - .../static/js/5784.e7ca6951.chunk.js | 1 + .../static/js/6080.9c6fc16e.chunk.js | 2 + .../js/6080.9c6fc16e.chunk.js.LICENSE.txt | 12 ++ ...8c1432.chunk.js => 6329.59062058.chunk.js} | 4 +- ...txt => 6329.59062058.chunk.js.LICENSE.txt} | 0 ...7d0ec8.chunk.js => 6390.e2ce5781.chunk.js} | 4 +- ...txt => 6390.e2ce5781.chunk.js.LICENSE.txt} | 0 .../static/js/6531.fbd78a3e.chunk.js | 1 - ...ed9ccc.chunk.js => 6919.9498ad4c.chunk.js} | 4 +- ...txt => 6919.9498ad4c.chunk.js.LICENSE.txt} | 0 .../static/js/7045.53fa1aaf.chunk.js | 1 - .../static/js/730.ba7afb84.chunk.js | 1 + ...45d6ac.chunk.js => 7520.448940d2.chunk.js} | 4 +- ...txt => 7520.448940d2.chunk.js.LICENSE.txt} | 0 ...f87a9a.chunk.js => 7529.e5632b15.chunk.js} | 4 +- ...txt => 7529.e5632b15.chunk.js.LICENSE.txt} | 0 .../static/js/7542.d61fc913.chunk.js | 1 - ...cfd3ba.chunk.js => 7543.8ff5adcd.chunk.js} | 4 +- ...txt => 7543.8ff5adcd.chunk.js.LICENSE.txt} | 0 .../static/js/783.95eb5b37.chunk.js | 1 - ...6ef449.chunk.js => 8065.386c007a.chunk.js} | 4 +- ...txt => 8065.386c007a.chunk.js.LICENSE.txt} | 0 .../static/js/8076.dac0f4f1.chunk.js | 1 - .../static/js/8424.5b5c42b5.chunk.js | 2 - .../static/js/8429.b285ce5a.chunk.js | 1 - .../static/js/8579.c75692bf.chunk.js | 1 + .../static/js/8667.658f61f2.chunk.js | 1 + .../static/js/9079.ebd6b11c.chunk.js | 1 + ...d773f2.chunk.js => 9173.9e31ad84.chunk.js} | 4 +- ...txt => 9173.9e31ad84.chunk.js.LICENSE.txt} | 0 .../static/js/9220.f5d70b51.chunk.js | 1 + ...2befbc.chunk.js => 9371.d135b76e.chunk.js} | 4 +- .../js/9371.d135b76e.chunk.js.LICENSE.txt | 6 + .../static/js/9399.abafeca2.chunk.js | 1 + .../static/js/9759.7dadb893.chunk.js | 1 - .../static/js/9818.24b1ff88.chunk.js | 1 - .../static/js/9818.d737cdca.chunk.js | 1 + ...36d1f5.chunk.js => 9876.c4e3221a.chunk.js} | 4 +- .../js/9876.c4e3221a.chunk.js.LICENSE.txt | 6 + .../monitoring/static/js/main.40147fad.js | 2 + ...CENSE.txt => main.40147fad.js.LICENSE.txt} | 0 .../monitoring/static/js/main.9fede0a0.js | 2 - ydb/core/viewer/monitoring/ts.worker.js | 2 + .../monitoring/ts.worker.js.LICENSE.txt | 14 ++ ydb/core/viewer/ya.make | 149 +++++++++--------- 115 files changed, 262 insertions(+), 206 deletions(-) create mode 100644 ydb/core/viewer/monitoring/css.worker.js rename ydb/core/viewer/monitoring/{static/js/1478.5044be66.chunk.js.LICENSE.txt => css.worker.js.LICENSE.txt} (100%) create mode 100644 ydb/core/viewer/monitoring/editor.worker.js create mode 100644 ydb/core/viewer/monitoring/html.worker.js rename ydb/core/viewer/monitoring/{static/js/2118.bc169874.chunk.js.LICENSE.txt => html.worker.js.LICENSE.txt} (100%) create mode 100644 ydb/core/viewer/monitoring/json.worker.js rename ydb/core/viewer/monitoring/{static/js/214.99a17949.chunk.js.LICENSE.txt => json.worker.js.LICENSE.txt} (100%) create mode 100644 ydb/core/viewer/monitoring/static/css/124.0621cff7.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/1551.d5e5efc2.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/163.1284ddc4.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/321.e12415cc.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/4142.20a17b90.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/445.faba3b31.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/5246.49d67ade.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/5246.94947d4e.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/5715.07ca45c9.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/5784.92ae005b.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/6080.697a0a70.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/7045.ebbcb0f7.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/7542.9a1fbaca.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/8076.5d26c70c.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/8424.308a04db.chunk.css delete mode 100644 ydb/core/viewer/monitoring/static/css/8429.ef9c97b7.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/8579.fa282830.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/8667.0fa1f184.chunk.css rename ydb/core/viewer/monitoring/static/css/{328.a726d329.chunk.css => 9399.a726d329.chunk.css} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/css/9818.3ebe673f.chunk.css create mode 100644 ydb/core/viewer/monitoring/static/css/9818.82d95e45.chunk.css rename ydb/core/viewer/monitoring/static/css/{main.efd6a3a5.css => main.bb5c1fe0.css} (51%) create mode 100644 ydb/core/viewer/monitoring/static/js/124.9a7bec12.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/1343.b5e020af.chunk.js rename ydb/core/viewer/monitoring/static/js/{1478.5044be66.chunk.js => 1478.631a9793.chunk.js} (96%) rename ydb/core/viewer/monitoring/static/js/{2403.82cd0025.chunk.js.LICENSE.txt => 1478.631a9793.chunk.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/1551.2e8e3e50.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/1551.2e8e3e50.chunk.js.LICENSE.txt delete mode 100644 ydb/core/viewer/monitoring/static/js/163.2451f8a1.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/163.eea01641.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/1745.9f25b511.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/1962.2fc0a4aa.chunk.js rename ydb/core/viewer/monitoring/static/js/{2118.bc169874.chunk.js => 2118.48c265c8.chunk.js} (99%) rename ydb/core/viewer/monitoring/static/js/{4635.ffa9b6b7.chunk.js.LICENSE.txt => 2118.48c265c8.chunk.js.LICENSE.txt} (100%) rename ydb/core/viewer/monitoring/static/js/{214.99a17949.chunk.js => 214.88d2c59f.chunk.js} (99%) rename ydb/core/viewer/monitoring/static/js/{6329.d78c1432.chunk.js.LICENSE.txt => 214.88d2c59f.chunk.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/2180.adcde51c.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/2335.ea670fe7.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/2335.eb54f5e5.chunk.js rename ydb/core/viewer/monitoring/static/js/{2403.82cd0025.chunk.js => 2403.5de440f9.chunk.js} (97%) rename ydb/core/viewer/monitoring/static/js/{6390.497d0ec8.chunk.js.LICENSE.txt => 2403.5de440f9.chunk.js.LICENSE.txt} (100%) rename ydb/core/viewer/monitoring/static/js/{2418.b7bfcc5a.chunk.js => 2418.4f82eefe.chunk.js} (50%) create mode 100644 ydb/core/viewer/monitoring/static/js/2535.4df017df.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/2937.09da0354.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/321.9a9868e1.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/321.9a9868e1.chunk.js.LICENSE.txt delete mode 100644 ydb/core/viewer/monitoring/static/js/328.4a94f418.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/3757.7c534899.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/4073.4c05eb94.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/4073.4c05eb94.chunk.js.LICENSE.txt create mode 100644 ydb/core/viewer/monitoring/static/js/4142.0cece594.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/445.a5414de4.chunk.js rename ydb/core/viewer/monitoring/static/js/{4635.ffa9b6b7.chunk.js => 4635.a36f9ad7.chunk.js} (96%) rename ydb/core/viewer/monitoring/static/js/{6919.84ed9ccc.chunk.js.LICENSE.txt => 4635.a36f9ad7.chunk.js.LICENSE.txt} (100%) rename ydb/core/viewer/monitoring/static/js/{4789.d52069de.chunk.js => 4789.4cbcac01.chunk.js} (99%) delete mode 100644 ydb/core/viewer/monitoring/static/js/4842.57182d38.chunk.js rename ydb/core/viewer/monitoring/static/js/{496.5964f8aa.chunk.js => 496.0cdc7358.chunk.js} (99%) delete mode 100644 ydb/core/viewer/monitoring/static/js/5246.224ba018.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/5246.f2a6e571.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/530.582a0d34.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/5715.0941d934.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/5784.e7ca6951.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/6080.9c6fc16e.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/6080.9c6fc16e.chunk.js.LICENSE.txt rename ydb/core/viewer/monitoring/static/js/{6329.d78c1432.chunk.js => 6329.59062058.chunk.js} (97%) rename ydb/core/viewer/monitoring/static/js/{7520.d245d6ac.chunk.js.LICENSE.txt => 6329.59062058.chunk.js.LICENSE.txt} (100%) rename ydb/core/viewer/monitoring/static/js/{6390.497d0ec8.chunk.js => 6390.e2ce5781.chunk.js} (96%) rename ydb/core/viewer/monitoring/static/js/{7529.ddf87a9a.chunk.js.LICENSE.txt => 6390.e2ce5781.chunk.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/6531.fbd78a3e.chunk.js rename ydb/core/viewer/monitoring/static/js/{6919.84ed9ccc.chunk.js => 6919.9498ad4c.chunk.js} (97%) rename ydb/core/viewer/monitoring/static/js/{7543.3fcfd3ba.chunk.js.LICENSE.txt => 6919.9498ad4c.chunk.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/7045.53fa1aaf.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/730.ba7afb84.chunk.js rename ydb/core/viewer/monitoring/static/js/{7520.d245d6ac.chunk.js => 7520.448940d2.chunk.js} (99%) rename ydb/core/viewer/monitoring/static/js/{8065.666ef449.chunk.js.LICENSE.txt => 7520.448940d2.chunk.js.LICENSE.txt} (100%) rename ydb/core/viewer/monitoring/static/js/{7529.ddf87a9a.chunk.js => 7529.e5632b15.chunk.js} (96%) rename ydb/core/viewer/monitoring/static/js/{8424.5b5c42b5.chunk.js.LICENSE.txt => 7529.e5632b15.chunk.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/7542.d61fc913.chunk.js rename ydb/core/viewer/monitoring/static/js/{7543.3fcfd3ba.chunk.js => 7543.8ff5adcd.chunk.js} (94%) rename ydb/core/viewer/monitoring/static/js/{9173.71d773f2.chunk.js.LICENSE.txt => 7543.8ff5adcd.chunk.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/783.95eb5b37.chunk.js rename ydb/core/viewer/monitoring/static/js/{8065.666ef449.chunk.js => 8065.386c007a.chunk.js} (99%) rename ydb/core/viewer/monitoring/static/js/{9371.b42befbc.chunk.js.LICENSE.txt => 8065.386c007a.chunk.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/8076.dac0f4f1.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/8424.5b5c42b5.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/8429.b285ce5a.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/8579.c75692bf.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/8667.658f61f2.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/9079.ebd6b11c.chunk.js rename ydb/core/viewer/monitoring/static/js/{9173.71d773f2.chunk.js => 9173.9e31ad84.chunk.js} (98%) rename ydb/core/viewer/monitoring/static/js/{9876.b336d1f5.chunk.js.LICENSE.txt => 9173.9e31ad84.chunk.js.LICENSE.txt} (100%) create mode 100644 ydb/core/viewer/monitoring/static/js/9220.f5d70b51.chunk.js rename ydb/core/viewer/monitoring/static/js/{9371.b42befbc.chunk.js => 9371.d135b76e.chunk.js} (98%) create mode 100644 ydb/core/viewer/monitoring/static/js/9371.d135b76e.chunk.js.LICENSE.txt create mode 100644 ydb/core/viewer/monitoring/static/js/9399.abafeca2.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/9759.7dadb893.chunk.js delete mode 100644 ydb/core/viewer/monitoring/static/js/9818.24b1ff88.chunk.js create mode 100644 ydb/core/viewer/monitoring/static/js/9818.d737cdca.chunk.js rename ydb/core/viewer/monitoring/static/js/{9876.b336d1f5.chunk.js => 9876.c4e3221a.chunk.js} (97%) create mode 100644 ydb/core/viewer/monitoring/static/js/9876.c4e3221a.chunk.js.LICENSE.txt create mode 100644 ydb/core/viewer/monitoring/static/js/main.40147fad.js rename ydb/core/viewer/monitoring/static/js/{main.9fede0a0.js.LICENSE.txt => main.40147fad.js.LICENSE.txt} (100%) delete mode 100644 ydb/core/viewer/monitoring/static/js/main.9fede0a0.js create mode 100644 ydb/core/viewer/monitoring/ts.worker.js create mode 100644 ydb/core/viewer/monitoring/ts.worker.js.LICENSE.txt diff --git a/ydb/core/viewer/monitoring/asset-manifest.json b/ydb/core/viewer/monitoring/asset-manifest.json index 6a9aaa284f5c..d60e101fcdc9 100644 --- a/ydb/core/viewer/monitoring/asset-manifest.json +++ b/ydb/core/viewer/monitoring/asset-manifest.json @@ -1,7 +1,7 @@ { "files": { - "main.css": "./static/css/main.efd6a3a5.css", - "main.js": "./static/js/main.9fede0a0.js", + "main.css": "./static/css/main.bb5c1fe0.css", + "main.js": "./static/js/main.40147fad.js", "static/js/3457.b193afe6.chunk.js": "./static/js/3457.b193afe6.chunk.js", "static/js/6876.867b698c.chunk.js": "./static/js/6876.867b698c.chunk.js", "static/js/2435.092e8d7f.chunk.js": "./static/js/2435.092e8d7f.chunk.js", @@ -106,7 +106,7 @@ "static/js/4964.c7c75eb0.chunk.js": "./static/js/4964.c7c75eb0.chunk.js", "static/js/1869.d6661a03.chunk.js": "./static/js/1869.d6661a03.chunk.js", "static/js/9917.67d792e3.chunk.js": "./static/js/9917.67d792e3.chunk.js", - "static/js/1343.b5e020af.chunk.js": "./static/js/1343.b5e020af.chunk.js", + "static/js/163.eea01641.chunk.js": "./static/js/163.eea01641.chunk.js", "static/js/3025.7e536c57.chunk.js": "./static/js/3025.7e536c57.chunk.js", "static/js/6156.0c562627.chunk.js": "./static/js/6156.0c562627.chunk.js", "static/js/6361.a9f11e7a.chunk.js": "./static/js/6361.a9f11e7a.chunk.js", @@ -151,29 +151,27 @@ "static/js/3898.1fec42e6.chunk.js": "./static/js/3898.1fec42e6.chunk.js", "static/js/1616.8a217b93.chunk.js": "./static/js/1616.8a217b93.chunk.js", "static/js/7522.1a0f9c02.chunk.js": "./static/js/7522.1a0f9c02.chunk.js", - "static/css/8429.ef9c97b7.chunk.css": "./static/css/8429.ef9c97b7.chunk.css", - "static/js/8429.b285ce5a.chunk.js": "./static/js/8429.b285ce5a.chunk.js", - "static/css/163.1284ddc4.chunk.css": "./static/css/163.1284ddc4.chunk.css", - "static/js/163.2451f8a1.chunk.js": "./static/js/163.2451f8a1.chunk.js", - "static/css/7045.ebbcb0f7.chunk.css": "./static/css/7045.ebbcb0f7.chunk.css", - "static/js/7045.53fa1aaf.chunk.js": "./static/js/7045.53fa1aaf.chunk.js", - "static/css/8076.5d26c70c.chunk.css": "./static/css/8076.5d26c70c.chunk.css", - "static/js/8076.dac0f4f1.chunk.js": "./static/js/8076.dac0f4f1.chunk.js", - "static/css/5715.07ca45c9.chunk.css": "./static/css/5715.07ca45c9.chunk.css", - "static/js/5715.0941d934.chunk.js": "./static/js/5715.0941d934.chunk.js", - "static/css/7542.9a1fbaca.chunk.css": "./static/css/7542.9a1fbaca.chunk.css", - "static/js/7542.d61fc913.chunk.js": "./static/js/7542.d61fc913.chunk.js", - "static/css/5246.49d67ade.chunk.css": "./static/css/5246.49d67ade.chunk.css", - "static/js/5246.224ba018.chunk.js": "./static/js/5246.224ba018.chunk.js", - "static/css/9818.3ebe673f.chunk.css": "./static/css/9818.3ebe673f.chunk.css", - "static/js/9818.24b1ff88.chunk.js": "./static/js/9818.24b1ff88.chunk.js", - "static/css/328.a726d329.chunk.css": "./static/css/328.a726d329.chunk.css", - "static/js/328.4a94f418.chunk.js": "./static/js/328.4a94f418.chunk.js", + "static/css/8667.0fa1f184.chunk.css": "./static/css/8667.0fa1f184.chunk.css", + "static/js/8667.658f61f2.chunk.js": "./static/js/8667.658f61f2.chunk.js", + "static/css/5784.92ae005b.chunk.css": "./static/css/5784.92ae005b.chunk.css", + "static/js/5784.e7ca6951.chunk.js": "./static/js/5784.e7ca6951.chunk.js", + "static/css/445.faba3b31.chunk.css": "./static/css/445.faba3b31.chunk.css", + "static/js/445.a5414de4.chunk.js": "./static/js/445.a5414de4.chunk.js", + "static/css/8579.fa282830.chunk.css": "./static/css/8579.fa282830.chunk.css", + "static/js/8579.c75692bf.chunk.js": "./static/js/8579.c75692bf.chunk.js", + "static/css/124.0621cff7.chunk.css": "./static/css/124.0621cff7.chunk.css", + "static/js/124.9a7bec12.chunk.js": "./static/js/124.9a7bec12.chunk.js", + "static/css/4142.20a17b90.chunk.css": "./static/css/4142.20a17b90.chunk.css", + "static/js/4142.0cece594.chunk.js": "./static/js/4142.0cece594.chunk.js", + "static/css/5246.94947d4e.chunk.css": "./static/css/5246.94947d4e.chunk.css", + "static/js/5246.f2a6e571.chunk.js": "./static/js/5246.f2a6e571.chunk.js", + "static/css/9818.82d95e45.chunk.css": "./static/css/9818.82d95e45.chunk.css", + "static/js/9818.d737cdca.chunk.js": "./static/js/9818.d737cdca.chunk.js", + "static/css/9399.a726d329.chunk.css": "./static/css/9399.a726d329.chunk.css", + "static/js/9399.abafeca2.chunk.js": "./static/js/9399.abafeca2.chunk.js", "static/js/599.c58caf58.chunk.js": "./static/js/599.c58caf58.chunk.js", - "static/js/6531.fbd78a3e.chunk.js": "./static/js/6531.fbd78a3e.chunk.js", + "static/js/9079.ebd6b11c.chunk.js": "./static/js/9079.ebd6b11c.chunk.js", "static/css/4983.5c3e5de4.chunk.css": "./static/css/4983.5c3e5de4.chunk.css", - "static/js/3757.7c534899.chunk.js": "./static/js/3757.7c534899.chunk.js", - "static/js/4842.57182d38.chunk.js": "./static/js/4842.57182d38.chunk.js", "static/js/1155.4fce1854.chunk.js": "./static/js/1155.4fce1854.chunk.js", "static/js/6230.8e64216a.chunk.js": "./static/js/6230.8e64216a.chunk.js", "static/js/337.b6fc715e.chunk.js": "./static/js/337.b6fc715e.chunk.js", @@ -181,36 +179,39 @@ "static/js/2322.29255c22.chunk.js": "./static/js/2322.29255c22.chunk.js", "static/js/4123.64882a16.chunk.js": "./static/js/4123.64882a16.chunk.js", "static/js/6289.51f8741e.chunk.js": "./static/js/6289.51f8741e.chunk.js", - "static/js/4635.ffa9b6b7.chunk.js": "./static/js/4635.ffa9b6b7.chunk.js", + "static/js/4635.a36f9ad7.chunk.js": "./static/js/4635.a36f9ad7.chunk.js", "static/js/4345.9238776d.chunk.js": "./static/js/4345.9238776d.chunk.js", "static/js/9319.40f9e46a.chunk.js": "./static/js/9319.40f9e46a.chunk.js", "static/js/924.382f18b1.chunk.js": "./static/js/924.382f18b1.chunk.js", "static/js/6795.5ec0c96a.chunk.js": "./static/js/6795.5ec0c96a.chunk.js", + "static/js/2118.48c265c8.chunk.js": "./static/js/2118.48c265c8.chunk.js", "static/js/2302.7e7a2fb4.chunk.js": "./static/js/2302.7e7a2fb4.chunk.js", "static/js/4388.edb51304.chunk.js": "./static/js/4388.edb51304.chunk.js", "static/js/4046.5dac72a9.chunk.js": "./static/js/4046.5dac72a9.chunk.js", "static/js/2190.27f354f5.chunk.js": "./static/js/2190.27f354f5.chunk.js", "static/js/3358.c777fe1f.chunk.js": "./static/js/3358.c777fe1f.chunk.js", "static/js/6142.b2452554.chunk.js": "./static/js/6142.b2452554.chunk.js", + "static/js/214.88d2c59f.chunk.js": "./static/js/214.88d2c59f.chunk.js", "static/js/2962.66e01691.chunk.js": "./static/js/2962.66e01691.chunk.js", - "static/js/214.99a17949.chunk.js": "./static/js/214.99a17949.chunk.js", "static/js/8791.b209de42.chunk.js": "./static/js/8791.b209de42.chunk.js", "static/js/6898.5580b941.chunk.js": "./static/js/6898.5580b941.chunk.js", - "static/js/9173.71d773f2.chunk.js": "./static/js/9173.71d773f2.chunk.js", + "static/js/9173.9e31ad84.chunk.js": "./static/js/9173.9e31ad84.chunk.js", "static/js/2532.30bb087d.chunk.js": "./static/js/2532.30bb087d.chunk.js", - "static/js/6329.d78c1432.chunk.js": "./static/js/6329.d78c1432.chunk.js", + "static/js/6329.59062058.chunk.js": "./static/js/6329.59062058.chunk.js", + "static/js/8065.386c007a.chunk.js": "./static/js/8065.386c007a.chunk.js", "static/js/2840.b69eb597.chunk.js": "./static/js/2840.b69eb597.chunk.js", "static/js/5311.a500a1ea.chunk.js": "./static/js/5311.a500a1ea.chunk.js", - "static/js/2403.82cd0025.chunk.js": "./static/js/2403.82cd0025.chunk.js", + "static/js/2403.5de440f9.chunk.js": "./static/js/2403.5de440f9.chunk.js", + "static/js/7520.448940d2.chunk.js": "./static/js/7520.448940d2.chunk.js", "static/js/1747.b4331799.chunk.js": "./static/js/1747.b4331799.chunk.js", "static/js/3498.c7d39060.chunk.js": "./static/js/3498.c7d39060.chunk.js", "static/js/185.7d51fcfa.chunk.js": "./static/js/185.7d51fcfa.chunk.js", "static/js/8450.baf3a89d.chunk.js": "./static/js/8450.baf3a89d.chunk.js", + "static/js/7529.e5632b15.chunk.js": "./static/js/7529.e5632b15.chunk.js", "static/js/3771.764124c3.chunk.js": "./static/js/3771.764124c3.chunk.js", - "static/js/7529.ddf87a9a.chunk.js": "./static/js/7529.ddf87a9a.chunk.js", "static/js/785.d2eae69c.chunk.js": "./static/js/785.d2eae69c.chunk.js", "static/js/5107.8cac6a03.chunk.js": "./static/js/5107.8cac6a03.chunk.js", - "static/js/6919.84ed9ccc.chunk.js": "./static/js/6919.84ed9ccc.chunk.js", + "static/js/6919.9498ad4c.chunk.js": "./static/js/6919.9498ad4c.chunk.js", "static/js/2104.4f22ecac.chunk.js": "./static/js/2104.4f22ecac.chunk.js", "static/js/9433.7ce648d0.chunk.js": "./static/js/9433.7ce648d0.chunk.js", "static/js/1956.0205a5bb.chunk.js": "./static/js/1956.0205a5bb.chunk.js", @@ -226,10 +227,10 @@ "static/js/3644.aeda46ca.chunk.js": "./static/js/3644.aeda46ca.chunk.js", "static/js/8797.f8f0ce13.chunk.js": "./static/js/8797.f8f0ce13.chunk.js", "static/js/2521.21bdfab9.chunk.js": "./static/js/2521.21bdfab9.chunk.js", - "static/js/1478.5044be66.chunk.js": "./static/js/1478.5044be66.chunk.js", + "static/js/1478.631a9793.chunk.js": "./static/js/1478.631a9793.chunk.js", "static/js/6300.dca75d45.chunk.js": "./static/js/6300.dca75d45.chunk.js", "static/js/3074.bbb8aaef.chunk.js": "./static/js/3074.bbb8aaef.chunk.js", - "static/js/9371.b42befbc.chunk.js": "./static/js/9371.b42befbc.chunk.js", + "static/js/9371.d135b76e.chunk.js": "./static/js/9371.d135b76e.chunk.js", "static/js/9923.270f0a19.chunk.js": "./static/js/9923.270f0a19.chunk.js", "static/js/358.d6300019.chunk.js": "./static/js/358.d6300019.chunk.js", "static/js/86.ad271bdc.chunk.js": "./static/js/86.ad271bdc.chunk.js", @@ -249,32 +250,33 @@ "static/js/2876.afe7e47f.chunk.js": "./static/js/2876.afe7e47f.chunk.js", "static/js/5868.be04313a.chunk.js": "./static/js/5868.be04313a.chunk.js", "static/js/2553.5faabf5a.chunk.js": "./static/js/2553.5faabf5a.chunk.js", - "static/js/9876.b336d1f5.chunk.js": "./static/js/9876.b336d1f5.chunk.js", + "static/js/9876.c4e3221a.chunk.js": "./static/js/9876.c4e3221a.chunk.js", + "static/js/4789.4cbcac01.chunk.js": "./static/js/4789.4cbcac01.chunk.js", "static/js/5378.86805fba.chunk.js": "./static/js/5378.86805fba.chunk.js", "static/js/2183.e2318c37.chunk.js": "./static/js/2183.e2318c37.chunk.js", - "static/js/7543.3fcfd3ba.chunk.js": "./static/js/7543.3fcfd3ba.chunk.js", - "static/js/6390.497d0ec8.chunk.js": "./static/js/6390.497d0ec8.chunk.js", - "static/js/2118.bc169874.chunk.js": "./static/js/2118.bc169874.chunk.js", - "static/js/8065.666ef449.chunk.js": "./static/js/8065.666ef449.chunk.js", - "static/js/7520.d245d6ac.chunk.js": "./static/js/7520.d245d6ac.chunk.js", - "static/js/4789.d52069de.chunk.js": "./static/js/4789.d52069de.chunk.js", + "static/js/7543.8ff5adcd.chunk.js": "./static/js/7543.8ff5adcd.chunk.js", + "static/js/6390.e2ce5781.chunk.js": "./static/js/6390.e2ce5781.chunk.js", + "static/js/9220.f5d70b51.chunk.js": "./static/js/9220.f5d70b51.chunk.js", + "static/js/2937.09da0354.chunk.js": "./static/js/2937.09da0354.chunk.js", + "static/js/1962.2fc0a4aa.chunk.js": "./static/js/1962.2fc0a4aa.chunk.js", "static/js/8607.e8952666.chunk.js": "./static/js/8607.e8952666.chunk.js", "static/css/2418.3ce054a3.chunk.css": "./static/css/2418.3ce054a3.chunk.css", - "static/js/2418.b7bfcc5a.chunk.js": "./static/js/2418.b7bfcc5a.chunk.js", - "static/css/1551.d5e5efc2.chunk.css": "./static/css/1551.d5e5efc2.chunk.css", - "static/js/1551.2e8e3e50.chunk.js": "./static/js/1551.2e8e3e50.chunk.js", - "static/css/8424.308a04db.chunk.css": "./static/css/8424.308a04db.chunk.css", - "static/js/8424.5b5c42b5.chunk.js": "./static/js/8424.5b5c42b5.chunk.js", - "static/js/530.582a0d34.chunk.js": "./static/js/530.582a0d34.chunk.js", - "static/css/321.e12415cc.chunk.css": "./static/css/321.e12415cc.chunk.css", - "static/js/321.9a9868e1.chunk.js": "./static/js/321.9a9868e1.chunk.js", + "static/js/2418.4f82eefe.chunk.js": "./static/js/2418.4f82eefe.chunk.js", + "static/js/4073.4c05eb94.chunk.js": "./static/js/4073.4c05eb94.chunk.js", + "static/js/730.ba7afb84.chunk.js": "./static/js/730.ba7afb84.chunk.js", + "static/css/6080.697a0a70.chunk.css": "./static/css/6080.697a0a70.chunk.css", + "static/js/6080.9c6fc16e.chunk.js": "./static/js/6080.9c6fc16e.chunk.js", "static/js/7645.6565454c.chunk.js": "./static/js/7645.6565454c.chunk.js", - "static/js/496.5964f8aa.chunk.js": "./static/js/496.5964f8aa.chunk.js", + "static/js/496.0cdc7358.chunk.js": "./static/js/496.0cdc7358.chunk.js", "static/js/7418.8548a710.chunk.js": "./static/js/7418.8548a710.chunk.js", - "static/js/9759.7dadb893.chunk.js": "./static/js/9759.7dadb893.chunk.js", - "static/js/2180.adcde51c.chunk.js": "./static/js/2180.adcde51c.chunk.js", - "static/js/2335.eb54f5e5.chunk.js": "./static/js/2335.eb54f5e5.chunk.js", - "static/js/783.95eb5b37.chunk.js": "./static/js/783.95eb5b37.chunk.js", + "static/js/2535.4df017df.chunk.js": "./static/js/2535.4df017df.chunk.js", + "static/js/1745.9f25b511.chunk.js": "./static/js/1745.9f25b511.chunk.js", + "static/js/2335.ea670fe7.chunk.js": "./static/js/2335.ea670fe7.chunk.js", + "ts.worker.js": "./ts.worker.js", + "css.worker.js": "./css.worker.js", + "html.worker.js": "./html.worker.js", + "json.worker.js": "./json.worker.js", + "editor.worker.js": "./editor.worker.js", "static/media/codicon.ttf": "./static/media/codicon.762fced46d6cddbda272.ttf", "static/media/thumbsUp.svg": "./static/media/thumbsUp.d4a03fbaa64ce85a0045bf8ba77f8e2b.svg", "static/media/error.svg": "./static/media/error.ca9e31d5d3dc34da07e11a00f7af0842.svg", @@ -282,7 +284,7 @@ "index.html": "./index.html" }, "entrypoints": [ - "static/css/main.efd6a3a5.css", - "static/js/main.9fede0a0.js" + "static/css/main.bb5c1fe0.css", + "static/js/main.40147fad.js" ] } \ No newline at end of file diff --git a/ydb/core/viewer/monitoring/css.worker.js b/ydb/core/viewer/monitoring/css.worker.js new file mode 100644 index 000000000000..1e913335a4a5 --- /dev/null +++ b/ydb/core/viewer/monitoring/css.worker.js @@ -0,0 +1,2 @@ +/*! For license information please see css.worker.js.LICENSE.txt */ +(()=>{"use strict";const e=new class{constructor(){this.listeners=[],this.unexpectedErrorHandler=function(e){setTimeout((()=>{if(e.stack){if(a.isErrorNoTelemetry(e))throw new a(e.message+"\n\n"+e.stack);throw new Error(e.message+"\n\n"+e.stack)}throw e}),0)}}emit(e){this.listeners.forEach((t=>{t(e)}))}onUnexpectedError(e){this.unexpectedErrorHandler(e),this.emit(e)}onUnexpectedExternalError(e){this.unexpectedErrorHandler(e)}};function t(t){r(t)||e.onUnexpectedError(t)}function n(e){if(e instanceof Error){const{name:t,message:n}=e;return{$isError:!0,name:t,message:n,stack:e.stacktrace||e.stack,noTelemetry:a.isErrorNoTelemetry(e)}}return e}const i="Canceled";function r(e){return e instanceof o||e instanceof Error&&e.name===i&&e.message===i}class o extends Error{constructor(){super(i),this.name=this.message}}class s extends Error{constructor(e){super("NotSupported"),e&&(this.message=e)}}class a extends Error{constructor(e){super(e),this.name="CodeExpectedError"}static fromError(e){if(e instanceof a)return e;const t=new a;return t.message=e.message,t.stack=e.stack,t}static isErrorNoTelemetry(e){return"CodeExpectedError"===e.name}}class l extends Error{constructor(e){super(e||"An unexpected bug occurred."),Object.setPrototypeOf(this,l.prototype)}}function c(e,t){const n=this;let i,r=!1;return function(){if(r)return i;if(r=!0,t)try{i=e.apply(n,arguments)}finally{t()}else i=e.apply(n,arguments);return i}}var h;!function(e){function t(e){return e&&"object"===typeof e&&"function"===typeof e[Symbol.iterator]}e.is=t;const n=Object.freeze([]);function*i(e){yield e}e.empty=function(){return n},e.single=i,e.wrap=function(e){return t(e)?e:i(e)},e.from=function(e){return e||n},e.reverse=function*(e){for(let t=e.length-1;t>=0;t--)yield e[t]},e.isEmpty=function(e){return!e||!0===e[Symbol.iterator]().next().done},e.first=function(e){return e[Symbol.iterator]().next().value},e.some=function(e,t){for(const n of e)if(t(n))return!0;return!1},e.find=function(e,t){for(const n of e)if(t(n))return n},e.filter=function*(e,t){for(const n of e)t(n)&&(yield n)},e.map=function*(e,t){let n=0;for(const i of e)yield t(i,n++)},e.concat=function*(){for(var e=arguments.length,t=new Array(e),n=0;n2&&void 0!==arguments[2]?arguments[2]:e.length;return function*(){for(t<0&&(t+=e.length),n<0?n+=e.length:n>e.length&&(n=e.length);t1&&void 0!==arguments[1]?arguments[1]:Number.POSITIVE_INFINITY;const i=[];if(0===n)return[i,t];const r=t[Symbol.iterator]();for(let o=0;or}]},e.asyncToArray=async function(e){const t=[];for await(const n of e)t.push(n);return Promise.resolve(t)}}(h||(h={}));let d=null;function p(e){return null===d||void 0===d||d.trackDisposable(e),e}function u(e){null===d||void 0===d||d.markAsDisposed(e)}function m(e,t){null===d||void 0===d||d.setParent(e,t)}function f(e){if(h.is(e)){const n=[];for(const i of e)if(i)try{i.dispose()}catch(t){n.push(t)}if(1===n.length)throw n[0];if(n.length>1)throw new AggregateError(n,"Encountered errors while disposing of store");return Array.isArray(e)?[]:e}if(e)return e.dispose(),e}function g(){for(var e=arguments.length,t=new Array(e),n=0;nf(t)));return function(e,t){if(d)for(const n of e)d.setParent(n,t)}(t,i),i}function b(e){const t=p({dispose:c((()=>{u(t),e()}))});return t}class v{constructor(){this._toDispose=new Set,this._isDisposed=!1,p(this)}dispose(){this._isDisposed||(u(this),this._isDisposed=!0,this.clear())}get isDisposed(){return this._isDisposed}clear(){if(0!==this._toDispose.size)try{f(this._toDispose)}finally{this._toDispose.clear()}}add(e){if(!e)return e;if(e===this)throw new Error("Cannot register a disposable on itself!");return m(e,this),this._isDisposed?v.DISABLE_DISPOSED_WARNING||console.warn(new Error("Trying to add a disposable to a DisposableStore that has already been disposed of. The added object will be leaked!").stack):this._toDispose.add(e),e}deleteAndLeak(e){e&&this._toDispose.has(e)&&(this._toDispose.delete(e),m(e,null))}}v.DISABLE_DISPOSED_WARNING=!1;class y{constructor(){this._store=new v,p(this),m(this._store,this)}dispose(){u(this),this._store.dispose()}_register(e){if(e===this)throw new Error("Cannot register a disposable on itself!");return this._store.add(e)}}y.None=Object.freeze({dispose(){}});class w{constructor(){this._store=new Map,this._isDisposed=!1,p(this)}dispose(){u(this),this._isDisposed=!0,this.clearAndDisposeAll()}clearAndDisposeAll(){if(this._store.size)try{f(this._store.values())}finally{this._store.clear()}}get(e){return this._store.get(e)}set(e,t){let n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];var i;this._isDisposed&&console.warn(new Error("Trying to add a disposable to a DisposableMap that has already been disposed of. The added object will be leaked!").stack),n||null===this._store.get(e)||void 0===i||i.dispose(),this._store.set(e,t)}deleteAndDispose(e){var t;null===this._store.get(e)||void 0===t||t.dispose(),this._store.delete(e)}[Symbol.iterator](){return this._store[Symbol.iterator]()}}class x{constructor(e){this.element=e,this.next=x.Undefined,this.prev=x.Undefined}}x.Undefined=new x(void 0);class S{constructor(){this._first=x.Undefined,this._last=x.Undefined,this._size=0}get size(){return this._size}isEmpty(){return this._first===x.Undefined}clear(){let e=this._first;for(;e!==x.Undefined;){const t=e.next;e.prev=x.Undefined,e.next=x.Undefined,e=t}this._first=x.Undefined,this._last=x.Undefined,this._size=0}unshift(e){return this._insert(e,!1)}push(e){return this._insert(e,!0)}_insert(e,t){const n=new x(e);if(this._first===x.Undefined)this._first=n,this._last=n;else if(t){const e=this._last;this._last=n,n.prev=e,e.next=n}else{const e=this._first;this._first=n,n.next=e,e.prev=n}this._size+=1;let i=!1;return()=>{i||(i=!0,this._remove(n))}}shift(){if(this._first!==x.Undefined){const e=this._first.element;return this._remove(this._first),e}}pop(){if(this._last!==x.Undefined){const e=this._last.element;return this._remove(this._last),e}}_remove(e){if(e.prev!==x.Undefined&&e.next!==x.Undefined){const t=e.prev;t.next=e.next,e.next.prev=t}else e.prev===x.Undefined&&e.next===x.Undefined?(this._first=x.Undefined,this._last=x.Undefined):e.next===x.Undefined?(this._last=this._last.prev,this._last.next=x.Undefined):e.prev===x.Undefined&&(this._first=this._first.next,this._first.prev=x.Undefined);this._size-=1}*[Symbol.iterator](){let e=this._first;for(;e!==x.Undefined;)yield e.element,e=e.next}}const C=globalThis.performance&&"function"===typeof globalThis.performance.now;class _{static create(e){return new _(e)}constructor(e){this._now=C&&!1===e?Date.now:globalThis.performance.now.bind(globalThis.performance),this._startTime=this._now(),this._stopTime=-1}stop(){this._stopTime=this._now()}reset(){this._startTime=this._now(),this._stopTime=-1}elapsed(){return-1!==this._stopTime?this._stopTime-this._startTime:this._now()-this._startTime}}const k=!1;var E;!function(e){function t(e){if(k){const{onDidAddListener:t}=e,n=F.create();let i=0;e.onDidAddListener=()=>{2===++i&&(console.warn("snapshotted emitter LIKELY used public and SHOULD HAVE BEEN created with DisposableStore. snapshotted here"),n.print()),null===t||void 0===t||t()}}}function n(e){return function(t){let n,i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null,r=!1;return n=e((e=>{if(!r)return n?n.dispose():r=!0,t.call(i,e)}),null,arguments.length>2?arguments[2]:void 0),r&&n.dispose(),n}}function i(e,t,n){return o((function(n){let i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null;return e((e=>n.call(i,t(e))),null,arguments.length>2?arguments[2]:void 0)}),n)}function r(e,t,n){return o((function(n){let i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null;return e((e=>t(e)&&n.call(i,e)),null,arguments.length>2?arguments[2]:void 0)}),n)}function o(e,n){let i;const r={onWillAddFirstListener(){i=e(o.fire,o)},onDidRemoveLastListener(){null===i||void 0===i||i.dispose()}};n||t(r);const o=new A(r);return null===n||void 0===n||n.add(o),o.event}function s(e,n){let i,r,o,s,a=arguments.length>2&&void 0!==arguments[2]?arguments[2]:100,l=arguments.length>3&&void 0!==arguments[3]&&arguments[3],c=arguments.length>4&&void 0!==arguments[4]&&arguments[4],h=arguments.length>6?arguments[6]:void 0,d=0;const p={leakWarningThreshold:arguments.length>5?arguments[5]:void 0,onWillAddFirstListener(){i=e((e=>{d++,r=n(r,e),l&&!o&&(u.fire(r),r=void 0),s=()=>{const e=r;r=void 0,o=void 0,(!l||d>1)&&u.fire(e),d=0},"number"===typeof a?(clearTimeout(o),o=setTimeout(s,a)):void 0===o&&(o=0,queueMicrotask(s))}))},onWillRemoveListener(){c&&d>0&&(null===s||void 0===s||s())},onDidRemoveLastListener(){s=void 0,i.dispose()}};h||t(p);const u=new A(p);return null===h||void 0===h||h.add(u),u.event}e.None=()=>y.None,e.defer=function(e,t){return s(e,(()=>{}),0,void 0,!0,void 0,t)},e.once=n,e.map=i,e.forEach=function(e,t,n){return o((function(n){let i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null;return e((e=>{t(e),n.call(i,e)}),null,arguments.length>2?arguments[2]:void 0)}),n)},e.filter=r,e.signal=function(e){return e},e.any=function(){for(var e=arguments.length,t=new Array(e),n=0;n1&&void 0!==arguments[1]?arguments[1]:null,i=arguments.length>2?arguments[2]:void 0;return function(e,t){t instanceof Array?t.push(e):t&&t.add(e);return e}(g(...t.map((t=>t((t=>e.call(n,t)))))),i)}},e.reduce=function(e,t,n,r){let o=n;return i(e,(e=>(o=t(o,e),o)),r)},e.debounce=s,e.accumulate=function(t){let n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0,i=arguments.length>2?arguments[2]:void 0;return e.debounce(t,((e,t)=>e?(e.push(t),e):[t]),n,void 0,!0,void 0,i)},e.latch=function(e){let t,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(e,t)=>e===t,i=!0;return r(e,(e=>{const r=i||!n(e,t);return i=!1,t=e,r}),arguments.length>2?arguments[2]:void 0)},e.split=function(t,n,i){return[e.filter(t,n,i),e.filter(t,(e=>!n(e)),i)]},e.buffer=function(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1],n=arguments.length>3?arguments[3]:void 0,i=(arguments.length>2&&void 0!==arguments[2]?arguments[2]:[]).slice(),r=e((e=>{i?i.push(e):s.fire(e)}));n&&n.add(r);const o=()=>{null===i||void 0===i||i.forEach((e=>s.fire(e))),i=null},s=new A({onWillAddFirstListener(){r||(r=e((e=>s.fire(e))),n&&n.add(r))},onDidAddFirstListener(){i&&(t?setTimeout(o):o())},onDidRemoveLastListener(){r&&r.dispose(),r=null}});return n&&n.add(s),s.event},e.chain=function(e,t){return(n,i,r)=>{const o=t(new l);return e((function(e){const t=o.evaluate(e);t!==a&&n.call(i,t)}),void 0,r)}};const a=Symbol("HaltChainable");class l{constructor(){this.steps=[]}map(e){return this.steps.push(e),this}forEach(e){return this.steps.push((t=>(e(t),t))),this}filter(e){return this.steps.push((t=>e(t)?t:a)),this}reduce(e,t){let n=t;return this.steps.push((t=>(n=e(n,t),n))),this}latch(){let e,t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:(e,t)=>e===t,n=!0;return this.steps.push((i=>{const r=n||!t(i,e);return n=!1,e=i,r?i:a})),this}evaluate(e){for(const t of this.steps)if((e=t(e))===a)break;return e}}e.fromNodeEventEmitter=function(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:e=>e;const i=function(){return r.fire(n(...arguments))},r=new A({onWillAddFirstListener:()=>e.on(t,i),onDidRemoveLastListener:()=>e.removeListener(t,i)});return r.event},e.fromDOMEventEmitter=function(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:e=>e;const i=function(){return r.fire(n(...arguments))},r=new A({onWillAddFirstListener:()=>e.addEventListener(t,i),onDidRemoveLastListener:()=>e.removeEventListener(t,i)});return r.event},e.toPromise=function(e){return new Promise((t=>n(e)(t)))},e.fromPromise=function(e){const t=new A;return e.then((e=>{t.fire(e)}),(()=>{t.fire(void 0)})).finally((()=>{t.dispose()})),t.event},e.runAndSubscribe=function(e,t,n){return t(n),e((e=>t(e)))};class c{constructor(e,n){this._observable=e,this._counter=0,this._hasChanged=!1;const i={onWillAddFirstListener:()=>{e.addObserver(this)},onDidRemoveLastListener:()=>{e.removeObserver(this)}};n||t(i),this.emitter=new A(i),n&&n.add(this.emitter)}beginUpdate(e){this._counter++}handlePossibleChange(e){}handleChange(e,t){this._hasChanged=!0}endUpdate(e){this._counter--,0===this._counter&&(this._observable.reportChanges(),this._hasChanged&&(this._hasChanged=!1,this.emitter.fire(this._observable.get())))}}e.fromObservable=function(e,t){return new c(e,t).emitter.event},e.fromObservableLight=function(e){return(t,n,i)=>{let r=0,o=!1;const s={beginUpdate(){r++},endUpdate(){r--,0===r&&(e.reportChanges(),o&&(o=!1,t.call(n)))},handlePossibleChange(){},handleChange(){o=!0}};e.addObserver(s),e.reportChanges();const a={dispose(){e.removeObserver(s)}};return i instanceof v?i.add(a):Array.isArray(i)&&i.push(a),a}}}(E||(E={}));class R{constructor(e){this.listenerCount=0,this.invocationCount=0,this.elapsedOverall=0,this.durations=[],this.name="".concat(e,"_").concat(R._idPool++),R.all.add(this)}start(e){this._stopWatch=new _,this.listenerCount=e}stop(){if(this._stopWatch){const e=this._stopWatch.elapsed();this.durations.push(e),this.elapsedOverall+=e,this.invocationCount+=1,this._stopWatch=void 0}}}R.all=new Set,R._idPool=0;class N{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:Math.random().toString(18).slice(2,5);this.threshold=e,this.name=t,this._warnCountdown=0}dispose(){var e;null===(e=this._stacks)||void 0===e||e.clear()}check(e,t){const n=this.threshold;if(n<=0||t{const t=this._stacks.get(e.value)||0;this._stacks.set(e.value,t-1)}}}class F{static create(){var e;return new F(null!==(e=(new Error).stack)&&void 0!==e?e:"")}constructor(e){this.value=e}print(){console.warn(this.value.split("\n").slice(2).join("\n"))}}class D{constructor(e){this.value=e}}const T=void 0;class A{constructor(e){var t,n,i,r,o;this._size=0,this._options=e,this._leakageMon=(null===(t=this._options)||void 0===t?void 0:t.leakWarningThreshold)?new N(null!==(i=null===(n=this._options)||void 0===n?void 0:n.leakWarningThreshold)&&void 0!==i?i:-1):void 0,this._perfMon=(null===(r=this._options)||void 0===r?void 0:r._profName)?new R(this._options._profName):void 0,this._deliveryQueue=null===(o=this._options)||void 0===o?void 0:o.deliveryQueue}dispose(){var e,t,n,i;this._disposed||(this._disposed=!0,(null===(e=this._deliveryQueue)||void 0===e?void 0:e.current)===this&&this._deliveryQueue.reset(),this._listeners&&(this._listeners=void 0,this._size=0),null===(n=null===(t=this._options)||void 0===t?void 0:t.onDidRemoveLastListener)||void 0===n||n.call(t),null===(i=this._leakageMon)||void 0===i||i.dispose())}get event(){var e;return null!==(e=this._event)&&void 0!==e||(this._event=(e,t,n)=>{var i,r,o,s,a;if(this._leakageMon&&this._size>3*this._leakageMon.threshold)return console.warn("[".concat(this._leakageMon.name,"] REFUSES to accept new listeners because it exceeded its threshold by far")),y.None;if(this._disposed)return y.None;t&&(e=e.bind(t));const l=new D(e);let c;this._leakageMon&&this._size>=Math.ceil(.2*this._leakageMon.threshold)&&(l.stack=F.create(),c=this._leakageMon.check(l.stack,this._size+1)),this._listeners?this._listeners instanceof D?(null!==(a=this._deliveryQueue)&&void 0!==a||(this._deliveryQueue=new M),this._listeners=[this._listeners,l]):this._listeners.push(l):(null===(r=null===(i=this._options)||void 0===i?void 0:i.onWillAddFirstListener)||void 0===r||r.call(i,this),this._listeners=l,null===(s=null===(o=this._options)||void 0===o?void 0:o.onDidAddFirstListener)||void 0===s||s.call(o,this)),this._size++;const h=b((()=>{null===T||void 0===T||T.unregister(h),null===c||void 0===c||c(),this._removeListener(l)}));if(n instanceof v?n.add(h):Array.isArray(n)&&n.push(h),T){const e=(new Error).stack.split("\n").slice(2).join("\n").trim();T.register(h,e,h)}return h}),this._event}_removeListener(e){var t,n,i,r;if(null===(n=null===(t=this._options)||void 0===t?void 0:t.onWillRemoveListener)||void 0===n||n.call(t,this),!this._listeners)return;if(1===this._size)return this._listeners=void 0,null===(r=null===(i=this._options)||void 0===i?void 0:i.onDidRemoveLastListener)||void 0===r||r.call(i,this),void(this._size=0);const o=this._listeners,s=o.indexOf(e);if(-1===s)throw console.log("disposed?",this._disposed),console.log("size?",this._size),console.log("arr?",JSON.stringify(this._listeners)),new Error("Attempted to dispose unknown listener");this._size--,o[s]=void 0;const a=this._deliveryQueue.current===this;if(2*this._size<=o.length){let e=0;for(let t=0;t0}}class M{constructor(){this.i=-1,this.end=0}enqueue(e,t,n){this.i=0,this.end=n,this.current=e,this.value=t}reset(){this.i=this.end,this.current=void 0,this.value=void 0}}Object.prototype.hasOwnProperty;function z(e){const t=[];for(const n of function(e){let t=[];for(;Object.prototype!==e;)t=t.concat(Object.getOwnPropertyNames(e)),e=Object.getPrototypeOf(e);return t}(e))"function"===typeof e[n]&&t.push(n);return t}let I="undefined"!==typeof document&&document.location&&document.location.hash.indexOf("pseudo=true")>=0;function L(e,t){let n;return n=0===t.length?e:e.replace(/\{(\d+)\}/g,((e,n)=>{const i=n[0],r=t[i];let o=e;return"string"===typeof r?o=r:"number"!==typeof r&&"boolean"!==typeof r&&void 0!==r&&null!==r||(o=String(r)),o})),I&&(n="\uff3b"+n.replace(/[aouei]/g,"$&$&")+"\uff3d"),n}function P(e,t){for(var n=arguments.length,i=new Array(n>2?n-2:0),r=2;r=0,B=K.indexOf("Macintosh")>=0,X=(K.indexOf("Macintosh")>=0||K.indexOf("iPad")>=0||K.indexOf("iPhone")>=0)&&!!navigator.maxTouchPoints&&navigator.maxTouchPoints>0,j=K.indexOf("Linux")>=0,Q=(null===K||void 0===K?void 0:K.indexOf("Mobi"))>=0,G=!0;U=void P({key:"ensureLoaderPluginIsLoaded",comment:["{Locked}"]},"_")||V,Z=U,ee=navigator.language}let se=0;B?se=1:q?se=3:j&&(se=2);const ae=q,le=B,ce=(G&&"function"===typeof ne.importScripts&&ne.origin,K),he="function"===typeof ne.postMessage&&!ne.importScripts;(()=>{if(he){const e=[];ne.addEventListener("message",(t=>{if(t.data&&t.data.vscodeScheduleAsyncWork)for(let n=0,i=e.length;n{const i=++t;e.push({id:i,callback:n}),ne.postMessage({vscodeScheduleAsyncWork:i},"*")}}})();const de=!!(ce&&ce.indexOf("Chrome")>=0);ce&&ce.indexOf("Firefox"),!de&&ce&&ce.indexOf("Safari"),ce&&ce.indexOf("Edg/"),ce&&ce.indexOf("Android");class pe{constructor(e){this.executor=e,this._didRun=!1}get value(){if(!this._didRun)try{this._value=this.executor()}catch(e){this._error=e}finally{this._didRun=!0}if(this._error)throw this._error;return this._value}get rawValue(){return this._value}}var ue;function me(e){return e.replace(/[\\\{\}\*\+\?\|\^\$\.\[\]\(\)]/g,"\\$&")}function fe(e){return e>=65&&e<=90}function ge(e){return 55296<=e&&e<=56319}function be(e){return 56320<=e&&e<=57343}function ve(e,t){return t-56320+(e-55296<<10)+65536}function ye(e,t,n){const i=e.charCodeAt(n);if(ge(i)&&n+1t[3*i+1]))return t[3*i+2];i=2*i+1}return 0}}xe._INSTANCE=null;class Se{static getInstance(e){return ue.cache.get(Array.from(e))}static getLocales(){return ue._locales.value}constructor(e){this.confusableDictionary=e}isAmbiguous(e){return this.confusableDictionary.has(e)}getPrimaryConfusable(e){return this.confusableDictionary.get(e)}getConfusableCodePoints(){return new Set(this.confusableDictionary.keys())}}ue=Se,Se.ambiguousCharacterData=new pe((()=>JSON.parse('{"_common":[8232,32,8233,32,5760,32,8192,32,8193,32,8194,32,8195,32,8196,32,8197,32,8198,32,8200,32,8201,32,8202,32,8287,32,8199,32,8239,32,2042,95,65101,95,65102,95,65103,95,8208,45,8209,45,8210,45,65112,45,1748,45,8259,45,727,45,8722,45,10134,45,11450,45,1549,44,1643,44,8218,44,184,44,42233,44,894,59,2307,58,2691,58,1417,58,1795,58,1796,58,5868,58,65072,58,6147,58,6153,58,8282,58,1475,58,760,58,42889,58,8758,58,720,58,42237,58,451,33,11601,33,660,63,577,63,2429,63,5038,63,42731,63,119149,46,8228,46,1793,46,1794,46,42510,46,68176,46,1632,46,1776,46,42232,46,1373,96,65287,96,8219,96,8242,96,1370,96,1523,96,8175,96,65344,96,900,96,8189,96,8125,96,8127,96,8190,96,697,96,884,96,712,96,714,96,715,96,756,96,699,96,701,96,700,96,702,96,42892,96,1497,96,2036,96,2037,96,5194,96,5836,96,94033,96,94034,96,65339,91,10088,40,10098,40,12308,40,64830,40,65341,93,10089,41,10099,41,12309,41,64831,41,10100,123,119060,123,10101,125,65342,94,8270,42,1645,42,8727,42,66335,42,5941,47,8257,47,8725,47,8260,47,9585,47,10187,47,10744,47,119354,47,12755,47,12339,47,11462,47,20031,47,12035,47,65340,92,65128,92,8726,92,10189,92,10741,92,10745,92,119311,92,119355,92,12756,92,20022,92,12034,92,42872,38,708,94,710,94,5869,43,10133,43,66203,43,8249,60,10094,60,706,60,119350,60,5176,60,5810,60,5120,61,11840,61,12448,61,42239,61,8250,62,10095,62,707,62,119351,62,5171,62,94015,62,8275,126,732,126,8128,126,8764,126,65372,124,65293,45,120784,50,120794,50,120804,50,120814,50,120824,50,130034,50,42842,50,423,50,1000,50,42564,50,5311,50,42735,50,119302,51,120785,51,120795,51,120805,51,120815,51,120825,51,130035,51,42923,51,540,51,439,51,42858,51,11468,51,1248,51,94011,51,71882,51,120786,52,120796,52,120806,52,120816,52,120826,52,130036,52,5070,52,71855,52,120787,53,120797,53,120807,53,120817,53,120827,53,130037,53,444,53,71867,53,120788,54,120798,54,120808,54,120818,54,120828,54,130038,54,11474,54,5102,54,71893,54,119314,55,120789,55,120799,55,120809,55,120819,55,120829,55,130039,55,66770,55,71878,55,2819,56,2538,56,2666,56,125131,56,120790,56,120800,56,120810,56,120820,56,120830,56,130040,56,547,56,546,56,66330,56,2663,57,2920,57,2541,57,3437,57,120791,57,120801,57,120811,57,120821,57,120831,57,130041,57,42862,57,11466,57,71884,57,71852,57,71894,57,9082,97,65345,97,119834,97,119886,97,119938,97,119990,97,120042,97,120094,97,120146,97,120198,97,120250,97,120302,97,120354,97,120406,97,120458,97,593,97,945,97,120514,97,120572,97,120630,97,120688,97,120746,97,65313,65,119808,65,119860,65,119912,65,119964,65,120016,65,120068,65,120120,65,120172,65,120224,65,120276,65,120328,65,120380,65,120432,65,913,65,120488,65,120546,65,120604,65,120662,65,120720,65,5034,65,5573,65,42222,65,94016,65,66208,65,119835,98,119887,98,119939,98,119991,98,120043,98,120095,98,120147,98,120199,98,120251,98,120303,98,120355,98,120407,98,120459,98,388,98,5071,98,5234,98,5551,98,65314,66,8492,66,119809,66,119861,66,119913,66,120017,66,120069,66,120121,66,120173,66,120225,66,120277,66,120329,66,120381,66,120433,66,42932,66,914,66,120489,66,120547,66,120605,66,120663,66,120721,66,5108,66,5623,66,42192,66,66178,66,66209,66,66305,66,65347,99,8573,99,119836,99,119888,99,119940,99,119992,99,120044,99,120096,99,120148,99,120200,99,120252,99,120304,99,120356,99,120408,99,120460,99,7428,99,1010,99,11429,99,43951,99,66621,99,128844,67,71922,67,71913,67,65315,67,8557,67,8450,67,8493,67,119810,67,119862,67,119914,67,119966,67,120018,67,120174,67,120226,67,120278,67,120330,67,120382,67,120434,67,1017,67,11428,67,5087,67,42202,67,66210,67,66306,67,66581,67,66844,67,8574,100,8518,100,119837,100,119889,100,119941,100,119993,100,120045,100,120097,100,120149,100,120201,100,120253,100,120305,100,120357,100,120409,100,120461,100,1281,100,5095,100,5231,100,42194,100,8558,68,8517,68,119811,68,119863,68,119915,68,119967,68,120019,68,120071,68,120123,68,120175,68,120227,68,120279,68,120331,68,120383,68,120435,68,5024,68,5598,68,5610,68,42195,68,8494,101,65349,101,8495,101,8519,101,119838,101,119890,101,119942,101,120046,101,120098,101,120150,101,120202,101,120254,101,120306,101,120358,101,120410,101,120462,101,43826,101,1213,101,8959,69,65317,69,8496,69,119812,69,119864,69,119916,69,120020,69,120072,69,120124,69,120176,69,120228,69,120280,69,120332,69,120384,69,120436,69,917,69,120492,69,120550,69,120608,69,120666,69,120724,69,11577,69,5036,69,42224,69,71846,69,71854,69,66182,69,119839,102,119891,102,119943,102,119995,102,120047,102,120099,102,120151,102,120203,102,120255,102,120307,102,120359,102,120411,102,120463,102,43829,102,42905,102,383,102,7837,102,1412,102,119315,70,8497,70,119813,70,119865,70,119917,70,120021,70,120073,70,120125,70,120177,70,120229,70,120281,70,120333,70,120385,70,120437,70,42904,70,988,70,120778,70,5556,70,42205,70,71874,70,71842,70,66183,70,66213,70,66853,70,65351,103,8458,103,119840,103,119892,103,119944,103,120048,103,120100,103,120152,103,120204,103,120256,103,120308,103,120360,103,120412,103,120464,103,609,103,7555,103,397,103,1409,103,119814,71,119866,71,119918,71,119970,71,120022,71,120074,71,120126,71,120178,71,120230,71,120282,71,120334,71,120386,71,120438,71,1292,71,5056,71,5107,71,42198,71,65352,104,8462,104,119841,104,119945,104,119997,104,120049,104,120101,104,120153,104,120205,104,120257,104,120309,104,120361,104,120413,104,120465,104,1211,104,1392,104,5058,104,65320,72,8459,72,8460,72,8461,72,119815,72,119867,72,119919,72,120023,72,120179,72,120231,72,120283,72,120335,72,120387,72,120439,72,919,72,120494,72,120552,72,120610,72,120668,72,120726,72,11406,72,5051,72,5500,72,42215,72,66255,72,731,105,9075,105,65353,105,8560,105,8505,105,8520,105,119842,105,119894,105,119946,105,119998,105,120050,105,120102,105,120154,105,120206,105,120258,105,120310,105,120362,105,120414,105,120466,105,120484,105,618,105,617,105,953,105,8126,105,890,105,120522,105,120580,105,120638,105,120696,105,120754,105,1110,105,42567,105,1231,105,43893,105,5029,105,71875,105,65354,106,8521,106,119843,106,119895,106,119947,106,119999,106,120051,106,120103,106,120155,106,120207,106,120259,106,120311,106,120363,106,120415,106,120467,106,1011,106,1112,106,65322,74,119817,74,119869,74,119921,74,119973,74,120025,74,120077,74,120129,74,120181,74,120233,74,120285,74,120337,74,120389,74,120441,74,42930,74,895,74,1032,74,5035,74,5261,74,42201,74,119844,107,119896,107,119948,107,120000,107,120052,107,120104,107,120156,107,120208,107,120260,107,120312,107,120364,107,120416,107,120468,107,8490,75,65323,75,119818,75,119870,75,119922,75,119974,75,120026,75,120078,75,120130,75,120182,75,120234,75,120286,75,120338,75,120390,75,120442,75,922,75,120497,75,120555,75,120613,75,120671,75,120729,75,11412,75,5094,75,5845,75,42199,75,66840,75,1472,108,8739,73,9213,73,65512,73,1633,108,1777,73,66336,108,125127,108,120783,73,120793,73,120803,73,120813,73,120823,73,130033,73,65321,73,8544,73,8464,73,8465,73,119816,73,119868,73,119920,73,120024,73,120128,73,120180,73,120232,73,120284,73,120336,73,120388,73,120440,73,65356,108,8572,73,8467,108,119845,108,119897,108,119949,108,120001,108,120053,108,120105,73,120157,73,120209,73,120261,73,120313,73,120365,73,120417,73,120469,73,448,73,120496,73,120554,73,120612,73,120670,73,120728,73,11410,73,1030,73,1216,73,1493,108,1503,108,1575,108,126464,108,126592,108,65166,108,65165,108,1994,108,11599,73,5825,73,42226,73,93992,73,66186,124,66313,124,119338,76,8556,76,8466,76,119819,76,119871,76,119923,76,120027,76,120079,76,120131,76,120183,76,120235,76,120287,76,120339,76,120391,76,120443,76,11472,76,5086,76,5290,76,42209,76,93974,76,71843,76,71858,76,66587,76,66854,76,65325,77,8559,77,8499,77,119820,77,119872,77,119924,77,120028,77,120080,77,120132,77,120184,77,120236,77,120288,77,120340,77,120392,77,120444,77,924,77,120499,77,120557,77,120615,77,120673,77,120731,77,1018,77,11416,77,5047,77,5616,77,5846,77,42207,77,66224,77,66321,77,119847,110,119899,110,119951,110,120003,110,120055,110,120107,110,120159,110,120211,110,120263,110,120315,110,120367,110,120419,110,120471,110,1400,110,1404,110,65326,78,8469,78,119821,78,119873,78,119925,78,119977,78,120029,78,120081,78,120185,78,120237,78,120289,78,120341,78,120393,78,120445,78,925,78,120500,78,120558,78,120616,78,120674,78,120732,78,11418,78,42208,78,66835,78,3074,111,3202,111,3330,111,3458,111,2406,111,2662,111,2790,111,3046,111,3174,111,3302,111,3430,111,3664,111,3792,111,4160,111,1637,111,1781,111,65359,111,8500,111,119848,111,119900,111,119952,111,120056,111,120108,111,120160,111,120212,111,120264,111,120316,111,120368,111,120420,111,120472,111,7439,111,7441,111,43837,111,959,111,120528,111,120586,111,120644,111,120702,111,120760,111,963,111,120532,111,120590,111,120648,111,120706,111,120764,111,11423,111,4351,111,1413,111,1505,111,1607,111,126500,111,126564,111,126596,111,65259,111,65260,111,65258,111,65257,111,1726,111,64428,111,64429,111,64427,111,64426,111,1729,111,64424,111,64425,111,64423,111,64422,111,1749,111,3360,111,4125,111,66794,111,71880,111,71895,111,66604,111,1984,79,2534,79,2918,79,12295,79,70864,79,71904,79,120782,79,120792,79,120802,79,120812,79,120822,79,130032,79,65327,79,119822,79,119874,79,119926,79,119978,79,120030,79,120082,79,120134,79,120186,79,120238,79,120290,79,120342,79,120394,79,120446,79,927,79,120502,79,120560,79,120618,79,120676,79,120734,79,11422,79,1365,79,11604,79,4816,79,2848,79,66754,79,42227,79,71861,79,66194,79,66219,79,66564,79,66838,79,9076,112,65360,112,119849,112,119901,112,119953,112,120005,112,120057,112,120109,112,120161,112,120213,112,120265,112,120317,112,120369,112,120421,112,120473,112,961,112,120530,112,120544,112,120588,112,120602,112,120646,112,120660,112,120704,112,120718,112,120762,112,120776,112,11427,112,65328,80,8473,80,119823,80,119875,80,119927,80,119979,80,120031,80,120083,80,120187,80,120239,80,120291,80,120343,80,120395,80,120447,80,929,80,120504,80,120562,80,120620,80,120678,80,120736,80,11426,80,5090,80,5229,80,42193,80,66197,80,119850,113,119902,113,119954,113,120006,113,120058,113,120110,113,120162,113,120214,113,120266,113,120318,113,120370,113,120422,113,120474,113,1307,113,1379,113,1382,113,8474,81,119824,81,119876,81,119928,81,119980,81,120032,81,120084,81,120188,81,120240,81,120292,81,120344,81,120396,81,120448,81,11605,81,119851,114,119903,114,119955,114,120007,114,120059,114,120111,114,120163,114,120215,114,120267,114,120319,114,120371,114,120423,114,120475,114,43847,114,43848,114,7462,114,11397,114,43905,114,119318,82,8475,82,8476,82,8477,82,119825,82,119877,82,119929,82,120033,82,120189,82,120241,82,120293,82,120345,82,120397,82,120449,82,422,82,5025,82,5074,82,66740,82,5511,82,42211,82,94005,82,65363,115,119852,115,119904,115,119956,115,120008,115,120060,115,120112,115,120164,115,120216,115,120268,115,120320,115,120372,115,120424,115,120476,115,42801,115,445,115,1109,115,43946,115,71873,115,66632,115,65331,83,119826,83,119878,83,119930,83,119982,83,120034,83,120086,83,120138,83,120190,83,120242,83,120294,83,120346,83,120398,83,120450,83,1029,83,1359,83,5077,83,5082,83,42210,83,94010,83,66198,83,66592,83,119853,116,119905,116,119957,116,120009,116,120061,116,120113,116,120165,116,120217,116,120269,116,120321,116,120373,116,120425,116,120477,116,8868,84,10201,84,128872,84,65332,84,119827,84,119879,84,119931,84,119983,84,120035,84,120087,84,120139,84,120191,84,120243,84,120295,84,120347,84,120399,84,120451,84,932,84,120507,84,120565,84,120623,84,120681,84,120739,84,11430,84,5026,84,42196,84,93962,84,71868,84,66199,84,66225,84,66325,84,119854,117,119906,117,119958,117,120010,117,120062,117,120114,117,120166,117,120218,117,120270,117,120322,117,120374,117,120426,117,120478,117,42911,117,7452,117,43854,117,43858,117,651,117,965,117,120534,117,120592,117,120650,117,120708,117,120766,117,1405,117,66806,117,71896,117,8746,85,8899,85,119828,85,119880,85,119932,85,119984,85,120036,85,120088,85,120140,85,120192,85,120244,85,120296,85,120348,85,120400,85,120452,85,1357,85,4608,85,66766,85,5196,85,42228,85,94018,85,71864,85,8744,118,8897,118,65366,118,8564,118,119855,118,119907,118,119959,118,120011,118,120063,118,120115,118,120167,118,120219,118,120271,118,120323,118,120375,118,120427,118,120479,118,7456,118,957,118,120526,118,120584,118,120642,118,120700,118,120758,118,1141,118,1496,118,71430,118,43945,118,71872,118,119309,86,1639,86,1783,86,8548,86,119829,86,119881,86,119933,86,119985,86,120037,86,120089,86,120141,86,120193,86,120245,86,120297,86,120349,86,120401,86,120453,86,1140,86,11576,86,5081,86,5167,86,42719,86,42214,86,93960,86,71840,86,66845,86,623,119,119856,119,119908,119,119960,119,120012,119,120064,119,120116,119,120168,119,120220,119,120272,119,120324,119,120376,119,120428,119,120480,119,7457,119,1121,119,1309,119,1377,119,71434,119,71438,119,71439,119,43907,119,71919,87,71910,87,119830,87,119882,87,119934,87,119986,87,120038,87,120090,87,120142,87,120194,87,120246,87,120298,87,120350,87,120402,87,120454,87,1308,87,5043,87,5076,87,42218,87,5742,120,10539,120,10540,120,10799,120,65368,120,8569,120,119857,120,119909,120,119961,120,120013,120,120065,120,120117,120,120169,120,120221,120,120273,120,120325,120,120377,120,120429,120,120481,120,5441,120,5501,120,5741,88,9587,88,66338,88,71916,88,65336,88,8553,88,119831,88,119883,88,119935,88,119987,88,120039,88,120091,88,120143,88,120195,88,120247,88,120299,88,120351,88,120403,88,120455,88,42931,88,935,88,120510,88,120568,88,120626,88,120684,88,120742,88,11436,88,11613,88,5815,88,42219,88,66192,88,66228,88,66327,88,66855,88,611,121,7564,121,65369,121,119858,121,119910,121,119962,121,120014,121,120066,121,120118,121,120170,121,120222,121,120274,121,120326,121,120378,121,120430,121,120482,121,655,121,7935,121,43866,121,947,121,8509,121,120516,121,120574,121,120632,121,120690,121,120748,121,1199,121,4327,121,71900,121,65337,89,119832,89,119884,89,119936,89,119988,89,120040,89,120092,89,120144,89,120196,89,120248,89,120300,89,120352,89,120404,89,120456,89,933,89,978,89,120508,89,120566,89,120624,89,120682,89,120740,89,11432,89,1198,89,5033,89,5053,89,42220,89,94019,89,71844,89,66226,89,119859,122,119911,122,119963,122,120015,122,120067,122,120119,122,120171,122,120223,122,120275,122,120327,122,120379,122,120431,122,120483,122,7458,122,43923,122,71876,122,66293,90,71909,90,65338,90,8484,90,8488,90,119833,90,119885,90,119937,90,119989,90,120041,90,120197,90,120249,90,120301,90,120353,90,120405,90,120457,90,918,90,120493,90,120551,90,120609,90,120667,90,120725,90,5059,90,42204,90,71849,90,65282,34,65284,36,65285,37,65286,38,65290,42,65291,43,65294,46,65295,47,65296,48,65297,49,65298,50,65299,51,65300,52,65301,53,65302,54,65303,55,65304,56,65305,57,65308,60,65309,61,65310,62,65312,64,65316,68,65318,70,65319,71,65324,76,65329,81,65330,82,65333,85,65334,86,65335,87,65343,95,65346,98,65348,100,65350,102,65355,107,65357,109,65358,110,65361,113,65362,114,65364,116,65365,117,65367,119,65370,122,65371,123,65373,125,119846,109],"_default":[160,32,8211,45,65374,126,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"cs":[65374,126,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"de":[65374,126,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"es":[8211,45,65374,126,65306,58,65281,33,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"fr":[65374,126,65306,58,65281,33,8216,96,8245,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"it":[160,32,8211,45,65374,126,65306,58,65281,33,8216,96,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"ja":[8211,45,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65292,44,65307,59],"ko":[8211,45,65374,126,65306,58,65281,33,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"pl":[65374,126,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"pt-BR":[65374,126,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"qps-ploc":[160,32,8211,45,65374,126,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"ru":[65374,126,65306,58,65281,33,8216,96,8217,96,8245,96,180,96,12494,47,305,105,921,73,1009,112,215,120,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"tr":[160,32,8211,45,65374,126,65306,58,65281,33,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65288,40,65289,41,65292,44,65307,59,65311,63],"zh-hans":[65374,126,65306,58,65281,33,8245,96,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65288,40,65289,41],"zh-hant":[8211,45,65374,126,180,96,12494,47,1047,51,1073,54,1072,97,1040,65,1068,98,1042,66,1089,99,1057,67,1077,101,1045,69,1053,72,305,105,1050,75,921,73,1052,77,1086,111,1054,79,1009,112,1088,112,1056,80,1075,114,1058,84,215,120,1093,120,1061,88,1091,121,1059,89,65283,35,65307,59]}'))),Se.cache=new class{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:JSON.stringify;this.fn=e,this._computeKey=t,this.lastCache=void 0,this.lastArgKey=void 0}get(e){const t=this._computeKey(e);return this.lastArgKey!==t&&(this.lastArgKey=t,this.lastCache=this.fn(e)),this.lastCache}}((e=>{function t(e){const t=new Map;for(let n=0;n!e.startsWith("_")&&e in i));0===o.length&&(o=["_default"]);for(const a of o){r=n(r,t(i[a]))}const s=function(e,t){const n=new Map(e);for(const[i,r]of t)n.set(i,r);return n}(t(i._common),r);return new ue(s)})),Se._locales=new pe((()=>Object.keys(ue.ambiguousCharacterData.value).filter((e=>!e.startsWith("_")))));class Ce{static getRawData(){return JSON.parse("[9,10,11,12,13,32,127,160,173,847,1564,4447,4448,6068,6069,6155,6156,6157,6158,7355,7356,8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8203,8204,8205,8206,8207,8234,8235,8236,8237,8238,8239,8287,8288,8289,8290,8291,8292,8293,8294,8295,8296,8297,8298,8299,8300,8301,8302,8303,10240,12288,12644,65024,65025,65026,65027,65028,65029,65030,65031,65032,65033,65034,65035,65036,65037,65038,65039,65279,65440,65520,65521,65522,65523,65524,65525,65526,65527,65528,65532,78844,119155,119156,119157,119158,119159,119160,119161,119162,917504,917505,917506,917507,917508,917509,917510,917511,917512,917513,917514,917515,917516,917517,917518,917519,917520,917521,917522,917523,917524,917525,917526,917527,917528,917529,917530,917531,917532,917533,917534,917535,917536,917537,917538,917539,917540,917541,917542,917543,917544,917545,917546,917547,917548,917549,917550,917551,917552,917553,917554,917555,917556,917557,917558,917559,917560,917561,917562,917563,917564,917565,917566,917567,917568,917569,917570,917571,917572,917573,917574,917575,917576,917577,917578,917579,917580,917581,917582,917583,917584,917585,917586,917587,917588,917589,917590,917591,917592,917593,917594,917595,917596,917597,917598,917599,917600,917601,917602,917603,917604,917605,917606,917607,917608,917609,917610,917611,917612,917613,917614,917615,917616,917617,917618,917619,917620,917621,917622,917623,917624,917625,917626,917627,917628,917629,917630,917631,917760,917761,917762,917763,917764,917765,917766,917767,917768,917769,917770,917771,917772,917773,917774,917775,917776,917777,917778,917779,917780,917781,917782,917783,917784,917785,917786,917787,917788,917789,917790,917791,917792,917793,917794,917795,917796,917797,917798,917799,917800,917801,917802,917803,917804,917805,917806,917807,917808,917809,917810,917811,917812,917813,917814,917815,917816,917817,917818,917819,917820,917821,917822,917823,917824,917825,917826,917827,917828,917829,917830,917831,917832,917833,917834,917835,917836,917837,917838,917839,917840,917841,917842,917843,917844,917845,917846,917847,917848,917849,917850,917851,917852,917853,917854,917855,917856,917857,917858,917859,917860,917861,917862,917863,917864,917865,917866,917867,917868,917869,917870,917871,917872,917873,917874,917875,917876,917877,917878,917879,917880,917881,917882,917883,917884,917885,917886,917887,917888,917889,917890,917891,917892,917893,917894,917895,917896,917897,917898,917899,917900,917901,917902,917903,917904,917905,917906,917907,917908,917909,917910,917911,917912,917913,917914,917915,917916,917917,917918,917919,917920,917921,917922,917923,917924,917925,917926,917927,917928,917929,917930,917931,917932,917933,917934,917935,917936,917937,917938,917939,917940,917941,917942,917943,917944,917945,917946,917947,917948,917949,917950,917951,917952,917953,917954,917955,917956,917957,917958,917959,917960,917961,917962,917963,917964,917965,917966,917967,917968,917969,917970,917971,917972,917973,917974,917975,917976,917977,917978,917979,917980,917981,917982,917983,917984,917985,917986,917987,917988,917989,917990,917991,917992,917993,917994,917995,917996,917997,917998,917999]")}static getData(){return this._data||(this._data=new Set(Ce.getRawData())),this._data}static isInvisibleCharacter(e){return Ce.getData().has(e)}static get codePoints(){return Ce.getData()}}Ce._data=void 0;const _e="$initialize";let ke;class Ee{constructor(e,t,n,i){this.vsWorker=e,this.req=t,this.method=n,this.args=i,this.type=0}}class Re{constructor(e,t,n,i){this.vsWorker=e,this.seq=t,this.res=n,this.err=i,this.type=1}}class Ne{constructor(e,t,n,i){this.vsWorker=e,this.req=t,this.eventName=n,this.arg=i,this.type=2}}class Fe{constructor(e,t,n){this.vsWorker=e,this.req=t,this.event=n,this.type=3}}class De{constructor(e,t){this.vsWorker=e,this.req=t,this.type=4}}class Te{constructor(e){this._workerId=-1,this._handler=e,this._lastSentReq=0,this._pendingReplies=Object.create(null),this._pendingEmitters=new Map,this._pendingEvents=new Map}setWorkerId(e){this._workerId=e}sendMessage(e,t){const n=String(++this._lastSentReq);return new Promise(((i,r)=>{this._pendingReplies[n]={resolve:i,reject:r},this._send(new Ee(this._workerId,n,e,t))}))}listen(e,t){let n=null;const i=new A({onWillAddFirstListener:()=>{n=String(++this._lastSentReq),this._pendingEmitters.set(n,i),this._send(new Ne(this._workerId,n,e,t))},onDidRemoveLastListener:()=>{this._pendingEmitters.delete(n),this._send(new De(this._workerId,n)),n=null}});return i.event}handleMessage(e){e&&e.vsWorker&&(-1!==this._workerId&&e.vsWorker!==this._workerId||this._handleMessage(e))}_handleMessage(e){switch(e.type){case 1:return this._handleReplyMessage(e);case 0:return this._handleRequestMessage(e);case 2:return this._handleSubscribeEventMessage(e);case 3:return this._handleEventMessage(e);case 4:return this._handleUnsubscribeEventMessage(e)}}_handleReplyMessage(e){if(!this._pendingReplies[e.seq])return void console.warn("Got reply to unknown seq");const t=this._pendingReplies[e.seq];if(delete this._pendingReplies[e.seq],e.err){let n=e.err;return e.err.$isError&&(n=new Error,n.name=e.err.name,n.message=e.err.message,n.stack=e.err.stack),void t.reject(n)}t.resolve(e.res)}_handleRequestMessage(e){const t=e.req;this._handler.handleMessage(e.method,e.args).then((e=>{this._send(new Re(this._workerId,t,e,void 0))}),(e=>{e.detail instanceof Error&&(e.detail=n(e.detail)),this._send(new Re(this._workerId,t,void 0,n(e)))}))}_handleSubscribeEventMessage(e){const t=e.req,n=this._handler.handleEvent(e.eventName,e.arg)((e=>{this._send(new Fe(this._workerId,t,e))}));this._pendingEvents.set(t,n)}_handleEventMessage(e){this._pendingEmitters.has(e.req)?this._pendingEmitters.get(e.req).fire(e.event):console.warn("Got event for unknown req")}_handleUnsubscribeEventMessage(e){this._pendingEvents.has(e.req)?(this._pendingEvents.get(e.req).dispose(),this._pendingEvents.delete(e.req)):console.warn("Got unsubscribe for unknown req")}_send(e){const t=[];if(0===e.type)for(let n=0;nfunction(){const n=Array.prototype.slice.call(arguments,0);return t(e,n)},r=e=>function(t){return n(e,t)},o={};for(const s of e)Me(s)?o[s]=r(s):Ae(s)?o[s]=n(s,void 0):o[s]=i(s);return o}class Ie{constructor(e,t){this._requestHandlerFactory=t,this._requestHandler=null,this._protocol=new Te({sendMessage:(t,n)=>{e(t,n)},handleMessage:(e,t)=>this._handleMessage(e,t),handleEvent:(e,t)=>this._handleEvent(e,t)})}onmessage(e){this._protocol.handleMessage(e)}_handleMessage(e,t){if(e===_e)return this.initialize(t[0],t[1],t[2],t[3]);if(!this._requestHandler||"function"!==typeof this._requestHandler[e])return Promise.reject(new Error("Missing requestHandler or method: "+e));try{return Promise.resolve(this._requestHandler[e].apply(this._requestHandler,t))}catch(Du){return Promise.reject(Du)}}_handleEvent(e,t){if(!this._requestHandler)throw new Error("Missing requestHandler");if(Me(e)){const n=this._requestHandler[e].call(this._requestHandler,t);if("function"!==typeof n)throw new Error("Missing dynamic event ".concat(e," on request handler."));return n}if(Ae(e)){const t=this._requestHandler[e];if("function"!==typeof t)throw new Error("Missing event ".concat(e," on request handler."));return t}throw new Error("Malformed event name ".concat(e))}initialize(e,t,n,i){this._protocol.setWorkerId(e);const r=ze(i,((e,t)=>this._protocol.sendMessage(e,t)),((e,t)=>this._protocol.listen(e,t)));return this._requestHandlerFactory?(this._requestHandler=this._requestHandlerFactory(r),Promise.resolve(z(this._requestHandler))):(t&&("undefined"!==typeof t.baseUrl&&delete t.baseUrl,"undefined"!==typeof t.paths&&"undefined"!==typeof t.paths.vs&&delete t.paths.vs,"undefined"!==typeof t.trustedTypesPolicy&&delete t.trustedTypesPolicy,t.catchError=!0,globalThis.require.config(t)),new Promise(((e,t)=>{(0,globalThis.require)([n],(n=>{this._requestHandler=n.create(r),this._requestHandler?e(z(this._requestHandler)):t(new Error("No RequestHandler!"))}),t)})))}}class Le{constructor(e,t,n,i){this.originalStart=e,this.originalLength=t,this.modifiedStart=n,this.modifiedLength=i}getOriginalEnd(){return this.originalStart+this.originalLength}getModifiedEnd(){return this.modifiedStart+this.modifiedLength}}function Pe(e,t){return(t<<5)-t+e|0}function Oe(e,t){t=Pe(149417,t);for(let n=0,i=e.length;n2&&void 0!==arguments[2]?arguments[2]:32)-t;return(e<>>n)>>>0}function Ve(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0,n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:e.byteLength,i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:0;for(let r=0;r1&&void 0!==arguments[1]?arguments[1]:32;return e instanceof ArrayBuffer?Array.from(new Uint8Array(e)).map((e=>e.toString(16).padStart(2,"0"))).join(""):function(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"0";for(;e.length>>0).toString(16),t/4)}class Ke{constructor(){this._h0=1732584193,this._h1=4023233417,this._h2=2562383102,this._h3=271733878,this._h4=3285377520,this._buff=new Uint8Array(67),this._buffDV=new DataView(this._buff.buffer),this._buffLen=0,this._totalLen=0,this._leftoverHighSurrogate=0,this._finished=!1}update(e){const t=e.length;if(0===t)return;const n=this._buff;let i,r,o=this._buffLen,s=this._leftoverHighSurrogate;for(0!==s?(i=s,r=-1,s=0):(i=e.charCodeAt(0),r=0);;){let a=i;if(ge(i)){if(!(r+1>>6,e[t++]=128|(63&n)>>>0):n<65536?(e[t++]=224|(61440&n)>>>12,e[t++]=128|(4032&n)>>>6,e[t++]=128|(63&n)>>>0):(e[t++]=240|(1835008&n)>>>18,e[t++]=128|(258048&n)>>>12,e[t++]=128|(4032&n)>>>6,e[t++]=128|(63&n)>>>0),t>=64&&(this._step(),t-=64,this._totalLen+=64,e[0]=e[64],e[1]=e[65],e[2]=e[66]),t}digest(){return this._finished||(this._finished=!0,this._leftoverHighSurrogate&&(this._leftoverHighSurrogate=0,this._buffLen=this._push(this._buff,this._buffLen,65533)),this._totalLen+=this._buffLen,this._wrapUp()),Ue(this._h0)+Ue(this._h1)+Ue(this._h2)+Ue(this._h3)+Ue(this._h4)}_wrapUp(){this._buff[this._buffLen++]=128,Ve(this._buff,this._buffLen),this._buffLen>56&&(this._step(),Ve(this._buff));const e=8*this._totalLen;this._buffDV.setUint32(56,Math.floor(e/4294967296),!1),this._buffDV.setUint32(60,e%4294967296,!1),this._step()}_step(){const e=Ke._bigBlock32,t=this._buffDV;for(let h=0;h<64;h+=4)e.setUint32(h,t.getUint32(h,!1),!1);for(let h=64;h<320;h+=4)e.setUint32(h,We(e.getUint32(h-12,!1)^e.getUint32(h-32,!1)^e.getUint32(h-56,!1)^e.getUint32(h-64,!1),1),!1);let n,i,r,o=this._h0,s=this._h1,a=this._h2,l=this._h3,c=this._h4;for(let h=0;h<80;h++)h<20?(n=s&a|~s&l,i=1518500249):h<40?(n=s^a^l,i=1859775393):h<60?(n=s&a|s&l|a&l,i=2400959708):(n=s^a^l,i=3395469782),r=We(o,5)+n+c+i+e.getUint32(4*h,!1)&4294967295,c=l,l=a,a=We(s,30),s=o,o=r;this._h0=this._h0+o&4294967295,this._h1=this._h1+s&4294967295,this._h2=this._h2+a&4294967295,this._h3=this._h3+l&4294967295,this._h4=this._h4+c&4294967295}}Ke._bigBlock32=new DataView(new ArrayBuffer(320));class qe{constructor(e){this.source=e}getElements(){const e=this.source,t=new Int32Array(e.length);for(let n=0,i=e.length;n0||this.m_modifiedCount>0)&&this.m_changes.push(new Le(this.m_originalStart,this.m_originalCount,this.m_modifiedStart,this.m_modifiedCount)),this.m_originalCount=0,this.m_modifiedCount=0,this.m_originalStart=1073741824,this.m_modifiedStart=1073741824}AddOriginalElement(e,t){this.m_originalStart=Math.min(this.m_originalStart,e),this.m_modifiedStart=Math.min(this.m_modifiedStart,t),this.m_originalCount++}AddModifiedElement(e,t){this.m_originalStart=Math.min(this.m_originalStart,e),this.m_modifiedStart=Math.min(this.m_modifiedStart,t),this.m_modifiedCount++}getChanges(){return(this.m_originalCount>0||this.m_modifiedCount>0)&&this.MarkNextChange(),this.m_changes}getReverseChanges(){return(this.m_originalCount>0||this.m_modifiedCount>0)&&this.MarkNextChange(),this.m_changes.reverse(),this.m_changes}}class Ge{constructor(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:null;this.ContinueProcessingPredicate=n,this._originalSequence=e,this._modifiedSequence=t;const[i,r,o]=Ge._getElements(e),[s,a,l]=Ge._getElements(t);this._hasStrings=o&&l,this._originalStringElements=i,this._originalElementsOrHash=r,this._modifiedStringElements=s,this._modifiedElementsOrHash=a,this.m_forwardHistory=[],this.m_reverseHistory=[]}static _isStringArray(e){return e.length>0&&"string"===typeof e[0]}static _getElements(e){const t=e.getElements();if(Ge._isStringArray(t)){const e=new Int32Array(t.length);for(let n=0,i=t.length;n=e&&i>=n&&this.ElementsAreEqual(t,i);)t--,i--;if(e>t||n>i){let r;return n<=i?(je.Assert(e===t+1,"originalStart should only be one more than originalEnd"),r=[new Le(e,0,n,i-n+1)]):e<=t?(je.Assert(n===i+1,"modifiedStart should only be one more than modifiedEnd"),r=[new Le(e,t-e+1,n,0)]):(je.Assert(e===t+1,"originalStart should only be one more than originalEnd"),je.Assert(n===i+1,"modifiedStart should only be one more than modifiedEnd"),r=[]),r}const o=[0],s=[0],a=this.ComputeRecursionPoint(e,t,n,i,o,s,r),l=o[0],c=s[0];if(null!==a)return a;if(!r[0]){const o=this.ComputeDiffRecursive(e,l,n,c,r);let s=[];return s=r[0]?[new Le(l+1,t-(l+1)+1,c+1,i-(c+1)+1)]:this.ComputeDiffRecursive(l+1,t,c+1,i,r),this.ConcatenateChanges(o,s)}return[new Le(e,t-e+1,n,i-n+1)]}WALKTRACE(e,t,n,i,r,o,s,a,l,c,h,d,p,u,m,f,g,b){let v=null,y=null,w=new He,x=t,S=n,C=p[0]-f[0]-i,_=-1073741824,k=this.m_forwardHistory.length-1;do{const t=C+e;t===x||t=0&&(e=(l=this.m_forwardHistory[k])[0],x=1,S=l.length-1)}while(--k>=-1);if(v=w.getReverseChanges(),b[0]){let e=p[0]+1,t=f[0]+1;if(null!==v&&v.length>0){const n=v[v.length-1];e=Math.max(e,n.getOriginalEnd()),t=Math.max(t,n.getModifiedEnd())}y=[new Le(e,d-e+1,t,m-t+1)]}else{w=new He,x=o,S=s,C=p[0]-f[0]-a,_=1073741824,k=g?this.m_reverseHistory.length-1:this.m_reverseHistory.length-2;do{const e=C+r;e===x||e=c[e+1]?(u=(h=c[e+1]-1)-C-a,h>_&&w.MarkNextChange(),_=h+1,w.AddOriginalElement(h+1,u+1),C=e+1-r):(u=(h=c[e-1])-C-a,h>_&&w.MarkNextChange(),_=h,w.AddModifiedElement(h+1,u+1),C=e-1-r),k>=0&&(r=(c=this.m_reverseHistory[k])[0],x=1,S=c.length-1)}while(--k>=-1);y=w.getChanges()}return this.ConcatenateChanges(v,y)}ComputeRecursionPoint(e,t,n,i,r,o,s){let a=0,l=0,c=0,h=0,d=0,p=0;e--,n--,r[0]=0,o[0]=0,this.m_forwardHistory=[],this.m_reverseHistory=[];const u=t-e+(i-n),m=u+1,f=new Int32Array(m),g=new Int32Array(m),b=i-n,v=t-e,y=e-n,w=t-i,x=(v-b)%2===0;f[b]=e,g[v]=t,s[0]=!1;for(let S=1;S<=u/2+1;S++){let u=0,C=0;c=this.ClipDiagonalBound(b-S,S,b,m),h=this.ClipDiagonalBound(b+S,S,b,m);for(let e=c;e<=h;e+=2){a=e===c||eu+C&&(u=a,C=l),!x&&Math.abs(e-v)<=S-1&&a>=g[e])return r[0]=a,o[0]=l,n<=g[e]&&S<=1448?this.WALKTRACE(b,c,h,y,v,d,p,w,f,g,a,t,r,l,i,o,x,s):null}const _=(u-e+(C-n)-S)/2;if(null!==this.ContinueProcessingPredicate&&!this.ContinueProcessingPredicate(u,_))return s[0]=!0,r[0]=u,o[0]=C,_>0&&S<=1448?this.WALKTRACE(b,c,h,y,v,d,p,w,f,g,a,t,r,l,i,o,x,s):(e++,n++,[new Le(e,t-e+1,n,i-n+1)]);d=this.ClipDiagonalBound(v-S,S,v,m),p=this.ClipDiagonalBound(v+S,S,v,m);for(let m=d;m<=p;m+=2){a=m===d||m=g[m+1]?g[m+1]-1:g[m-1],l=a-(m-v)-w;const u=a;for(;a>e&&l>n&&this.ElementsAreEqual(a,l);)a--,l--;if(g[m]=a,x&&Math.abs(m-b)<=S&&a<=f[m])return r[0]=a,o[0]=l,u>=f[m]&&S<=1448?this.WALKTRACE(b,c,h,y,v,d,p,w,f,g,a,t,r,l,i,o,x,s):null}if(S<=1447){let e=new Int32Array(h-c+2);e[0]=b-c+1,$e.Copy2(f,c,e,1,h-c+1),this.m_forwardHistory.push(e),e=new Int32Array(p-d+2),e[0]=v-d+1,$e.Copy2(g,d,e,1,p-d+1),this.m_reverseHistory.push(e)}}return this.WALKTRACE(b,c,h,y,v,d,p,w,f,g,a,t,r,l,i,o,x,s)}PrettifyChanges(e){for(let t=0;t0,s=n.modifiedLength>0;for(;n.originalStart+n.originalLength=0;t--){const n=e[t];let i=0,r=0;if(t>0){const n=e[t-1];i=n.originalStart+n.originalLength,r=n.modifiedStart+n.modifiedLength}const o=n.originalLength>0,s=n.modifiedLength>0;let a=0,l=this._boundaryScore(n.originalStart,n.originalLength,n.modifiedStart,n.modifiedLength);for(let e=1;;e++){const t=n.originalStart-e,c=n.modifiedStart-e;if(tl&&(l=h,a=e)}n.originalStart-=a,n.modifiedStart-=a;const c=[null];t>0&&this.ChangesOverlap(e[t-1],e[t],c)&&(e[t-1]=c[0],e.splice(t,1),t++)}if(this._hasStrings)for(let t=1,n=e.length;t0&&t>a&&(a=t,l=h,c=e)}return a>0?[l,c]:null}_contiguousSequenceScore(e,t,n){let i=0;for(let r=0;r=this._originalElementsOrHash.length-1||this._hasStrings&&/^\s*$/.test(this._originalStringElements[e])}_OriginalRegionIsBoundary(e,t){if(this._OriginalIsBoundary(e)||this._OriginalIsBoundary(e-1))return!0;if(t>0){const n=e+t;if(this._OriginalIsBoundary(n-1)||this._OriginalIsBoundary(n))return!0}return!1}_ModifiedIsBoundary(e){return e<=0||e>=this._modifiedElementsOrHash.length-1||this._hasStrings&&/^\s*$/.test(this._modifiedStringElements[e])}_ModifiedRegionIsBoundary(e,t){if(this._ModifiedIsBoundary(e)||this._ModifiedIsBoundary(e-1))return!0;if(t>0){const n=e+t;if(this._ModifiedIsBoundary(n-1)||this._ModifiedIsBoundary(n))return!0}return!1}_boundaryScore(e,t,n,i){return(this._OriginalRegionIsBoundary(e,t)?1:0)+(this._ModifiedRegionIsBoundary(n,i)?1:0)}ConcatenateChanges(e,t){const n=[];if(0===e.length||0===t.length)return t.length>0?t:e;if(this.ChangesOverlap(e[e.length-1],t[0],n)){const i=new Array(e.length+t.length-1);return $e.Copy(e,0,i,0,e.length-1),i[e.length-1]=n[0],$e.Copy(t,1,i,e.length,t.length-1),i}{const n=new Array(e.length+t.length);return $e.Copy(e,0,n,0,e.length),$e.Copy(t,0,n,e.length,t.length),n}}ChangesOverlap(e,t,n){if(je.Assert(e.originalStart<=t.originalStart,"Left change is not less than or equal to right change"),je.Assert(e.modifiedStart<=t.modifiedStart,"Left change is not less than or equal to right change"),e.originalStart+e.originalLength>=t.originalStart||e.modifiedStart+e.modifiedLength>=t.modifiedStart){const i=e.originalStart;let r=e.originalLength;const o=e.modifiedStart;let s=e.modifiedLength;return e.originalStart+e.originalLength>=t.originalStart&&(r=t.originalStart+t.originalLength-e.originalStart),e.modifiedStart+e.modifiedLength>=t.modifiedStart&&(s=t.modifiedStart+t.modifiedLength-e.modifiedStart),n[0]=new Le(i,r,o,s),!0}return n[0]=null,!1}ClipDiagonalBound(e,t,n,i){if(e>=0&&ee.cwd()}}else ke="undefined"!==typeof process?{get platform(){return process.platform},get arch(){return process.arch},get env(){return{NODE_ENV:"production",PUBLIC_URL:".",WDS_SOCKET_HOST:void 0,WDS_SOCKET_PATH:void 0,WDS_SOCKET_PORT:void 0,FAST_REFRESH:!0,REACT_APP_BACKEND:"http://localhost:8765",REACT_APP_META_BACKEND:"undefined"}},cwd:()=>({NODE_ENV:"production",PUBLIC_URL:".",WDS_SOCKET_HOST:void 0,WDS_SOCKET_PATH:void 0,WDS_SOCKET_PORT:void 0,FAST_REFRESH:!0,REACT_APP_BACKEND:"http://localhost:8765",REACT_APP_META_BACKEND:"undefined"}.VSCODE_CWD||process.cwd())}:{get platform(){return ae?"win32":le?"darwin":"linux"},get arch(){},get env(){return{}},cwd:()=>"/"};const Xe=ke.cwd,Ye=(ke.env,ke.platform),Qe=65,Ze=97,et=90,tt=122,nt=46,it=47,rt=92,ot=58;class st extends Error{constructor(e,t,n){let i;"string"===typeof t&&0===t.indexOf("not ")?(i="must not be",t=t.replace(/^not /,"")):i="must be";const r=-1!==e.indexOf(".")?"property":"argument";let o='The "'.concat(e,'" ').concat(r," ").concat(i," of type ").concat(t);o+=". Received type ".concat(typeof n),super(o),this.code="ERR_INVALID_ARG_TYPE"}}function at(e,t){if("string"!==typeof e)throw new st(t,"string",e)}const lt="win32"===Ye;function ct(e){return e===it||e===rt}function ht(e){return e===it}function dt(e){return e>=Qe&&e<=et||e>=Ze&&e<=tt}function pt(e,t,n,i){let r="",o=0,s=-1,a=0,l=0;for(let c=0;c<=e.length;++c){if(c2){const e=r.lastIndexOf(n);-1===e?(r="",o=0):(r=r.slice(0,e),o=r.length-1-r.lastIndexOf(n)),s=c,a=0;continue}if(0!==r.length){r="",o=0,s=c,a=0;continue}}t&&(r+=r.length>0?"".concat(n,".."):"..",o=2)}else r.length>0?r+="".concat(n).concat(e.slice(s+1,c)):r=e.slice(s+1,c),o=c-s-1;s=c,a=0}else l===nt&&-1!==a?++a:a=-1}return r}function ut(e,t){!function(e,t){if(null===e||"object"!==typeof e)throw new st(t,"Object",e)}(t,"pathObject");const n=t.dir||t.root,i=t.base||"".concat(t.name||"").concat(t.ext||"");return n?n===t.root?"".concat(n).concat(i):"".concat(n).concat(e).concat(i):i}const mt={resolve(){let e="",t="",n=!1;for(let i=arguments.length-1;i>=-1;i--){let r;if(i>=0){if(r=i<0||arguments.length<=i?void 0:arguments[i],at(r,"path"),0===r.length)continue}else 0===e.length?r=Xe():(r={NODE_ENV:"production",PUBLIC_URL:".",WDS_SOCKET_HOST:void 0,WDS_SOCKET_PATH:void 0,WDS_SOCKET_PORT:void 0,FAST_REFRESH:!0,REACT_APP_BACKEND:"http://localhost:8765",REACT_APP_META_BACKEND:"undefined"}["=".concat(e)]||Xe(),(void 0===r||r.slice(0,2).toLowerCase()!==e.toLowerCase()&&r.charCodeAt(2)===rt)&&(r="".concat(e,"\\")));const o=r.length;let s=0,a="",l=!1;const c=r.charCodeAt(0);if(1===o)ct(c)&&(s=1,l=!0);else if(ct(c))if(l=!0,ct(r.charCodeAt(1))){let e=2,t=e;for(;e2&&ct(r.charCodeAt(2))&&(l=!0,s=3));if(a.length>0)if(e.length>0){if(a.toLowerCase()!==e.toLowerCase())continue}else e=a;if(n){if(e.length>0)break}else if(t="".concat(r.slice(s),"\\").concat(t),n=l,l&&e.length>0)break}return t=pt(t,!n,"\\",ct),n?"".concat(e,"\\").concat(t):"".concat(e).concat(t)||"."},normalize(e){at(e,"path");const t=e.length;if(0===t)return".";let n,i=0,r=!1;const o=e.charCodeAt(0);if(1===t)return ht(o)?"\\":e;if(ct(o))if(r=!0,ct(e.charCodeAt(1))){let r=2,o=r;for(;r2&&ct(e.charCodeAt(2))&&(r=!0,i=3));let s=i0&&ct(e.charCodeAt(t-1))&&(s+="\\"),void 0===n?r?"\\".concat(s):s:r?"".concat(n,"\\").concat(s):"".concat(n).concat(s)},isAbsolute(e){at(e,"path");const t=e.length;if(0===t)return!1;const n=e.charCodeAt(0);return ct(n)||t>2&&dt(n)&&e.charCodeAt(1)===ot&&ct(e.charCodeAt(2))},join(){if(0===arguments.length)return".";let e,t;for(let r=0;r0&&(void 0===e?e=t=n:e+="\\".concat(n))}if(void 0===e)return".";let n=!0,i=0;if("string"===typeof t&&ct(t.charCodeAt(0))){++i;const e=t.length;e>1&&ct(t.charCodeAt(1))&&(++i,e>2&&(ct(t.charCodeAt(2))?++i:n=!1))}if(n){for(;i=2&&(e="\\".concat(e.slice(i)))}return mt.normalize(e)},relative(e,t){if(at(e,"from"),at(t,"to"),e===t)return"";const n=mt.resolve(e),i=mt.resolve(t);if(n===i)return"";if((e=n.toLowerCase())===(t=i.toLowerCase()))return"";let r=0;for(;rr&&e.charCodeAt(o-1)===rt;)o--;const s=o-r;let a=0;for(;aa&&t.charCodeAt(l-1)===rt;)l--;const c=l-a,h=sh){if(t.charCodeAt(a+p)===rt)return i.slice(a+p+1);if(2===p)return i.slice(a+p)}s>h&&(e.charCodeAt(r+p)===rt?d=p:2===p&&(d=3)),-1===d&&(d=0)}let u="";for(p=r+d+1;p<=o;++p)p!==o&&e.charCodeAt(p)!==rt||(u+=0===u.length?"..":"\\..");return a+=d,u.length>0?"".concat(u).concat(i.slice(a,l)):(i.charCodeAt(a)===rt&&++a,i.slice(a,l))},toNamespacedPath(e){if("string"!==typeof e||0===e.length)return e;const t=mt.resolve(e);if(t.length<=2)return e;if(t.charCodeAt(0)===rt){if(t.charCodeAt(1)===rt){const e=t.charCodeAt(2);if(63!==e&&e!==nt)return"\\\\?\\UNC\\".concat(t.slice(2))}}else if(dt(t.charCodeAt(0))&&t.charCodeAt(1)===ot&&t.charCodeAt(2)===rt)return"\\\\?\\".concat(t);return e},dirname(e){at(e,"path");const t=e.length;if(0===t)return".";let n=-1,i=0;const r=e.charCodeAt(0);if(1===t)return ct(r)?e:".";if(ct(r)){if(n=i=1,ct(e.charCodeAt(1))){let r=2,o=r;for(;r2&&ct(e.charCodeAt(2))?3:2,i=n);let o=-1,s=!0;for(let a=t-1;a>=i;--a)if(ct(e.charCodeAt(a))){if(!s){o=a;break}}else s=!1;if(-1===o){if(-1===n)return".";o=n}return e.slice(0,o)},basename(e,t){void 0!==t&&at(t,"ext"),at(e,"path");let n,i=0,r=-1,o=!0;if(e.length>=2&&dt(e.charCodeAt(0))&&e.charCodeAt(1)===ot&&(i=2),void 0!==t&&t.length>0&&t.length<=e.length){if(t===e)return"";let s=t.length-1,a=-1;for(n=e.length-1;n>=i;--n){const l=e.charCodeAt(n);if(ct(l)){if(!o){i=n+1;break}}else-1===a&&(o=!1,a=n+1),s>=0&&(l===t.charCodeAt(s)?-1===--s&&(r=n):(s=-1,r=a))}return i===r?r=a:-1===r&&(r=e.length),e.slice(i,r)}for(n=e.length-1;n>=i;--n)if(ct(e.charCodeAt(n))){if(!o){i=n+1;break}}else-1===r&&(o=!1,r=n+1);return-1===r?"":e.slice(i,r)},extname(e){at(e,"path");let t=0,n=-1,i=0,r=-1,o=!0,s=0;e.length>=2&&e.charCodeAt(1)===ot&&dt(e.charCodeAt(0))&&(t=i=2);for(let a=e.length-1;a>=t;--a){const t=e.charCodeAt(a);if(ct(t)){if(!o){i=a+1;break}}else-1===r&&(o=!1,r=a+1),t===nt?-1===n?n=a:1!==s&&(s=1):-1!==n&&(s=-1)}return-1===n||-1===r||0===s||1===s&&n===r-1&&n===i+1?"":e.slice(n,r)},format:ut.bind(null,"\\"),parse(e){at(e,"path");const t={root:"",dir:"",base:"",ext:"",name:""};if(0===e.length)return t;const n=e.length;let i=0,r=e.charCodeAt(0);if(1===n)return ct(r)?(t.root=t.dir=e,t):(t.base=t.name=e,t);if(ct(r)){if(i=1,ct(e.charCodeAt(1))){let t=2,r=t;for(;t0&&(t.root=e.slice(0,i));let o=-1,s=i,a=-1,l=!0,c=e.length-1,h=0;for(;c>=i;--c)if(r=e.charCodeAt(c),ct(r)){if(!l){s=c+1;break}}else-1===a&&(l=!1,a=c+1),r===nt?-1===o?o=c:1!==h&&(h=1):-1!==o&&(h=-1);return-1!==a&&(-1===o||0===h||1===h&&o===a-1&&o===s+1?t.base=t.name=e.slice(s,a):(t.name=e.slice(s,o),t.base=e.slice(s,a),t.ext=e.slice(o,a))),t.dir=s>0&&s!==i?e.slice(0,s-1):t.root,t},sep:"\\",delimiter:";",win32:null,posix:null},ft=(()=>{if(lt){const e=/\\/g;return()=>{const t=Xe().replace(e,"/");return t.slice(t.indexOf("/"))}}return()=>Xe()})(),gt={resolve(){let e="",t=!1;for(let n=arguments.length-1;n>=-1&&!t;n--){const i=n>=0?n<0||arguments.length<=n?void 0:arguments[n]:ft();at(i,"path"),0!==i.length&&(e="".concat(i,"/").concat(e),t=i.charCodeAt(0)===it)}return e=pt(e,!t,"/",ht),t?"/".concat(e):e.length>0?e:"."},normalize(e){if(at(e,"path"),0===e.length)return".";const t=e.charCodeAt(0)===it,n=e.charCodeAt(e.length-1)===it;return 0===(e=pt(e,!t,"/",ht)).length?t?"/":n?"./":".":(n&&(e+="/"),t?"/".concat(e):e)},isAbsolute:e=>(at(e,"path"),e.length>0&&e.charCodeAt(0)===it),join(){if(0===arguments.length)return".";let e;for(let t=0;t0&&(void 0===e?e=n:e+="/".concat(n))}return void 0===e?".":gt.normalize(e)},relative(e,t){if(at(e,"from"),at(t,"to"),e===t)return"";if((e=gt.resolve(e))===(t=gt.resolve(t)))return"";const n=e.length,i=n-1,r=t.length-1,o=io){if(t.charCodeAt(1+a)===it)return t.slice(1+a+1);if(0===a)return t.slice(1+a)}else i>o&&(e.charCodeAt(1+a)===it?s=a:0===a&&(s=0));let l="";for(a=1+s+1;a<=n;++a)a!==n&&e.charCodeAt(a)!==it||(l+=0===l.length?"..":"/..");return"".concat(l).concat(t.slice(1+s))},toNamespacedPath:e=>e,dirname(e){if(at(e,"path"),0===e.length)return".";const t=e.charCodeAt(0)===it;let n=-1,i=!0;for(let r=e.length-1;r>=1;--r)if(e.charCodeAt(r)===it){if(!i){n=r;break}}else i=!1;return-1===n?t?"/":".":t&&1===n?"//":e.slice(0,n)},basename(e,t){void 0!==t&&at(t,"ext"),at(e,"path");let n,i=0,r=-1,o=!0;if(void 0!==t&&t.length>0&&t.length<=e.length){if(t===e)return"";let s=t.length-1,a=-1;for(n=e.length-1;n>=0;--n){const l=e.charCodeAt(n);if(l===it){if(!o){i=n+1;break}}else-1===a&&(o=!1,a=n+1),s>=0&&(l===t.charCodeAt(s)?-1===--s&&(r=n):(s=-1,r=a))}return i===r?r=a:-1===r&&(r=e.length),e.slice(i,r)}for(n=e.length-1;n>=0;--n)if(e.charCodeAt(n)===it){if(!o){i=n+1;break}}else-1===r&&(o=!1,r=n+1);return-1===r?"":e.slice(i,r)},extname(e){at(e,"path");let t=-1,n=0,i=-1,r=!0,o=0;for(let s=e.length-1;s>=0;--s){const a=e.charCodeAt(s);if(a!==it)-1===i&&(r=!1,i=s+1),a===nt?-1===t?t=s:1!==o&&(o=1):-1!==t&&(o=-1);else if(!r){n=s+1;break}}return-1===t||-1===i||0===o||1===o&&t===i-1&&t===n+1?"":e.slice(t,i)},format:ut.bind(null,"/"),parse(e){at(e,"path");const t={root:"",dir:"",base:"",ext:"",name:""};if(0===e.length)return t;const n=e.charCodeAt(0)===it;let i;n?(t.root="/",i=1):i=0;let r=-1,o=0,s=-1,a=!0,l=e.length-1,c=0;for(;l>=i;--l){const t=e.charCodeAt(l);if(t!==it)-1===s&&(a=!1,s=l+1),t===nt?-1===r?r=l:1!==c&&(c=1):-1!==r&&(c=-1);else if(!a){o=l+1;break}}if(-1!==s){const i=0===o&&n?1:o;-1===r||0===c||1===c&&r===s-1&&r===o+1?t.base=t.name=e.slice(i,s):(t.name=e.slice(i,r),t.base=e.slice(i,s),t.ext=e.slice(r,s))}return o>0?t.dir=e.slice(0,o-1):n&&(t.dir="/"),t},sep:"/",delimiter:":",win32:null,posix:null};gt.win32=mt.win32=mt,gt.posix=mt.posix=gt;lt?mt.normalize:gt.normalize,lt?mt.resolve:gt.resolve,lt?mt.relative:gt.relative,lt?mt.dirname:gt.dirname,lt?mt.basename:gt.basename,lt?mt.extname:gt.extname,lt?mt.sep:gt.sep;const bt=/^\w[\w\d+.-]*$/,vt=/^\//,yt=/^\/\//;const wt="",xt="/",St=/^(([^:/?#]+?):)?(\/\/([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?/;class Ct{static isUri(e){return e instanceof Ct||!!e&&("string"===typeof e.authority&&"string"===typeof e.fragment&&"string"===typeof e.path&&"string"===typeof e.query&&"string"===typeof e.scheme&&"string"===typeof e.fsPath&&"function"===typeof e.with&&"function"===typeof e.toString)}constructor(e,t,n,i,r){let o=arguments.length>5&&void 0!==arguments[5]&&arguments[5];"object"===typeof e?(this.scheme=e.scheme||wt,this.authority=e.authority||wt,this.path=e.path||wt,this.query=e.query||wt,this.fragment=e.fragment||wt):(this.scheme=function(e,t){return e||t?e:"file"}(e,o),this.authority=t||wt,this.path=function(e,t){switch(e){case"https":case"http":case"file":t?t[0]!==xt&&(t=xt+t):t=xt}return t}(this.scheme,n||wt),this.query=i||wt,this.fragment=r||wt,function(e,t){if(!e.scheme&&t)throw new Error('[UriError]: Scheme is missing: {scheme: "", authority: "'.concat(e.authority,'", path: "').concat(e.path,'", query: "').concat(e.query,'", fragment: "').concat(e.fragment,'"}'));if(e.scheme&&!bt.test(e.scheme))throw new Error("[UriError]: Scheme contains illegal characters.");if(e.path)if(e.authority){if(!vt.test(e.path))throw new Error('[UriError]: If a URI contains an authority component, then the path component must either be empty or begin with a slash ("/") character')}else if(yt.test(e.path))throw new Error('[UriError]: If a URI does not contain an authority component, then the path cannot begin with two slash characters ("//")')}(this,o))}get fsPath(){return Ft(this,!1)}with(e){if(!e)return this;let{scheme:t,authority:n,path:i,query:r,fragment:o}=e;return void 0===t?t=this.scheme:null===t&&(t=wt),void 0===n?n=this.authority:null===n&&(n=wt),void 0===i?i=this.path:null===i&&(i=wt),void 0===r?r=this.query:null===r&&(r=wt),void 0===o?o=this.fragment:null===o&&(o=wt),t===this.scheme&&n===this.authority&&i===this.path&&r===this.query&&o===this.fragment?this:new kt(t,n,i,r,o)}static parse(e){let t=arguments.length>1&&void 0!==arguments[1]&&arguments[1];const n=St.exec(e);return n?new kt(n[2]||wt,Mt(n[4]||wt),Mt(n[5]||wt),Mt(n[7]||wt),Mt(n[9]||wt),t):new kt(wt,wt,wt,wt,wt)}static file(e){let t=wt;if(ae&&(e=e.replace(/\\/g,xt)),e[0]===xt&&e[1]===xt){const n=e.indexOf(xt,2);-1===n?(t=e.substring(2),e=xt):(t=e.substring(2,n),e=e.substring(n)||xt)}return new kt("file",t,e,wt,wt)}static from(e,t){return new kt(e.scheme,e.authority,e.path,e.query,e.fragment,t)}static joinPath(e){if(!e.path)throw new Error("[UriError]: cannot call joinPath on URI without path");let t;for(var n=arguments.length,i=new Array(n>1?n-1:0),r=1;r0&&void 0!==arguments[0]&&arguments[0])}toJSON(){return this}static revive(e){var t,n;if(e){if(e instanceof Ct)return e;{const i=new kt(e);return i._formatted=null!==(t=e.external)&&void 0!==t?t:null,i._fsPath=e._sep===_t&&null!==(n=e.fsPath)&&void 0!==n?n:null,i}}return e}}const _t=ae?1:void 0;class kt extends Ct{constructor(){super(...arguments),this._formatted=null,this._fsPath=null}get fsPath(){return this._fsPath||(this._fsPath=Ft(this,!1)),this._fsPath}toString(){return arguments.length>0&&void 0!==arguments[0]&&arguments[0]?Dt(this,!0):(this._formatted||(this._formatted=Dt(this,!1)),this._formatted)}toJSON(){const e={$mid:1};return this._fsPath&&(e.fsPath=this._fsPath,e._sep=_t),this._formatted&&(e.external=this._formatted),this.path&&(e.path=this.path),this.scheme&&(e.scheme=this.scheme),this.authority&&(e.authority=this.authority),this.query&&(e.query=this.query),this.fragment&&(e.fragment=this.fragment),e}}const Et={58:"%3A",47:"%2F",63:"%3F",35:"%23",91:"%5B",93:"%5D",64:"%40",33:"%21",36:"%24",38:"%26",39:"%27",40:"%28",41:"%29",42:"%2A",43:"%2B",44:"%2C",59:"%3B",61:"%3D",32:"%20"};function Rt(e,t,n){let i,r=-1;for(let o=0;o=97&&s<=122||s>=65&&s<=90||s>=48&&s<=57||45===s||46===s||95===s||126===s||t&&47===s||n&&91===s||n&&93===s||n&&58===s)-1!==r&&(i+=encodeURIComponent(e.substring(r,o)),r=-1),void 0!==i&&(i+=e.charAt(o));else{void 0===i&&(i=e.substr(0,o));const t=Et[s];void 0!==t?(-1!==r&&(i+=encodeURIComponent(e.substring(r,o)),r=-1),i+=t):-1===r&&(r=o)}}return-1!==r&&(i+=encodeURIComponent(e.substring(r))),void 0!==i?i:e}function Nt(e){let t;for(let n=0;n1&&"file"===e.scheme?"//".concat(e.authority).concat(e.path):47===e.path.charCodeAt(0)&&(e.path.charCodeAt(1)>=65&&e.path.charCodeAt(1)<=90||e.path.charCodeAt(1)>=97&&e.path.charCodeAt(1)<=122)&&58===e.path.charCodeAt(2)?t?e.path.substr(1):e.path[1].toLowerCase()+e.path.substr(2):e.path,ae&&(n=n.replace(/\//g,"\\")),n}function Dt(e,t){const n=t?Nt:Rt;let i="",{scheme:r,authority:o,path:s,query:a,fragment:l}=e;if(r&&(i+=r,i+=":"),(o||"file"===r)&&(i+=xt,i+=xt),o){let e=o.indexOf("@");if(-1!==e){const t=o.substr(0,e);o=o.substr(e+1),e=t.lastIndexOf(":"),-1===e?i+=n(t,!1,!1):(i+=n(t.substr(0,e),!1,!1),i+=":",i+=n(t.substr(e+1),!1,!0)),i+="@"}o=o.toLowerCase(),e=o.lastIndexOf(":"),-1===e?i+=n(o,!1,!0):(i+=n(o.substr(0,e),!1,!0),i+=o.substr(e))}if(s){if(s.length>=3&&47===s.charCodeAt(0)&&58===s.charCodeAt(2)){const e=s.charCodeAt(1);e>=65&&e<=90&&(s="/".concat(String.fromCharCode(e+32),":").concat(s.substr(3)))}else if(s.length>=2&&58===s.charCodeAt(1)){const e=s.charCodeAt(0);e>=65&&e<=90&&(s="".concat(String.fromCharCode(e+32),":").concat(s.substr(2)))}i+=n(s,!0,!1)}return a&&(i+="?",i+=n(a,!1,!1)),l&&(i+="#",i+=t?l:Rt(l,!1,!1)),i}function Tt(e){try{return decodeURIComponent(e)}catch(O){return e.length>3?e.substr(0,3)+Tt(e.substr(3)):e}}const At=/(%[0-9A-Za-z][0-9A-Za-z])+/g;function Mt(e){return e.match(At)?e.replace(At,(e=>Tt(e))):e}class zt{constructor(e,t){this.lineNumber=e,this.column=t}with(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:this.lineNumber,t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:this.column;return e===this.lineNumber&&t===this.column?this:new zt(e,t)}delta(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:0,t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0;return this.with(this.lineNumber+e,this.column+t)}equals(e){return zt.equals(this,e)}static equals(e,t){return!e&&!t||!!e&&!!t&&e.lineNumber===t.lineNumber&&e.column===t.column}isBefore(e){return zt.isBefore(this,e)}static isBefore(e,t){return e.lineNumbern||e===n&&t>i?(this.startLineNumber=n,this.startColumn=i,this.endLineNumber=e,this.endColumn=t):(this.startLineNumber=e,this.startColumn=t,this.endLineNumber=n,this.endColumn=i)}isEmpty(){return It.isEmpty(this)}static isEmpty(e){return e.startLineNumber===e.endLineNumber&&e.startColumn===e.endColumn}containsPosition(e){return It.containsPosition(this,e)}static containsPosition(e,t){return!(t.lineNumbere.endLineNumber)&&(!(t.lineNumber===e.startLineNumber&&t.columne.endColumn))}static strictContainsPosition(e,t){return!(t.lineNumbere.endLineNumber)&&(!(t.lineNumber===e.startLineNumber&&t.column<=e.startColumn)&&!(t.lineNumber===e.endLineNumber&&t.column>=e.endColumn))}containsRange(e){return It.containsRange(this,e)}static containsRange(e,t){return!(t.startLineNumbere.endLineNumber||t.endLineNumber>e.endLineNumber)&&(!(t.startLineNumber===e.startLineNumber&&t.startColumne.endColumn)))}strictContainsRange(e){return It.strictContainsRange(this,e)}static strictContainsRange(e,t){return!(t.startLineNumbere.endLineNumber||t.endLineNumber>e.endLineNumber)&&(!(t.startLineNumber===e.startLineNumber&&t.startColumn<=e.startColumn)&&!(t.endLineNumber===e.endLineNumber&&t.endColumn>=e.endColumn)))}plusRange(e){return It.plusRange(this,e)}static plusRange(e,t){let n,i,r,o;return t.startLineNumbere.endLineNumber?(r=t.endLineNumber,o=t.endColumn):t.endLineNumber===e.endLineNumber?(r=t.endLineNumber,o=Math.max(t.endColumn,e.endColumn)):(r=e.endLineNumber,o=e.endColumn),new It(n,i,r,o)}intersectRanges(e){return It.intersectRanges(this,e)}static intersectRanges(e,t){let n=e.startLineNumber,i=e.startColumn,r=e.endLineNumber,o=e.endColumn;const s=t.startLineNumber,a=t.startColumn,l=t.endLineNumber,c=t.endColumn;return nl?(r=l,o=c):r===l&&(o=Math.min(o,c)),n>r||n===r&&i>o?null:new It(n,i,r,o)}equalsRange(e){return It.equalsRange(this,e)}static equalsRange(e,t){return!e&&!t||!!e&&!!t&&e.startLineNumber===t.startLineNumber&&e.startColumn===t.startColumn&&e.endLineNumber===t.endLineNumber&&e.endColumn===t.endColumn}getEndPosition(){return It.getEndPosition(this)}static getEndPosition(e){return new zt(e.endLineNumber,e.endColumn)}getStartPosition(){return It.getStartPosition(this)}static getStartPosition(e){return new zt(e.startLineNumber,e.startColumn)}toString(){return"["+this.startLineNumber+","+this.startColumn+" -> "+this.endLineNumber+","+this.endColumn+"]"}setEndPosition(e,t){return new It(this.startLineNumber,this.startColumn,e,t)}setStartPosition(e,t){return new It(e,t,this.endLineNumber,this.endColumn)}collapseToStart(){return It.collapseToStart(this)}static collapseToStart(e){return new It(e.startLineNumber,e.startColumn,e.startLineNumber,e.startColumn)}collapseToEnd(){return It.collapseToEnd(this)}static collapseToEnd(e){return new It(e.endLineNumber,e.endColumn,e.endLineNumber,e.endColumn)}delta(e){return new It(this.startLineNumber+e,this.startColumn,this.endLineNumber+e,this.endColumn)}static fromPositions(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:e;return new It(e.lineNumber,e.column,t.lineNumber,t.column)}static lift(e){return e?new It(e.startLineNumber,e.startColumn,e.endLineNumber,e.endColumn):null}static isIRange(e){return e&&"number"===typeof e.startLineNumber&&"number"===typeof e.startColumn&&"number"===typeof e.endLineNumber&&"number"===typeof e.endColumn}static areIntersectingOrTouching(e,t){return!(e.endLineNumbere.startLineNumber}toJSON(){return this}}var Lt;function Pt(e,t){return(n,i)=>t(e(n),e(i))}!function(e){e.isLessThan=function(e){return e<0},e.isLessThanOrEqual=function(e){return e<=0},e.isGreaterThan=function(e){return e>0},e.isNeitherLessOrGreaterThan=function(e){return 0===e},e.greaterThan=1,e.lessThan=-1,e.neitherLessOrGreaterThan=0}(Lt||(Lt={}));const Ot=(e,t)=>e-t;class Wt{constructor(e){this.iterate=e}toArray(){const e=[];return this.iterate((t=>(e.push(t),!0))),e}filter(e){return new Wt((t=>this.iterate((n=>!e(n)||t(n)))))}map(e){return new Wt((t=>this.iterate((n=>t(e(n))))))}findLast(e){let t;return this.iterate((n=>(e(n)&&(t=n),!0))),t}findLastMaxBy(e){let t,n=!0;return this.iterate((i=>((n||Lt.isGreaterThan(e(i,t)))&&(n=!1,t=i),!0))),t}}Wt.empty=new Wt((e=>{}));function Vt(e){return e<0?0:e>255?255:0|e}function Ut(e){return e<0?0:e>4294967295?4294967295:0|e}class Kt{constructor(e){this.values=e,this.prefixSum=new Uint32Array(e.length),this.prefixSumValidIndex=new Int32Array(1),this.prefixSumValidIndex[0]=-1}insertValues(e,t){e=Ut(e);const n=this.values,i=this.prefixSum,r=t.length;return 0!==r&&(this.values=new Uint32Array(n.length+r),this.values.set(n.subarray(0,e),0),this.values.set(n.subarray(e),e+r),this.values.set(t,e),e-1=0&&this.prefixSum.set(i.subarray(0,this.prefixSumValidIndex[0]+1)),!0)}setValue(e,t){return e=Ut(e),t=Ut(t),this.values[e]!==t&&(this.values[e]=t,e-1=n.length)return!1;const r=n.length-e;return t>=r&&(t=r),0!==t&&(this.values=new Uint32Array(n.length-t),this.values.set(n.subarray(0,e),0),this.values.set(n.subarray(e+t),e),this.prefixSum=new Uint32Array(this.values.length),e-1=0&&this.prefixSum.set(i.subarray(0,this.prefixSumValidIndex[0]+1)),!0)}getTotalSum(){return 0===this.values.length?0:this._getPrefixSum(this.values.length-1)}getPrefixSum(e){return e<0?0:(e=Ut(e),this._getPrefixSum(e))}_getPrefixSum(e){if(e<=this.prefixSumValidIndex[0])return this.prefixSum[e];let t=this.prefixSumValidIndex[0]+1;0===t&&(this.prefixSum[0]=this.values[0],t++),e>=this.values.length&&(e=this.values.length-1);for(let n=t;n<=e;n++)this.prefixSum[n]=this.prefixSum[n-1]+this.values[n];return this.prefixSumValidIndex[0]=Math.max(this.prefixSumValidIndex[0],e),this.prefixSum[e]}getIndexOf(e){e=Math.floor(e),this.getTotalSum();let t=0,n=this.values.length-1,i=0,r=0,o=0;for(;t<=n;)if(i=t+(n-t)/2|0,r=this.prefixSum[i],o=r-this.values[i],e=r))break;t=i+1}return new qt(i,e-o)}}class qt{constructor(e,t){this.index=e,this.remainder=t,this._prefixSumIndexOfResultBrand=void 0,this.index=e,this.remainder=t}}class Bt{constructor(e,t,n,i){this._uri=e,this._lines=t,this._eol=n,this._versionId=i,this._lineStarts=null,this._cachedTextValue=null}dispose(){this._lines.length=0}get version(){return this._versionId}getText(){return null===this._cachedTextValue&&(this._cachedTextValue=this._lines.join(this._eol)),this._cachedTextValue}onEvents(e){e.eol&&e.eol!==this._eol&&(this._eol=e.eol,this._lineStarts=null);const t=e.changes;for(const n of t)this._acceptDeleteRange(n.range),this._acceptInsertText(new zt(n.range.startLineNumber,n.range.startColumn),n.text);this._versionId=e.versionId,this._cachedTextValue=null}_ensureLineStarts(){if(!this._lineStarts){const e=this._eol.length,t=this._lines.length,n=new Uint32Array(t);for(let i=0;i/?";const $t=function(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t="(-?\\d*\\.\\d\\w*)|([^";for(const n of jt)e.indexOf(n)>=0||(t+="\\"+n);return t+="\\s]+)",new RegExp(t,"g")}();function Ht(e){let t=$t;if(e&&e instanceof RegExp)if(e.global)t=e;else{let n="g";e.ignoreCase&&(n+="i"),e.multiline&&(n+="m"),e.unicode&&(n+="u"),t=new RegExp(e.source,n)}return t.lastIndex=0,t}const Gt=new S;function Jt(e,t,n,i,r){if(t=Ht(t),r||(r=h.first(Gt)),n.length>r.maxLen){let o=e-r.maxLen/2;return o<0?o=0:i+=o,Jt(e,t,n=n.substring(o,e+r.maxLen/2),i,r)}const o=Date.now(),s=e-1-i;let a=-1,l=null;for(let c=1;!(Date.now()-o>=r.timeBudget);c++){const e=s-r.windowSize*c;t.lastIndex=Math.max(0,e);const i=Xt(t,n,s,a);if(!i&&l)break;if(l=i,e<=0)break;a=e}if(l){const e={word:l[0],startColumn:i+1+l.index,endColumn:i+1+l.index+l[0].length};return t.lastIndex=0,e}return null}function Xt(e,t,n,i){let r;for(;r=e.exec(t);){const t=r.index||0;if(t<=n&&e.lastIndex>=n)return r;if(i>0&&t>i)return null}return null}Gt.unshift({maxLen:1e3,windowSize:15,timeBudget:150});class Yt{constructor(e){const t=Vt(e);this._defaultValue=t,this._asciiMap=Yt._createAsciiMap(t),this._map=new Map}static _createAsciiMap(e){const t=new Uint8Array(256);return t.fill(e),t}set(e,t){const n=Vt(t);e>=0&&e<256?this._asciiMap[e]=n:this._map.set(e,n)}get(e){return e>=0&&e<256?this._asciiMap[e]:this._map.get(e)||this._defaultValue}clear(){this._asciiMap.fill(this._defaultValue),this._map.clear()}}class Qt{constructor(e,t,n){const i=new Uint8Array(e*t);for(let r=0,o=e*t;rt&&(t=o),i>n&&(n=i),s>n&&(n=s)}t++,n++;const i=new Qt(n,t,0);for(let r=0,o=e.length;r=this._maxCharCode?0:this._states.get(e,t)}}let en=null;let tn=null;class nn{static _createLink(e,t,n,i,r){let o=r-1;do{const n=t.charCodeAt(o);if(2!==e.get(n))break;o--}while(o>i);if(i>0){const e=t.charCodeAt(i-1),n=t.charCodeAt(o);(40===e&&41===n||91===e&&93===n||123===e&&125===n)&&o--}return{range:{startLineNumber:n,startColumn:i+1,endLineNumber:n,endColumn:o+2},url:t.substring(i,o+1)}}static computeLinks(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:(null===en&&(en=new Zt([[1,104,2],[1,72,2],[1,102,6],[1,70,6],[2,116,3],[2,84,3],[3,116,4],[3,84,4],[4,112,5],[4,80,5],[5,115,9],[5,83,9],[5,58,10],[6,105,7],[6,73,7],[7,108,8],[7,76,8],[8,101,9],[8,69,9],[9,58,10],[10,47,11],[11,47,12]])),en);const n=function(){if(null===tn){tn=new Yt(0);const e=" \t<>'\"\u3001\u3002\uff61\uff64\uff0c\uff0e\uff1a\uff1b\u2018\u3008\u300c\u300e\u3014\uff08\uff3b\uff5b\uff62\uff63\uff5d\uff3d\uff09\u3015\u300f\u300d\u3009\u2019\uff40\uff5e\u2026";for(let n=0;n=0?(i+=n?1:-1,i<0?i=e.length-1:i%=e.length,e[i]):null}}rn.INSTANCE=new rn;const on=Object.freeze((function(e,t){const n=setTimeout(e.bind(t),0);return{dispose(){clearTimeout(n)}}}));var sn;!function(e){e.isCancellationToken=function(t){return t===e.None||t===e.Cancelled||(t instanceof an||!(!t||"object"!==typeof t)&&("boolean"===typeof t.isCancellationRequested&&"function"===typeof t.onCancellationRequested))},e.None=Object.freeze({isCancellationRequested:!1,onCancellationRequested:E.None}),e.Cancelled=Object.freeze({isCancellationRequested:!0,onCancellationRequested:on})}(sn||(sn={}));class an{constructor(){this._isCancelled=!1,this._emitter=null}cancel(){this._isCancelled||(this._isCancelled=!0,this._emitter&&(this._emitter.fire(void 0),this.dispose()))}get isCancellationRequested(){return this._isCancelled}get onCancellationRequested(){return this._isCancelled?on:(this._emitter||(this._emitter=new A),this._emitter.event)}dispose(){this._emitter&&(this._emitter.dispose(),this._emitter=null)}}class ln{constructor(e){this._token=void 0,this._parentListener=void 0,this._parentListener=e&&e.onCancellationRequested(this.cancel,this)}get token(){return this._token||(this._token=new an),this._token}cancel(){this._token?this._token instanceof an&&this._token.cancel():this._token=sn.Cancelled}dispose(){var e;arguments.length>0&&void 0!==arguments[0]&&arguments[0]&&this.cancel(),null===(e=this._parentListener)||void 0===e||e.dispose(),this._token?this._token instanceof an&&this._token.dispose():this._token=sn.None}}class cn{constructor(){this._keyCodeToStr=[],this._strToKeyCode=Object.create(null)}define(e,t){this._keyCodeToStr[e]=t,this._strToKeyCode[t.toLowerCase()]=e}keyCodeToStr(e){return this._keyCodeToStr[e]}strToKeyCode(e){return this._strToKeyCode[e.toLowerCase()]||0}}const hn=new cn,dn=new cn,pn=new cn,un=new Array(230),mn={},fn=[],gn=Object.create(null),bn=Object.create(null),vn=[],yn=[];for(let Tu=0;Tu<=193;Tu++)vn[Tu]=-1;for(let Tu=0;Tu<=132;Tu++)yn[Tu]=-1;var wn;!function(){const e="",t=[[1,0,"None",0,"unknown",0,"VK_UNKNOWN",e,e],[1,1,"Hyper",0,e,0,e,e,e],[1,2,"Super",0,e,0,e,e,e],[1,3,"Fn",0,e,0,e,e,e],[1,4,"FnLock",0,e,0,e,e,e],[1,5,"Suspend",0,e,0,e,e,e],[1,6,"Resume",0,e,0,e,e,e],[1,7,"Turbo",0,e,0,e,e,e],[1,8,"Sleep",0,e,0,"VK_SLEEP",e,e],[1,9,"WakeUp",0,e,0,e,e,e],[0,10,"KeyA",31,"A",65,"VK_A",e,e],[0,11,"KeyB",32,"B",66,"VK_B",e,e],[0,12,"KeyC",33,"C",67,"VK_C",e,e],[0,13,"KeyD",34,"D",68,"VK_D",e,e],[0,14,"KeyE",35,"E",69,"VK_E",e,e],[0,15,"KeyF",36,"F",70,"VK_F",e,e],[0,16,"KeyG",37,"G",71,"VK_G",e,e],[0,17,"KeyH",38,"H",72,"VK_H",e,e],[0,18,"KeyI",39,"I",73,"VK_I",e,e],[0,19,"KeyJ",40,"J",74,"VK_J",e,e],[0,20,"KeyK",41,"K",75,"VK_K",e,e],[0,21,"KeyL",42,"L",76,"VK_L",e,e],[0,22,"KeyM",43,"M",77,"VK_M",e,e],[0,23,"KeyN",44,"N",78,"VK_N",e,e],[0,24,"KeyO",45,"O",79,"VK_O",e,e],[0,25,"KeyP",46,"P",80,"VK_P",e,e],[0,26,"KeyQ",47,"Q",81,"VK_Q",e,e],[0,27,"KeyR",48,"R",82,"VK_R",e,e],[0,28,"KeyS",49,"S",83,"VK_S",e,e],[0,29,"KeyT",50,"T",84,"VK_T",e,e],[0,30,"KeyU",51,"U",85,"VK_U",e,e],[0,31,"KeyV",52,"V",86,"VK_V",e,e],[0,32,"KeyW",53,"W",87,"VK_W",e,e],[0,33,"KeyX",54,"X",88,"VK_X",e,e],[0,34,"KeyY",55,"Y",89,"VK_Y",e,e],[0,35,"KeyZ",56,"Z",90,"VK_Z",e,e],[0,36,"Digit1",22,"1",49,"VK_1",e,e],[0,37,"Digit2",23,"2",50,"VK_2",e,e],[0,38,"Digit3",24,"3",51,"VK_3",e,e],[0,39,"Digit4",25,"4",52,"VK_4",e,e],[0,40,"Digit5",26,"5",53,"VK_5",e,e],[0,41,"Digit6",27,"6",54,"VK_6",e,e],[0,42,"Digit7",28,"7",55,"VK_7",e,e],[0,43,"Digit8",29,"8",56,"VK_8",e,e],[0,44,"Digit9",30,"9",57,"VK_9",e,e],[0,45,"Digit0",21,"0",48,"VK_0",e,e],[1,46,"Enter",3,"Enter",13,"VK_RETURN",e,e],[1,47,"Escape",9,"Escape",27,"VK_ESCAPE",e,e],[1,48,"Backspace",1,"Backspace",8,"VK_BACK",e,e],[1,49,"Tab",2,"Tab",9,"VK_TAB",e,e],[1,50,"Space",10,"Space",32,"VK_SPACE",e,e],[0,51,"Minus",88,"-",189,"VK_OEM_MINUS","-","OEM_MINUS"],[0,52,"Equal",86,"=",187,"VK_OEM_PLUS","=","OEM_PLUS"],[0,53,"BracketLeft",92,"[",219,"VK_OEM_4","[","OEM_4"],[0,54,"BracketRight",94,"]",221,"VK_OEM_6","]","OEM_6"],[0,55,"Backslash",93,"\\",220,"VK_OEM_5","\\","OEM_5"],[0,56,"IntlHash",0,e,0,e,e,e],[0,57,"Semicolon",85,";",186,"VK_OEM_1",";","OEM_1"],[0,58,"Quote",95,"'",222,"VK_OEM_7","'","OEM_7"],[0,59,"Backquote",91,"`",192,"VK_OEM_3","`","OEM_3"],[0,60,"Comma",87,",",188,"VK_OEM_COMMA",",","OEM_COMMA"],[0,61,"Period",89,".",190,"VK_OEM_PERIOD",".","OEM_PERIOD"],[0,62,"Slash",90,"/",191,"VK_OEM_2","/","OEM_2"],[1,63,"CapsLock",8,"CapsLock",20,"VK_CAPITAL",e,e],[1,64,"F1",59,"F1",112,"VK_F1",e,e],[1,65,"F2",60,"F2",113,"VK_F2",e,e],[1,66,"F3",61,"F3",114,"VK_F3",e,e],[1,67,"F4",62,"F4",115,"VK_F4",e,e],[1,68,"F5",63,"F5",116,"VK_F5",e,e],[1,69,"F6",64,"F6",117,"VK_F6",e,e],[1,70,"F7",65,"F7",118,"VK_F7",e,e],[1,71,"F8",66,"F8",119,"VK_F8",e,e],[1,72,"F9",67,"F9",120,"VK_F9",e,e],[1,73,"F10",68,"F10",121,"VK_F10",e,e],[1,74,"F11",69,"F11",122,"VK_F11",e,e],[1,75,"F12",70,"F12",123,"VK_F12",e,e],[1,76,"PrintScreen",0,e,0,e,e,e],[1,77,"ScrollLock",84,"ScrollLock",145,"VK_SCROLL",e,e],[1,78,"Pause",7,"PauseBreak",19,"VK_PAUSE",e,e],[1,79,"Insert",19,"Insert",45,"VK_INSERT",e,e],[1,80,"Home",14,"Home",36,"VK_HOME",e,e],[1,81,"PageUp",11,"PageUp",33,"VK_PRIOR",e,e],[1,82,"Delete",20,"Delete",46,"VK_DELETE",e,e],[1,83,"End",13,"End",35,"VK_END",e,e],[1,84,"PageDown",12,"PageDown",34,"VK_NEXT",e,e],[1,85,"ArrowRight",17,"RightArrow",39,"VK_RIGHT","Right",e],[1,86,"ArrowLeft",15,"LeftArrow",37,"VK_LEFT","Left",e],[1,87,"ArrowDown",18,"DownArrow",40,"VK_DOWN","Down",e],[1,88,"ArrowUp",16,"UpArrow",38,"VK_UP","Up",e],[1,89,"NumLock",83,"NumLock",144,"VK_NUMLOCK",e,e],[1,90,"NumpadDivide",113,"NumPad_Divide",111,"VK_DIVIDE",e,e],[1,91,"NumpadMultiply",108,"NumPad_Multiply",106,"VK_MULTIPLY",e,e],[1,92,"NumpadSubtract",111,"NumPad_Subtract",109,"VK_SUBTRACT",e,e],[1,93,"NumpadAdd",109,"NumPad_Add",107,"VK_ADD",e,e],[1,94,"NumpadEnter",3,e,0,e,e,e],[1,95,"Numpad1",99,"NumPad1",97,"VK_NUMPAD1",e,e],[1,96,"Numpad2",100,"NumPad2",98,"VK_NUMPAD2",e,e],[1,97,"Numpad3",101,"NumPad3",99,"VK_NUMPAD3",e,e],[1,98,"Numpad4",102,"NumPad4",100,"VK_NUMPAD4",e,e],[1,99,"Numpad5",103,"NumPad5",101,"VK_NUMPAD5",e,e],[1,100,"Numpad6",104,"NumPad6",102,"VK_NUMPAD6",e,e],[1,101,"Numpad7",105,"NumPad7",103,"VK_NUMPAD7",e,e],[1,102,"Numpad8",106,"NumPad8",104,"VK_NUMPAD8",e,e],[1,103,"Numpad9",107,"NumPad9",105,"VK_NUMPAD9",e,e],[1,104,"Numpad0",98,"NumPad0",96,"VK_NUMPAD0",e,e],[1,105,"NumpadDecimal",112,"NumPad_Decimal",110,"VK_DECIMAL",e,e],[0,106,"IntlBackslash",97,"OEM_102",226,"VK_OEM_102",e,e],[1,107,"ContextMenu",58,"ContextMenu",93,e,e,e],[1,108,"Power",0,e,0,e,e,e],[1,109,"NumpadEqual",0,e,0,e,e,e],[1,110,"F13",71,"F13",124,"VK_F13",e,e],[1,111,"F14",72,"F14",125,"VK_F14",e,e],[1,112,"F15",73,"F15",126,"VK_F15",e,e],[1,113,"F16",74,"F16",127,"VK_F16",e,e],[1,114,"F17",75,"F17",128,"VK_F17",e,e],[1,115,"F18",76,"F18",129,"VK_F18",e,e],[1,116,"F19",77,"F19",130,"VK_F19",e,e],[1,117,"F20",78,"F20",131,"VK_F20",e,e],[1,118,"F21",79,"F21",132,"VK_F21",e,e],[1,119,"F22",80,"F22",133,"VK_F22",e,e],[1,120,"F23",81,"F23",134,"VK_F23",e,e],[1,121,"F24",82,"F24",135,"VK_F24",e,e],[1,122,"Open",0,e,0,e,e,e],[1,123,"Help",0,e,0,e,e,e],[1,124,"Select",0,e,0,e,e,e],[1,125,"Again",0,e,0,e,e,e],[1,126,"Undo",0,e,0,e,e,e],[1,127,"Cut",0,e,0,e,e,e],[1,128,"Copy",0,e,0,e,e,e],[1,129,"Paste",0,e,0,e,e,e],[1,130,"Find",0,e,0,e,e,e],[1,131,"AudioVolumeMute",117,"AudioVolumeMute",173,"VK_VOLUME_MUTE",e,e],[1,132,"AudioVolumeUp",118,"AudioVolumeUp",175,"VK_VOLUME_UP",e,e],[1,133,"AudioVolumeDown",119,"AudioVolumeDown",174,"VK_VOLUME_DOWN",e,e],[1,134,"NumpadComma",110,"NumPad_Separator",108,"VK_SEPARATOR",e,e],[0,135,"IntlRo",115,"ABNT_C1",193,"VK_ABNT_C1",e,e],[1,136,"KanaMode",0,e,0,e,e,e],[0,137,"IntlYen",0,e,0,e,e,e],[1,138,"Convert",0,e,0,e,e,e],[1,139,"NonConvert",0,e,0,e,e,e],[1,140,"Lang1",0,e,0,e,e,e],[1,141,"Lang2",0,e,0,e,e,e],[1,142,"Lang3",0,e,0,e,e,e],[1,143,"Lang4",0,e,0,e,e,e],[1,144,"Lang5",0,e,0,e,e,e],[1,145,"Abort",0,e,0,e,e,e],[1,146,"Props",0,e,0,e,e,e],[1,147,"NumpadParenLeft",0,e,0,e,e,e],[1,148,"NumpadParenRight",0,e,0,e,e,e],[1,149,"NumpadBackspace",0,e,0,e,e,e],[1,150,"NumpadMemoryStore",0,e,0,e,e,e],[1,151,"NumpadMemoryRecall",0,e,0,e,e,e],[1,152,"NumpadMemoryClear",0,e,0,e,e,e],[1,153,"NumpadMemoryAdd",0,e,0,e,e,e],[1,154,"NumpadMemorySubtract",0,e,0,e,e,e],[1,155,"NumpadClear",131,"Clear",12,"VK_CLEAR",e,e],[1,156,"NumpadClearEntry",0,e,0,e,e,e],[1,0,e,5,"Ctrl",17,"VK_CONTROL",e,e],[1,0,e,4,"Shift",16,"VK_SHIFT",e,e],[1,0,e,6,"Alt",18,"VK_MENU",e,e],[1,0,e,57,"Meta",91,"VK_COMMAND",e,e],[1,157,"ControlLeft",5,e,0,"VK_LCONTROL",e,e],[1,158,"ShiftLeft",4,e,0,"VK_LSHIFT",e,e],[1,159,"AltLeft",6,e,0,"VK_LMENU",e,e],[1,160,"MetaLeft",57,e,0,"VK_LWIN",e,e],[1,161,"ControlRight",5,e,0,"VK_RCONTROL",e,e],[1,162,"ShiftRight",4,e,0,"VK_RSHIFT",e,e],[1,163,"AltRight",6,e,0,"VK_RMENU",e,e],[1,164,"MetaRight",57,e,0,"VK_RWIN",e,e],[1,165,"BrightnessUp",0,e,0,e,e,e],[1,166,"BrightnessDown",0,e,0,e,e,e],[1,167,"MediaPlay",0,e,0,e,e,e],[1,168,"MediaRecord",0,e,0,e,e,e],[1,169,"MediaFastForward",0,e,0,e,e,e],[1,170,"MediaRewind",0,e,0,e,e,e],[1,171,"MediaTrackNext",124,"MediaTrackNext",176,"VK_MEDIA_NEXT_TRACK",e,e],[1,172,"MediaTrackPrevious",125,"MediaTrackPrevious",177,"VK_MEDIA_PREV_TRACK",e,e],[1,173,"MediaStop",126,"MediaStop",178,"VK_MEDIA_STOP",e,e],[1,174,"Eject",0,e,0,e,e,e],[1,175,"MediaPlayPause",127,"MediaPlayPause",179,"VK_MEDIA_PLAY_PAUSE",e,e],[1,176,"MediaSelect",128,"LaunchMediaPlayer",181,"VK_MEDIA_LAUNCH_MEDIA_SELECT",e,e],[1,177,"LaunchMail",129,"LaunchMail",180,"VK_MEDIA_LAUNCH_MAIL",e,e],[1,178,"LaunchApp2",130,"LaunchApp2",183,"VK_MEDIA_LAUNCH_APP2",e,e],[1,179,"LaunchApp1",0,e,0,"VK_MEDIA_LAUNCH_APP1",e,e],[1,180,"SelectTask",0,e,0,e,e,e],[1,181,"LaunchScreenSaver",0,e,0,e,e,e],[1,182,"BrowserSearch",120,"BrowserSearch",170,"VK_BROWSER_SEARCH",e,e],[1,183,"BrowserHome",121,"BrowserHome",172,"VK_BROWSER_HOME",e,e],[1,184,"BrowserBack",122,"BrowserBack",166,"VK_BROWSER_BACK",e,e],[1,185,"BrowserForward",123,"BrowserForward",167,"VK_BROWSER_FORWARD",e,e],[1,186,"BrowserStop",0,e,0,"VK_BROWSER_STOP",e,e],[1,187,"BrowserRefresh",0,e,0,"VK_BROWSER_REFRESH",e,e],[1,188,"BrowserFavorites",0,e,0,"VK_BROWSER_FAVORITES",e,e],[1,189,"ZoomToggle",0,e,0,e,e,e],[1,190,"MailReply",0,e,0,e,e,e],[1,191,"MailForward",0,e,0,e,e,e],[1,192,"MailSend",0,e,0,e,e,e],[1,0,e,114,"KeyInComposition",229,e,e,e],[1,0,e,116,"ABNT_C2",194,"VK_ABNT_C2",e,e],[1,0,e,96,"OEM_8",223,"VK_OEM_8",e,e],[1,0,e,0,e,0,"VK_KANA",e,e],[1,0,e,0,e,0,"VK_HANGUL",e,e],[1,0,e,0,e,0,"VK_JUNJA",e,e],[1,0,e,0,e,0,"VK_FINAL",e,e],[1,0,e,0,e,0,"VK_HANJA",e,e],[1,0,e,0,e,0,"VK_KANJI",e,e],[1,0,e,0,e,0,"VK_CONVERT",e,e],[1,0,e,0,e,0,"VK_NONCONVERT",e,e],[1,0,e,0,e,0,"VK_ACCEPT",e,e],[1,0,e,0,e,0,"VK_MODECHANGE",e,e],[1,0,e,0,e,0,"VK_SELECT",e,e],[1,0,e,0,e,0,"VK_PRINT",e,e],[1,0,e,0,e,0,"VK_EXECUTE",e,e],[1,0,e,0,e,0,"VK_SNAPSHOT",e,e],[1,0,e,0,e,0,"VK_HELP",e,e],[1,0,e,0,e,0,"VK_APPS",e,e],[1,0,e,0,e,0,"VK_PROCESSKEY",e,e],[1,0,e,0,e,0,"VK_PACKET",e,e],[1,0,e,0,e,0,"VK_DBE_SBCSCHAR",e,e],[1,0,e,0,e,0,"VK_DBE_DBCSCHAR",e,e],[1,0,e,0,e,0,"VK_ATTN",e,e],[1,0,e,0,e,0,"VK_CRSEL",e,e],[1,0,e,0,e,0,"VK_EXSEL",e,e],[1,0,e,0,e,0,"VK_EREOF",e,e],[1,0,e,0,e,0,"VK_PLAY",e,e],[1,0,e,0,e,0,"VK_ZOOM",e,e],[1,0,e,0,e,0,"VK_NONAME",e,e],[1,0,e,0,e,0,"VK_PA1",e,e],[1,0,e,0,e,0,"VK_OEM_CLEAR",e,e]],n=[],i=[];for(const r of t){const[e,t,o,s,a,l,c,h,d]=r;if(i[t]||(i[t]=!0,fn[t]=o,gn[o]=t,bn[o.toLowerCase()]=t,e&&(vn[t]=s,0!==s&&3!==s&&5!==s&&4!==s&&6!==s&&57!==s&&(yn[s]=t))),!n[s]){if(n[s]=!0,!a)throw new Error("String representation missing for key code ".concat(s," around scan code ").concat(o));hn.define(s,a),dn.define(s,h||a),pn.define(s,d||h||a)}l&&(un[l]=s),c&&(mn[c]=s)}yn[3]=46}(),function(e){e.toString=function(e){return hn.keyCodeToStr(e)},e.fromString=function(e){return hn.strToKeyCode(e)},e.toUserSettingsUS=function(e){return dn.keyCodeToStr(e)},e.toUserSettingsGeneral=function(e){return pn.keyCodeToStr(e)},e.fromUserSettings=function(e){return dn.strToKeyCode(e)||pn.strToKeyCode(e)},e.toElectronAccelerator=function(e){if(e>=98&&e<=113)return null;switch(e){case 16:return"Up";case 18:return"Down";case 15:return"Left";case 17:return"Right"}return hn.keyCodeToStr(e)}}(wn||(wn={}));class xn extends It{constructor(e,t,n,i){super(e,t,n,i),this.selectionStartLineNumber=e,this.selectionStartColumn=t,this.positionLineNumber=n,this.positionColumn=i}toString(){return"["+this.selectionStartLineNumber+","+this.selectionStartColumn+" -> "+this.positionLineNumber+","+this.positionColumn+"]"}equalsSelection(e){return xn.selectionsEqual(this,e)}static selectionsEqual(e,t){return e.selectionStartLineNumber===t.selectionStartLineNumber&&e.selectionStartColumn===t.selectionStartColumn&&e.positionLineNumber===t.positionLineNumber&&e.positionColumn===t.positionColumn}getDirection(){return this.selectionStartLineNumber===this.startLineNumber&&this.selectionStartColumn===this.startColumn?0:1}setEndPosition(e,t){return 0===this.getDirection()?new xn(this.startLineNumber,this.startColumn,e,t):new xn(e,t,this.startLineNumber,this.startColumn)}getPosition(){return new zt(this.positionLineNumber,this.positionColumn)}getSelectionStart(){return new zt(this.selectionStartLineNumber,this.selectionStartColumn)}setStartPosition(e,t){return 0===this.getDirection()?new xn(e,t,this.endLineNumber,this.endColumn):new xn(this.endLineNumber,this.endColumn,e,t)}static fromPositions(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:e;return new xn(e.lineNumber,e.column,t.lineNumber,t.column)}static fromRange(e,t){return 0===t?new xn(e.startLineNumber,e.startColumn,e.endLineNumber,e.endColumn):new xn(e.endLineNumber,e.endColumn,e.startLineNumber,e.startColumn)}static liftSelection(e){return new xn(e.selectionStartLineNumber,e.selectionStartColumn,e.positionLineNumber,e.positionColumn)}static selectionsArrEqual(e,t){if(e&&!t||!e&&t)return!1;if(!e&&!t)return!0;if(e.length!==t.length)return!1;for(let n=0,i=e.length;n{this._tokenizationSupports.get(e)===t&&(this._tokenizationSupports.delete(e),this.handleChange([e]))}))}get(e){return this._tokenizationSupports.get(e)||null}registerFactory(e,t){var n;null===(n=this._factories.get(e))||void 0===n||n.dispose();const i=new En(this,e,t);return this._factories.set(e,i),b((()=>{const t=this._factories.get(e);t&&t===i&&(this._factories.delete(e),t.dispose())}))}async getOrCreate(e){const t=this.get(e);if(t)return t;const n=this._factories.get(e);return!n||n.isResolved?null:(await n.resolve(),this.get(e))}isResolved(e){if(this.get(e))return!0;const t=this._factories.get(e);return!(t&&!t.isResolved)}setColorMap(e){this._colorMap=e,this._onDidChange.fire({changedLanguages:Array.from(this._tokenizationSupports.keys()),changedColorMap:!0})}getColorMap(){return this._colorMap}getDefaultBackground(){return this._colorMap&&this._colorMap.length>2?this._colorMap[2]:null}};var On,Wn,Vn,Un,Kn,qn,Bn,jn,$n,Hn,Gn,Jn,Xn,Yn,Qn,Zn,ei,ti,ni,ii,ri,oi,si,ai,li,ci,hi,di,pi,ui,mi,fi,gi,bi,vi,yi,wi,xi,Si,Ci,_i,ki,Ei,Ri,Ni,Fi,Di;!function(e){e[e.Invoke=0]="Invoke",e[e.Automatic=1]="Automatic"}(On||(On={})),function(e){e[e.Unknown=0]="Unknown",e[e.Disabled=1]="Disabled",e[e.Enabled=2]="Enabled"}(Wn||(Wn={})),function(e){e[e.Invoke=1]="Invoke",e[e.Auto=2]="Auto"}(Vn||(Vn={})),function(e){e[e.None=0]="None",e[e.KeepWhitespace=1]="KeepWhitespace",e[e.InsertAsSnippet=4]="InsertAsSnippet"}(Un||(Un={})),function(e){e[e.Method=0]="Method",e[e.Function=1]="Function",e[e.Constructor=2]="Constructor",e[e.Field=3]="Field",e[e.Variable=4]="Variable",e[e.Class=5]="Class",e[e.Struct=6]="Struct",e[e.Interface=7]="Interface",e[e.Module=8]="Module",e[e.Property=9]="Property",e[e.Event=10]="Event",e[e.Operator=11]="Operator",e[e.Unit=12]="Unit",e[e.Value=13]="Value",e[e.Constant=14]="Constant",e[e.Enum=15]="Enum",e[e.EnumMember=16]="EnumMember",e[e.Keyword=17]="Keyword",e[e.Text=18]="Text",e[e.Color=19]="Color",e[e.File=20]="File",e[e.Reference=21]="Reference",e[e.Customcolor=22]="Customcolor",e[e.Folder=23]="Folder",e[e.TypeParameter=24]="TypeParameter",e[e.User=25]="User",e[e.Issue=26]="Issue",e[e.Snippet=27]="Snippet"}(Kn||(Kn={})),function(e){e[e.Deprecated=1]="Deprecated"}(qn||(qn={})),function(e){e[e.Invoke=0]="Invoke",e[e.TriggerCharacter=1]="TriggerCharacter",e[e.TriggerForIncompleteCompletions=2]="TriggerForIncompleteCompletions"}(Bn||(Bn={})),function(e){e[e.EXACT=0]="EXACT",e[e.ABOVE=1]="ABOVE",e[e.BELOW=2]="BELOW"}(jn||(jn={})),function(e){e[e.NotSet=0]="NotSet",e[e.ContentFlush=1]="ContentFlush",e[e.RecoverFromMarkers=2]="RecoverFromMarkers",e[e.Explicit=3]="Explicit",e[e.Paste=4]="Paste",e[e.Undo=5]="Undo",e[e.Redo=6]="Redo"}($n||($n={})),function(e){e[e.LF=1]="LF",e[e.CRLF=2]="CRLF"}(Hn||(Hn={})),function(e){e[e.Text=0]="Text",e[e.Read=1]="Read",e[e.Write=2]="Write"}(Gn||(Gn={})),function(e){e[e.None=0]="None",e[e.Keep=1]="Keep",e[e.Brackets=2]="Brackets",e[e.Advanced=3]="Advanced",e[e.Full=4]="Full"}(Jn||(Jn={})),function(e){e[e.acceptSuggestionOnCommitCharacter=0]="acceptSuggestionOnCommitCharacter",e[e.acceptSuggestionOnEnter=1]="acceptSuggestionOnEnter",e[e.accessibilitySupport=2]="accessibilitySupport",e[e.accessibilityPageSize=3]="accessibilityPageSize",e[e.ariaLabel=4]="ariaLabel",e[e.ariaRequired=5]="ariaRequired",e[e.autoClosingBrackets=6]="autoClosingBrackets",e[e.autoClosingComments=7]="autoClosingComments",e[e.screenReaderAnnounceInlineSuggestion=8]="screenReaderAnnounceInlineSuggestion",e[e.autoClosingDelete=9]="autoClosingDelete",e[e.autoClosingOvertype=10]="autoClosingOvertype",e[e.autoClosingQuotes=11]="autoClosingQuotes",e[e.autoIndent=12]="autoIndent",e[e.automaticLayout=13]="automaticLayout",e[e.autoSurround=14]="autoSurround",e[e.bracketPairColorization=15]="bracketPairColorization",e[e.guides=16]="guides",e[e.codeLens=17]="codeLens",e[e.codeLensFontFamily=18]="codeLensFontFamily",e[e.codeLensFontSize=19]="codeLensFontSize",e[e.colorDecorators=20]="colorDecorators",e[e.colorDecoratorsLimit=21]="colorDecoratorsLimit",e[e.columnSelection=22]="columnSelection",e[e.comments=23]="comments",e[e.contextmenu=24]="contextmenu",e[e.copyWithSyntaxHighlighting=25]="copyWithSyntaxHighlighting",e[e.cursorBlinking=26]="cursorBlinking",e[e.cursorSmoothCaretAnimation=27]="cursorSmoothCaretAnimation",e[e.cursorStyle=28]="cursorStyle",e[e.cursorSurroundingLines=29]="cursorSurroundingLines",e[e.cursorSurroundingLinesStyle=30]="cursorSurroundingLinesStyle",e[e.cursorWidth=31]="cursorWidth",e[e.disableLayerHinting=32]="disableLayerHinting",e[e.disableMonospaceOptimizations=33]="disableMonospaceOptimizations",e[e.domReadOnly=34]="domReadOnly",e[e.dragAndDrop=35]="dragAndDrop",e[e.dropIntoEditor=36]="dropIntoEditor",e[e.emptySelectionClipboard=37]="emptySelectionClipboard",e[e.experimentalWhitespaceRendering=38]="experimentalWhitespaceRendering",e[e.extraEditorClassName=39]="extraEditorClassName",e[e.fastScrollSensitivity=40]="fastScrollSensitivity",e[e.find=41]="find",e[e.fixedOverflowWidgets=42]="fixedOverflowWidgets",e[e.folding=43]="folding",e[e.foldingStrategy=44]="foldingStrategy",e[e.foldingHighlight=45]="foldingHighlight",e[e.foldingImportsByDefault=46]="foldingImportsByDefault",e[e.foldingMaximumRegions=47]="foldingMaximumRegions",e[e.unfoldOnClickAfterEndOfLine=48]="unfoldOnClickAfterEndOfLine",e[e.fontFamily=49]="fontFamily",e[e.fontInfo=50]="fontInfo",e[e.fontLigatures=51]="fontLigatures",e[e.fontSize=52]="fontSize",e[e.fontWeight=53]="fontWeight",e[e.fontVariations=54]="fontVariations",e[e.formatOnPaste=55]="formatOnPaste",e[e.formatOnType=56]="formatOnType",e[e.glyphMargin=57]="glyphMargin",e[e.gotoLocation=58]="gotoLocation",e[e.hideCursorInOverviewRuler=59]="hideCursorInOverviewRuler",e[e.hover=60]="hover",e[e.inDiffEditor=61]="inDiffEditor",e[e.inlineSuggest=62]="inlineSuggest",e[e.inlineEdit=63]="inlineEdit",e[e.letterSpacing=64]="letterSpacing",e[e.lightbulb=65]="lightbulb",e[e.lineDecorationsWidth=66]="lineDecorationsWidth",e[e.lineHeight=67]="lineHeight",e[e.lineNumbers=68]="lineNumbers",e[e.lineNumbersMinChars=69]="lineNumbersMinChars",e[e.linkedEditing=70]="linkedEditing",e[e.links=71]="links",e[e.matchBrackets=72]="matchBrackets",e[e.minimap=73]="minimap",e[e.mouseStyle=74]="mouseStyle",e[e.mouseWheelScrollSensitivity=75]="mouseWheelScrollSensitivity",e[e.mouseWheelZoom=76]="mouseWheelZoom",e[e.multiCursorMergeOverlapping=77]="multiCursorMergeOverlapping",e[e.multiCursorModifier=78]="multiCursorModifier",e[e.multiCursorPaste=79]="multiCursorPaste",e[e.multiCursorLimit=80]="multiCursorLimit",e[e.occurrencesHighlight=81]="occurrencesHighlight",e[e.overviewRulerBorder=82]="overviewRulerBorder",e[e.overviewRulerLanes=83]="overviewRulerLanes",e[e.padding=84]="padding",e[e.pasteAs=85]="pasteAs",e[e.parameterHints=86]="parameterHints",e[e.peekWidgetDefaultFocus=87]="peekWidgetDefaultFocus",e[e.definitionLinkOpensInPeek=88]="definitionLinkOpensInPeek",e[e.quickSuggestions=89]="quickSuggestions",e[e.quickSuggestionsDelay=90]="quickSuggestionsDelay",e[e.readOnly=91]="readOnly",e[e.readOnlyMessage=92]="readOnlyMessage",e[e.renameOnType=93]="renameOnType",e[e.renderControlCharacters=94]="renderControlCharacters",e[e.renderFinalNewline=95]="renderFinalNewline",e[e.renderLineHighlight=96]="renderLineHighlight",e[e.renderLineHighlightOnlyWhenFocus=97]="renderLineHighlightOnlyWhenFocus",e[e.renderValidationDecorations=98]="renderValidationDecorations",e[e.renderWhitespace=99]="renderWhitespace",e[e.revealHorizontalRightPadding=100]="revealHorizontalRightPadding",e[e.roundedSelection=101]="roundedSelection",e[e.rulers=102]="rulers",e[e.scrollbar=103]="scrollbar",e[e.scrollBeyondLastColumn=104]="scrollBeyondLastColumn",e[e.scrollBeyondLastLine=105]="scrollBeyondLastLine",e[e.scrollPredominantAxis=106]="scrollPredominantAxis",e[e.selectionClipboard=107]="selectionClipboard",e[e.selectionHighlight=108]="selectionHighlight",e[e.selectOnLineNumbers=109]="selectOnLineNumbers",e[e.showFoldingControls=110]="showFoldingControls",e[e.showUnused=111]="showUnused",e[e.snippetSuggestions=112]="snippetSuggestions",e[e.smartSelect=113]="smartSelect",e[e.smoothScrolling=114]="smoothScrolling",e[e.stickyScroll=115]="stickyScroll",e[e.stickyTabStops=116]="stickyTabStops",e[e.stopRenderingLineAfter=117]="stopRenderingLineAfter",e[e.suggest=118]="suggest",e[e.suggestFontSize=119]="suggestFontSize",e[e.suggestLineHeight=120]="suggestLineHeight",e[e.suggestOnTriggerCharacters=121]="suggestOnTriggerCharacters",e[e.suggestSelection=122]="suggestSelection",e[e.tabCompletion=123]="tabCompletion",e[e.tabIndex=124]="tabIndex",e[e.unicodeHighlighting=125]="unicodeHighlighting",e[e.unusualLineTerminators=126]="unusualLineTerminators",e[e.useShadowDOM=127]="useShadowDOM",e[e.useTabStops=128]="useTabStops",e[e.wordBreak=129]="wordBreak",e[e.wordSegmenterLocales=130]="wordSegmenterLocales",e[e.wordSeparators=131]="wordSeparators",e[e.wordWrap=132]="wordWrap",e[e.wordWrapBreakAfterCharacters=133]="wordWrapBreakAfterCharacters",e[e.wordWrapBreakBeforeCharacters=134]="wordWrapBreakBeforeCharacters",e[e.wordWrapColumn=135]="wordWrapColumn",e[e.wordWrapOverride1=136]="wordWrapOverride1",e[e.wordWrapOverride2=137]="wordWrapOverride2",e[e.wrappingIndent=138]="wrappingIndent",e[e.wrappingStrategy=139]="wrappingStrategy",e[e.showDeprecated=140]="showDeprecated",e[e.inlayHints=141]="inlayHints",e[e.editorClassName=142]="editorClassName",e[e.pixelRatio=143]="pixelRatio",e[e.tabFocusMode=144]="tabFocusMode",e[e.layoutInfo=145]="layoutInfo",e[e.wrappingInfo=146]="wrappingInfo",e[e.defaultColorDecorators=147]="defaultColorDecorators",e[e.colorDecoratorsActivatedOn=148]="colorDecoratorsActivatedOn",e[e.inlineCompletionsAccessibilityVerbose=149]="inlineCompletionsAccessibilityVerbose"}(Xn||(Xn={})),function(e){e[e.TextDefined=0]="TextDefined",e[e.LF=1]="LF",e[e.CRLF=2]="CRLF"}(Yn||(Yn={})),function(e){e[e.LF=0]="LF",e[e.CRLF=1]="CRLF"}(Qn||(Qn={})),function(e){e[e.Left=1]="Left",e[e.Center=2]="Center",e[e.Right=3]="Right"}(Zn||(Zn={})),function(e){e[e.None=0]="None",e[e.Indent=1]="Indent",e[e.IndentOutdent=2]="IndentOutdent",e[e.Outdent=3]="Outdent"}(ei||(ei={})),function(e){e[e.Both=0]="Both",e[e.Right=1]="Right",e[e.Left=2]="Left",e[e.None=3]="None"}(ti||(ti={})),function(e){e[e.Type=1]="Type",e[e.Parameter=2]="Parameter"}(ni||(ni={})),function(e){e[e.Automatic=0]="Automatic",e[e.Explicit=1]="Explicit"}(ii||(ii={})),function(e){e[e.Invoke=0]="Invoke",e[e.Automatic=1]="Automatic"}(ri||(ri={})),function(e){e[e.DependsOnKbLayout=-1]="DependsOnKbLayout",e[e.Unknown=0]="Unknown",e[e.Backspace=1]="Backspace",e[e.Tab=2]="Tab",e[e.Enter=3]="Enter",e[e.Shift=4]="Shift",e[e.Ctrl=5]="Ctrl",e[e.Alt=6]="Alt",e[e.PauseBreak=7]="PauseBreak",e[e.CapsLock=8]="CapsLock",e[e.Escape=9]="Escape",e[e.Space=10]="Space",e[e.PageUp=11]="PageUp",e[e.PageDown=12]="PageDown",e[e.End=13]="End",e[e.Home=14]="Home",e[e.LeftArrow=15]="LeftArrow",e[e.UpArrow=16]="UpArrow",e[e.RightArrow=17]="RightArrow",e[e.DownArrow=18]="DownArrow",e[e.Insert=19]="Insert",e[e.Delete=20]="Delete",e[e.Digit0=21]="Digit0",e[e.Digit1=22]="Digit1",e[e.Digit2=23]="Digit2",e[e.Digit3=24]="Digit3",e[e.Digit4=25]="Digit4",e[e.Digit5=26]="Digit5",e[e.Digit6=27]="Digit6",e[e.Digit7=28]="Digit7",e[e.Digit8=29]="Digit8",e[e.Digit9=30]="Digit9",e[e.KeyA=31]="KeyA",e[e.KeyB=32]="KeyB",e[e.KeyC=33]="KeyC",e[e.KeyD=34]="KeyD",e[e.KeyE=35]="KeyE",e[e.KeyF=36]="KeyF",e[e.KeyG=37]="KeyG",e[e.KeyH=38]="KeyH",e[e.KeyI=39]="KeyI",e[e.KeyJ=40]="KeyJ",e[e.KeyK=41]="KeyK",e[e.KeyL=42]="KeyL",e[e.KeyM=43]="KeyM",e[e.KeyN=44]="KeyN",e[e.KeyO=45]="KeyO",e[e.KeyP=46]="KeyP",e[e.KeyQ=47]="KeyQ",e[e.KeyR=48]="KeyR",e[e.KeyS=49]="KeyS",e[e.KeyT=50]="KeyT",e[e.KeyU=51]="KeyU",e[e.KeyV=52]="KeyV",e[e.KeyW=53]="KeyW",e[e.KeyX=54]="KeyX",e[e.KeyY=55]="KeyY",e[e.KeyZ=56]="KeyZ",e[e.Meta=57]="Meta",e[e.ContextMenu=58]="ContextMenu",e[e.F1=59]="F1",e[e.F2=60]="F2",e[e.F3=61]="F3",e[e.F4=62]="F4",e[e.F5=63]="F5",e[e.F6=64]="F6",e[e.F7=65]="F7",e[e.F8=66]="F8",e[e.F9=67]="F9",e[e.F10=68]="F10",e[e.F11=69]="F11",e[e.F12=70]="F12",e[e.F13=71]="F13",e[e.F14=72]="F14",e[e.F15=73]="F15",e[e.F16=74]="F16",e[e.F17=75]="F17",e[e.F18=76]="F18",e[e.F19=77]="F19",e[e.F20=78]="F20",e[e.F21=79]="F21",e[e.F22=80]="F22",e[e.F23=81]="F23",e[e.F24=82]="F24",e[e.NumLock=83]="NumLock",e[e.ScrollLock=84]="ScrollLock",e[e.Semicolon=85]="Semicolon",e[e.Equal=86]="Equal",e[e.Comma=87]="Comma",e[e.Minus=88]="Minus",e[e.Period=89]="Period",e[e.Slash=90]="Slash",e[e.Backquote=91]="Backquote",e[e.BracketLeft=92]="BracketLeft",e[e.Backslash=93]="Backslash",e[e.BracketRight=94]="BracketRight",e[e.Quote=95]="Quote",e[e.OEM_8=96]="OEM_8",e[e.IntlBackslash=97]="IntlBackslash",e[e.Numpad0=98]="Numpad0",e[e.Numpad1=99]="Numpad1",e[e.Numpad2=100]="Numpad2",e[e.Numpad3=101]="Numpad3",e[e.Numpad4=102]="Numpad4",e[e.Numpad5=103]="Numpad5",e[e.Numpad6=104]="Numpad6",e[e.Numpad7=105]="Numpad7",e[e.Numpad8=106]="Numpad8",e[e.Numpad9=107]="Numpad9",e[e.NumpadMultiply=108]="NumpadMultiply",e[e.NumpadAdd=109]="NumpadAdd",e[e.NUMPAD_SEPARATOR=110]="NUMPAD_SEPARATOR",e[e.NumpadSubtract=111]="NumpadSubtract",e[e.NumpadDecimal=112]="NumpadDecimal",e[e.NumpadDivide=113]="NumpadDivide",e[e.KEY_IN_COMPOSITION=114]="KEY_IN_COMPOSITION",e[e.ABNT_C1=115]="ABNT_C1",e[e.ABNT_C2=116]="ABNT_C2",e[e.AudioVolumeMute=117]="AudioVolumeMute",e[e.AudioVolumeUp=118]="AudioVolumeUp",e[e.AudioVolumeDown=119]="AudioVolumeDown",e[e.BrowserSearch=120]="BrowserSearch",e[e.BrowserHome=121]="BrowserHome",e[e.BrowserBack=122]="BrowserBack",e[e.BrowserForward=123]="BrowserForward",e[e.MediaTrackNext=124]="MediaTrackNext",e[e.MediaTrackPrevious=125]="MediaTrackPrevious",e[e.MediaStop=126]="MediaStop",e[e.MediaPlayPause=127]="MediaPlayPause",e[e.LaunchMediaPlayer=128]="LaunchMediaPlayer",e[e.LaunchMail=129]="LaunchMail",e[e.LaunchApp2=130]="LaunchApp2",e[e.Clear=131]="Clear",e[e.MAX_VALUE=132]="MAX_VALUE"}(oi||(oi={})),function(e){e[e.Hint=1]="Hint",e[e.Info=2]="Info",e[e.Warning=4]="Warning",e[e.Error=8]="Error"}(si||(si={})),function(e){e[e.Unnecessary=1]="Unnecessary",e[e.Deprecated=2]="Deprecated"}(ai||(ai={})),function(e){e[e.Inline=1]="Inline",e[e.Gutter=2]="Gutter"}(li||(li={})),function(e){e[e.Normal=1]="Normal",e[e.Underlined=2]="Underlined"}(ci||(ci={})),function(e){e[e.UNKNOWN=0]="UNKNOWN",e[e.TEXTAREA=1]="TEXTAREA",e[e.GUTTER_GLYPH_MARGIN=2]="GUTTER_GLYPH_MARGIN",e[e.GUTTER_LINE_NUMBERS=3]="GUTTER_LINE_NUMBERS",e[e.GUTTER_LINE_DECORATIONS=4]="GUTTER_LINE_DECORATIONS",e[e.GUTTER_VIEW_ZONE=5]="GUTTER_VIEW_ZONE",e[e.CONTENT_TEXT=6]="CONTENT_TEXT",e[e.CONTENT_EMPTY=7]="CONTENT_EMPTY",e[e.CONTENT_VIEW_ZONE=8]="CONTENT_VIEW_ZONE",e[e.CONTENT_WIDGET=9]="CONTENT_WIDGET",e[e.OVERVIEW_RULER=10]="OVERVIEW_RULER",e[e.SCROLLBAR=11]="SCROLLBAR",e[e.OVERLAY_WIDGET=12]="OVERLAY_WIDGET",e[e.OUTSIDE_EDITOR=13]="OUTSIDE_EDITOR"}(hi||(hi={})),function(e){e[e.AIGenerated=1]="AIGenerated"}(di||(di={})),function(e){e[e.TOP_RIGHT_CORNER=0]="TOP_RIGHT_CORNER",e[e.BOTTOM_RIGHT_CORNER=1]="BOTTOM_RIGHT_CORNER",e[e.TOP_CENTER=2]="TOP_CENTER"}(pi||(pi={})),function(e){e[e.Left=1]="Left",e[e.Center=2]="Center",e[e.Right=4]="Right",e[e.Full=7]="Full"}(ui||(ui={})),function(e){e[e.Word=0]="Word",e[e.Line=1]="Line",e[e.Suggest=2]="Suggest"}(mi||(mi={})),function(e){e[e.Left=0]="Left",e[e.Right=1]="Right",e[e.None=2]="None",e[e.LeftOfInjectedText=3]="LeftOfInjectedText",e[e.RightOfInjectedText=4]="RightOfInjectedText"}(fi||(fi={})),function(e){e[e.Off=0]="Off",e[e.On=1]="On",e[e.Relative=2]="Relative",e[e.Interval=3]="Interval",e[e.Custom=4]="Custom"}(gi||(gi={})),function(e){e[e.None=0]="None",e[e.Text=1]="Text",e[e.Blocks=2]="Blocks"}(bi||(bi={})),function(e){e[e.Smooth=0]="Smooth",e[e.Immediate=1]="Immediate"}(vi||(vi={})),function(e){e[e.Auto=1]="Auto",e[e.Hidden=2]="Hidden",e[e.Visible=3]="Visible"}(yi||(yi={})),function(e){e[e.LTR=0]="LTR",e[e.RTL=1]="RTL"}(wi||(wi={})),function(e){e.Off="off",e.OnCode="onCode",e.On="on"}(xi||(xi={})),function(e){e[e.Invoke=1]="Invoke",e[e.TriggerCharacter=2]="TriggerCharacter",e[e.ContentChange=3]="ContentChange"}(Si||(Si={})),function(e){e[e.File=0]="File",e[e.Module=1]="Module",e[e.Namespace=2]="Namespace",e[e.Package=3]="Package",e[e.Class=4]="Class",e[e.Method=5]="Method",e[e.Property=6]="Property",e[e.Field=7]="Field",e[e.Constructor=8]="Constructor",e[e.Enum=9]="Enum",e[e.Interface=10]="Interface",e[e.Function=11]="Function",e[e.Variable=12]="Variable",e[e.Constant=13]="Constant",e[e.String=14]="String",e[e.Number=15]="Number",e[e.Boolean=16]="Boolean",e[e.Array=17]="Array",e[e.Object=18]="Object",e[e.Key=19]="Key",e[e.Null=20]="Null",e[e.EnumMember=21]="EnumMember",e[e.Struct=22]="Struct",e[e.Event=23]="Event",e[e.Operator=24]="Operator",e[e.TypeParameter=25]="TypeParameter"}(Ci||(Ci={})),function(e){e[e.Deprecated=1]="Deprecated"}(_i||(_i={})),function(e){e[e.Hidden=0]="Hidden",e[e.Blink=1]="Blink",e[e.Smooth=2]="Smooth",e[e.Phase=3]="Phase",e[e.Expand=4]="Expand",e[e.Solid=5]="Solid"}(ki||(ki={})),function(e){e[e.Line=1]="Line",e[e.Block=2]="Block",e[e.Underline=3]="Underline",e[e.LineThin=4]="LineThin",e[e.BlockOutline=5]="BlockOutline",e[e.UnderlineThin=6]="UnderlineThin"}(Ei||(Ei={})),function(e){e[e.AlwaysGrowsWhenTypingAtEdges=0]="AlwaysGrowsWhenTypingAtEdges",e[e.NeverGrowsWhenTypingAtEdges=1]="NeverGrowsWhenTypingAtEdges",e[e.GrowsOnlyWhenTypingBefore=2]="GrowsOnlyWhenTypingBefore",e[e.GrowsOnlyWhenTypingAfter=3]="GrowsOnlyWhenTypingAfter"}(Ri||(Ri={})),function(e){e[e.None=0]="None",e[e.Same=1]="Same",e[e.Indent=2]="Indent",e[e.DeepIndent=3]="DeepIndent"}(Ni||(Ni={}));class Ti{static chord(e,t){return function(e,t){return(e|(65535&t)<<16>>>0)>>>0}(e,t)}}Ti.CtrlCmd=2048,Ti.Shift=1024,Ti.Alt=512,Ti.WinCtrl=256;class Ai{constructor(e,t){this.uri=e,this.value=t}}class Mi{constructor(e,t){if(this[Fi]="ResourceMap",e instanceof Mi)this.map=new Map(e.map),this.toKey=null!==t&&void 0!==t?t:Mi.defaultToKey;else if(function(e){return Array.isArray(e)}(e)){this.map=new Map,this.toKey=null!==t&&void 0!==t?t:Mi.defaultToKey;for(const[t,n]of e)this.set(t,n)}else this.map=new Map,this.toKey=null!==e&&void 0!==e?e:Mi.defaultToKey}set(e,t){return this.map.set(this.toKey(e),new Ai(e,t)),this}get(e){var t;return null===(t=this.map.get(this.toKey(e)))||void 0===t?void 0:t.value}has(e){return this.map.has(this.toKey(e))}get size(){return this.map.size}clear(){this.map.clear()}delete(e){return this.map.delete(this.toKey(e))}forEach(e,t){"undefined"!==typeof t&&(e=e.bind(t));for(const[n,i]of this.map)e(i.value,i.uri,this)}*values(){for(const e of this.map.values())yield e.value}*keys(){for(const e of this.map.values())yield e.uri}*entries(){for(const e of this.map.values())yield[e.uri,e.value]}*[(Fi=Symbol.toStringTag,Symbol.iterator)](){for(const[,e]of this.map)yield[e.uri,e.value]}}Mi.defaultToKey=e=>e.toString();class zi{constructor(){this[Di]="LinkedMap",this._map=new Map,this._head=void 0,this._tail=void 0,this._size=0,this._state=0}clear(){this._map.clear(),this._head=void 0,this._tail=void 0,this._size=0,this._state++}isEmpty(){return!this._head&&!this._tail}get size(){return this._size}get first(){var e;return null===(e=this._head)||void 0===e?void 0:e.value}get last(){var e;return null===(e=this._tail)||void 0===e?void 0:e.value}has(e){return this._map.has(e)}get(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0;const n=this._map.get(e);if(n)return 0!==t&&this.touch(n,t),n.value}set(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,i=this._map.get(e);if(i)i.value=t,0!==n&&this.touch(i,n);else{switch(i={key:e,value:t,next:void 0,previous:void 0},n){case 0:case 2:default:this.addItemLast(i);break;case 1:this.addItemFirst(i)}this._map.set(e,i),this._size++}return this}delete(e){return!!this.remove(e)}remove(e){const t=this._map.get(e);if(t)return this._map.delete(e),this.removeItem(t),this._size--,t.value}shift(){if(!this._head&&!this._tail)return;if(!this._head||!this._tail)throw new Error("Invalid list");const e=this._head;return this._map.delete(e.key),this.removeItem(e),this._size--,e.value}forEach(e,t){const n=this._state;let i=this._head;for(;i;){if(t?e.bind(t)(i.value,i.key,this):e(i.value,i.key,this),this._state!==n)throw new Error("LinkedMap got modified during iteration.");i=i.next}}keys(){const e=this,t=this._state;let n=this._head;const i={[Symbol.iterator]:()=>i,next(){if(e._state!==t)throw new Error("LinkedMap got modified during iteration.");if(n){const e={value:n.key,done:!1};return n=n.next,e}return{value:void 0,done:!0}}};return i}values(){const e=this,t=this._state;let n=this._head;const i={[Symbol.iterator]:()=>i,next(){if(e._state!==t)throw new Error("LinkedMap got modified during iteration.");if(n){const e={value:n.value,done:!1};return n=n.next,e}return{value:void 0,done:!0}}};return i}entries(){const e=this,t=this._state;let n=this._head;const i={[Symbol.iterator]:()=>i,next(){if(e._state!==t)throw new Error("LinkedMap got modified during iteration.");if(n){const e={value:[n.key,n.value],done:!1};return n=n.next,e}return{value:void 0,done:!0}}};return i}[(Di=Symbol.toStringTag,Symbol.iterator)](){return this.entries()}trimOld(e){if(e>=this.size)return;if(0===e)return void this.clear();let t=this._head,n=this.size;for(;t&&n>e;)this._map.delete(t.key),t=t.next,n--;this._head=t,this._size=n,t&&(t.previous=void 0),this._state++}addItemFirst(e){if(this._head||this._tail){if(!this._head)throw new Error("Invalid list");e.next=this._head,this._head.previous=e}else this._tail=e;this._head=e,this._state++}addItemLast(e){if(this._head||this._tail){if(!this._tail)throw new Error("Invalid list");e.previous=this._tail,this._tail.next=e}else this._head=e;this._tail=e,this._state++}removeItem(e){if(e===this._head&&e===this._tail)this._head=void 0,this._tail=void 0;else if(e===this._head){if(!e.next)throw new Error("Invalid list");e.next.previous=void 0,this._head=e.next}else if(e===this._tail){if(!e.previous)throw new Error("Invalid list");e.previous.next=void 0,this._tail=e.previous}else{const t=e.next,n=e.previous;if(!t||!n)throw new Error("Invalid list");t.previous=n,n.next=t}e.next=void 0,e.previous=void 0,this._state++}touch(e,t){if(!this._head||!this._tail)throw new Error("Invalid list");if(1===t||2===t)if(1===t){if(e===this._head)return;const t=e.next,n=e.previous;e===this._tail?(n.next=void 0,this._tail=n):(t.previous=n,n.next=t),e.previous=void 0,e.next=this._head,this._head.previous=e,this._head=e,this._state++}else if(2===t){if(e===this._tail)return;const t=e.next,n=e.previous;e===this._head?(t.previous=void 0,this._head=t):(t.previous=n,n.next=t),e.next=void 0,e.previous=this._tail,this._tail.next=e,this._tail=e,this._state++}}toJSON(){const e=[];return this.forEach(((t,n)=>{e.push([n,t])})),e}fromJSON(e){this.clear();for(const[t,n]of e)this.set(t,n)}}class Ii{constructor(){this.map=new Map}add(e,t){let n=this.map.get(e);n||(n=new Set,this.map.set(e,n)),n.add(t)}delete(e,t){const n=this.map.get(e);n&&(n.delete(t),0===n.size&&this.map.delete(e))}forEach(e,t){const n=this.map.get(e);n&&n.forEach(t)}get(e){const t=this.map.get(e);return t||new Set}}new class extends zi{constructor(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:1;super(),this._limit=e,this._ratio=Math.min(Math.max(0,t),1)}get limit(){return this._limit}set limit(e){this._limit=e,this.checkTrim()}get(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:2;return super.get(e,t)}peek(e){return super.get(e,0)}set(e,t){return super.set(e,t,2),this.checkTrim(),this}checkTrim(){this.size>this._limit&&this.trimOld(Math.round(this._limit*this._ratio))}}(10);var Li,Pi,Oi;!function(e){e[e.Left=1]="Left",e[e.Center=2]="Center",e[e.Right=4]="Right",e[e.Full=7]="Full"}(Li||(Li={})),function(e){e[e.Left=1]="Left",e[e.Center=2]="Center",e[e.Right=3]="Right"}(Pi||(Pi={})),function(e){e[e.Both=0]="Both",e[e.Right=1]="Right",e[e.Left=2]="Left",e[e.None=3]="None"}(Oi||(Oi={}));function Wi(e,t,n,i,r){return function(e,t,n,i,r){if(0===i)return!0;const o=t.charCodeAt(i-1);if(0!==e.get(o))return!0;if(13===o||10===o)return!0;if(r>0){const n=t.charCodeAt(i);if(0!==e.get(n))return!0}return!1}(e,t,0,i,r)&&function(e,t,n,i,r){if(i+r===n)return!0;const o=t.charCodeAt(i+r);if(0!==e.get(o))return!0;if(13===o||10===o)return!0;if(r>0){const n=t.charCodeAt(i+r-1);if(0!==e.get(n))return!0}return!1}(e,t,n,i,r)}class Vi{constructor(e,t){this._wordSeparators=e,this._searchRegex=t,this._prevMatchStartIndex=-1,this._prevMatchLength=0}reset(e){this._searchRegex.lastIndex=e,this._prevMatchStartIndex=-1,this._prevMatchLength=0}next(e){const t=e.length;let n;do{if(this._prevMatchStartIndex+this._prevMatchLength===t)return null;if(n=this._searchRegex.exec(e),!n)return null;const i=n.index,r=n[0].length;if(i===this._prevMatchStartIndex&&r===this._prevMatchLength){if(0===r){ye(e,t,this._searchRegex.lastIndex)>65535?this._searchRegex.lastIndex+=2:this._searchRegex.lastIndex+=1;continue}return null}if(this._prevMatchStartIndex=i,this._prevMatchLength=r,!this._wordSeparators||Wi(this._wordSeparators,e,t,i,r))return n}while(n);return null}}function Ui(e){throw new Error(arguments.length>1&&void 0!==arguments[1]?arguments[1]:"Unreachable")}function Ki(e){e()||(e(),t(new l("Assertion Failed")))}function qi(e,t){let n=0;for(;nString.fromCodePoint(e))).join("")),"]"))),"g");const c=new Vi(null,a),h=[];let d,p=!1,u=0,m=0,f=0;e:for(let g=i,b=r;g<=b;g++){const t=e.getLineContent(g),n=t.length;c.reset(0);do{if(d=c.next(t),d){let e=d.index,i=d.index+d[0].length;if(e>0){ge(t.charCodeAt(e-1))&&e--}if(i+1=t){p=!0;break e}h.push(new It(g,e+1,g,i+1))}}}while(d)}return{ranges:h,hasMore:p,ambiguousCharacterCount:u,invisibleCharacterCount:m,nonBasicAsciiCharacterCount:f}}static computeUnicodeHighlightReason(e,t){const n=new ji(t);switch(n.shouldHighlightNonBasicASCII(e,null)){case 0:return null;case 2:return{kind:1};case 3:{const i=e.codePointAt(0),r=n.ambiguousCharacters.getPrimaryConfusable(i),o=Se.getLocales().filter((e=>!Se.getInstance(new Set([...t.allowedLocales,e])).isAmbiguous(i)));return{kind:0,confusableWith:String.fromCodePoint(r),notAmbiguousInLocales:o}}case 1:return{kind:2}}}}class ji{constructor(e){this.options=e,this.allowedCodePoints=new Set(e.allowedCodePoints),this.ambiguousCharacters=Se.getInstance(new Set(e.allowedLocales))}getCandidateCodePoints(){if(this.options.nonBasicASCII)return"allNonBasicAscii";const e=new Set;if(this.options.invisibleCharacters)for(const t of Ce.codePoints)$i(String.fromCodePoint(t))||e.add(t);if(this.options.ambiguousCharacters)for(const t of this.ambiguousCharacters.getConfusableCodePoints())e.add(t);for(const t of this.allowedCodePoints)e.delete(t);return e}shouldHighlightNonBasicASCII(e,t){const n=e.codePointAt(0);if(this.allowedCodePoints.has(n))return 0;if(this.options.nonBasicASCII)return 1;let i=!1,r=!1;if(t)for(const s of t){const e=s.codePointAt(0),t=(o=s,we.test(o));i=i||t,t||this.ambiguousCharacters.isAmbiguous(e)||Ce.isInvisibleCharacter(e)||(r=!0)}var o;return!i&&r?0:this.options.invisibleCharacters&&!$i(e)&&Ce.isInvisibleCharacter(n)?2:this.options.ambiguousCharacters&&this.ambiguousCharacters.isAmbiguous(n)?3:0}}function $i(e){return" "===e||"\n"===e||"\t"===e}class Hi{constructor(e,t,n){this.changes=e,this.moves=t,this.hitTimeout=n}}class Gi{constructor(e,t){this.lineRangeMapping=e,this.changes=t}}class Ji{static addRange(e,t){let n=0;for(;nt))return new Ji(e,t)}static ofLength(e){return new Ji(0,e)}static ofStartAndLength(e,t){return new Ji(e,e+t)}constructor(e,t){if(this.start=e,this.endExclusive=t,e>t)throw new l("Invalid range: ".concat(this.toString()))}get isEmpty(){return this.start===this.endExclusive}delta(e){return new Ji(this.start+e,this.endExclusive+e)}deltaStart(e){return new Ji(this.start+e,this.endExclusive)}deltaEnd(e){return new Ji(this.start,this.endExclusive+e)}get length(){return this.endExclusive-this.start}toString(){return"[".concat(this.start,", ").concat(this.endExclusive,")")}contains(e){return this.start<=e&&e=e.endExclusive}slice(e){return e.slice(this.start,this.endExclusive)}substring(e){return e.substring(this.start,this.endExclusive)}clip(e){if(this.isEmpty)throw new l("Invalid clipping range: ".concat(this.toString()));return Math.max(this.start,Math.min(this.endExclusive-1,e))}clipCyclic(e){if(this.isEmpty)throw new l("Invalid clipping range: ".concat(this.toString()));return e=this.endExclusive?this.start+(e-this.start)%this.length:e}forEach(e){for(let t=this.start;t2&&void 0!==arguments[2]?arguments[2]:0,i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:e.length;for(;n2&&void 0!==arguments[2]?arguments[2]:0,i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:e.length;for(;nt)throw new l("startLineNumber ".concat(e," cannot be after endLineNumberExclusive ").concat(t));this.startLineNumber=e,this.endLineNumberExclusive=t}contains(e){return this.startLineNumber<=e&&e0&&void 0!==arguments[0]?arguments[0]:[];this._normalizedRanges=e}get ranges(){return this._normalizedRanges}addRange(e){if(0===e.length)return;const t=Qi(this._normalizedRanges,(t=>t.endLineNumberExclusive>=e.startLineNumber)),n=Yi(this._normalizedRanges,(t=>t.startLineNumber<=e.endLineNumberExclusive))+1;if(t===n)this._normalizedRanges.splice(t,0,e);else if(t===n-1){const n=this._normalizedRanges[t];this._normalizedRanges[t]=n.join(e)}else{const i=this._normalizedRanges[t].join(this._normalizedRanges[n-1]).join(e);this._normalizedRanges.splice(t,n-t,i)}}contains(e){const t=Xi(this._normalizedRanges,(t=>t.startLineNumber<=e));return!!t&&t.endLineNumberExclusive>e}intersects(e){const t=Xi(this._normalizedRanges,(t=>t.startLineNumbere.startLineNumber}getUnion(e){if(0===this._normalizedRanges.length)return e;if(0===e._normalizedRanges.length)return this;const t=[];let n=0,i=0,r=null;for(;n=o.startLineNumber?r=new er(r.startLineNumber,Math.max(r.endLineNumberExclusive,o.endLineNumberExclusive)):(t.push(r),r=o)}return null!==r&&t.push(r),new tr(t)}subtractFrom(e){const t=Qi(this._normalizedRanges,(t=>t.endLineNumberExclusive>=e.startLineNumber)),n=Yi(this._normalizedRanges,(t=>t.startLineNumber<=e.endLineNumberExclusive))+1;if(t===n)return new tr([e]);const i=[];let r=e.startLineNumber;for(let o=t;or&&i.push(new er(r,e.startLineNumber)),r=e.endLineNumberExclusive}return re.toString())).join(", ")}getIntersection(e){const t=[];let n=0,i=0;for(;nt.delta(e))))}}class nr{static inverse(e,t,n){const i=[];let r=1,o=1;for(const a of e){const e=new nr(new er(r,a.original.startLineNumber),new er(o,a.modified.startLineNumber));e.modified.isEmpty||i.push(e),r=a.original.endLineNumberExclusive,o=a.modified.endLineNumberExclusive}const s=new nr(new er(r,t+1),new er(o,n+1));return s.modified.isEmpty||i.push(s),i}static clip(e,t,n){const i=[];for(const r of e){const e=r.original.intersect(t),o=r.modified.intersect(n);e&&!e.isEmpty&&o&&!o.isEmpty&&i.push(new nr(e,o))}return i}constructor(e,t){this.original=e,this.modified=t}toString(){return"{".concat(this.original.toString(),"->").concat(this.modified.toString(),"}")}flip(){return new nr(this.modified,this.original)}join(e){return new nr(this.original.join(e.original),this.modified.join(e.modified))}}class ir extends nr{static fromRangeMappings(e){const t=er.join(e.map((e=>er.fromRangeInclusive(e.originalRange)))),n=er.join(e.map((e=>er.fromRangeInclusive(e.modifiedRange))));return new ir(t,n,e)}constructor(e,t,n){super(e,t),this.innerChanges=n}flip(){var e;return new ir(this.modified,this.original,null===(e=this.innerChanges)||void 0===e?void 0:e.map((e=>e.flip())))}withInnerChangesFromLineRanges(){return new ir(this.original,this.modified,[new rr(this.original.toExclusiveRange(),this.modified.toExclusiveRange())])}}class rr{constructor(e,t){this.originalRange=e,this.modifiedRange=t}toString(){return"{".concat(this.originalRange.toString(),"->").concat(this.modifiedRange.toString(),"}")}flip(){return new rr(this.modifiedRange,this.originalRange)}}const or=3;class sr{computeDiff(e,t,n){var i;const r=new pr(e,t,{maxComputationTime:n.maxComputationTimeMs,shouldIgnoreTrimWhitespace:n.ignoreTrimWhitespace,shouldComputeCharChanges:!0,shouldMakePrettyDiff:!0,shouldPostProcessCharChanges:!0}).computeDiff(),o=[];let s=null;for(const a of r.changes){let e,t;e=0===a.originalEndLineNumber?new er(a.originalStartLineNumber+1,a.originalStartLineNumber+1):new er(a.originalStartLineNumber,a.originalEndLineNumber+1),t=0===a.modifiedEndLineNumber?new er(a.modifiedStartLineNumber+1,a.modifiedStartLineNumber+1):new er(a.modifiedStartLineNumber,a.modifiedEndLineNumber+1);let n=new ir(e,t,null===(i=a.charChanges)||void 0===i?void 0:i.map((e=>new rr(new It(e.originalStartLineNumber,e.originalStartColumn,e.originalEndLineNumber,e.originalEndColumn),new It(e.modifiedStartLineNumber,e.modifiedStartColumn,e.modifiedEndLineNumber,e.modifiedEndColumn)))));s&&(s.modified.endLineNumberExclusive!==n.modified.startLineNumber&&s.original.endLineNumberExclusive!==n.original.startLineNumber||(n=new ir(s.original.join(n.original),s.modified.join(n.modified),s.innerChanges&&n.innerChanges?s.innerChanges.concat(n.innerChanges):void 0),o.pop())),o.push(n),s=n}return Ki((()=>qi(o,((e,t)=>t.original.startLineNumber-e.original.endLineNumberExclusive===t.modified.startLineNumber-e.modified.endLineNumberExclusive&&e.original.endLineNumberExclusive(10===e?"\\n":String.fromCharCode(e))+"-(".concat(this._lineNumbers[t],",").concat(this._columns[t],")"))).join(", ")+"]"}_assertIndex(e,t){if(e<0||e>=t.length)throw new Error("Illegal index")}getElements(){return this._charCodes}getStartLineNumber(e){return e>0&&e===this._lineNumbers.length?this.getEndLineNumber(e-1):(this._assertIndex(e,this._lineNumbers),this._lineNumbers[e])}getEndLineNumber(e){return-1===e?this.getStartLineNumber(e+1):(this._assertIndex(e,this._lineNumbers),10===this._charCodes[e]?this._lineNumbers[e]+1:this._lineNumbers[e])}getStartColumn(e){return e>0&&e===this._columns.length?this.getEndColumn(e-1):(this._assertIndex(e,this._columns),this._columns[e])}getEndColumn(e){return-1===e?this.getStartColumn(e+1):(this._assertIndex(e,this._columns),10===this._charCodes[e]?1:this._columns[e]+1)}}class hr{constructor(e,t,n,i,r,o,s,a){this.originalStartLineNumber=e,this.originalStartColumn=t,this.originalEndLineNumber=n,this.originalEndColumn=i,this.modifiedStartLineNumber=r,this.modifiedStartColumn=o,this.modifiedEndLineNumber=s,this.modifiedEndColumn=a}static createFromDiffChange(e,t,n){const i=t.getStartLineNumber(e.originalStart),r=t.getStartColumn(e.originalStart),o=t.getEndLineNumber(e.originalStart+e.originalLength-1),s=t.getEndColumn(e.originalStart+e.originalLength-1),a=n.getStartLineNumber(e.modifiedStart),l=n.getStartColumn(e.modifiedStart),c=n.getEndLineNumber(e.modifiedStart+e.modifiedLength-1),h=n.getEndColumn(e.modifiedStart+e.modifiedLength-1);return new hr(i,r,o,s,a,l,c,h)}}class dr{constructor(e,t,n,i,r){this.originalStartLineNumber=e,this.originalEndLineNumber=t,this.modifiedStartLineNumber=n,this.modifiedEndLineNumber=i,this.charChanges=r}static createFromDiffResult(e,t,n,i,r,o,s){let a,l,c,h,d;if(0===t.originalLength?(a=n.getStartLineNumber(t.originalStart)-1,l=0):(a=n.getStartLineNumber(t.originalStart),l=n.getEndLineNumber(t.originalStart+t.originalLength-1)),0===t.modifiedLength?(c=i.getStartLineNumber(t.modifiedStart)-1,h=0):(c=i.getStartLineNumber(t.modifiedStart),h=i.getEndLineNumber(t.modifiedStart+t.modifiedLength-1)),o&&t.originalLength>0&&t.originalLength<20&&t.modifiedLength>0&&t.modifiedLength<20&&r()){const o=n.createCharSequence(e,t.originalStart,t.originalStart+t.originalLength-1),a=i.createCharSequence(e,t.modifiedStart,t.modifiedStart+t.modifiedLength-1);if(o.getElements().length>0&&a.getElements().length>0){let e=ar(o,a,r,!0).changes;s&&(e=function(e){if(e.length<=1)return e;const t=[e[0]];let n=t[0];for(let i=1,r=e.length;i1&&s>1;){if(e.charCodeAt(n-2)!==t.charCodeAt(s-2))break;n--,s--}(n>1||s>1)&&this._pushTrimWhitespaceCharChange(i,r+1,1,n,o+1,1,s)}{let n=mr(e,1),s=mr(t,1);const a=e.length+1,l=t.length+1;for(;n1&&void 0!==arguments[1]?arguments[1]:e.length-1;t>=0;t--){const n=e.charCodeAt(t);if(32!==n&&9!==n)return t}return-1}(e);return-1===n?t:n+2}function fr(e){if(0===e)return()=>!0;const t=Date.now();return()=>Date.now()-t{n.push(br.fromOffsetPairs(e?e.getEndExclusives():vr.zero,i?i.getStarts():new vr(t,(e?e.seq2Range.endExclusive-e.seq1Range.endExclusive:0)+t)))})),n}static fromOffsetPairs(e,t){return new br(new Ji(e.offset1,t.offset1),new Ji(e.offset2,t.offset2))}constructor(e,t){this.seq1Range=e,this.seq2Range=t}swap(){return new br(this.seq2Range,this.seq1Range)}toString(){return"".concat(this.seq1Range," <-> ").concat(this.seq2Range)}join(e){return new br(this.seq1Range.join(e.seq1Range),this.seq2Range.join(e.seq2Range))}delta(e){return 0===e?this:new br(this.seq1Range.delta(e),this.seq2Range.delta(e))}deltaStart(e){return 0===e?this:new br(this.seq1Range.deltaStart(e),this.seq2Range.deltaStart(e))}deltaEnd(e){return 0===e?this:new br(this.seq1Range.deltaEnd(e),this.seq2Range.deltaEnd(e))}intersect(e){const t=this.seq1Range.intersect(e.seq1Range),n=this.seq2Range.intersect(e.seq2Range);if(t&&n)return new br(t,n)}getStarts(){return new vr(this.seq1Range.start,this.seq2Range.start)}getEndExclusives(){return new vr(this.seq1Range.endExclusive,this.seq2Range.endExclusive)}}class vr{constructor(e,t){this.offset1=e,this.offset2=t}toString(){return"".concat(this.offset1," <-> ").concat(this.offset2)}delta(e){return 0===e?this:new vr(this.offset1+e,this.offset2+e)}equals(e){return this.offset1===e.offset1&&this.offset2===e.offset2}}vr.zero=new vr(0,0),vr.max=new vr(Number.MAX_SAFE_INTEGER,Number.MAX_SAFE_INTEGER);class yr{isValid(){return!0}}yr.instance=new yr;class wr{constructor(e){if(this.timeout=e,this.startTime=Date.now(),this.valid=!0,e<=0)throw new l("timeout must be positive")}isValid(){return!(Date.now()-this.startTime2&&void 0!==arguments[2]?arguments[2]:yr.instance,i=arguments.length>3?arguments[3]:void 0;if(0===e.length||0===t.length)return gr.trivial(e,t);const r=new xr(e.length,t.length),o=new xr(e.length,t.length),s=new xr(e.length,t.length);for(let u=0;u0&&a>0&&3===o.get(u-1,a-1)&&(h+=s.get(u-1,a-1)),h+=i?i(u,a):1):h=-1;const d=Math.max(l,c,h);if(d===h){const e=u>0&&a>0?s.get(u-1,a-1):0;s.set(u,a,e+1),o.set(u,a,3)}else d===l?(s.set(u,a,0),o.set(u,a,1)):d===c&&(s.set(u,a,0),o.set(u,a,2));r.set(u,a,d)}const a=[];let l=e.length,c=t.length;function h(e,t){e+1===l&&t+1===c||a.push(new br(new Ji(e+1,l),new Ji(t+1,c))),l=e,c=t}let d=e.length-1,p=t.length-1;for(;d>=0&&p>=0;)3===o.get(d,p)?(h(d,p),d--,p--):1===o.get(d,p)?d--:p--;return h(-1,-1),a.reverse(),new gr(a,!1)}}class kr{compute(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:yr.instance;if(0===e.length||0===t.length)return gr.trivial(e,t);const i=e,r=t;function o(e,t){for(;ei.length||p>r.length)continue;const u=o(d,p);a.set(c,u);const m=d===s?l.get(c+1):l.get(c-1);if(l.set(c,u!==d?new Er(m,d,p,u-d):m),a.get(c)===i.length&&a.get(c)-c===r.length)break e}}let h=l.get(c);const d=[];let p=i.length,u=r.length;for(;;){const e=h?h.x+h.length:0,t=h?h.y+h.length:0;if(e===p&&t===u||d.push(new br(new Ji(e,p),new Ji(t,u))),!h)break;p=h.x,u=h.y,h=h.prev}return d.reverse(),new gr(d,!1)}}class Er{constructor(e,t,n,i){this.prev=e,this.x=t,this.y=n,this.length=i}}class Rr{constructor(){this.positiveArr=new Int32Array(10),this.negativeArr=new Int32Array(10)}get(e){return e<0?(e=-e-1,this.negativeArr[e]):this.positiveArr[e]}set(e,t){if(e<0){if((e=-e-1)>=this.negativeArr.length){const e=this.negativeArr;this.negativeArr=new Int32Array(2*e.length),this.negativeArr.set(e)}this.negativeArr[e]=t}else{if(e>=this.positiveArr.length){const e=this.positiveArr;this.positiveArr=new Int32Array(2*e.length),this.positiveArr.set(e)}this.positiveArr[e]=t}}}class Nr{constructor(){this.positiveArr=[],this.negativeArr=[]}get(e){return e<0?(e=-e-1,this.negativeArr[e]):this.positiveArr[e]}set(e,t){e<0?(e=-e-1,this.negativeArr[e]=t):this.positiveArr[e]=t}}class Fr{constructor(e,t,n){this.lines=e,this.considerWhitespaceChanges=n,this.elements=[],this.firstCharOffsetByLine=[],this.additionalOffsetByLine=[];let i=!1;t.start>0&&t.endExclusive>=e.length&&(t=new Ji(t.start-1,t.endExclusive),i=!0),this.lineRange=t,this.firstCharOffsetByLine[0]=0;for(let r=this.lineRange.start;rString.fromCharCode(e))).join("")}getElement(e){return this.elements[e]}get length(){return this.elements.length}getBoundaryScore(e){const t=Mr(e>0?this.elements[e-1]:-1),n=Mr(et<=e));return new zt(this.lineRange.start+t+1,e-this.firstCharOffsetByLine[t]+this.additionalOffsetByLine[t]+1)}translateRange(e){return It.fromPositions(this.translateOffset(e.start),this.translateOffset(e.endExclusive))}findWordContaining(e){if(e<0||e>=this.elements.length)return;if(!Dr(this.elements[e]))return;let t=e;for(;t>0&&Dr(this.elements[t-1]);)t--;let n=e;for(;nt<=e.start)))&&void 0!==t?t:0,r=null!==(n=function(e,t){const n=Qi(e,t);return n===e.length?void 0:e[n]}(this.firstCharOffsetByLine,(t=>e.endExclusive<=t)))&&void 0!==n?n:this.elements.length;return new Ji(i,r)}}function Dr(e){return e>=97&&e<=122||e>=65&&e<=90||e>=48&&e<=57}const Tr={0:0,1:0,2:0,3:10,4:2,5:30,6:3,7:10,8:10};function Ar(e){return Tr[e]}function Mr(e){return 10===e?8:13===e?7:Sr(e)?6:e>=97&&e<=122?0:e>=65&&e<=90?1:e>=48&&e<=57?2:-1===e?3:44===e||59===e?5:4}function zr(e,t,n,i,r,o){let{moves:s,excludedChanges:a}=function(e,t,n,i){const r=[],o=e.filter((e=>e.modified.isEmpty&&e.original.length>=3)).map((e=>new Cr(e.original,t,e))),s=new Set(e.filter((e=>e.original.isEmpty&&e.modified.length>=3)).map((e=>new Cr(e.modified,n,e)))),a=new Set;for(const l of o){let e,t=-1;for(const n of s){const i=l.computeSimilarity(n);i>t&&(t=i,e=n)}if(t>.9&&e&&(s.delete(e),r.push(new nr(l.range,e.range)),a.add(l.source),a.add(e.source)),!i.isValid())return{moves:r,excludedChanges:a}}return{moves:r,excludedChanges:a}}(e,t,n,o);if(!o.isValid())return[];const l=function(e,t,n,i,r,o){const s=[],a=new Ii;for(const u of e)for(let e=u.original.startLineNumber;ee.modified.startLineNumber),Ot));for(const u of e){let e=[];for(let t=u.modified.startLineNumber;t{let{range:n}=t;for(const s of e)if(s.originalLineRange.endLineNumberExclusive+1===n.endLineNumberExclusive&&s.modifiedLineRange.endLineNumberExclusive+1===r.endLineNumberExclusive)return s.originalLineRange=new er(s.originalLineRange.startLineNumber,n.endLineNumberExclusive),s.modifiedLineRange=new er(s.modifiedLineRange.startLineNumber,r.endLineNumberExclusive),void o.push(s);const i={modifiedLineRange:r,originalLineRange:n};l.push(i),o.push(i)})),e=o}if(!o.isValid())return[]}l.sort((c=Pt((e=>e.modifiedLineRange.length),Ot),(e,t)=>-c(e,t)));var c;const h=new tr,d=new tr;for(const u of l){const e=u.modifiedLineRange.startLineNumber-u.originalLineRange.startLineNumber,t=h.subtractFrom(u.modifiedLineRange),n=d.subtractFrom(u.originalLineRange).getWithDelta(e),i=t.getIntersection(n);for(const r of i.ranges){if(r.length<3)continue;const t=r,n=r.delta(-e);s.push(new nr(n,t)),h.addRange(t),d.addRange(n)}}s.sort(Pt((e=>e.original.startLineNumber),Ot));const p=new Zi(e);for(let u=0;ue.original.startLineNumber<=t.original.startLineNumber)),a=Xi(e,(e=>e.modified.startLineNumber<=t.modified.startLineNumber)),l=Math.max(t.original.startLineNumber-n.original.startLineNumber,t.modified.startLineNumber-a.modified.startLineNumber),c=p.findLastMonotonous((e=>e.original.startLineNumbere.modified.startLineNumberi.length||n>r.length)break;if(h.contains(n)||d.contains(e))break;if(!Ir(i[e-1],r[n-1],o))break}for(g>0&&(d.addRange(new er(t.original.startLineNumber-g,t.original.startLineNumber)),h.addRange(new er(t.modified.startLineNumber-g,t.modified.startLineNumber))),b=0;bi.length||n>r.length)break;if(h.contains(n)||d.contains(e))break;if(!Ir(i[e-1],r[n-1],o))break}b>0&&(d.addRange(new er(t.original.endLineNumberExclusive,t.original.endLineNumberExclusive+b)),h.addRange(new er(t.modified.endLineNumberExclusive,t.modified.endLineNumberExclusive+b))),(g>0||b>0)&&(s[u]=new nr(new er(t.original.startLineNumber-g,t.original.endLineNumberExclusive+b),new er(t.modified.startLineNumber-g,t.modified.endLineNumberExclusive+b)))}return s}(e.filter((e=>!a.has(e))),i,r,t,n,o);return function(e,t){for(const n of t)e.push(n)}(s,l),s=function(e){if(0===e.length)return e;e.sort(Pt((e=>e.original.startLineNumber),Ot));const t=[e[0]];for(let n=1;n=0&&s>=0&&o+s<=2?t[t.length-1]=i.join(r):t.push(r)}return t}(s),s=s.filter((e=>{const n=e.original.toOffsetRange().slice(t).map((e=>e.trim()));return n.join("\n").length>=15&&function(e,t){let n=0;for(const i of e)t(i)&&n++;return n}(n,(e=>e.length>=2))>=2})),s=function(e,t){const n=new Zi(e);return t=t.filter((t=>(n.findLastMonotonous((e=>e.original.startLineNumbere.modified.startLineNumber300&&t.length>300)return!1;const i=(new kr).compute(new Fr([e],new Ji(0,1),!1),new Fr([t],new Ji(0,1),!1),n);let r=0;const o=br.invert(i.diffs,e.length);for(const a of o)a.seq1Range.forEach((t=>{Sr(e.charCodeAt(t))||r++}));const s=function(t){let n=0;for(let i=0;it.length?e:t);return r/s>.6&&s>10}function Lr(e,t,n){let i=n;return i=Pr(e,t,i),i=Pr(e,t,i),i=function(e,t,n){if(!e.getBoundaryScore||!t.getBoundaryScore)return n;for(let i=0;i0?n[i-1]:void 0,o=n[i],s=i+10&&(s=s.delta(a))}r.push(s)}return i.length>0&&r.push(i[i.length-1]),r}function Or(e,t,n,i,r){let o=1;for(;e.seq1Range.start-o>=i.start&&e.seq2Range.start-o>=r.start&&n.isStronglyEqual(e.seq2Range.start-o,e.seq2Range.endExclusive-o)&&o<100;)o++;o--;let s=0;for(;e.seq1Range.start+sl&&(l=s,a=c)}return e.delta(a)}class Wr{constructor(e,t){this.trimmedHash=e,this.lines=t}getElement(e){return this.trimmedHash[e]}get length(){return this.trimmedHash.length}getBoundaryScore(e){return 1e3-((0===e?0:Vr(this.lines[e-1]))+(e===this.lines.length?0:Vr(this.lines[e])))}getText(e){return this.lines.slice(e.start,e.endExclusive).join("\n")}isStronglyEqual(e,t){return this.lines[e]===this.lines[t]}}function Vr(e){let t=0;for(;t2&&void 0!==arguments[2]?arguments[2]:(e,t)=>e===t;if(e===t)return!0;if(!e||!t)return!1;if(e.length!==t.length)return!1;for(let i=0,r=e.length;ie===t)))return new Hi([],[],!1);if(1===e.length&&0===e[0].length||1===t.length&&0===t[0].length)return new Hi([new ir(new er(1,e.length+1),new er(1,t.length+1),[new rr(new It(1,1,e.length,e[0].length+1),new It(1,1,t.length,t[0].length+1))])],[],!1);const i=0===n.maxComputationTimeMs?yr.instance:new wr(n.maxComputationTimeMs),r=!n.ignoreTrimWhitespace,o=new Map;function s(e){let t=o.get(e);return void 0===t&&(t=o.size,o.set(e,t)),t}const a=e.map((e=>s(e.trim()))),l=t.map((e=>s(e.trim()))),c=new Wr(a,e),h=new Wr(l,t),d=(()=>c.length+h.length<1700?this.dynamicProgrammingDiffing.compute(c,h,i,((n,i)=>e[n]===t[i]?0===t[i].length?.1:1+Math.log(1+t[i].length):.99)):this.myersDiffingAlgorithm.compute(c,h))();let p=d.diffs,u=d.hitTimeout;p=Lr(c,h,p),p=function(e,t,n){let i=n;if(0===i.length)return i;let r,o=0;do{r=!1;const s=[i[0]];for(let a=1;a5||n.seq1Range.length+n.seq2Range.length>5)}h(c,l)?(r=!0,s[s.length-1]=s[s.length-1].join(l)):s.push(l)}i=s}while(o++<10&&r);return i}(c,0,p);const m=[],f=n=>{if(r)for(let o=0;ow.seq1Range.start-g===w.seq2Range.start-b));f(w.seq1Range.start-g),g=w.seq1Range.endExclusive,b=w.seq2Range.endExclusive;const n=this.refineDiff(e,t,w,i,r);n.hitTimeout&&(u=!0);for(const e of n.mappings)m.push(e)}f(e.length-g);const v=Kr(m,e,t);let y=[];return n.computeMoves&&(y=this.computeMoves(v,e,t,a,l,i,r)),Ki((()=>{function n(e,t){if(e.lineNumber<1||e.lineNumber>t.length)return!1;const n=t[e.lineNumber-1];return!(e.column<1||e.column>n.length+1)}function i(e,t){return!(e.startLineNumber<1||e.startLineNumber>t.length+1)&&!(e.endLineNumberExclusive<1||e.endLineNumberExclusive>t.length+1)}for(const r of v){if(!r.innerChanges)return!1;for(const i of r.innerChanges){if(!(n(i.modifiedRange.getStartPosition(),t)&&n(i.modifiedRange.getEndPosition(),t)&&n(i.originalRange.getStartPosition(),e)&&n(i.originalRange.getEndPosition(),e)))return!1}if(!i(r.modified,t)||!i(r.original,e))return!1}return!0})),new Hi(v,y,u)}computeMoves(e,t,n,i,r,o,s){return zr(e,t,n,i,r,o).map((e=>{const i=Kr(this.refineDiff(t,n,new br(e.original.toOffsetRange(),e.modified.toOffsetRange()),o,s).mappings,t,n,!0);return new Gi(e,i)}))}refineDiff(e,t,n,i,r){const o=new Fr(e,n.seq1Range,r),s=new Fr(t,n.seq2Range,r),a=o.length+s.length<500?this.dynamicProgrammingDiffing.compute(o,s,i):this.myersDiffingAlgorithm.compute(o,s,i);let l=a.diffs;l=Lr(o,s,l),l=function(e,t,n){const i=br.invert(n,e.length),r=[];let o=new vr(0,0);function s(n,s){if(n.offset10;){const n=i[0];if(!n.seq1Range.intersects(c.seq1Range)&&!n.seq2Range.intersects(c.seq2Range))break;const r=e.findWordContaining(n.seq1Range.start),o=t.findWordContaining(n.seq2Range.start),s=new br(r,o),a=s.intersect(n);if(d+=a.seq1Range.length,p+=a.seq2Range.length,c=c.join(s),!(c.seq1Range.endExclusive>=n.seq1Range.endExclusive))break;i.shift()}d+p<2*(c.seq1Range.length+c.seq2Range.length)/3&&r.push(c),o=c.getEndExclusives()}for(;i.length>0;){const e=i.shift();e.seq1Range.isEmpty||(s(e.getStarts(),e),s(e.getEndExclusives().delta(-1),e))}return function(e,t){const n=[];for(;e.length>0||t.length>0;){const i=e[0],r=t[0];let o;o=i&&(!r||i.seq1Range.start0&&n[n.length-1].seq1Range.endExclusive>=o.seq1Range.start?n[n.length-1]=n[n.length-1].join(o):n.push(o)}return n}(n,r)}(o,s,l),l=function(e,t,n){const i=[];for(const r of n){const e=i[i.length-1];e&&(r.seq1Range.start-e.seq1Range.endExclusive<=2||r.seq2Range.start-e.seq2Range.endExclusive<=2)?i[i.length-1]=new br(e.seq1Range.join(r.seq1Range),e.seq2Range.join(r.seq2Range)):i.push(r)}return i}(0,0,l),l=function(e,t,n){let i=n;if(0===i.length)return i;let r,o=0;do{r=!1;const a=[i[0]];for(let l=1;l5||r.length>500)return!1;const o=e.getText(r).trim();if(o.length>20||o.split(/\r\n|\r|\n/).length>1)return!1;const s=e.countLinesIn(n.seq1Range),a=n.seq1Range.length,l=t.countLinesIn(n.seq2Range),d=n.seq2Range.length,p=e.countLinesIn(i.seq1Range),u=i.seq1Range.length,m=t.countLinesIn(i.seq2Range),f=i.seq2Range.length,g=130;function b(e){return Math.min(e,g)}return Math.pow(Math.pow(b(40*s+a),1.5)+Math.pow(b(40*l+d),1.5),1.5)+Math.pow(Math.pow(b(40*p+u),1.5)+Math.pow(b(40*m+f),1.5),1.5)>(g**1.5)**1.5*1.3}d(h,c)?(r=!0,a[a.length-1]=a[a.length-1].join(c)):a.push(c)}i=a}while(o++<10&&r);const s=[];return function(e,t){for(let n=0;n{let r=n;function o(e){return e.length>0&&e.trim().length<=3&&n.seq1Range.length+n.seq2Range.length>100}const a=e.extendToFullLines(n.seq1Range),l=e.getText(new Ji(a.start,n.seq1Range.start));o(l)&&(r=r.deltaStart(-l.length));const c=e.getText(new Ji(n.seq1Range.endExclusive,a.endExclusive));o(c)&&(r=r.deltaEnd(c.length));const h=br.fromOffsetPairs(t?t.getEndExclusives():vr.zero,i?i.getStarts():vr.max),d=r.intersect(h);s.length>0&&d.getStarts().equals(s[s.length-1].getEndExclusives())?s[s.length-1]=s[s.length-1].join(d):s.push(d)})),s}(o,s,l);return{mappings:l.map((e=>new rr(o.translateRange(e.seq1Range),s.translateRange(e.seq2Range)))),hitTimeout:a.hitTimeout}}}function Kr(e,t,n){let i=arguments.length>3&&void 0!==arguments[3]&&arguments[3];const r=[];for(const o of function*(e,t){let n,i;for(const r of e)void 0!==i&&t(i,r)?n.push(r):(n&&(yield n),n=[r]),i=r;n&&(yield n)}(e.map((e=>function(e,t,n){let i=0,r=0;1===e.modifiedRange.endColumn&&1===e.originalRange.endColumn&&e.originalRange.startLineNumber+i<=e.originalRange.endLineNumber&&e.modifiedRange.startLineNumber+i<=e.modifiedRange.endLineNumber&&(r=-1);e.modifiedRange.startColumn-1>=n[e.modifiedRange.startLineNumber-1].length&&e.originalRange.startColumn-1>=t[e.originalRange.startLineNumber-1].length&&e.originalRange.startLineNumber<=e.originalRange.endLineNumber+r&&e.modifiedRange.startLineNumber<=e.modifiedRange.endLineNumber+r&&(i=1);const o=new er(e.originalRange.startLineNumber+i,e.originalRange.endLineNumber+1+r),s=new er(e.modifiedRange.startLineNumber+i,e.modifiedRange.endLineNumber+1+r);return new ir(o,s,[e])}(e,t,n))),((e,t)=>e.original.overlapOrTouch(t.original)||e.modified.overlapOrTouch(t.modified)))){const e=o[0],t=o[o.length-1];r.push(new ir(e.original.join(t.original),e.modified.join(t.modified),o.map((e=>e.innerChanges[0]))))}return Ki((()=>{if(!i&&r.length>0){if(r[0].modified.startLineNumber!==r[0].original.startLineNumber)return!1;if(n.length-r[r.length-1].modified.endLineNumberExclusive!==t.length-r[r.length-1].original.endLineNumberExclusive)return!1}return qi(r,((e,t)=>t.original.startLineNumber-e.original.endLineNumberExclusive===t.modified.startLineNumber-e.modified.endLineNumberExclusive&&e.original.endLineNumberExclusivenew sr,Br=()=>new Ur;function jr(e,t){const n=Math.pow(10,t);return Math.round(e*n)/n}class $r{constructor(e,t,n){let i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;this._rgbaBrand=void 0,this.r=0|Math.min(255,Math.max(0,e)),this.g=0|Math.min(255,Math.max(0,t)),this.b=0|Math.min(255,Math.max(0,n)),this.a=jr(Math.max(Math.min(1,i),0),3)}static equals(e,t){return e.r===t.r&&e.g===t.g&&e.b===t.b&&e.a===t.a}}class Hr{constructor(e,t,n,i){this._hslaBrand=void 0,this.h=0|Math.max(Math.min(360,e),0),this.s=jr(Math.max(Math.min(1,t),0),3),this.l=jr(Math.max(Math.min(1,n),0),3),this.a=jr(Math.max(Math.min(1,i),0),3)}static equals(e,t){return e.h===t.h&&e.s===t.s&&e.l===t.l&&e.a===t.a}static fromRGBA(e){const t=e.r/255,n=e.g/255,i=e.b/255,r=e.a,o=Math.max(t,n,i),s=Math.min(t,n,i);let a=0,l=0;const c=(s+o)/2,h=o-s;if(h>0){switch(l=Math.min(c<=.5?h/(2*c):h/(2-2*c),1),o){case t:a=(n-i)/h+(n1&&(n-=1),n<1/6?e+6*(t-e)*n:n<.5?t:n<2/3?e+(t-e)*(2/3-n)*6:e}static toRGBA(e){const t=e.h/360,{s:n,l:i,a:r}=e;let o,s,a;if(0===n)o=s=a=i;else{const e=i<.5?i*(1+n):i+n-i*n,r=2*i-e;o=Hr._hue2rgb(r,e,t+1/3),s=Hr._hue2rgb(r,e,t),a=Hr._hue2rgb(r,e,t-1/3)}return new $r(Math.round(255*o),Math.round(255*s),Math.round(255*a),r)}}class Gr{constructor(e,t,n,i){this._hsvaBrand=void 0,this.h=0|Math.max(Math.min(360,e),0),this.s=jr(Math.max(Math.min(1,t),0),3),this.v=jr(Math.max(Math.min(1,n),0),3),this.a=jr(Math.max(Math.min(1,i),0),3)}static equals(e,t){return e.h===t.h&&e.s===t.s&&e.v===t.v&&e.a===t.a}static fromRGBA(e){const t=e.r/255,n=e.g/255,i=e.b/255,r=Math.max(t,n,i),o=r-Math.min(t,n,i),s=0===r?0:o/r;let a;return a=0===o?0:r===t?((n-i)/o%6+6)%6:r===n?(i-t)/o+2:(t-n)/o+4,new Gr(Math.round(60*a),s,r,e.a)}static toRGBA(e){const{h:t,s:n,v:i,a:r}=e,o=i*n,s=o*(1-Math.abs(t/60%2-1)),a=i-o;let[l,c,h]=[0,0,0];return t<60?(l=o,c=s):t<120?(l=s,c=o):t<180?(c=o,h=s):t<240?(c=s,h=o):t<300?(l=s,h=o):t<=360&&(l=o,h=s),l=Math.round(255*(l+a)),c=Math.round(255*(c+a)),h=Math.round(255*(h+a)),new $r(l,c,h,r)}}class Jr{static fromHex(e){return Jr.Format.CSS.parseHex(e)||Jr.red}static equals(e,t){return!e&&!t||!(!e||!t)&&e.equals(t)}get hsla(){return this._hsla?this._hsla:Hr.fromRGBA(this.rgba)}get hsva(){return this._hsva?this._hsva:Gr.fromRGBA(this.rgba)}constructor(e){if(!e)throw new Error("Color needs a value");if(e instanceof $r)this.rgba=e;else if(e instanceof Hr)this._hsla=e,this.rgba=Hr.toRGBA(e);else{if(!(e instanceof Gr))throw new Error("Invalid color ctor argument");this._hsva=e,this.rgba=Gr.toRGBA(e)}}equals(e){return!!e&&$r.equals(this.rgba,e.rgba)&&Hr.equals(this.hsla,e.hsla)&&Gr.equals(this.hsva,e.hsva)}getRelativeLuminance(){return jr(.2126*Jr._relativeLuminanceForComponent(this.rgba.r)+.7152*Jr._relativeLuminanceForComponent(this.rgba.g)+.0722*Jr._relativeLuminanceForComponent(this.rgba.b),4)}static _relativeLuminanceForComponent(e){const t=e/255;return t<=.03928?t/12.92:Math.pow((t+.055)/1.055,2.4)}isLighter(){return(299*this.rgba.r+587*this.rgba.g+114*this.rgba.b)/1e3>=128}isLighterThan(e){return this.getRelativeLuminance()>e.getRelativeLuminance()}isDarkerThan(e){return this.getRelativeLuminance()0)for(const i of n){const n=i.filter((e=>void 0!==e)),r=n[1],o=n[2];if(!o)continue;let s;if("rgb"===r){const t=/^\(\s*(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\s*,\s*(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\s*,\s*(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\s*\)$/gm;s=eo(Qr(e,i),no(o,t),!1)}else if("rgba"===r){const t=/^\(\s*(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\s*,\s*(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\s*,\s*(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])\s*,\s*(0[.][0-9]+|[.][0-9]+|[01][.]|[01])\s*\)$/gm;s=eo(Qr(e,i),no(o,t),!0)}else if("hsl"===r){const t=/^\(\s*(36[0]|3[0-5][0-9]|[12][0-9][0-9]|[1-9]?[0-9])\s*,\s*(100|\d{1,2}[.]\d*|\d{1,2})%\s*,\s*(100|\d{1,2}[.]\d*|\d{1,2})%\s*\)$/gm;s=to(Qr(e,i),no(o,t),!1)}else if("hsla"===r){const t=/^\(\s*(36[0]|3[0-5][0-9]|[12][0-9][0-9]|[1-9]?[0-9])\s*,\s*(100|\d{1,2}[.]\d*|\d{1,2})%\s*,\s*(100|\d{1,2}[.]\d*|\d{1,2})%\s*,\s*(0[.][0-9]+|[.][0-9]+|[01][.]|[01])\s*\)$/gm;s=to(Qr(e,i),no(o,t),!0)}else"#"===r&&(s=Zr(Qr(e,i),r+o));s&&t.push(s)}return t}(e):[]}Jr.white=new Jr(new $r(255,255,255,1)),Jr.black=new Jr(new $r(0,0,0,1)),Jr.red=new Jr(new $r(255,0,0,1)),Jr.blue=new Jr(new $r(0,0,255,1)),Jr.green=new Jr(new $r(0,255,0,1)),Jr.cyan=new Jr(new $r(0,255,255,1)),Jr.lightgrey=new Jr(new $r(211,211,211,1)),Jr.transparent=new Jr(new $r(0,0,0,0)),function(e){let t;!function(t){let n;!function(t){function n(e){const t=e.toString(16);return 2!==t.length?"0"+t:t}function i(e){switch(e){case 48:return 0;case 49:return 1;case 50:return 2;case 51:return 3;case 52:return 4;case 53:return 5;case 54:return 6;case 55:return 7;case 56:return 8;case 57:return 9;case 97:case 65:return 10;case 98:case 66:return 11;case 99:case 67:return 12;case 100:case 68:return 13;case 101:case 69:return 14;case 102:case 70:return 15}return 0}t.formatRGB=function(t){return 1===t.rgba.a?"rgb(".concat(t.rgba.r,", ").concat(t.rgba.g,", ").concat(t.rgba.b,")"):e.Format.CSS.formatRGBA(t)},t.formatRGBA=function(e){return"rgba(".concat(e.rgba.r,", ").concat(e.rgba.g,", ").concat(e.rgba.b,", ").concat(+e.rgba.a.toFixed(2),")")},t.formatHSL=function(t){return 1===t.hsla.a?"hsl(".concat(t.hsla.h,", ").concat((100*t.hsla.s).toFixed(2),"%, ").concat((100*t.hsla.l).toFixed(2),"%)"):e.Format.CSS.formatHSLA(t)},t.formatHSLA=function(e){return"hsla(".concat(e.hsla.h,", ").concat((100*e.hsla.s).toFixed(2),"%, ").concat((100*e.hsla.l).toFixed(2),"%, ").concat(e.hsla.a.toFixed(2),")")},t.formatHex=function(e){return"#".concat(n(e.rgba.r)).concat(n(e.rgba.g)).concat(n(e.rgba.b))},t.formatHexA=function(t){return arguments.length>1&&void 0!==arguments[1]&&arguments[1]&&1===t.rgba.a?e.Format.CSS.formatHex(t):"#".concat(n(t.rgba.r)).concat(n(t.rgba.g)).concat(n(t.rgba.b)).concat(n(Math.round(255*t.rgba.a)))},t.format=function(t){return t.isOpaque()?e.Format.CSS.formatHex(t):e.Format.CSS.formatRGBA(t)},t.parseHex=function(t){const n=t.length;if(0===n)return null;if(35!==t.charCodeAt(0))return null;if(7===n){const n=16*i(t.charCodeAt(1))+i(t.charCodeAt(2)),r=16*i(t.charCodeAt(3))+i(t.charCodeAt(4)),o=16*i(t.charCodeAt(5))+i(t.charCodeAt(6));return new e(new $r(n,r,o,1))}if(9===n){const n=16*i(t.charCodeAt(1))+i(t.charCodeAt(2)),r=16*i(t.charCodeAt(3))+i(t.charCodeAt(4)),o=16*i(t.charCodeAt(5))+i(t.charCodeAt(6)),s=16*i(t.charCodeAt(7))+i(t.charCodeAt(8));return new e(new $r(n,r,o,s/255))}if(4===n){const n=i(t.charCodeAt(1)),r=i(t.charCodeAt(2)),o=i(t.charCodeAt(3));return new e(new $r(16*n+n,16*r+r,16*o+o))}if(5===n){const n=i(t.charCodeAt(1)),r=i(t.charCodeAt(2)),o=i(t.charCodeAt(3)),s=i(t.charCodeAt(4));return new e(new $r(16*n+n,16*r+r,16*o+o,(16*s+s)/255))}return null}}(n=t.CSS||(t.CSS={}))}(t=e.Format||(e.Format={}))}(Jr||(Jr={}));const ro=/\bMARK:\s*(.*)$/d,oo=/^-+|-+$/g;function so(e,t){var n;let i=[];if(t.findRegionSectionHeaders&&(null===(n=t.foldingRules)||void 0===n?void 0:n.markers)){const n=function(e,t){const n=[],i=e.getLineCount();for(let r=1;r<=i;r++){const i=e.getLineContent(r),o=i.match(t.foldingRules.markers.start);if(o){const e={startLineNumber:r,startColumn:o[0].length+1,endLineNumber:r,endColumn:i.length+1};if(e.endColumn>e.startColumn){const t={range:e,...lo(i.substring(o[0].length)),shouldBeInComments:!1};(t.text||t.hasSeparatorLine)&&n.push(t)}}}return n}(e,t);i=i.concat(n)}if(t.findMarkSectionHeaders){const t=function(e){const t=[],n=e.getLineCount();for(let i=1;i<=n;i++){ao(e.getLineContent(i),i,t)}return t}(e);i=i.concat(t)}return i}function ao(e,t,n){ro.lastIndex=0;const i=ro.exec(e);if(i){const e={startLineNumber:t,startColumn:i.indices[1][0]+1,endLineNumber:t,endColumn:i.indices[1][1]+1};if(e.endColumn>e.startColumn){const t={range:e,...lo(i[1]),shouldBeInComments:!0};(t.text||t.hasSeparatorLine)&&n.push(t)}}}function lo(e){const t=(e=e.trim()).startsWith("-");return{text:e=e.replace(oo,""),hasSeparatorLine:t}}class co extends Bt{get uri(){return this._uri}get eol(){return this._eol}getValue(){return this.getText()}findMatches(e){const t=[];for(let n=0;nthis._lines.length)t=this._lines.length,n=this._lines[t-1].length+1,i=!0;else{const e=this._lines[t-1].length+1;n<1?(n=1,i=!0):n>e&&(n=e,i=!0)}return i?{lineNumber:t,column:n}:e}}class ho{constructor(e,t){this._host=e,this._models=Object.create(null),this._foreignModuleFactory=t,this._foreignModule=null}dispose(){this._models=Object.create(null)}_getModel(e){return this._models[e]}_getModels(){const e=[];return Object.keys(this._models).forEach((t=>e.push(this._models[t]))),e}acceptNewModel(e){this._models[e.url]=new co(Ct.parse(e.url),e.lines,e.EOL,e.versionId)}acceptModelChanged(e,t){if(!this._models[e])return;this._models[e].onEvents(t)}acceptRemovedModel(e){this._models[e]&&delete this._models[e]}async computeUnicodeHighlights(e,t,n){const i=this._getModel(e);return i?Bi.computeUnicodeHighlights(i,t,n):{ranges:[],hasMore:!1,ambiguousCharacterCount:0,invisibleCharacterCount:0,nonBasicAsciiCharacterCount:0}}async findSectionHeaders(e,t){const n=this._getModel(e);return n?so(n,t):[]}async computeDiff(e,t,n,i){const r=this._getModel(e),o=this._getModel(t);if(!r||!o)return null;return ho.computeDiff(r,o,n,i)}static computeDiff(e,t,n,i){const r="advanced"===i?Br():qr(),o=e.getLinesContent(),s=t.getLinesContent(),a=r.computeDiff(o,s,n);function l(e){return e.map((e=>{var t;return[e.original.startLineNumber,e.original.endLineNumberExclusive,e.modified.startLineNumber,e.modified.endLineNumberExclusive,null===(t=e.innerChanges)||void 0===t?void 0:t.map((e=>[e.originalRange.startLineNumber,e.originalRange.startColumn,e.originalRange.endLineNumber,e.originalRange.endColumn,e.modifiedRange.startLineNumber,e.modifiedRange.startColumn,e.modifiedRange.endLineNumber,e.modifiedRange.endColumn]))]}))}return{identical:!(a.changes.length>0)&&this._modelsAreIdentical(e,t),quitEarly:a.hitTimeout,changes:l(a.changes),moves:a.moves.map((e=>[e.lineRangeMapping.original.startLineNumber,e.lineRangeMapping.original.endLineNumberExclusive,e.lineRangeMapping.modified.startLineNumber,e.lineRangeMapping.modified.endLineNumberExclusive,l(e.changes)]))}}static _modelsAreIdentical(e,t){const n=e.getLineCount();if(n!==t.getLineCount())return!1;for(let i=1;i<=n;i++){if(e.getLineContent(i)!==t.getLineContent(i))return!1}return!0}async computeMoreMinimalEdits(e,t,n){const i=this._getModel(e);if(!i)return t;const r=[];let o;t=t.slice(0).sort(((e,t)=>{if(e.range&&t.range)return It.compareRangesUsingStarts(e.range,t.range);return(e.range?0:1)-(t.range?0:1)}));let s=0;for(let a=1;aho._diffLimit){r.push({range:a,text:l});continue}const t=Be(e,l,n),s=i.offsetAt(It.lift(a).getStartPosition());for(const n of t){const e=i.positionAt(s+n.originalStart),t=i.positionAt(s+n.originalStart+n.originalLength),o={text:l.substr(n.modifiedStart,n.modifiedLength),range:{startLineNumber:e.lineNumber,startColumn:e.column,endLineNumber:t.lineNumber,endColumn:t.column}};i.getValueInRange(o.range)!==o.text&&r.push(o)}}return"number"===typeof o&&r.push({eol:o,text:"",range:{startLineNumber:0,startColumn:0,endLineNumber:0,endColumn:0}}),r}async computeLinks(e){const t=this._getModel(e);return t?function(e){return e&&"function"===typeof e.getLineCount&&"function"===typeof e.getLineContent?nn.computeLinks(e):[]}(t):null}async computeDefaultDocumentColors(e){const t=this._getModel(e);return t?io(t):null}async textualSuggest(e,t,n,i){const r=new _,o=new RegExp(n,i),s=new Set;e:for(const a of e){const e=this._getModel(a);if(e)for(const n of e.words(o))if(n!==t&&isNaN(Number(n))&&(s.add(n),s.size>ho._suggestionsLimit))break e}return{words:Array.from(s),duration:r.elapsed()}}async computeWordRanges(e,t,n,i){const r=this._getModel(e);if(!r)return Object.create(null);const o=new RegExp(n,i),s=Object.create(null);for(let a=t.startLineNumber;afunction(){const n=Array.prototype.slice.call(arguments,0);return t(e,n)},i={};for(const r of e)i[r]=n(r);return i}(n,((e,t)=>this._host.fhr(e,t))),r={host:i,getMirrorModels:()=>this._getModels()};return this._foreignModuleFactory?(this._foreignModule=this._foreignModuleFactory(r,t),Promise.resolve(z(this._foreignModule))):Promise.reject(new Error("Unexpected usage"))}fmr(e,t){if(!this._foreignModule||"function"!==typeof this._foreignModule[e])return Promise.reject(new Error("Missing requestHandler or method: "+e));try{return Promise.resolve(this._foreignModule[e].apply(this._foreignModule,t))}catch(Du){return Promise.reject(Du)}}}ho._diffLimit=1e5,ho._suggestionsLimit=1e4,"function"===typeof importScripts&&(globalThis.monaco={editor:void 0,languages:void 0,CancellationTokenSource:ln,Emitter:A,KeyCode:oi,KeyMod:Ti,Position:zt,Range:It,Selection:xn,SelectionDirection:wi,MarkerSeverity:si,MarkerTag:ai,Uri:Ct,Token:Rn});let po=!1;function uo(e){if(po)return;po=!0;const t=new Ie((e=>{globalThis.postMessage(e)}),(t=>new ho(t,e)));globalThis.onmessage=e=>{t.onmessage(e.data)}}var mo,fo;globalThis.onmessage=e=>{po||uo(null)},(fo=mo||(mo={}))[fo.Ident=0]="Ident",fo[fo.AtKeyword=1]="AtKeyword",fo[fo.String=2]="String",fo[fo.BadString=3]="BadString",fo[fo.UnquotedString=4]="UnquotedString",fo[fo.Hash=5]="Hash",fo[fo.Num=6]="Num",fo[fo.Percentage=7]="Percentage",fo[fo.Dimension=8]="Dimension",fo[fo.UnicodeRange=9]="UnicodeRange",fo[fo.CDO=10]="CDO",fo[fo.CDC=11]="CDC",fo[fo.Colon=12]="Colon",fo[fo.SemiColon=13]="SemiColon",fo[fo.CurlyL=14]="CurlyL",fo[fo.CurlyR=15]="CurlyR",fo[fo.ParenthesisL=16]="ParenthesisL",fo[fo.ParenthesisR=17]="ParenthesisR",fo[fo.BracketL=18]="BracketL",fo[fo.BracketR=19]="BracketR",fo[fo.Whitespace=20]="Whitespace",fo[fo.Includes=21]="Includes",fo[fo.Dashmatch=22]="Dashmatch",fo[fo.SubstringOperator=23]="SubstringOperator",fo[fo.PrefixOperator=24]="PrefixOperator",fo[fo.SuffixOperator=25]="SuffixOperator",fo[fo.Delim=26]="Delim",fo[fo.EMS=27]="EMS",fo[fo.EXS=28]="EXS",fo[fo.Length=29]="Length",fo[fo.Angle=30]="Angle",fo[fo.Time=31]="Time",fo[fo.Freq=32]="Freq",fo[fo.Exclamation=33]="Exclamation",fo[fo.Resolution=34]="Resolution",fo[fo.Comma=35]="Comma",fo[fo.Charset=36]="Charset",fo[fo.EscapedJavaScript=37]="EscapedJavaScript",fo[fo.BadEscapedJavaScript=38]="BadEscapedJavaScript",fo[fo.Comment=39]="Comment",fo[fo.SingleLineComment=40]="SingleLineComment",fo[fo.EOF=41]="EOF",fo[fo.CustomToken=42]="CustomToken";var go=function(){function e(e){this.source=e,this.len=e.length,this.position=0}return e.prototype.substring=function(e,t){return void 0===t&&(t=this.position),this.source.substring(e,t)},e.prototype.eos=function(){return this.len<=this.position},e.prototype.pos=function(){return this.position},e.prototype.goBackTo=function(e){this.position=e},e.prototype.goBack=function(e){this.position-=e},e.prototype.advance=function(e){this.position+=e},e.prototype.nextChar=function(){return this.source.charCodeAt(this.position++)||0},e.prototype.peekChar=function(e){return void 0===e&&(e=0),this.source.charCodeAt(this.position+e)||0},e.prototype.lookbackChar=function(e){return void 0===e&&(e=0),this.source.charCodeAt(this.position-e)||0},e.prototype.advanceIfChar=function(e){return e===this.source.charCodeAt(this.position)&&(this.position++,!0)},e.prototype.advanceIfChars=function(e){if(this.position+e.length>this.source.length)return!1;for(var t=0;t".charCodeAt(0),Po="@".charCodeAt(0),Oo="#".charCodeAt(0),Wo="$".charCodeAt(0),Vo="\\".charCodeAt(0),Uo="/".charCodeAt(0),Ko="\n".charCodeAt(0),qo="\r".charCodeAt(0),Bo="\f".charCodeAt(0),jo='"'.charCodeAt(0),$o="'".charCodeAt(0),Ho=" ".charCodeAt(0),Go="\t".charCodeAt(0),Jo=";".charCodeAt(0),Xo=":".charCodeAt(0),Yo="{".charCodeAt(0),Qo="}".charCodeAt(0),Zo="[".charCodeAt(0),es="]".charCodeAt(0),ts=",".charCodeAt(0),ns=".".charCodeAt(0),is="!".charCodeAt(0),rs="?".charCodeAt(0),os="+".charCodeAt(0),ss={};ss[Jo]=mo.SemiColon,ss[Xo]=mo.Colon,ss[Yo]=mo.CurlyL,ss[Qo]=mo.CurlyR,ss[es]=mo.BracketR,ss[Zo]=mo.BracketL,ss[Mo]=mo.ParenthesisL,ss[zo]=mo.ParenthesisR,ss[ts]=mo.Comma;var as={};as.em=mo.EMS,as.ex=mo.EXS,as.px=mo.Length,as.cm=mo.Length,as.mm=mo.Length,as.in=mo.Length,as.pt=mo.Length,as.pc=mo.Length,as.deg=mo.Angle,as.rad=mo.Angle,as.grad=mo.Angle,as.ms=mo.Time,as.s=mo.Time,as.hz=mo.Freq,as.khz=mo.Freq,as["%"]=mo.Percentage,as.fr=mo.Percentage,as.dpi=mo.Resolution,as.dpcm=mo.Resolution;var ls=function(){function e(){this.stream=new go(""),this.ignoreComment=!0,this.ignoreWhitespace=!0,this.inURL=!1}return e.prototype.setSource=function(e){this.stream=new go(e)},e.prototype.finishToken=function(e,t,n){return{offset:e,len:this.stream.pos()-e,type:t,text:n||this.stream.substring(e)}},e.prototype.substring=function(e,t){return this.stream.substring(e,e+t)},e.prototype.pos=function(){return this.stream.pos()},e.prototype.goBackTo=function(e){this.stream.goBackTo(e)},e.prototype.scanUnquotedString=function(){var e=this.stream.pos(),t=[];return this._unquotedString(t)?this.finishToken(e,mo.UnquotedString,t.join("")):null},e.prototype.scan=function(){var e=this.trivia();if(null!==e)return e;var t=this.stream.pos();return this.stream.eos()?this.finishToken(t,mo.EOF):this.scanNext(t)},e.prototype.tryScanUnicode=function(){var e=this.stream.pos();if(!this.stream.eos()&&this._unicodeRange())return this.finishToken(e,mo.UnicodeRange);this.stream.goBackTo(e)},e.prototype.scanNext=function(e){if(this.stream.advanceIfChars([Io,is,Fo,Fo]))return this.finishToken(e,mo.CDO);if(this.stream.advanceIfChars([Fo,Fo,Lo]))return this.finishToken(e,mo.CDC);var t=[];if(this.ident(t))return this.finishToken(e,mo.Ident,t.join(""));if(this.stream.advanceIfChar(Po)){if(t=["@"],this._name(t)){var n=t.join("");return"@charset"===n?this.finishToken(e,mo.Charset,n):this.finishToken(e,mo.AtKeyword,n)}return this.finishToken(e,mo.Delim)}if(this.stream.advanceIfChar(Oo))return t=["#"],this._name(t)?this.finishToken(e,mo.Hash,t.join("")):this.finishToken(e,mo.Delim);if(this.stream.advanceIfChar(is))return this.finishToken(e,mo.Exclamation);if(this._number()){var i=this.stream.pos();if(t=[this.stream.substring(e,i)],this.stream.advanceIfChar(To))return this.finishToken(e,mo.Percentage);if(this.ident(t)){var r=this.stream.substring(i).toLowerCase(),o=as[r];return"undefined"!==typeof o?this.finishToken(e,o,t.join("")):this.finishToken(e,mo.Dimension,t.join(""))}return this.finishToken(e,mo.Num)}t=[];var s=this._string(t);return null!==s?this.finishToken(e,s,t.join("")):"undefined"!==typeof(s=ss[this.stream.peekChar()])?(this.stream.advance(1),this.finishToken(e,s)):this.stream.peekChar(0)===ko&&this.stream.peekChar(1)===Ro?(this.stream.advance(2),this.finishToken(e,mo.Includes)):this.stream.peekChar(0)===No&&this.stream.peekChar(1)===Ro?(this.stream.advance(2),this.finishToken(e,mo.Dashmatch)):this.stream.peekChar(0)===Ao&&this.stream.peekChar(1)===Ro?(this.stream.advance(2),this.finishToken(e,mo.SubstringOperator)):this.stream.peekChar(0)===Eo&&this.stream.peekChar(1)===Ro?(this.stream.advance(2),this.finishToken(e,mo.PrefixOperator)):this.stream.peekChar(0)===Wo&&this.stream.peekChar(1)===Ro?(this.stream.advance(2),this.finishToken(e,mo.SuffixOperator)):(this.stream.nextChar(),this.finishToken(e,mo.Delim))},e.prototype.trivia=function(){for(;;){var e=this.stream.pos();if(this._whitespace()){if(!this.ignoreWhitespace)return this.finishToken(e,mo.Whitespace)}else{if(!this.comment())return null;if(!this.ignoreComment)return this.finishToken(e,mo.Comment)}}},e.prototype.comment=function(){if(this.stream.advanceIfChars([Uo,Ao])){var e=!1,t=!1;return this.stream.advanceWhileChar((function(n){return t&&n===Uo?(e=!0,!1):(t=n===Ao,!0)})),e&&this.stream.advance(1),!0}return!1},e.prototype._number=function(){var e,t=0;return this.stream.peekChar()===ns&&(t=1),(e=this.stream.peekChar(t))>=Co&&e<=_o&&(this.stream.advance(t+1),this.stream.advanceWhileChar((function(e){return e>=Co&&e<=_o||0===t&&e===ns})),!0)},e.prototype._newline=function(e){var t=this.stream.peekChar();switch(t){case qo:case Bo:case Ko:return this.stream.advance(1),e.push(String.fromCharCode(t)),t===qo&&this.stream.advanceIfChar(Ko)&&e.push("\n"),!0}return!1},e.prototype._escape=function(e,t){var n=this.stream.peekChar();if(n===Vo){this.stream.advance(1),n=this.stream.peekChar();for(var i=0;i<6&&(n>=Co&&n<=_o||n>=bo&&n<=vo||n>=wo&&n<=xo);)this.stream.advance(1),n=this.stream.peekChar(),i++;if(i>0){try{var r=parseInt(this.stream.substring(this.stream.pos()-i),16);r&&e.push(String.fromCharCode(r))}catch(Du){}return n===Ho||n===Go?this.stream.advance(1):this._newline([]),!0}if(n!==qo&&n!==Bo&&n!==Ko)return this.stream.advance(1),e.push(String.fromCharCode(n)),!0;if(t)return this._newline(e)}return!1},e.prototype._stringChar=function(e,t){var n=this.stream.peekChar();return 0!==n&&n!==e&&n!==Vo&&n!==qo&&n!==Bo&&n!==Ko&&(this.stream.advance(1),t.push(String.fromCharCode(n)),!0)},e.prototype._string=function(e){if(this.stream.peekChar()===$o||this.stream.peekChar()===jo){var t=this.stream.nextChar();for(e.push(String.fromCharCode(t));this._stringChar(t,e)||this._escape(e,!0););return this.stream.peekChar()===t?(this.stream.nextChar(),e.push(String.fromCharCode(t)),mo.String):mo.BadString}return null},e.prototype._unquotedChar=function(e){var t=this.stream.peekChar();return 0!==t&&t!==Vo&&t!==$o&&t!==jo&&t!==Mo&&t!==zo&&t!==Ho&&t!==Go&&t!==Ko&&t!==Bo&&t!==qo&&(this.stream.advance(1),e.push(String.fromCharCode(t)),!0)},e.prototype._unquotedString=function(e){for(var t=!1;this._unquotedChar(e)||this._escape(e);)t=!0;return t},e.prototype._whitespace=function(){return this.stream.advanceWhileChar((function(e){return e===Ho||e===Go||e===Ko||e===Bo||e===qo}))>0},e.prototype._name=function(e){for(var t=!1;this._identChar(e)||this._escape(e);)t=!0;return t},e.prototype.ident=function(e){var t=this.stream.pos();if(this._minus(e)){if(this._minus(e)||this._identFirstChar(e)||this._escape(e)){for(;this._identChar(e)||this._escape(e););return!0}}else if(this._identFirstChar(e)||this._escape(e)){for(;this._identChar(e)||this._escape(e););return!0}return this.stream.goBackTo(t),!1},e.prototype._identFirstChar=function(e){var t=this.stream.peekChar();return(t===Do||t>=bo&&t<=yo||t>=wo&&t<=So||t>=128&&t<=65535)&&(this.stream.advance(1),e.push(String.fromCharCode(t)),!0)},e.prototype._minus=function(e){var t=this.stream.peekChar();return t===Fo&&(this.stream.advance(1),e.push(String.fromCharCode(t)),!0)},e.prototype._identChar=function(e){var t=this.stream.peekChar();return(t===Do||t===Fo||t>=bo&&t<=yo||t>=wo&&t<=So||t>=Co&&t<=_o||t>=128&&t<=65535)&&(this.stream.advance(1),e.push(String.fromCharCode(t)),!0)},e.prototype._unicodeRange=function(){if(this.stream.advanceIfChar(os)){var e=function(e){return e>=Co&&e<=_o||e>=bo&&e<=vo||e>=wo&&e<=xo},t=this.stream.advanceWhileChar(e)+this.stream.advanceWhileChar((function(e){return e===rs}));if(t>=1&&t<=6){if(!this.stream.advanceIfChar(Fo))return!0;var n=this.stream.advanceWhileChar(e);if(n>=1&&n<=6)return!0}}return!1},e}();function cs(e,t){if(e.length0?e.lastIndexOf(t)===n:0===n&&e===t}function ds(e,t){return void 0===t&&(t=!0),e?e.length<140?e:e.slice(0,140)+(t?"\u2026":""):""}function ps(e,t){for(var n="";t>0;)1===(1&t)&&(n+=e),e+=e,t>>>=1;return n}var us,ms,fs,gs,bs=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])})(t,n)};return function(t,n){if("function"!==typeof n&&null!==n)throw new TypeError("Class extends value "+String(n)+" is not a constructor or null");function i(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(i.prototype=n.prototype,new i)}}();function vs(e,t){var n=null;return!e||te.end?null:(e.accept((function(e){return-1===e.offset&&-1===e.length||e.offset<=t&&e.end>=t&&(n?e.length<=n.length&&(n=e):n=e,!0)})),n)}function ys(e,t){for(var n=vs(e,t),i=[];n;)i.unshift(n),n=n.parent;return i}(ms=us||(us={}))[ms.Undefined=0]="Undefined",ms[ms.Identifier=1]="Identifier",ms[ms.Stylesheet=2]="Stylesheet",ms[ms.Ruleset=3]="Ruleset",ms[ms.Selector=4]="Selector",ms[ms.SimpleSelector=5]="SimpleSelector",ms[ms.SelectorInterpolation=6]="SelectorInterpolation",ms[ms.SelectorCombinator=7]="SelectorCombinator",ms[ms.SelectorCombinatorParent=8]="SelectorCombinatorParent",ms[ms.SelectorCombinatorSibling=9]="SelectorCombinatorSibling",ms[ms.SelectorCombinatorAllSiblings=10]="SelectorCombinatorAllSiblings",ms[ms.SelectorCombinatorShadowPiercingDescendant=11]="SelectorCombinatorShadowPiercingDescendant",ms[ms.Page=12]="Page",ms[ms.PageBoxMarginBox=13]="PageBoxMarginBox",ms[ms.ClassSelector=14]="ClassSelector",ms[ms.IdentifierSelector=15]="IdentifierSelector",ms[ms.ElementNameSelector=16]="ElementNameSelector",ms[ms.PseudoSelector=17]="PseudoSelector",ms[ms.AttributeSelector=18]="AttributeSelector",ms[ms.Declaration=19]="Declaration",ms[ms.Declarations=20]="Declarations",ms[ms.Property=21]="Property",ms[ms.Expression=22]="Expression",ms[ms.BinaryExpression=23]="BinaryExpression",ms[ms.Term=24]="Term",ms[ms.Operator=25]="Operator",ms[ms.Value=26]="Value",ms[ms.StringLiteral=27]="StringLiteral",ms[ms.URILiteral=28]="URILiteral",ms[ms.EscapedValue=29]="EscapedValue",ms[ms.Function=30]="Function",ms[ms.NumericValue=31]="NumericValue",ms[ms.HexColorValue=32]="HexColorValue",ms[ms.RatioValue=33]="RatioValue",ms[ms.MixinDeclaration=34]="MixinDeclaration",ms[ms.MixinReference=35]="MixinReference",ms[ms.VariableName=36]="VariableName",ms[ms.VariableDeclaration=37]="VariableDeclaration",ms[ms.Prio=38]="Prio",ms[ms.Interpolation=39]="Interpolation",ms[ms.NestedProperties=40]="NestedProperties",ms[ms.ExtendsReference=41]="ExtendsReference",ms[ms.SelectorPlaceholder=42]="SelectorPlaceholder",ms[ms.Debug=43]="Debug",ms[ms.If=44]="If",ms[ms.Else=45]="Else",ms[ms.For=46]="For",ms[ms.Each=47]="Each",ms[ms.While=48]="While",ms[ms.MixinContentReference=49]="MixinContentReference",ms[ms.MixinContentDeclaration=50]="MixinContentDeclaration",ms[ms.Media=51]="Media",ms[ms.Keyframe=52]="Keyframe",ms[ms.FontFace=53]="FontFace",ms[ms.Import=54]="Import",ms[ms.Namespace=55]="Namespace",ms[ms.Invocation=56]="Invocation",ms[ms.FunctionDeclaration=57]="FunctionDeclaration",ms[ms.ReturnStatement=58]="ReturnStatement",ms[ms.MediaQuery=59]="MediaQuery",ms[ms.MediaCondition=60]="MediaCondition",ms[ms.MediaFeature=61]="MediaFeature",ms[ms.FunctionParameter=62]="FunctionParameter",ms[ms.FunctionArgument=63]="FunctionArgument",ms[ms.KeyframeSelector=64]="KeyframeSelector",ms[ms.ViewPort=65]="ViewPort",ms[ms.Document=66]="Document",ms[ms.AtApplyRule=67]="AtApplyRule",ms[ms.CustomPropertyDeclaration=68]="CustomPropertyDeclaration",ms[ms.CustomPropertySet=69]="CustomPropertySet",ms[ms.ListEntry=70]="ListEntry",ms[ms.Supports=71]="Supports",ms[ms.SupportsCondition=72]="SupportsCondition",ms[ms.NamespacePrefix=73]="NamespacePrefix",ms[ms.GridLine=74]="GridLine",ms[ms.Plugin=75]="Plugin",ms[ms.UnknownAtRule=76]="UnknownAtRule",ms[ms.Use=77]="Use",ms[ms.ModuleConfiguration=78]="ModuleConfiguration",ms[ms.Forward=79]="Forward",ms[ms.ForwardVisibility=80]="ForwardVisibility",ms[ms.Module=81]="Module",ms[ms.UnicodeRange=82]="UnicodeRange",(gs=fs||(fs={}))[gs.Mixin=0]="Mixin",gs[gs.Rule=1]="Rule",gs[gs.Variable=2]="Variable",gs[gs.Function=3]="Function",gs[gs.Keyframe=4]="Keyframe",gs[gs.Unknown=5]="Unknown",gs[gs.Module=6]="Module",gs[gs.Forward=7]="Forward",gs[gs.ForwardVisibility=8]="ForwardVisibility";var ws,xs,Ss=function(){function e(e,t,n){void 0===e&&(e=-1),void 0===t&&(t=-1),this.parent=null,this.offset=e,this.length=t,n&&(this.nodeType=n)}return Object.defineProperty(e.prototype,"end",{get:function(){return this.offset+this.length},enumerable:!1,configurable:!0}),Object.defineProperty(e.prototype,"type",{get:function(){return this.nodeType||us.Undefined},set:function(e){this.nodeType=e},enumerable:!1,configurable:!0}),e.prototype.getTextProvider=function(){for(var e=this;e&&!e.textProvider;)e=e.parent;return e?e.textProvider:function(){return"unknown"}},e.prototype.getText=function(){return this.getTextProvider()(this.offset,this.length)},e.prototype.matches=function(e){return this.length===e.length&&this.getTextProvider()(this.offset,this.length)===e},e.prototype.startsWith=function(e){return this.length>=e.length&&this.getTextProvider()(this.offset,e.length)===e},e.prototype.endsWith=function(e){return this.length>=e.length&&this.getTextProvider()(this.end-e.length,e.length)===e},e.prototype.accept=function(e){if(e(this)&&this.children)for(var t=0,n=this.children;t=0&&e.parent.children.splice(n,1)}e.parent=this;var i=this.children;return i||(i=this.children=[]),-1!==t?i.splice(t,0,e):i.push(e),e},e.prototype.attachTo=function(e,t){return void 0===t&&(t=-1),e&&e.adoptChild(this,t),this},e.prototype.collectIssues=function(e){this.issues&&e.push.apply(e,this.issues)},e.prototype.addIssue=function(e){this.issues||(this.issues=[]),this.issues.push(e)},e.prototype.hasIssue=function(e){return Array.isArray(this.issues)&&this.issues.some((function(t){return t.getRule()===e}))},e.prototype.isErroneous=function(e){return void 0===e&&(e=!1),!!(this.issues&&this.issues.length>0)||e&&Array.isArray(this.children)&&this.children.some((function(e){return e.isErroneous(!0)}))},e.prototype.setNode=function(e,t,n){return void 0===n&&(n=-1),!!t&&(t.attachTo(this,n),this[e]=t,!0)},e.prototype.addChild=function(e){return!!e&&(this.children||(this.children=[]),e.attachTo(this),this.updateOffsetAndLength(e),!0)},e.prototype.updateOffsetAndLength=function(e){(e.offsetthis.end||-1===this.length)&&(this.length=t-this.offset)},e.prototype.hasChildren=function(){return!!this.children&&this.children.length>0},e.prototype.getChildren=function(){return this.children?this.children.slice(0):[]},e.prototype.getChild=function(e){return this.children&&e=0;n--)if((t=this.children[n]).offset<=e)return t;return null},e.prototype.findChildAtOffset=function(e,t){var n=this.findFirstChildBeforeOffset(e);return n&&n.end>=e?t&&n.findChildAtOffset(e,!0)||n:null},e.prototype.encloses=function(e){return this.offset<=e.offset&&this.offset+this.length>=e.offset+e.length},e.prototype.getParent=function(){for(var e=this.parent;e instanceof Cs;)e=e.parent;return e},e.prototype.findParent=function(e){for(var t=this;t&&t.type!==e;)t=t.parent;return t},e.prototype.findAParent=function(){for(var e=[],t=0;t2?n-2:0),r=2;r{let i=n[0];return"undefined"!==typeof t[i]?t[i]:e})),n}(t,i)}function Wa(e){return Oa}var Va,Ua,Ka,qa,Ba,ja,$a,Ha,Ga,Ja,Xa,Ya,Qa,Za,el,tl,nl,il,rl,ol,sl,al,ll,cl,hl,dl,pl,ul,ml,fl,gl,bl,vl,yl,wl,xl,Sl,Cl,_l,kl,El,Rl,Nl,Fl,Dl,Tl,Al,Ml,zl,Il=Wa(),Ll=function(){return function(e,t){this.id=e,this.message=t}}(),Pl={NumberExpected:new Ll("css-numberexpected",Il("expected.number","number expected")),ConditionExpected:new Ll("css-conditionexpected",Il("expected.condt","condition expected")),RuleOrSelectorExpected:new Ll("css-ruleorselectorexpected",Il("expected.ruleorselector","at-rule or selector expected")),DotExpected:new Ll("css-dotexpected",Il("expected.dot","dot expected")),ColonExpected:new Ll("css-colonexpected",Il("expected.colon","colon expected")),SemiColonExpected:new Ll("css-semicolonexpected",Il("expected.semicolon","semi-colon expected")),TermExpected:new Ll("css-termexpected",Il("expected.term","term expected")),ExpressionExpected:new Ll("css-expressionexpected",Il("expected.expression","expression expected")),OperatorExpected:new Ll("css-operatorexpected",Il("expected.operator","operator expected")),IdentifierExpected:new Ll("css-identifierexpected",Il("expected.ident","identifier expected")),PercentageExpected:new Ll("css-percentageexpected",Il("expected.percentage","percentage expected")),URIOrStringExpected:new Ll("css-uriorstringexpected",Il("expected.uriorstring","uri or string expected")),URIExpected:new Ll("css-uriexpected",Il("expected.uri","URI expected")),VariableNameExpected:new Ll("css-varnameexpected",Il("expected.varname","variable name expected")),VariableValueExpected:new Ll("css-varvalueexpected",Il("expected.varvalue","variable value expected")),PropertyValueExpected:new Ll("css-propertyvalueexpected",Il("expected.propvalue","property value expected")),LeftCurlyExpected:new Ll("css-lcurlyexpected",Il("expected.lcurly","{ expected")),RightCurlyExpected:new Ll("css-rcurlyexpected",Il("expected.rcurly","} expected")),LeftSquareBracketExpected:new Ll("css-rbracketexpected",Il("expected.lsquare","[ expected")),RightSquareBracketExpected:new Ll("css-lbracketexpected",Il("expected.rsquare","] expected")),LeftParenthesisExpected:new Ll("css-lparentexpected",Il("expected.lparen","( expected")),RightParenthesisExpected:new Ll("css-rparentexpected",Il("expected.rparent",") expected")),CommaExpected:new Ll("css-commaexpected",Il("expected.comma","comma expected")),PageDirectiveOrDeclarationExpected:new Ll("css-pagedirordeclexpected",Il("expected.pagedirordecl","page directive or declaraton expected")),UnknownAtRule:new Ll("css-unknownatrule",Il("unknown.atrule","at-rule unknown")),UnknownKeyword:new Ll("css-unknownkeyword",Il("unknown.keyword","unknown keyword")),SelectorExpected:new Ll("css-selectorexpected",Il("expected.selector","selector expected")),StringLiteralExpected:new Ll("css-stringliteralexpected",Il("expected.stringliteral","string literal expected")),WhitespaceExpected:new Ll("css-whitespaceexpected",Il("expected.whitespace","whitespace expected")),MediaQueryExpected:new Ll("css-mediaqueryexpected",Il("expected.mediaquery","media query expected")),IdentifierOrWildcardExpected:new Ll("css-idorwildcardexpected",Il("expected.idorwildcard","identifier or wildcard expected")),WildcardExpected:new Ll("css-wildcardexpected",Il("expected.wildcard","wildcard expected")),IdentifierOrVariableExpected:new Ll("css-idorvarexpected",Il("expected.idorvar","identifier or variable expected"))};(Ua=Va||(Va={})).MIN_VALUE=-2147483648,Ua.MAX_VALUE=2147483647,(qa=Ka||(Ka={})).MIN_VALUE=0,qa.MAX_VALUE=2147483647,(ja=Ba||(Ba={})).create=function(e,t){return e===Number.MAX_VALUE&&(e=Ka.MAX_VALUE),t===Number.MAX_VALUE&&(t=Ka.MAX_VALUE),{line:e,character:t}},ja.is=function(e){var t=e;return Pc.objectLiteral(t)&&Pc.uinteger(t.line)&&Pc.uinteger(t.character)},(Ha=$a||($a={})).create=function(e,t,n,i){if(Pc.uinteger(e)&&Pc.uinteger(t)&&Pc.uinteger(n)&&Pc.uinteger(i))return{start:Ba.create(e,t),end:Ba.create(n,i)};if(Ba.is(e)&&Ba.is(t))return{start:e,end:t};throw new Error("Range#create called with invalid arguments["+e+", "+t+", "+n+", "+i+"]")},Ha.is=function(e){var t=e;return Pc.objectLiteral(t)&&Ba.is(t.start)&&Ba.is(t.end)},(Ja=Ga||(Ga={})).create=function(e,t){return{uri:e,range:t}},Ja.is=function(e){var t=e;return Pc.defined(t)&&$a.is(t.range)&&(Pc.string(t.uri)||Pc.undefined(t.uri))},(Ya=Xa||(Xa={})).create=function(e,t,n,i){return{targetUri:e,targetRange:t,targetSelectionRange:n,originSelectionRange:i}},Ya.is=function(e){var t=e;return Pc.defined(t)&&$a.is(t.targetRange)&&Pc.string(t.targetUri)&&($a.is(t.targetSelectionRange)||Pc.undefined(t.targetSelectionRange))&&($a.is(t.originSelectionRange)||Pc.undefined(t.originSelectionRange))},(Za=Qa||(Qa={})).create=function(e,t,n,i){return{red:e,green:t,blue:n,alpha:i}},Za.is=function(e){var t=e;return Pc.numberRange(t.red,0,1)&&Pc.numberRange(t.green,0,1)&&Pc.numberRange(t.blue,0,1)&&Pc.numberRange(t.alpha,0,1)},(tl=el||(el={})).create=function(e,t){return{range:e,color:t}},tl.is=function(e){var t=e;return $a.is(t.range)&&Qa.is(t.color)},(il=nl||(nl={})).create=function(e,t,n){return{label:e,textEdit:t,additionalTextEdits:n}},il.is=function(e){var t=e;return Pc.string(t.label)&&(Pc.undefined(t.textEdit)||yl.is(t))&&(Pc.undefined(t.additionalTextEdits)||Pc.typedArray(t.additionalTextEdits,yl.is))},(ol=rl||(rl={})).Comment="comment",ol.Imports="imports",ol.Region="region",(al=sl||(sl={})).create=function(e,t,n,i,r){var o={startLine:e,endLine:t};return Pc.defined(n)&&(o.startCharacter=n),Pc.defined(i)&&(o.endCharacter=i),Pc.defined(r)&&(o.kind=r),o},al.is=function(e){var t=e;return Pc.uinteger(t.startLine)&&Pc.uinteger(t.startLine)&&(Pc.undefined(t.startCharacter)||Pc.uinteger(t.startCharacter))&&(Pc.undefined(t.endCharacter)||Pc.uinteger(t.endCharacter))&&(Pc.undefined(t.kind)||Pc.string(t.kind))},(cl=ll||(ll={})).create=function(e,t){return{location:e,message:t}},cl.is=function(e){var t=e;return Pc.defined(t)&&Ga.is(t.location)&&Pc.string(t.message)},(dl=hl||(hl={})).Error=1,dl.Warning=2,dl.Information=3,dl.Hint=4,(ul=pl||(pl={})).Unnecessary=1,ul.Deprecated=2,(ml||(ml={})).is=function(e){var t=e;return void 0!==t&&null!==t&&Pc.string(t.href)},(gl=fl||(fl={})).create=function(e,t,n,i,r,o){var s={range:e,message:t};return Pc.defined(n)&&(s.severity=n),Pc.defined(i)&&(s.code=i),Pc.defined(r)&&(s.source=r),Pc.defined(o)&&(s.relatedInformation=o),s},gl.is=function(e){var t,n=e;return Pc.defined(n)&&$a.is(n.range)&&Pc.string(n.message)&&(Pc.number(n.severity)||Pc.undefined(n.severity))&&(Pc.integer(n.code)||Pc.string(n.code)||Pc.undefined(n.code))&&(Pc.undefined(n.codeDescription)||Pc.string(null===(t=n.codeDescription)||void 0===t?void 0:t.href))&&(Pc.string(n.source)||Pc.undefined(n.source))&&(Pc.undefined(n.relatedInformation)||Pc.typedArray(n.relatedInformation,ll.is))},(vl=bl||(bl={})).create=function(e,t){for(var n=[],i=2;i0&&(r.arguments=n),r},vl.is=function(e){var t=e;return Pc.defined(t)&&Pc.string(t.title)&&Pc.string(t.command)},(wl=yl||(yl={})).replace=function(e,t){return{range:e,newText:t}},wl.insert=function(e,t){return{range:{start:e,end:e},newText:t}},wl.del=function(e){return{range:e,newText:""}},wl.is=function(e){var t=e;return Pc.objectLiteral(t)&&Pc.string(t.newText)&&$a.is(t.range)},(Sl=xl||(xl={})).create=function(e,t,n){var i={label:e};return void 0!==t&&(i.needsConfirmation=t),void 0!==n&&(i.description=n),i},Sl.is=function(e){var t=e;return void 0!==t&&Pc.objectLiteral(t)&&Pc.string(t.label)&&(Pc.boolean(t.needsConfirmation)||void 0===t.needsConfirmation)&&(Pc.string(t.description)||void 0===t.description)},(Cl||(Cl={})).is=function(e){return"string"===typeof e},(kl=_l||(_l={})).replace=function(e,t,n){return{range:e,newText:t,annotationId:n}},kl.insert=function(e,t,n){return{range:{start:e,end:e},newText:t,annotationId:n}},kl.del=function(e,t){return{range:e,newText:"",annotationId:t}},kl.is=function(e){var t=e;return yl.is(t)&&(xl.is(t.annotationId)||Cl.is(t.annotationId))},(Rl=El||(El={})).create=function(e,t){return{textDocument:e,edits:t}},Rl.is=function(e){var t=e;return Pc.defined(t)&&Kl.is(t.textDocument)&&Array.isArray(t.edits)},(Fl=Nl||(Nl={})).create=function(e,t,n){var i={kind:"create",uri:e};return void 0===t||void 0===t.overwrite&&void 0===t.ignoreIfExists||(i.options=t),void 0!==n&&(i.annotationId=n),i},Fl.is=function(e){var t=e;return t&&"create"===t.kind&&Pc.string(t.uri)&&(void 0===t.options||(void 0===t.options.overwrite||Pc.boolean(t.options.overwrite))&&(void 0===t.options.ignoreIfExists||Pc.boolean(t.options.ignoreIfExists)))&&(void 0===t.annotationId||Cl.is(t.annotationId))},(Tl=Dl||(Dl={})).create=function(e,t,n,i){var r={kind:"rename",oldUri:e,newUri:t};return void 0===n||void 0===n.overwrite&&void 0===n.ignoreIfExists||(r.options=n),void 0!==i&&(r.annotationId=i),r},Tl.is=function(e){var t=e;return t&&"rename"===t.kind&&Pc.string(t.oldUri)&&Pc.string(t.newUri)&&(void 0===t.options||(void 0===t.options.overwrite||Pc.boolean(t.options.overwrite))&&(void 0===t.options.ignoreIfExists||Pc.boolean(t.options.ignoreIfExists)))&&(void 0===t.annotationId||Cl.is(t.annotationId))},(Ml=Al||(Al={})).create=function(e,t,n){var i={kind:"delete",uri:e};return void 0===t||void 0===t.recursive&&void 0===t.ignoreIfNotExists||(i.options=t),void 0!==n&&(i.annotationId=n),i},Ml.is=function(e){var t=e;return t&&"delete"===t.kind&&Pc.string(t.uri)&&(void 0===t.options||(void 0===t.options.recursive||Pc.boolean(t.options.recursive))&&(void 0===t.options.ignoreIfNotExists||Pc.boolean(t.options.ignoreIfNotExists)))&&(void 0===t.annotationId||Cl.is(t.annotationId))},(zl||(zl={})).is=function(e){var t=e;return t&&(void 0!==t.changes||void 0!==t.documentChanges)&&(void 0===t.documentChanges||t.documentChanges.every((function(e){return Pc.string(e.kind)?Nl.is(e)||Dl.is(e)||Al.is(e):El.is(e)})))};var Ol,Wl,Vl,Ul,Kl,ql,Bl,jl,$l,Hl,Gl,Jl,Xl,Yl,Ql,Zl,ec,tc,nc,ic,rc,oc,sc,ac,lc,cc,hc,dc,pc,uc,mc,fc,gc,bc,vc,yc,wc,xc,Sc,Cc,_c,kc,Ec,Rc,Nc,Fc,Dc,Tc,Ac,Mc,zc,Ic=function(){function e(e,t){this.edits=e,this.changeAnnotations=t}return e.prototype.insert=function(e,t,n){var i,r;if(void 0===n?i=yl.insert(e,t):Cl.is(n)?(r=n,i=_l.insert(e,t,n)):(this.assertChangeAnnotations(this.changeAnnotations),r=this.changeAnnotations.manage(n),i=_l.insert(e,t,r)),this.edits.push(i),void 0!==r)return r},e.prototype.replace=function(e,t,n){var i,r;if(void 0===n?i=yl.replace(e,t):Cl.is(n)?(r=n,i=_l.replace(e,t,n)):(this.assertChangeAnnotations(this.changeAnnotations),r=this.changeAnnotations.manage(n),i=_l.replace(e,t,r)),this.edits.push(i),void 0!==r)return r},e.prototype.delete=function(e,t){var n,i;if(void 0===t?n=yl.del(e):Cl.is(t)?(i=t,n=_l.del(e,t)):(this.assertChangeAnnotations(this.changeAnnotations),i=this.changeAnnotations.manage(t),n=_l.del(e,i)),this.edits.push(n),void 0!==i)return i},e.prototype.add=function(e){this.edits.push(e)},e.prototype.all=function(){return this.edits},e.prototype.clear=function(){this.edits.splice(0,this.edits.length)},e.prototype.assertChangeAnnotations=function(e){if(void 0===e)throw new Error("Text edit change is not configured to manage change annotations.")},e}(),Lc=function(){function e(e){this._annotations=void 0===e?Object.create(null):e,this._counter=0,this._size=0}return e.prototype.all=function(){return this._annotations},Object.defineProperty(e.prototype,"size",{get:function(){return this._size},enumerable:!1,configurable:!0}),e.prototype.manage=function(e,t){var n;if(Cl.is(e)?n=e:(n=this.nextId(),t=e),void 0!==this._annotations[n])throw new Error("Id "+n+" is already in use.");if(void 0===t)throw new Error("No annotation provided for id "+n);return this._annotations[n]=t,this._size++,n},e.prototype.nextId=function(){return this._counter++,this._counter.toString()},e}();!function(){function e(e){var t=this;this._textEditChanges=Object.create(null),void 0!==e?(this._workspaceEdit=e,e.documentChanges?(this._changeAnnotations=new Lc(e.changeAnnotations),e.changeAnnotations=this._changeAnnotations.all(),e.documentChanges.forEach((function(e){if(El.is(e)){var n=new Ic(e.edits,t._changeAnnotations);t._textEditChanges[e.textDocument.uri]=n}}))):e.changes&&Object.keys(e.changes).forEach((function(n){var i=new Ic(e.changes[n]);t._textEditChanges[n]=i}))):this._workspaceEdit={}}Object.defineProperty(e.prototype,"edit",{get:function(){return this.initDocumentChanges(),void 0!==this._changeAnnotations&&(0===this._changeAnnotations.size?this._workspaceEdit.changeAnnotations=void 0:this._workspaceEdit.changeAnnotations=this._changeAnnotations.all()),this._workspaceEdit},enumerable:!1,configurable:!0}),e.prototype.getTextEditChange=function(e){if(Kl.is(e)){if(this.initDocumentChanges(),void 0===this._workspaceEdit.documentChanges)throw new Error("Workspace edit is not configured for document changes.");var t={uri:e.uri,version:e.version};if(!(i=this._textEditChanges[t.uri])){var n={textDocument:t,edits:r=[]};this._workspaceEdit.documentChanges.push(n),i=new Ic(r,this._changeAnnotations),this._textEditChanges[t.uri]=i}return i}if(this.initChanges(),void 0===this._workspaceEdit.changes)throw new Error("Workspace edit is not configured for normal text edit changes.");var i;if(!(i=this._textEditChanges[e])){var r=[];this._workspaceEdit.changes[e]=r,i=new Ic(r),this._textEditChanges[e]=i}return i},e.prototype.initDocumentChanges=function(){void 0===this._workspaceEdit.documentChanges&&void 0===this._workspaceEdit.changes&&(this._changeAnnotations=new Lc,this._workspaceEdit.documentChanges=[],this._workspaceEdit.changeAnnotations=this._changeAnnotations.all())},e.prototype.initChanges=function(){void 0===this._workspaceEdit.documentChanges&&void 0===this._workspaceEdit.changes&&(this._workspaceEdit.changes=Object.create(null))},e.prototype.createFile=function(e,t,n){if(this.initDocumentChanges(),void 0===this._workspaceEdit.documentChanges)throw new Error("Workspace edit is not configured for document changes.");var i,r,o;if(xl.is(t)||Cl.is(t)?i=t:n=t,void 0===i?r=Nl.create(e,n):(o=Cl.is(i)?i:this._changeAnnotations.manage(i),r=Nl.create(e,n,o)),this._workspaceEdit.documentChanges.push(r),void 0!==o)return o},e.prototype.renameFile=function(e,t,n,i){if(this.initDocumentChanges(),void 0===this._workspaceEdit.documentChanges)throw new Error("Workspace edit is not configured for document changes.");var r,o,s;if(xl.is(n)||Cl.is(n)?r=n:i=n,void 0===r?o=Dl.create(e,t,i):(s=Cl.is(r)?r:this._changeAnnotations.manage(r),o=Dl.create(e,t,i,s)),this._workspaceEdit.documentChanges.push(o),void 0!==s)return s},e.prototype.deleteFile=function(e,t,n){if(this.initDocumentChanges(),void 0===this._workspaceEdit.documentChanges)throw new Error("Workspace edit is not configured for document changes.");var i,r,o;if(xl.is(t)||Cl.is(t)?i=t:n=t,void 0===i?r=Al.create(e,n):(o=Cl.is(i)?i:this._changeAnnotations.manage(i),r=Al.create(e,n,o)),this._workspaceEdit.documentChanges.push(r),void 0!==o)return o}}();(Wl=Ol||(Ol={})).create=function(e){return{uri:e}},Wl.is=function(e){var t=e;return Pc.defined(t)&&Pc.string(t.uri)},(Ul=Vl||(Vl={})).create=function(e,t){return{uri:e,version:t}},Ul.is=function(e){var t=e;return Pc.defined(t)&&Pc.string(t.uri)&&Pc.integer(t.version)},(ql=Kl||(Kl={})).create=function(e,t){return{uri:e,version:t}},ql.is=function(e){var t=e;return Pc.defined(t)&&Pc.string(t.uri)&&(null===t.version||Pc.integer(t.version))},(jl=Bl||(Bl={})).create=function(e,t,n,i){return{uri:e,languageId:t,version:n,text:i}},jl.is=function(e){var t=e;return Pc.defined(t)&&Pc.string(t.uri)&&Pc.string(t.languageId)&&Pc.integer(t.version)&&Pc.string(t.text)},(Hl=$l||($l={})).PlainText="plaintext",Hl.Markdown="markdown",function(e){e.is=function(t){var n=t;return n===e.PlainText||n===e.Markdown}}($l||($l={})),(Gl||(Gl={})).is=function(e){var t=e;return Pc.objectLiteral(e)&&$l.is(t.kind)&&Pc.string(t.value)},(Xl=Jl||(Jl={})).Text=1,Xl.Method=2,Xl.Function=3,Xl.Constructor=4,Xl.Field=5,Xl.Variable=6,Xl.Class=7,Xl.Interface=8,Xl.Module=9,Xl.Property=10,Xl.Unit=11,Xl.Value=12,Xl.Enum=13,Xl.Keyword=14,Xl.Snippet=15,Xl.Color=16,Xl.File=17,Xl.Reference=18,Xl.Folder=19,Xl.EnumMember=20,Xl.Constant=21,Xl.Struct=22,Xl.Event=23,Xl.Operator=24,Xl.TypeParameter=25,(Ql=Yl||(Yl={})).PlainText=1,Ql.Snippet=2,(Zl||(Zl={})).Deprecated=1,(tc=ec||(ec={})).create=function(e,t,n){return{newText:e,insert:t,replace:n}},tc.is=function(e){var t=e;return t&&Pc.string(t.newText)&&$a.is(t.insert)&&$a.is(t.replace)},(ic=nc||(nc={})).asIs=1,ic.adjustIndentation=2,(rc||(rc={})).create=function(e){return{label:e}},(oc||(oc={})).create=function(e,t){return{items:e||[],isIncomplete:!!t}},(ac=sc||(sc={})).fromPlainText=function(e){return e.replace(/[\\`*_{}[\]()#+\-.!]/g,"\\$&")},ac.is=function(e){var t=e;return Pc.string(t)||Pc.objectLiteral(t)&&Pc.string(t.language)&&Pc.string(t.value)},(lc||(lc={})).is=function(e){var t=e;return!!t&&Pc.objectLiteral(t)&&(Gl.is(t.contents)||sc.is(t.contents)||Pc.typedArray(t.contents,sc.is))&&(void 0===e.range||$a.is(e.range))},(cc||(cc={})).create=function(e,t){return t?{label:e,documentation:t}:{label:e}},(hc||(hc={})).create=function(e,t){for(var n=[],i=2;i=0;s--){var a=r[s],l=e.offsetAt(a.range.start),c=e.offsetAt(a.range.end);if(!(c<=o))throw new Error("Overlapping edit");i=i.substring(0,l)+a.newText+i.substring(c,i.length),o=l}return i}}(zc||(zc={}));var Pc,Oc=function(){function e(e,t,n,i){this._uri=e,this._languageId=t,this._version=n,this._content=i,this._lineOffsets=void 0}return Object.defineProperty(e.prototype,"uri",{get:function(){return this._uri},enumerable:!1,configurable:!0}),Object.defineProperty(e.prototype,"languageId",{get:function(){return this._languageId},enumerable:!1,configurable:!0}),Object.defineProperty(e.prototype,"version",{get:function(){return this._version},enumerable:!1,configurable:!0}),e.prototype.getText=function(e){if(e){var t=this.offsetAt(e.start),n=this.offsetAt(e.end);return this._content.substring(t,n)}return this._content},e.prototype.update=function(e,t){this._content=e.text,this._version=t,this._lineOffsets=void 0},e.prototype.getLineOffsets=function(){if(void 0===this._lineOffsets){for(var e=[],t=this._content,n=!0,i=0;i0&&e.push(t.length),this._lineOffsets=e}return this._lineOffsets},e.prototype.positionAt=function(e){e=Math.max(Math.min(e,this._content.length),0);var t=this.getLineOffsets(),n=0,i=t.length;if(0===i)return Ba.create(0,e);for(;ne?i=r:n=r+1}var o=n-1;return Ba.create(o,e-t[o])},e.prototype.offsetAt=function(e){var t=this.getLineOffsets();if(e.line>=t.length)return this._content.length;if(e.line<0)return 0;var n=t[e.line],i=e.line+1e?i=r:n=r+1}let r=n-1;return{line:r,character:e-t[r]}}offsetAt(e){let t=this.getLineOffsets();if(e.line>=t.length)return this._content.length;if(e.line<0)return 0;let n=t[e.line],i=e.line+12&&void 0!==arguments[2]?arguments[2]:0;const i=t?[n]:[];for(let r=0;rn.line||t.line===n.line&&t.character>n.character?{start:n,end:t}:e}function Gc(e){const t=Hc(e.range);return t!==e.range?{newText:e.newText,range:t}:e}(Vc=Wc||(Wc={})).create=function(e,t,n,i){return new Bc(e,t,n,i)},Vc.update=function(e,t,n){if(e instanceof Bc)return e.update(t,n),e;throw new Error("TextDocument.update: document must be created by TextDocument.create")},Vc.applyEdits=function(e,t){let n=e.getText(),i=jc(t.map(Gc),((e,t)=>{let n=e.range.start.line-t.range.start.line;return 0===n?e.range.start.character-t.range.start.character:n})),r=0;const o=[];for(const s of i){let t=e.offsetAt(s.range.start);if(tr&&o.push(n.substring(r,t)),s.newText.length&&o.push(s.newText),r=e.offsetAt(s.range.end)}return o.push(n.substr(r)),o.join("")},(Uc||(Uc={})).LATEST={textDocument:{completion:{completionItem:{documentationFormat:[$l.Markdown,$l.PlainText]}},hover:{contentFormat:[$l.Markdown,$l.PlainText]}}},(qc=Kc||(Kc={}))[qc.Unknown=0]="Unknown",qc[qc.File=1]="File",qc[qc.Directory=2]="Directory",qc[qc.SymbolicLink=64]="SymbolicLink";var Jc={E:"Edge",FF:"Firefox",S:"Safari",C:"Chrome",IE:"IE",O:"Opera"};function Xc(e){switch(e){case"experimental":return"\u26a0\ufe0f Property is experimental. Be cautious when using it.\ufe0f\n\n";case"nonstandard":return"\ud83d\udea8\ufe0f Property is nonstandard. Avoid using it.\n\n";case"obsolete":return"\ud83d\udea8\ufe0f\ufe0f\ufe0f Property is obsolete. Avoid using it.\n\n";default:return""}}function Yc(e,t,n){var i;if(""!==(i=t?{kind:"markdown",value:eh(e,n)}:{kind:"plaintext",value:Zc(e,n)}).value)return i}function Qc(e){return(e=e.replace(/[\\`*_{}[\]()#+\-.!]/g,"\\$&")).replace(//g,">")}function Zc(e,t){if(!e.description||""===e.description)return"";if("string"!==typeof e.description)return e.description.value;var n="";if(!1!==(null===t||void 0===t?void 0:t.documentation)){e.status&&(n+=Xc(e.status)),n+=e.description;var i=th(e.browsers);i&&(n+="\n("+i+")"),"syntax"in e&&(n+="\n\nSyntax: ".concat(e.syntax))}return e.references&&e.references.length>0&&!1!==(null===t||void 0===t?void 0:t.references)&&(n.length>0&&(n+="\n\n"),n+=e.references.map((function(e){return"".concat(e.name,": ").concat(e.url)})).join(" | ")),n}function eh(e,t){if(!e.description||""===e.description)return"";var n="";if(!1!==(null===t||void 0===t?void 0:t.documentation)){e.status&&(n+=Xc(e.status)),"string"===typeof e.description?n+=Qc(e.description):n+=e.description.kind===$l.Markdown?e.description.value:Qc(e.description.value);var i=th(e.browsers);i&&(n+="\n\n("+Qc(i)+")"),"syntax"in e&&e.syntax&&(n+="\n\nSyntax: ".concat(Qc(e.syntax)))}return e.references&&e.references.length>0&&!1!==(null===t||void 0===t?void 0:t.references)&&(n.length>0&&(n+="\n\n"),n+=e.references.map((function(e){return"[".concat(e.name,"](").concat(e.url,")")})).join(" | ")),n}function th(e){return void 0===e&&(e=[]),0===e.length?null:e.map((function(e){var t="",n=e.match(/([A-Z]+)(\d+)?/),i=n[1],r=n[2];return i in Jc&&(t+=Jc[i]),r&&(t+=" "+r),t})).join(", ")}var nh=Wa(),ih=[{func:"rgb($red, $green, $blue)",desc:nh("css.builtin.rgb","Creates a Color from red, green, and blue values.")},{func:"rgba($red, $green, $blue, $alpha)",desc:nh("css.builtin.rgba","Creates a Color from red, green, blue, and alpha values.")},{func:"hsl($hue, $saturation, $lightness)",desc:nh("css.builtin.hsl","Creates a Color from hue, saturation, and lightness values.")},{func:"hsla($hue, $saturation, $lightness, $alpha)",desc:nh("css.builtin.hsla","Creates a Color from hue, saturation, lightness, and alpha values.")},{func:"hwb($hue $white $black)",desc:nh("css.builtin.hwb","Creates a Color from hue, white and black.")}],rh={aliceblue:"#f0f8ff",antiquewhite:"#faebd7",aqua:"#00ffff",aquamarine:"#7fffd4",azure:"#f0ffff",beige:"#f5f5dc",bisque:"#ffe4c4",black:"#000000",blanchedalmond:"#ffebcd",blue:"#0000ff",blueviolet:"#8a2be2",brown:"#a52a2a",burlywood:"#deb887",cadetblue:"#5f9ea0",chartreuse:"#7fff00",chocolate:"#d2691e",coral:"#ff7f50",cornflowerblue:"#6495ed",cornsilk:"#fff8dc",crimson:"#dc143c",cyan:"#00ffff",darkblue:"#00008b",darkcyan:"#008b8b",darkgoldenrod:"#b8860b",darkgray:"#a9a9a9",darkgrey:"#a9a9a9",darkgreen:"#006400",darkkhaki:"#bdb76b",darkmagenta:"#8b008b",darkolivegreen:"#556b2f",darkorange:"#ff8c00",darkorchid:"#9932cc",darkred:"#8b0000",darksalmon:"#e9967a",darkseagreen:"#8fbc8f",darkslateblue:"#483d8b",darkslategray:"#2f4f4f",darkslategrey:"#2f4f4f",darkturquoise:"#00ced1",darkviolet:"#9400d3",deeppink:"#ff1493",deepskyblue:"#00bfff",dimgray:"#696969",dimgrey:"#696969",dodgerblue:"#1e90ff",firebrick:"#b22222",floralwhite:"#fffaf0",forestgreen:"#228b22",fuchsia:"#ff00ff",gainsboro:"#dcdcdc",ghostwhite:"#f8f8ff",gold:"#ffd700",goldenrod:"#daa520",gray:"#808080",grey:"#808080",green:"#008000",greenyellow:"#adff2f",honeydew:"#f0fff0",hotpink:"#ff69b4",indianred:"#cd5c5c",indigo:"#4b0082",ivory:"#fffff0",khaki:"#f0e68c",lavender:"#e6e6fa",lavenderblush:"#fff0f5",lawngreen:"#7cfc00",lemonchiffon:"#fffacd",lightblue:"#add8e6",lightcoral:"#f08080",lightcyan:"#e0ffff",lightgoldenrodyellow:"#fafad2",lightgray:"#d3d3d3",lightgrey:"#d3d3d3",lightgreen:"#90ee90",lightpink:"#ffb6c1",lightsalmon:"#ffa07a",lightseagreen:"#20b2aa",lightskyblue:"#87cefa",lightslategray:"#778899",lightslategrey:"#778899",lightsteelblue:"#b0c4de",lightyellow:"#ffffe0",lime:"#00ff00",limegreen:"#32cd32",linen:"#faf0e6",magenta:"#ff00ff",maroon:"#800000",mediumaquamarine:"#66cdaa",mediumblue:"#0000cd",mediumorchid:"#ba55d3",mediumpurple:"#9370d8",mediumseagreen:"#3cb371",mediumslateblue:"#7b68ee",mediumspringgreen:"#00fa9a",mediumturquoise:"#48d1cc",mediumvioletred:"#c71585",midnightblue:"#191970",mintcream:"#f5fffa",mistyrose:"#ffe4e1",moccasin:"#ffe4b5",navajowhite:"#ffdead",navy:"#000080",oldlace:"#fdf5e6",olive:"#808000",olivedrab:"#6b8e23",orange:"#ffa500",orangered:"#ff4500",orchid:"#da70d6",palegoldenrod:"#eee8aa",palegreen:"#98fb98",paleturquoise:"#afeeee",palevioletred:"#d87093",papayawhip:"#ffefd5",peachpuff:"#ffdab9",peru:"#cd853f",pink:"#ffc0cb",plum:"#dda0dd",powderblue:"#b0e0e6",purple:"#800080",red:"#ff0000",rebeccapurple:"#663399",rosybrown:"#bc8f8f",royalblue:"#4169e1",saddlebrown:"#8b4513",salmon:"#fa8072",sandybrown:"#f4a460",seagreen:"#2e8b57",seashell:"#fff5ee",sienna:"#a0522d",silver:"#c0c0c0",skyblue:"#87ceeb",slateblue:"#6a5acd",slategray:"#708090",slategrey:"#708090",snow:"#fffafa",springgreen:"#00ff7f",steelblue:"#4682b4",tan:"#d2b48c",teal:"#008080",thistle:"#d8bfd8",tomato:"#ff6347",turquoise:"#40e0d0",violet:"#ee82ee",wheat:"#f5deb3",white:"#ffffff",whitesmoke:"#f5f5f5",yellow:"#ffff00",yellowgreen:"#9acd32"},oh={currentColor:"The value of the 'color' property. The computed value of the 'currentColor' keyword is the computed value of the 'color' property. If the 'currentColor' keyword is set on the 'color' property itself, it is treated as 'color:inherit' at parse time.",transparent:"Fully transparent. This keyword can be considered a shorthand for rgba(0,0,0,0) which is its computed value."};function sh(e,t){var n=e.getText().match(/^([-+]?[0-9]*\.?[0-9]+)(%?)$/);if(n){n[2]&&(t=100);var i=parseFloat(n[1])/t;if(i>=0&&i<=1)return i}throw new Error}function ah(e){var t=e.getText(),n=t.match(/^([-+]?[0-9]*\.?[0-9]+)(deg|rad|grad|turn)?$/);if(n)switch(n[2]){case"deg":return parseFloat(t)%360;case"rad":return 180*parseFloat(t)/Math.PI%360;case"grad":return.9*parseFloat(t)%360;case"turn":return 360*parseFloat(t)%360;default:if("undefined"===typeof n[2])return parseFloat(t)%360}throw new Error}var lh=48,ch=57,hh=65,dh=97,ph=102;function uh(e){return e=dh&&e<=ph?e-dh+10:0)}function mh(e){if("#"!==e[0])return null;switch(e.length){case 4:return{red:17*uh(e.charCodeAt(1))/255,green:17*uh(e.charCodeAt(2))/255,blue:17*uh(e.charCodeAt(3))/255,alpha:1};case 5:return{red:17*uh(e.charCodeAt(1))/255,green:17*uh(e.charCodeAt(2))/255,blue:17*uh(e.charCodeAt(3))/255,alpha:17*uh(e.charCodeAt(4))/255};case 7:return{red:(16*uh(e.charCodeAt(1))+uh(e.charCodeAt(2)))/255,green:(16*uh(e.charCodeAt(3))+uh(e.charCodeAt(4)))/255,blue:(16*uh(e.charCodeAt(5))+uh(e.charCodeAt(6)))/255,alpha:1};case 9:return{red:(16*uh(e.charCodeAt(1))+uh(e.charCodeAt(2)))/255,green:(16*uh(e.charCodeAt(3))+uh(e.charCodeAt(4)))/255,blue:(16*uh(e.charCodeAt(5))+uh(e.charCodeAt(6)))/255,alpha:(16*uh(e.charCodeAt(7))+uh(e.charCodeAt(8)))/255}}return null}function fh(e,t,n,i){if(void 0===i&&(i=1),0===t)return{red:n,green:n,blue:n,alpha:i};var r=function(e,t,n){for(;n<0;)n+=6;for(;n>=6;)n-=6;return n<1?(t-e)*n+e:n<3?t:n<4?(t-e)*(4-n)+e:e},o=n<=.5?n*(t+1):n+t-n*t,s=2*n-o;return{red:r(s,o,(e/=60)+2),green:r(s,o,e),blue:r(s,o,e-2),alpha:i}}function gh(e){var t=e.red,n=e.green,i=e.blue,r=e.alpha,o=Math.max(t,n,i),s=Math.min(t,n,i),a=0,l=0,c=(s+o)/2,h=o-s;if(h>0){switch(l=Math.min(c<=.5?h/(2*c):h/(2-2*c),1),o){case t:a=(n-i)/h+(n4)return null;try{var c=4===i.length?sh(i[3],1):1;if("rgb"===n||"rgba"===n)return{red:sh(i[0],255),green:sh(i[1],255),blue:sh(i[2],255),alpha:c};if("hsl"===n||"hsla"===n)return fh(ah(i[0]),sh(i[1],100),sh(i[2],100),c);if("hwb"===n)return function(e,t,n,i){if(void 0===i&&(i=1),t+n>=1){var r=t/(t+n);return{red:r,green:r,blue:r,alpha:i}}var o=fh(e,1,.5,i),s=o.red;s*=1-t-n,s+=t;var a=o.green;a*=1-t-n,a+=t;var l=o.blue;return l*=1-t-n,{red:s,green:a,blue:l+=t,alpha:i}}(ah(i[0]),sh(i[1],100),sh(i[2],100),c)}catch(Du){return null}}else if(e.type===us.Identifier){if(e.parent&&e.parent.type!==us.Term)return null;var h=e.parent;if(h&&h.parent&&h.parent.type===us.BinaryExpression){var d=h.parent;if(d.parent&&d.parent.type===us.ListEntry&&d.parent.key===d)return null}var p=e.getText().toLowerCase();if("none"===p)return null;var u=rh[p];if(u)return mh(u)}return null}var vh={bottom:"Computes to \u2018100%\u2019 for the vertical position if one or two values are given, otherwise specifies the bottom edge as the origin for the next offset.",center:"Computes to \u201850%\u2019 (\u2018left 50%\u2019) for the horizontal position if the horizontal position is not otherwise specified, or \u201850%\u2019 (\u2018top 50%\u2019) for the vertical position if it is.",left:"Computes to \u20180%\u2019 for the horizontal position if one or two values are given, otherwise specifies the left edge as the origin for the next offset.",right:"Computes to \u2018100%\u2019 for the horizontal position if one or two values are given, otherwise specifies the right edge as the origin for the next offset.",top:"Computes to \u20180%\u2019 for the vertical position if one or two values are given, otherwise specifies the top edge as the origin for the next offset."},yh={"no-repeat":"Placed once and not repeated in this direction.",repeat:"Repeated in this direction as often as needed to cover the background painting area.","repeat-x":"Computes to \u2018repeat no-repeat\u2019.","repeat-y":"Computes to \u2018no-repeat repeat\u2019.",round:"Repeated as often as will fit within the background positioning area. If it doesn\u2019t fit a whole number of times, it is rescaled so that it does.",space:"Repeated as often as will fit within the background positioning area without being clipped and then the images are spaced out to fill the area."},wh={dashed:"A series of square-ended dashes.",dotted:"A series of round dots.",double:"Two parallel solid lines with some space between them.",groove:"Looks as if it were carved in the canvas.",hidden:"Same as \u2018none\u2019, but has different behavior in the border conflict resolution rules for border-collapsed tables.",inset:"Looks as if the content on the inside of the border is sunken into the canvas.",none:"No border. Color and width are ignored.",outset:"Looks as if the content on the inside of the border is coming out of the canvas.",ridge:"Looks as if it were coming out of the canvas.",solid:"A single line segment."},xh=["medium","thick","thin"],Sh={"border-box":"The background is painted within (clipped to) the border box.","content-box":"The background is painted within (clipped to) the content box.","padding-box":"The background is painted within (clipped to) the padding box."},Ch={"margin-box":"Uses the margin box as reference box.","fill-box":"Uses the object bounding box as reference box.","stroke-box":"Uses the stroke bounding box as reference box.","view-box":"Uses the nearest SVG viewport as reference box."},_h={initial:"Represents the value specified as the property\u2019s initial value.",inherit:"Represents the computed value of the property on the element\u2019s parent.",unset:"Acts as either `inherit` or `initial`, depending on whether the property is inherited or not."},kh={"var()":"Evaluates the value of a custom variable.","calc()":"Evaluates an mathematical expression. The following operators can be used: + - * /."},Eh={"url()":"Reference an image file by URL","image()":"Provide image fallbacks and annotations.","-webkit-image-set()":"Provide multiple resolutions. Remember to use unprefixed image-set() in addition.","image-set()":"Provide multiple resolutions of an image and const the UA decide which is most appropriate in a given situation.","-moz-element()":"Use an element in the document as an image. Remember to use unprefixed element() in addition.","element()":"Use an element in the document as an image.","cross-fade()":"Indicates the two images to be combined and how far along in the transition the combination is.","-webkit-gradient()":"Deprecated. Use modern linear-gradient() or radial-gradient() instead.","-webkit-linear-gradient()":"Linear gradient. Remember to use unprefixed version in addition.","-moz-linear-gradient()":"Linear gradient. Remember to use unprefixed version in addition.","-o-linear-gradient()":"Linear gradient. Remember to use unprefixed version in addition.","linear-gradient()":"A linear gradient is created by specifying a straight gradient line, and then several colors placed along that line.","-webkit-repeating-linear-gradient()":"Repeating Linear gradient. Remember to use unprefixed version in addition.","-moz-repeating-linear-gradient()":"Repeating Linear gradient. Remember to use unprefixed version in addition.","-o-repeating-linear-gradient()":"Repeating Linear gradient. Remember to use unprefixed version in addition.","repeating-linear-gradient()":"Same as linear-gradient, except the color-stops are repeated infinitely in both directions, with their positions shifted by multiples of the difference between the last specified color-stop\u2019s position and the first specified color-stop\u2019s position.","-webkit-radial-gradient()":"Radial gradient. Remember to use unprefixed version in addition.","-moz-radial-gradient()":"Radial gradient. Remember to use unprefixed version in addition.","radial-gradient()":"Colors emerge from a single point and smoothly spread outward in a circular or elliptical shape.","-webkit-repeating-radial-gradient()":"Repeating radial gradient. Remember to use unprefixed version in addition.","-moz-repeating-radial-gradient()":"Repeating radial gradient. Remember to use unprefixed version in addition.","repeating-radial-gradient()":"Same as radial-gradient, except the color-stops are repeated infinitely in both directions, with their positions shifted by multiples of the difference between the last specified color-stop\u2019s position and the first specified color-stop\u2019s position."},Rh={ease:"Equivalent to cubic-bezier(0.25, 0.1, 0.25, 1.0).","ease-in":"Equivalent to cubic-bezier(0.42, 0, 1.0, 1.0).","ease-in-out":"Equivalent to cubic-bezier(0.42, 0, 0.58, 1.0).","ease-out":"Equivalent to cubic-bezier(0, 0, 0.58, 1.0).",linear:"Equivalent to cubic-bezier(0.0, 0.0, 1.0, 1.0).","step-end":"Equivalent to steps(1, end).","step-start":"Equivalent to steps(1, start).","steps()":"The first parameter specifies the number of intervals in the function. The second parameter, which is optional, is either the value \u201cstart\u201d or \u201cend\u201d.","cubic-bezier()":"Specifies a cubic-bezier curve. The four values specify points P1 and P2 of the curve as (x1, y1, x2, y2).","cubic-bezier(0.6, -0.28, 0.735, 0.045)":"Ease-in Back. Overshoots.","cubic-bezier(0.68, -0.55, 0.265, 1.55)":"Ease-in-out Back. Overshoots.","cubic-bezier(0.175, 0.885, 0.32, 1.275)":"Ease-out Back. Overshoots.","cubic-bezier(0.6, 0.04, 0.98, 0.335)":"Ease-in Circular. Based on half circle.","cubic-bezier(0.785, 0.135, 0.15, 0.86)":"Ease-in-out Circular. Based on half circle.","cubic-bezier(0.075, 0.82, 0.165, 1)":"Ease-out Circular. Based on half circle.","cubic-bezier(0.55, 0.055, 0.675, 0.19)":"Ease-in Cubic. Based on power of three.","cubic-bezier(0.645, 0.045, 0.355, 1)":"Ease-in-out Cubic. Based on power of three.","cubic-bezier(0.215, 0.610, 0.355, 1)":"Ease-out Cubic. Based on power of three.","cubic-bezier(0.95, 0.05, 0.795, 0.035)":"Ease-in Exponential. Based on two to the power ten.","cubic-bezier(1, 0, 0, 1)":"Ease-in-out Exponential. Based on two to the power ten.","cubic-bezier(0.19, 1, 0.22, 1)":"Ease-out Exponential. Based on two to the power ten.","cubic-bezier(0.47, 0, 0.745, 0.715)":"Ease-in Sine.","cubic-bezier(0.445, 0.05, 0.55, 0.95)":"Ease-in-out Sine.","cubic-bezier(0.39, 0.575, 0.565, 1)":"Ease-out Sine.","cubic-bezier(0.55, 0.085, 0.68, 0.53)":"Ease-in Quadratic. Based on power of two.","cubic-bezier(0.455, 0.03, 0.515, 0.955)":"Ease-in-out Quadratic. Based on power of two.","cubic-bezier(0.25, 0.46, 0.45, 0.94)":"Ease-out Quadratic. Based on power of two.","cubic-bezier(0.895, 0.03, 0.685, 0.22)":"Ease-in Quartic. Based on power of four.","cubic-bezier(0.77, 0, 0.175, 1)":"Ease-in-out Quartic. Based on power of four.","cubic-bezier(0.165, 0.84, 0.44, 1)":"Ease-out Quartic. Based on power of four.","cubic-bezier(0.755, 0.05, 0.855, 0.06)":"Ease-in Quintic. Based on power of five.","cubic-bezier(0.86, 0, 0.07, 1)":"Ease-in-out Quintic. Based on power of five.","cubic-bezier(0.23, 1, 0.320, 1)":"Ease-out Quintic. Based on power of five."},Nh={"circle()":"Defines a circle.","ellipse()":"Defines an ellipse.","inset()":"Defines an inset rectangle.","polygon()":"Defines a polygon."},Fh={length:["em","rem","ex","px","cm","mm","in","pt","pc","ch","vw","vh","vmin","vmax"],angle:["deg","rad","grad","turn"],time:["ms","s"],frequency:["Hz","kHz"],resolution:["dpi","dpcm","dppx"],percentage:["%","fr"]},Dh=["a","abbr","address","area","article","aside","audio","b","base","bdi","bdo","blockquote","body","br","button","canvas","caption","cite","code","col","colgroup","data","datalist","dd","del","details","dfn","dialog","div","dl","dt","em","embed","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","head","header","hgroup","hr","html","i","iframe","img","input","ins","kbd","keygen","label","legend","li","link","main","map","mark","menu","menuitem","meta","meter","nav","noscript","object","ol","optgroup","option","output","p","param","picture","pre","progress","q","rb","rp","rt","rtc","ruby","s","samp","script","section","select","small","source","span","strong","style","sub","summary","sup","table","tbody","td","template","textarea","tfoot","th","thead","time","title","tr","track","u","ul","const","video","wbr"],Th=["circle","clipPath","cursor","defs","desc","ellipse","feBlend","feColorMatrix","feComponentTransfer","feComposite","feConvolveMatrix","feDiffuseLighting","feDisplacementMap","feDistantLight","feDropShadow","feFlood","feFuncA","feFuncB","feFuncG","feFuncR","feGaussianBlur","feImage","feMerge","feMergeNode","feMorphology","feOffset","fePointLight","feSpecularLighting","feSpotLight","feTile","feTurbulence","filter","foreignObject","g","hatch","hatchpath","image","line","linearGradient","marker","mask","mesh","meshpatch","meshrow","metadata","mpath","path","pattern","polygon","polyline","radialGradient","rect","set","solidcolor","stop","svg","switch","symbol","text","textPath","tspan","use","view"],Ah=["@bottom-center","@bottom-left","@bottom-left-corner","@bottom-right","@bottom-right-corner","@left-bottom","@left-middle","@left-top","@right-bottom","@right-middle","@right-top","@top-center","@top-left","@top-left-corner","@top-right","@top-right-corner"];function Mh(e){return Object.keys(e).map((function(t){return e[t]}))}function zh(e){return"undefined"!==typeof e}var Ih=function(e,t,n){if(n||2===arguments.length)for(var i,r=0,o=t.length;re.offset?r-e.offset:0}return e},e.prototype.markError=function(e,t,n,i){this.token!==this.lastErrorToken&&(e.addIssue(new La(e,t,ws.Error,void 0,this.token.offset,this.token.len)),this.lastErrorToken=this.token),(n||i)&&this.resync(n,i)},e.prototype.parseStylesheet=function(e){var t=e.version,n=e.getText();return this.internalParse(n,this._parseStylesheet,(function(i,r){if(e.version!==t)throw new Error("Underlying model has changed, AST is no longer valid");return n.substr(i,r)}))},e.prototype.internalParse=function(e,t,n){this.scanner.setSource(e),this.token=this.scanner.scan();var i=t.bind(this)();return i&&(i.textProvider=n||function(t,n){return e.substr(t,n)}),i},e.prototype._parseStylesheet=function(){for(var e=this.create(Es);e.addChild(this._parseStylesheetStart()););var t=!1;do{var n=!1;do{n=!1;var i=this._parseStylesheetStatement();for(i&&(e.addChild(i),n=!0,t=!1,this.peek(mo.EOF)||!this._needsSemicolonAfter(i)||this.accept(mo.SemiColon)||this.markError(e,Pl.SemiColonExpected));this.accept(mo.SemiColon)||this.accept(mo.CDO)||this.accept(mo.CDC);)n=!0,t=!1}while(n);if(this.peek(mo.EOF))break;t||(this.peek(mo.AtKeyword)?this.markError(e,Pl.UnknownAtRule):this.markError(e,Pl.RuleOrSelectorExpected),t=!0),this.consumeToken()}while(!this.peek(mo.EOF));return this.finish(e)},e.prototype._parseStylesheetStart=function(){return this._parseCharset()},e.prototype._parseStylesheetStatement=function(e){return void 0===e&&(e=!1),this.peek(mo.AtKeyword)?this._parseStylesheetAtStatement(e):this._parseRuleset(e)},e.prototype._parseStylesheetAtStatement=function(e){return void 0===e&&(e=!1),this._parseImport()||this._parseMedia(e)||this._parsePage()||this._parseFontFace()||this._parseKeyframe()||this._parseSupports(e)||this._parseViewPort()||this._parseNamespace()||this._parseDocument()||this._parseUnknownAtRule()},e.prototype._tryParseRuleset=function(e){var t=this.mark();if(this._parseSelector(e)){for(;this.accept(mo.Comma)&&this._parseSelector(e););if(this.accept(mo.CurlyL))return this.restoreAtMark(t),this._parseRuleset(e)}return this.restoreAtMark(t),null},e.prototype._parseRuleset=function(e){void 0===e&&(e=!1);var t=this.create(Fs),n=t.getSelectors();if(!n.addChild(this._parseSelector(e)))return null;for(;this.accept(mo.Comma);)if(!n.addChild(this._parseSelector(e)))return this.finish(t,Pl.SelectorExpected);return this._parseBody(t,this._parseRuleSetDeclaration.bind(this))},e.prototype._parseRuleSetDeclarationAtStatement=function(){return this._parseUnknownAtRule()},e.prototype._parseRuleSetDeclaration=function(){return this.peek(mo.AtKeyword)?this._parseRuleSetDeclarationAtStatement():this._parseDeclaration()},e.prototype._needsSemicolonAfter=function(e){switch(e.type){case us.Keyframe:case us.ViewPort:case us.Media:case us.Ruleset:case us.Namespace:case us.If:case us.For:case us.Each:case us.While:case us.MixinDeclaration:case us.FunctionDeclaration:case us.MixinContentDeclaration:return!1;case us.ExtendsReference:case us.MixinContentReference:case us.ReturnStatement:case us.MediaQuery:case us.Debug:case us.Import:case us.AtApplyRule:case us.CustomPropertyDeclaration:return!0;case us.VariableDeclaration:return e.needsSemicolon;case us.MixinReference:return!e.getContent();case us.Declaration:return!e.getNestedProperties()}return!1},e.prototype._parseDeclarations=function(e){var t=this.create(Rs);if(!this.accept(mo.CurlyL))return null;for(var n=e();t.addChild(n)&&!this.peek(mo.CurlyR);){if(this._needsSemicolonAfter(n)&&!this.accept(mo.SemiColon))return this.finish(t,Pl.SemiColonExpected,[mo.SemiColon,mo.CurlyR]);for(n&&this.prevToken&&this.prevToken.type===mo.SemiColon&&(n.semicolonPosition=this.prevToken.offset);this.accept(mo.SemiColon););n=e()}return this.accept(mo.CurlyR)?this.finish(t):this.finish(t,Pl.RightCurlyExpected,[mo.CurlyR,mo.SemiColon])},e.prototype._parseBody=function(e,t){return e.setDeclarations(this._parseDeclarations(t))?this.finish(e):this.finish(e,Pl.LeftCurlyExpected,[mo.CurlyR,mo.SemiColon])},e.prototype._parseSelector=function(e){var t=this.create(Ds),n=!1;for(e&&(n=t.addChild(this._parseCombinator()));t.addChild(this._parseSimpleSelector());)n=!0,t.addChild(this._parseCombinator());return n?this.finish(t):null},e.prototype._parseDeclaration=function(e){var t=this._tryParseCustomPropertyDeclaration(e);if(t)return t;var n=this.create(zs);return n.setProperty(this._parseProperty())?this.accept(mo.Colon)?(this.prevToken&&(n.colonPosition=this.prevToken.offset),n.setValue(this._parseExpr())?(n.addChild(this._parsePrio()),this.peek(mo.SemiColon)&&(n.semicolonPosition=this.token.offset),this.finish(n)):this.finish(n,Pl.PropertyValueExpected)):this.finish(n,Pl.ColonExpected,[mo.Colon],e||[mo.SemiColon]):null},e.prototype._tryParseCustomPropertyDeclaration=function(e){if(!this.peekRegExp(mo.Ident,/^--/))return null;var t=this.create(Is);if(!t.setProperty(this._parseProperty()))return null;if(!this.accept(mo.Colon))return this.finish(t,Pl.ColonExpected,[mo.Colon]);this.prevToken&&(t.colonPosition=this.prevToken.offset);var n=this.mark();if(this.peek(mo.CurlyL)){var i=this.create(Ms),r=this._parseDeclarations(this._parseRuleSetDeclaration.bind(this));if(i.setDeclarations(r)&&!r.isErroneous(!0)&&(i.addChild(this._parsePrio()),this.peek(mo.SemiColon)))return this.finish(i),t.setPropertySet(i),t.semicolonPosition=this.token.offset,this.finish(t);this.restoreAtMark(n)}var o=this._parseExpr();return o&&!o.isErroneous(!0)&&(this._parsePrio(),this.peekOne.apply(this,Ih(Ih([],e||[],!1),[mo.SemiColon,mo.EOF],!1)))?(t.setValue(o),this.peek(mo.SemiColon)&&(t.semicolonPosition=this.token.offset),this.finish(t)):(this.restoreAtMark(n),t.addChild(this._parseCustomPropertyValue(e)),t.addChild(this._parsePrio()),zh(t.colonPosition)&&this.token.offset===t.colonPosition+1?this.finish(t,Pl.PropertyValueExpected):this.finish(t))},e.prototype._parseCustomPropertyValue=function(e){var t=this;void 0===e&&(e=[mo.CurlyR]);var n=this.create(Ss),i=function(){return 0===o&&0===s&&0===a},r=function(){return-1!==e.indexOf(t.token.type)},o=0,s=0,a=0;e:for(;;){switch(this.token.type){case mo.SemiColon:case mo.Exclamation:if(i())break e;break;case mo.CurlyL:o++;break;case mo.CurlyR:if(--o<0){if(r()&&0===s&&0===a)break e;return this.finish(n,Pl.LeftCurlyExpected)}break;case mo.ParenthesisL:s++;break;case mo.ParenthesisR:if(--s<0){if(r()&&0===a&&0===o)break e;return this.finish(n,Pl.LeftParenthesisExpected)}break;case mo.BracketL:a++;break;case mo.BracketR:if(--a<0)return this.finish(n,Pl.LeftSquareBracketExpected);break;case mo.BadString:break e;case mo.EOF:var l=Pl.RightCurlyExpected;return a>0?l=Pl.RightSquareBracketExpected:s>0&&(l=Pl.RightParenthesisExpected),this.finish(n,l)}this.consumeToken()}return this.finish(n)},e.prototype._tryToParseDeclaration=function(e){var t=this.mark();return this._parseProperty()&&this.accept(mo.Colon)?(this.restoreAtMark(t),this._parseDeclaration(e)):(this.restoreAtMark(t),null)},e.prototype._parseProperty=function(){var e=this.create(Ls),t=this.mark();return(this.acceptDelim("*")||this.acceptDelim("_"))&&this.hasWhitespace()?(this.restoreAtMark(t),null):e.setIdentifier(this._parsePropertyIdentifier())?this.finish(e):null},e.prototype._parsePropertyIdentifier=function(){return this._parseIdent()},e.prototype._parseCharset=function(){if(!this.peek(mo.Charset))return null;var e=this.create(Ss);return this.consumeToken(),this.accept(mo.String)?this.accept(mo.SemiColon)?this.finish(e):this.finish(e,Pl.SemiColonExpected):this.finish(e,Pl.IdentifierExpected)},e.prototype._parseImport=function(){if(!this.peekKeyword("@import"))return null;var e=this.create(Ys);return this.consumeToken(),e.addChild(this._parseURILiteral())||e.addChild(this._parseStringLiteral())?(this.peek(mo.SemiColon)||this.peek(mo.EOF)||e.setMedialist(this._parseMediaQueryList()),this.finish(e)):this.finish(e,Pl.URIOrStringExpected)},e.prototype._parseNamespace=function(){if(!this.peekKeyword("@namespace"))return null;var e=this.create(na);return this.consumeToken(),e.addChild(this._parseURILiteral())||(e.addChild(this._parseIdent()),e.addChild(this._parseURILiteral())||e.addChild(this._parseStringLiteral()))?this.accept(mo.SemiColon)?this.finish(e):this.finish(e,Pl.SemiColonExpected):this.finish(e,Pl.URIExpected,[mo.SemiColon])},e.prototype._parseFontFace=function(){if(!this.peekKeyword("@font-face"))return null;var e=this.create(Hs);return this.consumeToken(),this._parseBody(e,this._parseRuleSetDeclaration.bind(this))},e.prototype._parseViewPort=function(){if(!this.peekKeyword("@-ms-viewport")&&!this.peekKeyword("@-o-viewport")&&!this.peekKeyword("@viewport"))return null;var e=this.create($s);return this.consumeToken(),this._parseBody(e,this._parseRuleSetDeclaration.bind(this))},e.prototype._parseKeyframe=function(){if(!this.peekRegExp(mo.AtKeyword,this.keyframeRegex))return null;var e=this.create(Js),t=this.create(Ss);return this.consumeToken(),e.setKeyword(this.finish(t)),t.matches("@-ms-keyframes")&&this.markError(t,Pl.UnknownKeyword),e.setIdentifier(this._parseKeyframeIdent())?this._parseBody(e,this._parseKeyframeSelector.bind(this)):this.finish(e,Pl.IdentifierExpected,[mo.CurlyR])},e.prototype._parseKeyframeIdent=function(){return this._parseIdent([fs.Keyframe])},e.prototype._parseKeyframeSelector=function(){var e=this.create(Xs);if(!e.addChild(this._parseIdent())&&!this.accept(mo.Percentage))return null;for(;this.accept(mo.Comma);)if(!e.addChild(this._parseIdent())&&!this.accept(mo.Percentage))return this.finish(e,Pl.PercentageExpected);return this._parseBody(e,this._parseRuleSetDeclaration.bind(this))},e.prototype._tryParseKeyframeSelector=function(){var e=this.create(Xs),t=this.mark();if(!e.addChild(this._parseIdent())&&!this.accept(mo.Percentage))return null;for(;this.accept(mo.Comma);)if(!e.addChild(this._parseIdent())&&!this.accept(mo.Percentage))return this.restoreAtMark(t),null;return this.peek(mo.CurlyL)?this._parseBody(e,this._parseRuleSetDeclaration.bind(this)):(this.restoreAtMark(t),null)},e.prototype._parseSupports=function(e){if(void 0===e&&(e=!1),!this.peekKeyword("@supports"))return null;var t=this.create(ra);return this.consumeToken(),t.addChild(this._parseSupportsCondition()),this._parseBody(t,this._parseSupportsDeclaration.bind(this,e))},e.prototype._parseSupportsDeclaration=function(e){return void 0===e&&(e=!1),e?this._tryParseRuleset(!0)||this._tryToParseDeclaration()||this._parseStylesheetStatement(!0):this._parseStylesheetStatement(!1)},e.prototype._parseSupportsCondition=function(){var e=this.create(ha);if(this.acceptIdent("not"))e.addChild(this._parseSupportsConditionInParens());else if(e.addChild(this._parseSupportsConditionInParens()),this.peekRegExp(mo.Ident,/^(and|or)$/i))for(var t=this.token.text.toLowerCase();this.acceptIdent(t);)e.addChild(this._parseSupportsConditionInParens());return this.finish(e)},e.prototype._parseSupportsConditionInParens=function(){var e=this.create(ha);if(this.accept(mo.ParenthesisL))return this.prevToken&&(e.lParent=this.prevToken.offset),e.addChild(this._tryToParseDeclaration([mo.ParenthesisR]))||this._parseSupportsCondition()?this.accept(mo.ParenthesisR)?(this.prevToken&&(e.rParent=this.prevToken.offset),this.finish(e)):this.finish(e,Pl.RightParenthesisExpected,[mo.ParenthesisR],[]):this.finish(e,Pl.ConditionExpected);if(this.peek(mo.Ident)){var t=this.mark();if(this.consumeToken(),!this.hasWhitespace()&&this.accept(mo.ParenthesisL)){for(var n=1;this.token.type!==mo.EOF&&0!==n;)this.token.type===mo.ParenthesisL?n++:this.token.type===mo.ParenthesisR&&n--,this.consumeToken();return this.finish(e)}this.restoreAtMark(t)}return this.finish(e,Pl.LeftParenthesisExpected,[],[mo.ParenthesisL])},e.prototype._parseMediaDeclaration=function(e){return void 0===e&&(e=!1),e?this._tryParseRuleset(!0)||this._tryToParseDeclaration()||this._parseStylesheetStatement(!0):this._parseStylesheetStatement(!1)},e.prototype._parseMedia=function(e){if(void 0===e&&(e=!1),!this.peekKeyword("@media"))return null;var t=this.create(ia);return this.consumeToken(),t.addChild(this._parseMediaQueryList())?this._parseBody(t,this._parseMediaDeclaration.bind(this,e)):this.finish(t,Pl.MediaQueryExpected)},e.prototype._parseMediaQueryList=function(){var e=this.create(sa);if(!e.addChild(this._parseMediaQuery()))return this.finish(e,Pl.MediaQueryExpected);for(;this.accept(mo.Comma);)if(!e.addChild(this._parseMediaQuery()))return this.finish(e,Pl.MediaQueryExpected);return this.finish(e)},e.prototype._parseMediaQuery=function(){var e=this.create(aa),t=this.mark();if(this.acceptIdent("not"),this.peek(mo.ParenthesisL))this.restoreAtMark(t),e.addChild(this._parseMediaCondition());else{if(this.acceptIdent("only"),!e.addChild(this._parseIdent()))return null;this.acceptIdent("and")&&e.addChild(this._parseMediaCondition())}return this.finish(e)},e.prototype._parseRatio=function(){var e=this.mark(),t=this.create(va);return this._parseNumeric()?this.acceptDelim("/")?this._parseNumeric()?this.finish(t):this.finish(t,Pl.NumberExpected):(this.restoreAtMark(e),null):null},e.prototype._parseMediaCondition=function(){var e=this.create(la);this.acceptIdent("not");for(var t=!0;t;){if(!this.accept(mo.ParenthesisL))return this.finish(e,Pl.LeftParenthesisExpected,[],[mo.CurlyL]);if(this.peek(mo.ParenthesisL)||this.peekIdent("not")?e.addChild(this._parseMediaCondition()):e.addChild(this._parseMediaFeature()),!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected,[],[mo.CurlyL]);t=this.acceptIdent("and")||this.acceptIdent("or")}return this.finish(e)},e.prototype._parseMediaFeature=function(){var e=this,t=[mo.ParenthesisR],n=this.create(ca),i=function(){return e.acceptDelim("<")||e.acceptDelim(">")?(e.hasWhitespace()||e.acceptDelim("="),!0):!!e.acceptDelim("=")};if(n.addChild(this._parseMediaFeatureName())){if(this.accept(mo.Colon)){if(!n.addChild(this._parseMediaFeatureValue()))return this.finish(n,Pl.TermExpected,[],t)}else if(i()){if(!n.addChild(this._parseMediaFeatureValue()))return this.finish(n,Pl.TermExpected,[],t);if(i()&&!n.addChild(this._parseMediaFeatureValue()))return this.finish(n,Pl.TermExpected,[],t)}}else{if(!n.addChild(this._parseMediaFeatureValue()))return this.finish(n,Pl.IdentifierExpected,[],t);if(!i())return this.finish(n,Pl.OperatorExpected,[],t);if(!n.addChild(this._parseMediaFeatureName()))return this.finish(n,Pl.IdentifierExpected,[],t);if(i()&&!n.addChild(this._parseMediaFeatureValue()))return this.finish(n,Pl.TermExpected,[],t)}return this.finish(n)},e.prototype._parseMediaFeatureName=function(){return this._parseIdent()},e.prototype._parseMediaFeatureValue=function(){return this._parseRatio()||this._parseTermExpression()},e.prototype._parseMedium=function(){var e=this.create(Ss);return e.addChild(this._parseIdent())?this.finish(e):null},e.prototype._parsePageDeclaration=function(){return this._parsePageMarginBox()||this._parseRuleSetDeclaration()},e.prototype._parsePage=function(){if(!this.peekKeyword("@page"))return null;var e=this.create(da);if(this.consumeToken(),e.addChild(this._parsePageSelector()))for(;this.accept(mo.Comma);)if(!e.addChild(this._parsePageSelector()))return this.finish(e,Pl.IdentifierExpected);return this._parseBody(e,this._parsePageDeclaration.bind(this))},e.prototype._parsePageMarginBox=function(){if(!this.peek(mo.AtKeyword))return null;var e=this.create(pa);return this.acceptOneKeyword(Ah)||this.markError(e,Pl.UnknownAtRule,[],[mo.CurlyL]),this._parseBody(e,this._parseRuleSetDeclaration.bind(this))},e.prototype._parsePageSelector=function(){if(!this.peek(mo.Ident)&&!this.peek(mo.Colon))return null;var e=this.create(Ss);return e.addChild(this._parseIdent()),this.accept(mo.Colon)&&!e.addChild(this._parseIdent())?this.finish(e,Pl.IdentifierExpected):this.finish(e)},e.prototype._parseDocument=function(){if(!this.peekKeyword("@-moz-document"))return null;var e=this.create(oa);return this.consumeToken(),this.resync([],[mo.CurlyL]),this._parseBody(e,this._parseStylesheetStatement.bind(this))},e.prototype._parseUnknownAtRule=function(){if(!this.peek(mo.AtKeyword))return null;var e=this.create(Ta);e.addChild(this._parseUnknownAtRuleName());var t=0,n=0,i=0,r=0;e:for(;;){switch(this.token.type){case mo.SemiColon:if(0===n&&0===i&&0===r)break e;break;case mo.EOF:return n>0?this.finish(e,Pl.RightCurlyExpected):r>0?this.finish(e,Pl.RightSquareBracketExpected):i>0?this.finish(e,Pl.RightParenthesisExpected):this.finish(e);case mo.CurlyL:t++,n++;break;case mo.CurlyR:if(n--,t>0&&0===n){if(this.consumeToken(),r>0)return this.finish(e,Pl.RightSquareBracketExpected);if(i>0)return this.finish(e,Pl.RightParenthesisExpected);break e}if(n<0){if(0===i&&0===r)break e;return this.finish(e,Pl.LeftCurlyExpected)}break;case mo.ParenthesisL:i++;break;case mo.ParenthesisR:if(--i<0)return this.finish(e,Pl.LeftParenthesisExpected);break;case mo.BracketL:r++;break;case mo.BracketR:if(--r<0)return this.finish(e,Pl.LeftSquareBracketExpected)}this.consumeToken()}return e},e.prototype._parseUnknownAtRuleName=function(){var e=this.create(Ss);return this.accept(mo.AtKeyword)?this.finish(e):e},e.prototype._parseOperator=function(){if(this.peekDelim("/")||this.peekDelim("*")||this.peekDelim("+")||this.peekDelim("-")||this.peek(mo.Dashmatch)||this.peek(mo.Includes)||this.peek(mo.SubstringOperator)||this.peek(mo.PrefixOperator)||this.peek(mo.SuffixOperator)||this.peekDelim("=")){var e=this.createNode(us.Operator);return this.consumeToken(),this.finish(e)}return null},e.prototype._parseUnaryOperator=function(){if(!this.peekDelim("+")&&!this.peekDelim("-"))return null;var e=this.create(Ss);return this.consumeToken(),this.finish(e)},e.prototype._parseCombinator=function(){if(this.peekDelim(">")){var e=this.create(Ss);this.consumeToken();var t=this.mark();if(!this.hasWhitespace()&&this.acceptDelim(">")){if(!this.hasWhitespace()&&this.acceptDelim(">"))return e.type=us.SelectorCombinatorShadowPiercingDescendant,this.finish(e);this.restoreAtMark(t)}return e.type=us.SelectorCombinatorParent,this.finish(e)}if(this.peekDelim("+")){e=this.create(Ss);return this.consumeToken(),e.type=us.SelectorCombinatorSibling,this.finish(e)}if(this.peekDelim("~")){e=this.create(Ss);return this.consumeToken(),e.type=us.SelectorCombinatorAllSiblings,this.finish(e)}if(this.peekDelim("/")){e=this.create(Ss);this.consumeToken();t=this.mark();if(!this.hasWhitespace()&&this.acceptIdent("deep")&&!this.hasWhitespace()&&this.acceptDelim("/"))return e.type=us.SelectorCombinatorShadowPiercingDescendant,this.finish(e);this.restoreAtMark(t)}return null},e.prototype._parseSimpleSelector=function(){var e=this.create(Ts),t=0;for(e.addChild(this._parseElementName())&&t++;(0===t||!this.hasWhitespace())&&e.addChild(this._parseSimpleSelectorBody());)t++;return t>0?this.finish(e):null},e.prototype._parseSimpleSelectorBody=function(){return this._parsePseudo()||this._parseHash()||this._parseClass()||this._parseAttrib()},e.prototype._parseSelectorIdent=function(){return this._parseIdent()},e.prototype._parseHash=function(){if(!this.peek(mo.Hash)&&!this.peekDelim("#"))return null;var e=this.createNode(us.IdentifierSelector);if(this.acceptDelim("#")){if(this.hasWhitespace()||!e.addChild(this._parseSelectorIdent()))return this.finish(e,Pl.IdentifierExpected)}else this.consumeToken();return this.finish(e)},e.prototype._parseClass=function(){if(!this.peekDelim("."))return null;var e=this.createNode(us.ClassSelector);return this.consumeToken(),this.hasWhitespace()||!e.addChild(this._parseSelectorIdent())?this.finish(e,Pl.IdentifierExpected):this.finish(e)},e.prototype._parseElementName=function(){var e=this.mark(),t=this.createNode(us.ElementNameSelector);return t.addChild(this._parseNamespacePrefix()),t.addChild(this._parseSelectorIdent())||this.acceptDelim("*")?this.finish(t):(this.restoreAtMark(e),null)},e.prototype._parseNamespacePrefix=function(){var e=this.mark(),t=this.createNode(us.NamespacePrefix);return!t.addChild(this._parseIdent())&&this.acceptDelim("*"),this.acceptDelim("|")?this.finish(t):(this.restoreAtMark(e),null)},e.prototype._parseAttrib=function(){if(!this.peek(mo.BracketL))return null;var e=this.create(ga);return this.consumeToken(),e.setNamespacePrefix(this._parseNamespacePrefix()),e.setIdentifier(this._parseIdent())?(e.setOperator(this._parseOperator())&&(e.setValue(this._parseBinaryExpr()),this.acceptIdent("i"),this.acceptIdent("s")),this.accept(mo.BracketR)?this.finish(e):this.finish(e,Pl.RightSquareBracketExpected)):this.finish(e,Pl.IdentifierExpected)},e.prototype._parsePseudo=function(){var e=this,t=this._tryParsePseudoIdentifier();if(t){if(!this.hasWhitespace()&&this.accept(mo.ParenthesisL)){if(t.addChild(this.try((function(){var t=e.create(Ss);if(!t.addChild(e._parseSelector(!1)))return null;for(;e.accept(mo.Comma)&&t.addChild(e._parseSelector(!1)););return e.peek(mo.ParenthesisR)?e.finish(t):null}))||this._parseBinaryExpr()),!this.accept(mo.ParenthesisR))return this.finish(t,Pl.RightParenthesisExpected)}return this.finish(t)}return null},e.prototype._tryParsePseudoIdentifier=function(){if(!this.peek(mo.Colon))return null;var e=this.mark(),t=this.createNode(us.PseudoSelector);return this.consumeToken(),this.hasWhitespace()?(this.restoreAtMark(e),null):(this.accept(mo.Colon),this.hasWhitespace()||!t.addChild(this._parseIdent())?this.finish(t,Pl.IdentifierExpected):this.finish(t))},e.prototype._tryParsePrio=function(){var e=this.mark(),t=this._parsePrio();return t||(this.restoreAtMark(e),null)},e.prototype._parsePrio=function(){if(!this.peek(mo.Exclamation))return null;var e=this.createNode(us.Prio);return this.accept(mo.Exclamation)&&this.acceptIdent("important")?this.finish(e):null},e.prototype._parseExpr=function(e){void 0===e&&(e=!1);var t=this.create(ua);if(!t.addChild(this._parseBinaryExpr()))return null;for(;;){if(this.peek(mo.Comma)){if(e)return this.finish(t);this.consumeToken()}else if(!this.hasWhitespace())break;if(!t.addChild(this._parseBinaryExpr()))break}return this.finish(t)},e.prototype._parseUnicodeRange=function(){if(!this.peekIdent("u"))return null;var e=this.create(_s);return this.acceptUnicodeRange()?this.finish(e):null},e.prototype._parseNamedLine=function(){if(!this.peek(mo.BracketL))return null;var e=this.createNode(us.GridLine);for(this.consumeToken();e.addChild(this._parseIdent()););return this.accept(mo.BracketR)?this.finish(e):this.finish(e,Pl.RightSquareBracketExpected)},e.prototype._parseBinaryExpr=function(e,t){var n=this.create(ma);if(!n.setLeft(e||this._parseTerm()))return null;if(!n.setOperator(t||this._parseOperator()))return this.finish(n);if(!n.setRight(this._parseTerm()))return this.finish(n,Pl.TermExpected);n=this.finish(n);var i=this._parseOperator();return i&&(n=this._parseBinaryExpr(n,i)),this.finish(n)},e.prototype._parseTerm=function(){var e=this.create(fa);return e.setOperator(this._parseUnaryOperator()),e.setExpression(this._parseTermExpression())?this.finish(e):null},e.prototype._parseTermExpression=function(){return this._parseURILiteral()||this._parseUnicodeRange()||this._parseFunction()||this._parseIdent()||this._parseStringLiteral()||this._parseNumeric()||this._parseHexColor()||this._parseOperation()||this._parseNamedLine()},e.prototype._parseOperation=function(){if(!this.peek(mo.ParenthesisL))return null;var e=this.create(Ss);return this.consumeToken(),e.addChild(this._parseExpr()),this.accept(mo.ParenthesisR)?this.finish(e):this.finish(e,Pl.RightParenthesisExpected)},e.prototype._parseNumeric=function(){if(this.peek(mo.Num)||this.peek(mo.Percentage)||this.peek(mo.Resolution)||this.peek(mo.Length)||this.peek(mo.EMS)||this.peek(mo.EXS)||this.peek(mo.Angle)||this.peek(mo.Time)||this.peek(mo.Dimension)||this.peek(mo.Freq)){var e=this.create(Sa);return this.consumeToken(),this.finish(e)}return null},e.prototype._parseStringLiteral=function(){if(!this.peek(mo.String)&&!this.peek(mo.BadString))return null;var e=this.createNode(us.StringLiteral);return this.consumeToken(),this.finish(e)},e.prototype._parseURILiteral=function(){if(!this.peekRegExp(mo.Ident,/^url(-prefix)?$/i))return null;var e=this.mark(),t=this.createNode(us.URILiteral);return this.accept(mo.Ident),this.hasWhitespace()||!this.peek(mo.ParenthesisL)?(this.restoreAtMark(e),null):(this.scanner.inURL=!0,this.consumeToken(),t.addChild(this._parseURLArgument()),this.scanner.inURL=!1,this.accept(mo.ParenthesisR)?this.finish(t):this.finish(t,Pl.RightParenthesisExpected))},e.prototype._parseURLArgument=function(){var e=this.create(Ss);return this.accept(mo.String)||this.accept(mo.BadString)||this.acceptUnquotedString()?this.finish(e):null},e.prototype._parseIdent=function(e){if(!this.peek(mo.Ident))return null;var t=this.create(ks);return e&&(t.referenceTypes=e),t.isCustomProperty=this.peekRegExp(mo.Ident,/^--/),this.consumeToken(),this.finish(t)},e.prototype._parseFunction=function(){var e=this.mark(),t=this.create(Ps);if(!t.setIdentifier(this._parseFunctionIdentifier()))return null;if(this.hasWhitespace()||!this.accept(mo.ParenthesisL))return this.restoreAtMark(e),null;if(t.getArguments().addChild(this._parseFunctionArgument()))for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)t.getArguments().addChild(this._parseFunctionArgument())||this.markError(t,Pl.ExpressionExpected);return this.accept(mo.ParenthesisR)?this.finish(t):this.finish(t,Pl.RightParenthesisExpected)},e.prototype._parseFunctionIdentifier=function(){if(!this.peek(mo.Ident))return null;var e=this.create(ks);if(e.referenceTypes=[fs.Function],this.acceptIdent("progid")){if(this.accept(mo.Colon))for(;this.accept(mo.Ident)&&this.acceptDelim("."););return this.finish(e)}return this.consumeToken(),this.finish(e)},e.prototype._parseFunctionArgument=function(){var e=this.create(Ws);return e.setValue(this._parseExpr(!0))?this.finish(e):null},e.prototype._parseHexColor=function(){if(this.peekRegExp(mo.Hash,/^#([A-Fa-f0-9]{3}|[A-Fa-f0-9]{4}|[A-Fa-f0-9]{6}|[A-Fa-f0-9]{8})$/g)){var e=this.create(ba);return this.consumeToken(),this.finish(e)}return null},e}();function Ph(e,t){return-1!==e.indexOf(t)}function Oh(){for(var e=[],t=0;te+t||this.offset===e&&this.length===t?this.findInScope(e,t):null},e.prototype.findInScope=function(e,t){void 0===t&&(t=0);var n=e+t,i=function(e,t){var n=0,i=e.length;if(0===i)return 0;for(;nn}));if(0===i)return this;var r=this.children[i-1];return r.offset<=e&&r.offset+r.length>=e+t?r.findInScope(e,t):this},e.prototype.addSymbol=function(e){this.symbols.push(e)},e.prototype.getSymbol=function(e,t){for(var n=0;n{var e={470:e=>{function t(e){if("string"!=typeof e)throw new TypeError("Path must be a string. Received "+JSON.stringify(e))}function n(e,t){for(var n,i="",r=0,o=-1,s=0,a=0;a<=e.length;++a){if(a2){var l=i.lastIndexOf("/");if(l!==i.length-1){-1===l?(i="",r=0):r=(i=i.slice(0,l)).length-1-i.lastIndexOf("/"),o=a,s=0;continue}}else if(2===i.length||1===i.length){i="",r=0,o=a,s=0;continue}t&&(i.length>0?i+="/..":i="..",r=2)}else i.length>0?i+="/"+e.slice(o+1,a):i=e.slice(o+1,a),r=a-o-1;o=a,s=0}else 46===n&&-1!==s?++s:s=-1}return i}var i={resolve:function(){for(var e,i="",r=!1,o=arguments.length-1;o>=-1&&!r;o--){var s;o>=0?s=arguments[o]:(void 0===e&&(e=process.cwd()),s=e),t(s),0!==s.length&&(i=s+"/"+i,r=47===s.charCodeAt(0))}return i=n(i,!r),r?i.length>0?"/"+i:"/":i.length>0?i:"."},normalize:function(e){if(t(e),0===e.length)return".";var i=47===e.charCodeAt(0),r=47===e.charCodeAt(e.length-1);return 0!==(e=n(e,!i)).length||i||(e="."),e.length>0&&r&&(e+="/"),i?"/"+e:e},isAbsolute:function(e){return t(e),e.length>0&&47===e.charCodeAt(0)},join:function(){if(0===arguments.length)return".";for(var e,n=0;n0&&(void 0===e?e=r:e+="/"+r)}return void 0===e?".":i.normalize(e)},relative:function(e,n){if(t(e),t(n),e===n)return"";if((e=i.resolve(e))===(n=i.resolve(n)))return"";for(var r=1;rc){if(47===n.charCodeAt(a+d))return n.slice(a+d+1);if(0===d)return n.slice(a+d)}else s>c&&(47===e.charCodeAt(r+d)?h=d:0===d&&(h=0));break}var p=e.charCodeAt(r+d);if(p!==n.charCodeAt(a+d))break;47===p&&(h=d)}var u="";for(d=r+h+1;d<=o;++d)d!==o&&47!==e.charCodeAt(d)||(0===u.length?u+="..":u+="/..");return u.length>0?u+n.slice(a+h):(a+=h,47===n.charCodeAt(a)&&++a,n.slice(a))},_makeLong:function(e){return e},dirname:function(e){if(t(e),0===e.length)return".";for(var n=e.charCodeAt(0),i=47===n,r=-1,o=!0,s=e.length-1;s>=1;--s)if(47===(n=e.charCodeAt(s))){if(!o){r=s;break}}else o=!1;return-1===r?i?"/":".":i&&1===r?"//":e.slice(0,r)},basename:function(e,n){if(void 0!==n&&"string"!=typeof n)throw new TypeError('"ext" argument must be a string');t(e);var i,r=0,o=-1,s=!0;if(void 0!==n&&n.length>0&&n.length<=e.length){if(n.length===e.length&&n===e)return"";var a=n.length-1,l=-1;for(i=e.length-1;i>=0;--i){var c=e.charCodeAt(i);if(47===c){if(!s){r=i+1;break}}else-1===l&&(s=!1,l=i+1),a>=0&&(c===n.charCodeAt(a)?-1==--a&&(o=i):(a=-1,o=l))}return r===o?o=l:-1===o&&(o=e.length),e.slice(r,o)}for(i=e.length-1;i>=0;--i)if(47===e.charCodeAt(i)){if(!s){r=i+1;break}}else-1===o&&(s=!1,o=i+1);return-1===o?"":e.slice(r,o)},extname:function(e){t(e);for(var n=-1,i=0,r=-1,o=!0,s=0,a=e.length-1;a>=0;--a){var l=e.charCodeAt(a);if(47!==l)-1===r&&(o=!1,r=a+1),46===l?-1===n?n=a:1!==s&&(s=1):-1!==n&&(s=-1);else if(!o){i=a+1;break}}return-1===n||-1===r||0===s||1===s&&n===r-1&&n===i+1?"":e.slice(n,r)},format:function(e){if(null===e||"object"!=typeof e)throw new TypeError('The "pathObject" argument must be of type Object. Received type '+typeof e);return function(e,t){var n=t.dir||t.root,i=t.base||(t.name||"")+(t.ext||"");return n?n===t.root?n+i:n+"/"+i:i}(0,e)},parse:function(e){t(e);var n={root:"",dir:"",base:"",ext:"",name:""};if(0===e.length)return n;var i,r=e.charCodeAt(0),o=47===r;o?(n.root="/",i=1):i=0;for(var s=-1,a=0,l=-1,c=!0,h=e.length-1,d=0;h>=i;--h)if(47!==(r=e.charCodeAt(h)))-1===l&&(c=!1,l=h+1),46===r?-1===s?s=h:1!==d&&(d=1):-1!==s&&(d=-1);else if(!c){a=h+1;break}return-1===s||-1===l||0===d||1===d&&s===l-1&&s===a+1?-1!==l&&(n.base=n.name=0===a&&o?e.slice(1,l):e.slice(a,l)):(0===a&&o?(n.name=e.slice(1,s),n.base=e.slice(1,l)):(n.name=e.slice(a,s),n.base=e.slice(a,l)),n.ext=e.slice(s,l)),a>0?n.dir=e.slice(0,a-1):o&&(n.dir="/"),n},sep:"/",delimiter:":",win32:null,posix:null};i.posix=i,e.exports=i},447:(e,t,n)=>{var i;if(n.r(t),n.d(t,{URI:()=>f,Utils:()=>E}),"object"==typeof process)i="win32"===process.platform;else if("object"==typeof navigator){var r=navigator.userAgent;i=r.indexOf("Windows")>=0}var o,s,a=(o=function(e,t){return(o=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])})(e,t)},function(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Class extends value "+String(t)+" is not a constructor or null");function n(){this.constructor=e}o(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)}),l=/^\w[\w\d+.-]*$/,c=/^\//,h=/^\/\//;function d(e,t){if(!e.scheme&&t)throw new Error('[UriError]: Scheme is missing: {scheme: "", authority: "'.concat(e.authority,'", path: "').concat(e.path,'", query: "').concat(e.query,'", fragment: "').concat(e.fragment,'"}'));if(e.scheme&&!l.test(e.scheme))throw new Error("[UriError]: Scheme contains illegal characters.");if(e.path)if(e.authority){if(!c.test(e.path))throw new Error('[UriError]: If a URI contains an authority component, then the path component must either be empty or begin with a slash ("/") character')}else if(h.test(e.path))throw new Error('[UriError]: If a URI does not contain an authority component, then the path cannot begin with two slash characters ("//")')}var p="",u="/",m=/^(([^:/?#]+?):)?(\/\/([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?/,f=function(){function e(e,t,n,i,r,o){void 0===o&&(o=!1),"object"==typeof e?(this.scheme=e.scheme||p,this.authority=e.authority||p,this.path=e.path||p,this.query=e.query||p,this.fragment=e.fragment||p):(this.scheme=function(e,t){return e||t?e:"file"}(e,o),this.authority=t||p,this.path=function(e,t){switch(e){case"https":case"http":case"file":t?t[0]!==u&&(t=u+t):t=u}return t}(this.scheme,n||p),this.query=i||p,this.fragment=r||p,d(this,o))}return e.isUri=function(t){return t instanceof e||!!t&&"string"==typeof t.authority&&"string"==typeof t.fragment&&"string"==typeof t.path&&"string"==typeof t.query&&"string"==typeof t.scheme&&"string"==typeof t.fsPath&&"function"==typeof t.with&&"function"==typeof t.toString},Object.defineProperty(e.prototype,"fsPath",{get:function(){return x(this,!1)},enumerable:!1,configurable:!0}),e.prototype.with=function(e){if(!e)return this;var t=e.scheme,n=e.authority,i=e.path,r=e.query,o=e.fragment;return void 0===t?t=this.scheme:null===t&&(t=p),void 0===n?n=this.authority:null===n&&(n=p),void 0===i?i=this.path:null===i&&(i=p),void 0===r?r=this.query:null===r&&(r=p),void 0===o?o=this.fragment:null===o&&(o=p),t===this.scheme&&n===this.authority&&i===this.path&&r===this.query&&o===this.fragment?this:new b(t,n,i,r,o)},e.parse=function(e,t){void 0===t&&(t=!1);var n=m.exec(e);return n?new b(n[2]||p,k(n[4]||p),k(n[5]||p),k(n[7]||p),k(n[9]||p),t):new b(p,p,p,p,p)},e.file=function(e){var t=p;if(i&&(e=e.replace(/\\/g,u)),e[0]===u&&e[1]===u){var n=e.indexOf(u,2);-1===n?(t=e.substring(2),e=u):(t=e.substring(2,n),e=e.substring(n)||u)}return new b("file",t,e,p,p)},e.from=function(e){var t=new b(e.scheme,e.authority,e.path,e.query,e.fragment);return d(t,!0),t},e.prototype.toString=function(e){return void 0===e&&(e=!1),S(this,e)},e.prototype.toJSON=function(){return this},e.revive=function(t){if(t){if(t instanceof e)return t;var n=new b(t);return n._formatted=t.external,n._fsPath=t._sep===g?t.fsPath:null,n}return t},e}(),g=i?1:void 0,b=function(e){function t(){var t=null!==e&&e.apply(this,arguments)||this;return t._formatted=null,t._fsPath=null,t}return a(t,e),Object.defineProperty(t.prototype,"fsPath",{get:function(){return this._fsPath||(this._fsPath=x(this,!1)),this._fsPath},enumerable:!1,configurable:!0}),t.prototype.toString=function(e){return void 0===e&&(e=!1),e?S(this,!0):(this._formatted||(this._formatted=S(this,!1)),this._formatted)},t.prototype.toJSON=function(){var e={$mid:1};return this._fsPath&&(e.fsPath=this._fsPath,e._sep=g),this._formatted&&(e.external=this._formatted),this.path&&(e.path=this.path),this.scheme&&(e.scheme=this.scheme),this.authority&&(e.authority=this.authority),this.query&&(e.query=this.query),this.fragment&&(e.fragment=this.fragment),e},t}(f),v=((s={})[58]="%3A",s[47]="%2F",s[63]="%3F",s[35]="%23",s[91]="%5B",s[93]="%5D",s[64]="%40",s[33]="%21",s[36]="%24",s[38]="%26",s[39]="%27",s[40]="%28",s[41]="%29",s[42]="%2A",s[43]="%2B",s[44]="%2C",s[59]="%3B",s[61]="%3D",s[32]="%20",s);function y(e,t){for(var n=void 0,i=-1,r=0;r=97&&o<=122||o>=65&&o<=90||o>=48&&o<=57||45===o||46===o||95===o||126===o||t&&47===o)-1!==i&&(n+=encodeURIComponent(e.substring(i,r)),i=-1),void 0!==n&&(n+=e.charAt(r));else{void 0===n&&(n=e.substr(0,r));var s=v[o];void 0!==s?(-1!==i&&(n+=encodeURIComponent(e.substring(i,r)),i=-1),n+=s):-1===i&&(i=r)}}return-1!==i&&(n+=encodeURIComponent(e.substring(i))),void 0!==n?n:e}function w(e){for(var t=void 0,n=0;n1&&"file"===e.scheme?"//".concat(e.authority).concat(e.path):47===e.path.charCodeAt(0)&&(e.path.charCodeAt(1)>=65&&e.path.charCodeAt(1)<=90||e.path.charCodeAt(1)>=97&&e.path.charCodeAt(1)<=122)&&58===e.path.charCodeAt(2)?t?e.path.substr(1):e.path[1].toLowerCase()+e.path.substr(2):e.path,i&&(n=n.replace(/\//g,"\\")),n}function S(e,t){var n=t?w:y,i="",r=e.scheme,o=e.authority,s=e.path,a=e.query,l=e.fragment;if(r&&(i+=r,i+=":"),(o||"file"===r)&&(i+=u,i+=u),o){var c=o.indexOf("@");if(-1!==c){var h=o.substr(0,c);o=o.substr(c+1),-1===(c=h.indexOf(":"))?i+=n(h,!1):(i+=n(h.substr(0,c),!1),i+=":",i+=n(h.substr(c+1),!1)),i+="@"}-1===(c=(o=o.toLowerCase()).indexOf(":"))?i+=n(o,!1):(i+=n(o.substr(0,c),!1),i+=o.substr(c))}if(s){if(s.length>=3&&47===s.charCodeAt(0)&&58===s.charCodeAt(2))(d=s.charCodeAt(1))>=65&&d<=90&&(s="/".concat(String.fromCharCode(d+32),":").concat(s.substr(3)));else if(s.length>=2&&58===s.charCodeAt(1)){var d;(d=s.charCodeAt(0))>=65&&d<=90&&(s="".concat(String.fromCharCode(d+32),":").concat(s.substr(2)))}i+=n(s,!0)}return a&&(i+="?",i+=n(a,!1)),l&&(i+="#",i+=t?l:y(l,!1)),i}function C(e){try{return decodeURIComponent(e)}catch(t){return e.length>3?e.substr(0,3)+C(e.substr(3)):e}}var _=/(%[0-9A-Za-z][0-9A-Za-z])+/g;function k(e){return e.match(_)?e.replace(_,(function(e){return C(e)})):e}var E,R,N=n(470),F=function(e,t,n){if(n||2===arguments.length)for(var i,r=0,o=t.length;r{for(var i in t)n.o(t,i)&&!n.o(e,i)&&Object.defineProperty(e,i,{enumerable:!0,get:t[i]})},n.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),n.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},n(447)})();var{URI:$h,Utils:Hh}=Wh,Gh=function(e,t,n){if(n||2===arguments.length)for(var i,r=0,o=t.length;r0&&r[r.length-1])&&(6===o[0]||2===o[0])){s=0;continue}if(3===o[0]&&(!r||o[1]>r[0]&&o[1]0&&r[r.length-1])&&(6===o[0]||2===o[0])){s=0;continue}if(3===o[0]&&(!r||o[1]>r[0]&&o[1]=0&&-1===' \t\n\r":{[()]},*>+'.indexOf(i.charAt(n));)n--;return i.substring(n+1,t)}(e,this.offset),this.defaultReplaceRange=$a.create(Ba.create(this.position.line,this.position.character-this.currentWord.length),this.position),this.textDocument=e,this.styleSheet=n,this.documentSettings=i;try{var r={isIncomplete:!1,items:[]};this.nodePath=ys(this.styleSheet,this.offset);for(var o=this.nodePath.length-1;o>=0;o--){var s=this.nodePath[o];if(s instanceof Ls)this.getCompletionsForDeclarationProperty(s.getParent(),r);else if(s instanceof ua)s.parent instanceof _a?this.getVariableProposals(null,r):this.getCompletionsForExpression(s,r);else if(s instanceof Ts){var a=s.findAParent(us.ExtendsReference,us.Ruleset);if(a)if(a.type===us.ExtendsReference)this.getCompletionsForExtendsReference(a,s,r);else{var l=a;this.getCompletionsForSelector(l,l&&l.isNested(),r)}}else if(s instanceof Ws)this.getCompletionsForFunctionArgument(s,s.getParent(),r);else if(s instanceof Rs)this.getCompletionsForDeclarations(s,r);else if(s instanceof Ca)this.getCompletionsForVariableDeclaration(s,r);else if(s instanceof Fs)this.getCompletionsForRuleSet(s,r);else if(s instanceof _a)this.getCompletionsForInterpolation(s,r);else if(s instanceof js)this.getCompletionsForFunctionDeclaration(s,r);else if(s instanceof Fa)this.getCompletionsForMixinReference(s,r);else if(s instanceof Ps)this.getCompletionsForFunctionArgument(null,s,r);else if(s instanceof ra)this.getCompletionsForSupports(s,r);else if(s instanceof ha)this.getCompletionsForSupportsCondition(s,r);else if(s instanceof Ea)this.getCompletionsForExtendsReference(s,null,r);else if(s.type===us.URILiteral)this.getCompletionForUriLiteralValue(s,r);else if(null===s.parent)this.getCompletionForTopLevel(r);else{if(s.type!==us.StringLiteral||!this.isImportPathParent(s.parent.type))continue;this.getCompletionForImportPath(s,r)}if(r.items.length>0||this.offset>s.offset)return this.finalize(r)}return this.getCompletionsForStylesheet(r),0===r.items.length&&this.variablePrefix&&0===this.currentWord.indexOf(this.variablePrefix)&&this.getVariableProposals(null,r),this.finalize(r)}finally{this.position=null,this.currentWord=null,this.textDocument=null,this.styleSheet=null,this.symbolContext=null,this.defaultReplaceRange=null,this.nodePath=null}},e.prototype.isImportPathParent=function(e){return e===us.Import},e.prototype.finalize=function(e){return e},e.prototype.findInNodePath=function(){for(var e=[],t=0;t=0;n--){var i=this.nodePath[n];if(-1!==e.indexOf(i.type))return i}return null},e.prototype.getCompletionsForDeclarationProperty=function(e,t){return this.getPropertyProposals(e,t)},e.prototype.getPropertyProposals=function(e,t){var n=this,i=this.isTriggerPropertyValueCompletionEnabled,r=this.isCompletePropertyWithSemicolonEnabled;return this.cssDataManager.getProperties().forEach((function(o){var s,a,l=!1;e?(s=n.getCompletionRange(e.getProperty()),a=o.name,zh(e.colonPosition)||(a+=": ",l=!0)):(s=n.getCompletionRange(null),a=o.name+": ",l=!0),!e&&r&&(a+="$0;"),e&&!e.semicolonPosition&&r&&n.offset>=n.textDocument.offsetAt(s.end)&&(a+="$0;");var c={label:o.name,documentation:Yc(o,n.doesSupportMarkdown()),tags:ud(o)?[Zl.Deprecated]:[],textEdit:yl.replace(s,a),insertTextFormat:Yl.Snippet,kind:Jl.Property};o.restrictions||(l=!1),i&&l&&(c.command=dd);var h=(255-("number"===typeof o.relevance?Math.min(Math.max(o.relevance,0),99):50)).toString(16),d=cs(o.name,"-")?od.VendorPrefixed:od.Normal;c.sortText=d+"_"+h,t.items.push(c)})),this.completionParticipants.forEach((function(e){e.onCssProperty&&e.onCssProperty({propertyName:n.currentWord,range:n.defaultReplaceRange})})),t},Object.defineProperty(e.prototype,"isTriggerPropertyValueCompletionEnabled",{get:function(){var e,t;return null===(t=null===(e=this.documentSettings)||void 0===e?void 0:e.triggerPropertyValueCompletion)||void 0===t||t},enumerable:!1,configurable:!0}),Object.defineProperty(e.prototype,"isCompletePropertyWithSemicolonEnabled",{get:function(){var e,t;return null===(t=null===(e=this.documentSettings)||void 0===e?void 0:e.completePropertyWithSemicolon)||void 0===t||t},enumerable:!1,configurable:!0}),e.prototype.getCompletionsForDeclarationValue=function(e,t){for(var n=this,i=e.getFullPropertyName(),r=this.cssDataManager.getProperty(i),o=e.getValue()||null;o&&o.hasChildren();)o=o.findChildAtOffset(this.offset,!1);if(this.completionParticipants.forEach((function(e){e.onCssPropertyValue&&e.onCssPropertyValue({propertyName:i,propertyValue:n.currentWord,range:n.getCompletionRange(o)})})),r){if(r.restrictions)for(var s=0,a=r.restrictions;s=e.offset+2&&this.getVariableProposals(null,t),t},e.prototype.getVariableProposals=function(e,t){for(var n=0,i=this.getSymbolContext().findSymbolsAtOffset(this.offset,fs.Variable);n0){var r=this.currentWord.match(/^-?\d[\.\d+]*/);r&&(i=r[0],n.isIncomplete=i.length===this.currentWord.length)}else 0===this.currentWord.length&&(n.isIncomplete=!0);if(t&&t.parent&&t.parent.type===us.Term&&(t=t.getParent()),e.restrictions)for(var o=0,s=e.restrictions;o=n.end?this.getCompletionForTopLevel(t):!n||this.offset<=n.offset?this.getCompletionsForSelector(e,e.isNested(),t):this.getCompletionsForDeclarations(e.getDeclarations(),t)},e.prototype.getCompletionsForSelector=function(e,t,n){var i=this,r=this.findInNodePath(us.PseudoSelector,us.IdentifierSelector,us.ClassSelector,us.ElementNameSelector);if(!r&&this.hasCharacterAtPosition(this.offset-this.currentWord.length-1,":")&&(this.currentWord=":"+this.currentWord,this.hasCharacterAtPosition(this.offset-this.currentWord.length-1,":")&&(this.currentWord=":"+this.currentWord),this.defaultReplaceRange=$a.create(Ba.create(this.position.line,this.position.character-this.currentWord.length),this.position)),this.cssDataManager.getPseudoClasses().forEach((function(e){var t=fd(e.name),o={label:e.name,textEdit:yl.replace(i.getCompletionRange(r),t),documentation:Yc(e,i.doesSupportMarkdown()),tags:ud(e)?[Zl.Deprecated]:[],kind:Jl.Function,insertTextFormat:e.name!==t?hd:void 0};cs(e.name,":-")&&(o.sortText=od.VendorPrefixed),n.items.push(o)})),this.cssDataManager.getPseudoElements().forEach((function(e){var t=fd(e.name),o={label:e.name,textEdit:yl.replace(i.getCompletionRange(r),t),documentation:Yc(e,i.doesSupportMarkdown()),tags:ud(e)?[Zl.Deprecated]:[],kind:Jl.Function,insertTextFormat:e.name!==t?hd:void 0};cs(e.name,"::-")&&(o.sortText=od.VendorPrefixed),n.items.push(o)})),!t){for(var o=0,s=Dh;o0){var t=d.substr(e.offset,e.length);return"."!==t.charAt(0)||h[t]||(h[t]=!0,n.items.push({label:t,textEdit:yl.replace(i.getCompletionRange(r),t),kind:Jl.Keyword})),!1}return!0})),e&&e.isNested()){var p=e.getSelectors().findFirstChildBeforeOffset(this.offset);p&&0===e.getSelectors().getChildren().indexOf(p)&&this.getPropertyProposals(null,n)}return n},e.prototype.getCompletionsForDeclarations=function(e,t){if(!e||this.offset===e.offset)return t;var n=e.findFirstChildBeforeOffset(this.offset);if(!n)return this.getCompletionsForDeclarationProperty(null,t);if(n instanceof As){var i=n;if(!zh(i.colonPosition)||this.offset<=i.colonPosition)return this.getCompletionsForDeclarationProperty(i,t);if(zh(i.semicolonPosition)&&i.semicolonPositione.colonPosition&&this.getVariableProposals(e.getValue(),t),t},e.prototype.getCompletionsForExpression=function(e,t){var n=e.getParent();if(n instanceof Ws)return this.getCompletionsForFunctionArgument(n,n.getParent(),t),t;var i=e.findParent(us.Declaration);if(!i)return this.getTermProposals(void 0,null,t),t;var r=e.findChildAtOffset(this.offset,!0);return r?r instanceof Sa||r instanceof ks?this.getCompletionsForDeclarationValue(i,t):t:this.getCompletionsForDeclarationValue(i,t)},e.prototype.getCompletionsForFunctionArgument=function(e,t,n){var i=t.getIdentifier();return i&&i.matches("var")&&(t.getArguments().hasChildren()&&t.getArguments().getChild(0)!==e||this.getVariableProposalsForCSSVarFunction(n)),n},e.prototype.getCompletionsForFunctionDeclaration=function(e,t){var n=e.getDeclarations();return n&&this.offset>n.offset&&this.offsete.lParent&&(!zh(e.rParent)||this.offset<=e.rParent)?this.getCompletionsForDeclarationProperty(null,t):t},e.prototype.getCompletionsForSupports=function(e,t){var n=e.getDeclarations();if(!n||this.offset<=n.offset){var i=e.findFirstChildBeforeOffset(this.offset);return i instanceof ha?this.getCompletionsForSupportsCondition(i,t):t}return this.getCompletionForTopLevel(t)},e.prototype.getCompletionsForExtendsReference=function(e,t,n){return n},e.prototype.getCompletionForUriLiteralValue=function(e,t){var n,i,r;if(e.hasChildren()){var o=e.getChild(0);n=o.getText(),i=this.position,r=this.getCompletionRange(o)}else{n="",i=this.position;var s=this.textDocument.positionAt(e.offset+"url(".length);r=$a.create(s,s)}return this.completionParticipants.forEach((function(e){e.onCssURILiteralValue&&e.onCssURILiteralValue({uriValue:n,position:i,range:r})})),t},e.prototype.getCompletionForImportPath=function(e,t){var n=this;return this.completionParticipants.forEach((function(t){t.onCssImportPath&&t.onCssImportPath({pathValue:e.getText(),position:n.position,range:n.getCompletionRange(e)})})),t},e.prototype.hasCharacterAtPosition=function(e,t){var n=this.textDocument.getText();return e>=0&&e"),this.writeLine(t,i.join(""))}},e}();!function(e){function t(e){var t=e.match(/^['"](.*)["']$/);return t?t[1]:e}e.ensure=function(e,n){return n+t(e)+n},e.remove=t}(yd||(yd={}));var Ed=function(){return function(){this.id=0,this.attr=0,this.tag=0}}();function Rd(e,t){for(var n=new Sd,i=0,r=e.getChildren();i1){var l=t.cloneWithParent();n.addChild(l.findRoot()),n=l}n.append(s[a])}}break;case us.SelectorPlaceholder:if(o.matches("@at-root"))return n;case us.ElementNameSelector:var c=o.getText();n.addAttr("name","*"===c?"element":Nd(c));break;case us.ClassSelector:n.addAttr("class",Nd(o.getText().substring(1)));break;case us.IdentifierSelector:n.addAttr("id",Nd(o.getText().substring(1)));break;case us.MixinDeclaration:n.addAttr("class",o.getName());break;case us.PseudoSelector:n.addAttr(Nd(o.getText()),"");break;case us.AttributeSelector:var h=o,d=h.getIdentifier();if(d){var p=h.getValue(),u=h.getOperator(),m=void 0;if(p&&u)switch(Nd(u.getText())){case"|=":m="".concat(yd.remove(Nd(p.getText())),"-\u2026");break;case"^=":m="".concat(yd.remove(Nd(p.getText())),"\u2026");break;case"$=":m="\u2026".concat(yd.remove(Nd(p.getText())));break;case"~=":m=" \u2026 ".concat(yd.remove(Nd(p.getText()))," \u2026 ");break;case"*=":m="\u2026".concat(yd.remove(Nd(p.getText())),"\u2026");break;default:m=yd.remove(Nd(p.getText()))}n.addAttr(Nd(d.getText()),m)}}}return n}function Nd(e){var t=new ls;t.setSource(e);var n=t.scanUnquotedString();return n?n.text:e}var Fd=function(){function e(e){this.cssDataManager=e}return e.prototype.selectorToMarkedString=function(e){var t=function(e){if(e.matches("@at-root"))return null;var t=new Cd,n=[],i=e.getParent();if(i instanceof Fs)for(var r=i.getParent();r&&!Td(r);){if(r instanceof Fs){if(r.getSelectors().matches("@at-root"))break;n.push(r)}r=r.getParent()}for(var o=new Dd(t),s=n.length-1;s>=0;s--){var a=n[s].getSelectors().getChild(0);a&&o.processSelector(a)}return o.processSelector(e),t}(e);if(t){var n=new kd('"').print(t);return n.push(this.selectorToSpecificityMarkedString(e)),n}return[]},e.prototype.simpleSelectorToMarkedString=function(e){var t=Rd(e),n=new kd('"').print(t);return n.push(this.selectorToSpecificityMarkedString(e)),n},e.prototype.isPseudoElementIdentifier=function(e){var t=e.match(/^::?([\w-]+)/);return!!t&&!!this.cssDataManager.getPseudoElement("::"+t[1])},e.prototype.selectorToSpecificityMarkedString=function(e){var t=this,n=function(e){var i=new Ed;e:for(var r=0,o=e.getChildren();r0){for(var l=new Ed,c=0,h=s.getChildren();cl.id?l=f:f.idl.attr?l=f:f.attrl.tag&&(l=f))}}i.id+=l.id,i.attr+=l.attr,i.tag+=l.tag;continue e}i.attr++}if(s.getChildren().length>0){var f=n(s);i.id+=f.id,i.attr+=f.attr,i.tag+=f.tag}}return i},i=n(e);return xd("specificity","[Selector Specificity](https://developer.mozilla.org/en-US/docs/Web/CSS/Specificity): ({0}, {1}, {2})",i.id,i.attr,i.tag)},e}(),Dd=function(){function e(e){this.prev=null,this.element=e}return e.prototype.processSelector=function(e){var t=null;if(!(this.element instanceof Cd)&&e.getChildren().some((function(e){return e.hasChildren()&&e.getChild(0).type===us.SelectorCombinator}))){var n=this.element.findRoot();n.parent instanceof Cd&&(t=this.element,this.element=n.parent,this.element.removeChild(n),this.prev=null)}for(var i=0,r=e.getChildren();i0&&r[r.length-1])&&(6===o[0]||2===o[0])){s=0;continue}if(3===o[0]&&(!r||o[1]>r[0]&&o[1]n)return 0;var r,o,s=[],a=[];for(r=0;r=r.length/2&&o.push({property:e.name,score:t})})),o.sort((function(e,t){return t.score-e.score||e.property.localeCompare(t.property)}));for(var s=3,a=0,l=o;a=0;a--){var l=s[a];if(l instanceof zs){var c=l.getProperty();if(c&&c.offset===r&&c.end===o)return void this.getFixesForUnknownProperty(e,c,n,i)}}},e}(),Zd=function(){return function(e){this.fullPropertyName=e.getFullPropertyName().toLowerCase(),this.node=e}}();function ep(e,t,n,i){var r=e[t];r.value=n,n&&(Ph(r.properties,i)||r.properties.push(i))}function tp(e,t,n,i){"top"===t||"right"===t||"bottom"===t||"left"===t?ep(e,t,n,i):function(e,t,n){ep(e,"top",t,n),ep(e,"right",t,n),ep(e,"bottom",t,n),ep(e,"left",t,n)}(e,n,i)}function np(e,t,n){switch(t.length){case 1:tp(e,void 0,t[0],n);break;case 2:tp(e,"top",t[0],n),tp(e,"bottom",t[0],n),tp(e,"right",t[1],n),tp(e,"left",t[1],n);break;case 3:tp(e,"top",t[0],n),tp(e,"right",t[1],n),tp(e,"left",t[1],n),tp(e,"bottom",t[2],n);break;case 4:tp(e,"top",t[0],n),tp(e,"right",t[1],n),tp(e,"bottom",t[2],n),tp(e,"left",t[3],n)}}function ip(e,t){for(var n=0,i=t;n0)for(var m=this.fetch(i,"float"),f=0;f0)for(m=this.fetch(i,"vertical-align"),f=0;f1)for(var S=0;S")||this.peekDelim("<")||this.peekIdent("and")||this.peekIdent("or")||this.peekDelim("%")){var t=this.createNode(us.Operator);return this.consumeToken(),this.finish(t)}return e.prototype._parseOperator.call(this)},t.prototype._parseUnaryOperator=function(){if(this.peekIdent("not")){var t=this.create(Ss);return this.consumeToken(),this.finish(t)}return e.prototype._parseUnaryOperator.call(this)},t.prototype._parseRuleSetDeclaration=function(){return this.peek(mo.AtKeyword)?this._parseKeyframe()||this._parseImport()||this._parseMedia(!0)||this._parseFontFace()||this._parseWarnAndDebug()||this._parseControlStatement()||this._parseFunctionDeclaration()||this._parseExtends()||this._parseMixinReference()||this._parseMixinContent()||this._parseMixinDeclaration()||this._parseRuleset(!0)||this._parseSupports(!0)||e.prototype._parseRuleSetDeclarationAtStatement.call(this):this._parseVariableDeclaration()||this._tryParseRuleset(!0)||e.prototype._parseRuleSetDeclaration.call(this)},t.prototype._parseDeclaration=function(e){var t=this._tryParseCustomPropertyDeclaration(e);if(t)return t;var n=this.create(zs);if(!n.setProperty(this._parseProperty()))return null;if(!this.accept(mo.Colon))return this.finish(n,Pl.ColonExpected,[mo.Colon],e||[mo.SemiColon]);this.prevToken&&(n.colonPosition=this.prevToken.offset);var i=!1;if(n.setValue(this._parseExpr())&&(i=!0,n.addChild(this._parsePrio())),this.peek(mo.CurlyL))n.setNestedProperties(this._parseNestedProperties());else if(!i)return this.finish(n,Pl.PropertyValueExpected);return this.peek(mo.SemiColon)&&(n.semicolonPosition=this.token.offset),this.finish(n)},t.prototype._parseNestedProperties=function(){var e=this.create(Gs);return this._parseBody(e,this._parseDeclaration.bind(this))},t.prototype._parseExtends=function(){if(this.peekKeyword("@extend")){var e=this.create(Ea);if(this.consumeToken(),!e.getSelectors().addChild(this._parseSimpleSelector()))return this.finish(e,Pl.SelectorExpected);for(;this.accept(mo.Comma);)e.getSelectors().addChild(this._parseSimpleSelector());return this.accept(mo.Exclamation)&&!this.acceptIdent("optional")?this.finish(e,Pl.UnknownKeyword):this.finish(e)}return null},t.prototype._parseSimpleSelectorBody=function(){return this._parseSelectorCombinator()||this._parseSelectorPlaceholder()||e.prototype._parseSimpleSelectorBody.call(this)},t.prototype._parseSelectorCombinator=function(){if(this.peekDelim("&")){var e=this.createNode(us.SelectorCombinator);for(this.consumeToken();!this.hasWhitespace()&&(this.acceptDelim("-")||this.accept(mo.Num)||this.accept(mo.Dimension)||e.addChild(this._parseIdent())||this.acceptDelim("&")););return this.finish(e)}return null},t.prototype._parseSelectorPlaceholder=function(){if(this.peekDelim("%")){var e=this.createNode(us.SelectorPlaceholder);return this.consumeToken(),this._parseIdent(),this.finish(e)}if(this.peekKeyword("@at-root")){e=this.createNode(us.SelectorPlaceholder);return this.consumeToken(),this.finish(e)}return null},t.prototype._parseElementName=function(){var t=this.mark(),n=e.prototype._parseElementName.call(this);return n&&!this.hasWhitespace()&&this.peek(mo.ParenthesisL)?(this.restoreAtMark(t),null):n},t.prototype._tryParsePseudoIdentifier=function(){return this._parseInterpolation()||e.prototype._tryParsePseudoIdentifier.call(this)},t.prototype._parseWarnAndDebug=function(){if(!this.peekKeyword("@debug")&&!this.peekKeyword("@warn")&&!this.peekKeyword("@error"))return null;var e=this.createNode(us.Debug);return this.consumeToken(),e.addChild(this._parseExpr()),this.finish(e)},t.prototype._parseControlStatement=function(e){return void 0===e&&(e=this._parseRuleSetDeclaration.bind(this)),this.peek(mo.AtKeyword)?this._parseIfStatement(e)||this._parseForStatement(e)||this._parseEachStatement(e)||this._parseWhileStatement(e):null},t.prototype._parseIfStatement=function(e){return this.peekKeyword("@if")?this._internalParseIfStatement(e):null},t.prototype._internalParseIfStatement=function(e){var t=this.create(Vs);if(this.consumeToken(),!t.setExpression(this._parseExpr(!0)))return this.finish(t,Pl.ExpressionExpected);if(this._parseBody(t,e),this.acceptKeyword("@else"))if(this.peekIdent("if"))t.setElseClause(this._internalParseIfStatement(e));else if(this.peek(mo.CurlyL)){var n=this.create(Bs);this._parseBody(n,e),t.setElseClause(n)}return this.finish(t)},t.prototype._parseForStatement=function(e){if(!this.peekKeyword("@for"))return null;var t=this.create(Us);return this.consumeToken(),t.setVariable(this._parseVariable())?this.acceptIdent("from")?t.addChild(this._parseBinaryExpr())?this.acceptIdent("to")||this.acceptIdent("through")?t.addChild(this._parseBinaryExpr())?this._parseBody(t,e):this.finish(t,Pl.ExpressionExpected,[mo.CurlyR]):this.finish(t,Pp.ThroughOrToExpected,[mo.CurlyR]):this.finish(t,Pl.ExpressionExpected,[mo.CurlyR]):this.finish(t,Pp.FromExpected,[mo.CurlyR]):this.finish(t,Pl.VariableNameExpected,[mo.CurlyR])},t.prototype._parseEachStatement=function(e){if(!this.peekKeyword("@each"))return null;var t=this.create(Ks);this.consumeToken();var n=t.getVariables();if(!n.addChild(this._parseVariable()))return this.finish(t,Pl.VariableNameExpected,[mo.CurlyR]);for(;this.accept(mo.Comma);)if(!n.addChild(this._parseVariable()))return this.finish(t,Pl.VariableNameExpected,[mo.CurlyR]);return this.finish(n),this.acceptIdent("in")?t.addChild(this._parseExpr())?this._parseBody(t,e):this.finish(t,Pl.ExpressionExpected,[mo.CurlyR]):this.finish(t,Pp.InExpected,[mo.CurlyR])},t.prototype._parseWhileStatement=function(e){if(!this.peekKeyword("@while"))return null;var t=this.create(qs);return this.consumeToken(),t.addChild(this._parseBinaryExpr())?this._parseBody(t,e):this.finish(t,Pl.ExpressionExpected,[mo.CurlyR])},t.prototype._parseFunctionBodyDeclaration=function(){return this._parseVariableDeclaration()||this._parseReturnStatement()||this._parseWarnAndDebug()||this._parseControlStatement(this._parseFunctionBodyDeclaration.bind(this))},t.prototype._parseFunctionDeclaration=function(){if(!this.peekKeyword("@function"))return null;var e=this.create(js);if(this.consumeToken(),!e.setIdentifier(this._parseIdent([fs.Function])))return this.finish(e,Pl.IdentifierExpected,[mo.CurlyR]);if(!this.accept(mo.ParenthesisL))return this.finish(e,Pl.LeftParenthesisExpected,[mo.CurlyR]);if(e.getParameters().addChild(this._parseParameterDeclaration()))for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)if(!e.getParameters().addChild(this._parseParameterDeclaration()))return this.finish(e,Pl.VariableNameExpected);return this.accept(mo.ParenthesisR)?this._parseBody(e,this._parseFunctionBodyDeclaration.bind(this)):this.finish(e,Pl.RightParenthesisExpected,[mo.CurlyR])},t.prototype._parseReturnStatement=function(){if(!this.peekKeyword("@return"))return null;var e=this.createNode(us.ReturnStatement);return this.consumeToken(),e.addChild(this._parseExpr())?this.finish(e):this.finish(e,Pl.ExpressionExpected)},t.prototype._parseMixinDeclaration=function(){if(!this.peekKeyword("@mixin"))return null;var e=this.create(Da);if(this.consumeToken(),!e.setIdentifier(this._parseIdent([fs.Mixin])))return this.finish(e,Pl.IdentifierExpected,[mo.CurlyR]);if(this.accept(mo.ParenthesisL)){if(e.getParameters().addChild(this._parseParameterDeclaration()))for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)if(!e.getParameters().addChild(this._parseParameterDeclaration()))return this.finish(e,Pl.VariableNameExpected);if(!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected,[mo.CurlyR])}return this._parseBody(e,this._parseRuleSetDeclaration.bind(this))},t.prototype._parseParameterDeclaration=function(){var e=this.create(Os);return e.setIdentifier(this._parseVariable())?(this.accept(Mp),this.accept(mo.Colon)&&!e.setDefaultValue(this._parseExpr(!0))?this.finish(e,Pl.VariableValueExpected,[],[mo.Comma,mo.ParenthesisR]):this.finish(e)):null},t.prototype._parseMixinContent=function(){if(!this.peekKeyword("@content"))return null;var e=this.create(Ra);if(this.consumeToken(),this.accept(mo.ParenthesisL)){if(e.getArguments().addChild(this._parseFunctionArgument()))for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)if(!e.getArguments().addChild(this._parseFunctionArgument()))return this.finish(e,Pl.ExpressionExpected);if(!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected)}return this.finish(e)},t.prototype._parseMixinReference=function(){if(!this.peekKeyword("@include"))return null;var e=this.create(Fa);this.consumeToken();var t=this._parseIdent([fs.Mixin]);if(!e.setIdentifier(t))return this.finish(e,Pl.IdentifierExpected,[mo.CurlyR]);if(!this.hasWhitespace()&&this.acceptDelim(".")&&!this.hasWhitespace()){var n=this._parseIdent([fs.Mixin]);if(!n)return this.finish(e,Pl.IdentifierExpected,[mo.CurlyR]);var i=this.create(Ia);t.referenceTypes=[fs.Module],i.setIdentifier(t),e.setIdentifier(n),e.addChild(i)}if(this.accept(mo.ParenthesisL)){if(e.getArguments().addChild(this._parseFunctionArgument()))for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)if(!e.getArguments().addChild(this._parseFunctionArgument()))return this.finish(e,Pl.ExpressionExpected);if(!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected)}return(this.peekIdent("using")||this.peek(mo.CurlyL))&&e.setContent(this._parseMixinContentDeclaration()),this.finish(e)},t.prototype._parseMixinContentDeclaration=function(){var e=this.create(Na);if(this.acceptIdent("using")){if(!this.accept(mo.ParenthesisL))return this.finish(e,Pl.LeftParenthesisExpected,[mo.CurlyL]);if(e.getParameters().addChild(this._parseParameterDeclaration()))for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)if(!e.getParameters().addChild(this._parseParameterDeclaration()))return this.finish(e,Pl.VariableNameExpected);if(!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected,[mo.CurlyL])}return this.peek(mo.CurlyL)&&this._parseBody(e,this._parseMixinReferenceBodyStatement.bind(this)),this.finish(e)},t.prototype._parseMixinReferenceBodyStatement=function(){return this._tryParseKeyframeSelector()||this._parseRuleSetDeclaration()},t.prototype._parseFunctionArgument=function(){var e=this.create(Ws),t=this.mark(),n=this._parseVariable();if(n)if(this.accept(mo.Colon))e.setIdentifier(n);else{if(this.accept(Mp))return e.setValue(n),this.finish(e);this.restoreAtMark(t)}return e.setValue(this._parseExpr(!0))?(this.accept(Mp),e.addChild(this._parsePrio()),this.finish(e)):e.setValue(this._tryParsePrio())?this.finish(e):null},t.prototype._parseURLArgument=function(){var t=this.mark(),n=e.prototype._parseURLArgument.call(this);if(!n||!this.peek(mo.ParenthesisR)){this.restoreAtMark(t);var i=this.create(Ss);return i.addChild(this._parseBinaryExpr()),this.finish(i)}return n},t.prototype._parseOperation=function(){if(!this.peek(mo.ParenthesisL))return null;var e=this.create(Ss);for(this.consumeToken();e.addChild(this._parseListElement());)this.accept(mo.Comma);return this.accept(mo.ParenthesisR)?this.finish(e):this.finish(e,Pl.RightParenthesisExpected)},t.prototype._parseListElement=function(){var e=this.create(Aa),t=this._parseBinaryExpr();if(!t)return null;if(this.accept(mo.Colon)){if(e.setKey(t),!e.setValue(this._parseBinaryExpr()))return this.finish(e,Pl.ExpressionExpected)}else e.setValue(t);return this.finish(e)},t.prototype._parseUse=function(){if(!this.peekKeyword("@use"))return null;var e=this.create(Qs);if(this.consumeToken(),!e.addChild(this._parseStringLiteral()))return this.finish(e,Pl.StringLiteralExpected);if(!this.peek(mo.SemiColon)&&!this.peek(mo.EOF)){if(!this.peekRegExp(mo.Ident,/as|with/))return this.finish(e,Pl.UnknownKeyword);if(this.acceptIdent("as")&&!e.setIdentifier(this._parseIdent([fs.Module]))&&!this.acceptDelim("*"))return this.finish(e,Pl.IdentifierOrWildcardExpected);if(this.acceptIdent("with")){if(!this.accept(mo.ParenthesisL))return this.finish(e,Pl.LeftParenthesisExpected,[mo.ParenthesisR]);if(!e.getParameters().addChild(this._parseModuleConfigDeclaration()))return this.finish(e,Pl.VariableNameExpected);for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)if(!e.getParameters().addChild(this._parseModuleConfigDeclaration()))return this.finish(e,Pl.VariableNameExpected);if(!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected)}}return this.accept(mo.SemiColon)||this.accept(mo.EOF)?this.finish(e):this.finish(e,Pl.SemiColonExpected)},t.prototype._parseModuleConfigDeclaration=function(){var e=this.create(Zs);return e.setIdentifier(this._parseVariable())?this.accept(mo.Colon)&&e.setValue(this._parseExpr(!0))?!this.accept(mo.Exclamation)||!this.hasWhitespace()&&this.acceptIdent("default")?this.finish(e):this.finish(e,Pl.UnknownKeyword):this.finish(e,Pl.VariableValueExpected,[],[mo.Comma,mo.ParenthesisR]):null},t.prototype._parseForward=function(){if(!this.peekKeyword("@forward"))return null;var e=this.create(ea);if(this.consumeToken(),!e.addChild(this._parseStringLiteral()))return this.finish(e,Pl.StringLiteralExpected);if(this.acceptIdent("with")){if(!this.accept(mo.ParenthesisL))return this.finish(e,Pl.LeftParenthesisExpected,[mo.ParenthesisR]);if(!e.getParameters().addChild(this._parseModuleConfigDeclaration()))return this.finish(e,Pl.VariableNameExpected);for(;this.accept(mo.Comma)&&!this.peek(mo.ParenthesisR);)if(!e.getParameters().addChild(this._parseModuleConfigDeclaration()))return this.finish(e,Pl.VariableNameExpected);if(!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected)}if(!this.peek(mo.SemiColon)&&!this.peek(mo.EOF)){if(!this.peekRegExp(mo.Ident,/as|hide|show/))return this.finish(e,Pl.UnknownKeyword);if(this.acceptIdent("as")){var t=this._parseIdent([fs.Forward]);if(!e.setIdentifier(t))return this.finish(e,Pl.IdentifierExpected);if(this.hasWhitespace()||!this.acceptDelim("*"))return this.finish(e,Pl.WildcardExpected)}if((this.peekIdent("hide")||this.peekIdent("show"))&&!e.addChild(this._parseForwardVisibility()))return this.finish(e,Pl.IdentifierOrVariableExpected)}return this.accept(mo.SemiColon)||this.accept(mo.EOF)?this.finish(e):this.finish(e,Pl.SemiColonExpected)},t.prototype._parseForwardVisibility=function(){var e=this.create(ta);for(e.setIdentifier(this._parseIdent());e.addChild(this._parseVariable()||this._parseIdent());)this.accept(mo.Comma);return e.getChildren().length>1?e:null},t.prototype._parseSupportsCondition=function(){return this._parseInterpolation()||e.prototype._parseSupportsCondition.call(this)},t}(Lh),Vp=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])})(t,n)};return function(t,n){if("function"!==typeof n&&null!==n)throw new TypeError("Class extends value "+String(n)+" is not a constructor or null");function i(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(i.prototype=n.prototype,new i)}}(),Up=Wa(),Kp=function(e){function t(n,i){var r=e.call(this,"$",n,i)||this;return qp(t.scssModuleLoaders),qp(t.scssModuleBuiltIns),r}return Vp(t,e),t.prototype.isImportPathParent=function(t){return t===us.Forward||t===us.Use||e.prototype.isImportPathParent.call(this,t)},t.prototype.getCompletionForImportPath=function(n,i){var r=n.getParent().type;if(r===us.Forward||r===us.Use)for(var o=0,s=t.scssModuleBuiltIns;o0){var t="string"===typeof e.documentation?{kind:"markdown",value:e.documentation}:{kind:"markdown",value:e.documentation.value};t.value+="\n\n",t.value+=e.references.map((function(e){return"[".concat(e.name,"](").concat(e.url,")")})).join(" | "),e.documentation=t}}))}var Bp,jp=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])})(t,n)};return function(t,n){if("function"!==typeof n&&null!==n)throw new TypeError("Class extends value "+String(n)+" is not a constructor or null");function i(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(i.prototype=n.prototype,new i)}}(),$p="/".charCodeAt(0),Hp="\n".charCodeAt(0),Gp="\r".charCodeAt(0),Jp="\f".charCodeAt(0),Xp="`".charCodeAt(0),Yp=".".charCodeAt(0),Qp=mo.CustomToken,Zp=Qp++,eu=function(e){function t(){return null!==e&&e.apply(this,arguments)||this}return jp(t,e),t.prototype.scanNext=function(t){var n=this.escapedJavaScript();return null!==n?this.finishToken(t,n):this.stream.advanceIfChars([Yp,Yp,Yp])?this.finishToken(t,Zp):e.prototype.scanNext.call(this,t)},t.prototype.comment=function(){return!!e.prototype.comment.call(this)||!(this.inURL||!this.stream.advanceIfChars([$p,$p]))&&(this.stream.advanceWhileChar((function(e){switch(e){case Hp:case Gp:case Jp:return!1;default:return!0}})),!0)},t.prototype.escapedJavaScript=function(){return this.stream.peekChar()===Xp?(this.stream.advance(1),this.stream.advanceWhileChar((function(e){return e!==Xp})),this.stream.advanceIfChar(Xp)?mo.EscapedJavaScript:mo.BadEscapedJavaScript):null},t}(ls),tu=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])})(t,n)};return function(t,n){if("function"!==typeof n&&null!==n)throw new TypeError("Class extends value "+String(n)+" is not a constructor or null");function i(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(i.prototype=n.prototype,new i)}}(),nu=function(e){function t(){return e.call(this,new eu)||this}return tu(t,e),t.prototype._parseStylesheetStatement=function(t){return void 0===t&&(t=!1),this.peek(mo.AtKeyword)?this._parseVariableDeclaration()||this._parsePlugin()||e.prototype._parseStylesheetAtStatement.call(this,t):this._tryParseMixinDeclaration()||this._tryParseMixinReference()||this._parseFunction()||this._parseRuleset(!0)},t.prototype._parseImport=function(){if(!this.peekKeyword("@import")&&!this.peekKeyword("@import-once"))return null;var e=this.create(Ys);if(this.consumeToken(),this.accept(mo.ParenthesisL)){if(!this.accept(mo.Ident))return this.finish(e,Pl.IdentifierExpected,[mo.SemiColon]);do{if(!this.accept(mo.Comma))break}while(this.accept(mo.Ident));if(!this.accept(mo.ParenthesisR))return this.finish(e,Pl.RightParenthesisExpected,[mo.SemiColon])}return e.addChild(this._parseURILiteral())||e.addChild(this._parseStringLiteral())?(this.peek(mo.SemiColon)||this.peek(mo.EOF)||e.setMedialist(this._parseMediaQueryList()),this.finish(e)):this.finish(e,Pl.URIOrStringExpected,[mo.SemiColon])},t.prototype._parsePlugin=function(){if(!this.peekKeyword("@plugin"))return null;var e=this.createNode(us.Plugin);return this.consumeToken(),e.addChild(this._parseStringLiteral())?this.accept(mo.SemiColon)?this.finish(e):this.finish(e,Pl.SemiColonExpected):this.finish(e,Pl.StringLiteralExpected)},t.prototype._parseMediaQuery=function(){var t=e.prototype._parseMediaQuery.call(this);if(!t){var n=this.create(aa);return n.addChild(this._parseVariable())?this.finish(n):null}return t},t.prototype._parseMediaDeclaration=function(e){return void 0===e&&(e=!1),this._tryParseRuleset(e)||this._tryToParseDeclaration()||this._tryParseMixinDeclaration()||this._tryParseMixinReference()||this._parseDetachedRuleSetMixin()||this._parseStylesheetStatement(e)},t.prototype._parseMediaFeatureName=function(){return this._parseIdent()||this._parseVariable()},t.prototype._parseVariableDeclaration=function(e){void 0===e&&(e=[]);var t=this.create(Ca),n=this.mark();if(!t.setVariable(this._parseVariable(!0)))return null;if(!this.accept(mo.Colon))return this.restoreAtMark(n),null;if(this.prevToken&&(t.colonPosition=this.prevToken.offset),t.setValue(this._parseDetachedRuleSet()))t.needsSemicolon=!1;else if(!t.setValue(this._parseExpr()))return this.finish(t,Pl.VariableValueExpected,[],e);return t.addChild(this._parsePrio()),this.peek(mo.SemiColon)&&(t.semicolonPosition=this.token.offset),this.finish(t)},t.prototype._parseDetachedRuleSet=function(){var e=this.mark();if(this.peekDelim("#")||this.peekDelim(".")){if(this.consumeToken(),this.hasWhitespace()||!this.accept(mo.ParenthesisL))return this.restoreAtMark(e),null;var t=this.create(Da);if(t.getParameters().addChild(this._parseMixinParameter()))for(;(this.accept(mo.Comma)||this.accept(mo.SemiColon))&&!this.peek(mo.ParenthesisR);)t.getParameters().addChild(this._parseMixinParameter())||this.markError(t,Pl.IdentifierExpected,[],[mo.ParenthesisR]);if(!this.accept(mo.ParenthesisR))return this.restoreAtMark(e),null}if(!this.peek(mo.CurlyL))return null;var n=this.create(Ns);return this._parseBody(n,this._parseDetachedRuleSetBody.bind(this)),this.finish(n)},t.prototype._parseDetachedRuleSetBody=function(){return this._tryParseKeyframeSelector()||this._parseRuleSetDeclaration()},t.prototype._addLookupChildren=function(e){if(!e.addChild(this._parseLookupValue()))return!1;for(var t=!1;this.peek(mo.BracketL)&&(t=!0),e.addChild(this._parseLookupValue());)t=!1;return!t},t.prototype._parseLookupValue=function(){var e=this.create(Ss),t=this.mark();return this.accept(mo.BracketL)&&((e.addChild(this._parseVariable(!1,!0))||e.addChild(this._parsePropertyIdentifier()))&&this.accept(mo.BracketR)||this.accept(mo.BracketR))?e:(this.restoreAtMark(t),null)},t.prototype._parseVariable=function(e,t){void 0===e&&(e=!1),void 0===t&&(t=!1);var n=!e&&this.peekDelim("$");if(!this.peekDelim("@")&&!n&&!this.peek(mo.AtKeyword))return null;for(var i=this.create(ka),r=this.mark();this.acceptDelim("@")||!e&&this.acceptDelim("$");)if(this.hasWhitespace())return this.restoreAtMark(r),null;return(this.accept(mo.AtKeyword)||this.accept(mo.Ident))&&(t||!this.peek(mo.BracketL)||this._addLookupChildren(i))?i:(this.restoreAtMark(r),null)},t.prototype._parseTermExpression=function(){return this._parseVariable()||this._parseEscaped()||e.prototype._parseTermExpression.call(this)||this._tryParseMixinReference(!1)},t.prototype._parseEscaped=function(){if(this.peek(mo.EscapedJavaScript)||this.peek(mo.BadEscapedJavaScript)){var e=this.createNode(us.EscapedValue);return this.consumeToken(),this.finish(e)}if(this.peekDelim("~")){e=this.createNode(us.EscapedValue);return this.consumeToken(),this.accept(mo.String)||this.accept(mo.EscapedJavaScript)?this.finish(e):this.finish(e,Pl.TermExpected)}return null},t.prototype._parseOperator=function(){var t=this._parseGuardOperator();return t||e.prototype._parseOperator.call(this)},t.prototype._parseGuardOperator=function(){if(this.peekDelim(">")){var e=this.createNode(us.Operator);return this.consumeToken(),this.acceptDelim("="),e}if(this.peekDelim("=")){e=this.createNode(us.Operator);return this.consumeToken(),this.acceptDelim("<"),e}if(this.peekDelim("<")){e=this.createNode(us.Operator);return this.consumeToken(),this.acceptDelim("="),e}return null},t.prototype._parseRuleSetDeclaration=function(){return this.peek(mo.AtKeyword)?this._parseKeyframe()||this._parseMedia(!0)||this._parseImport()||this._parseSupports(!0)||this._parseDetachedRuleSetMixin()||this._parseVariableDeclaration()||e.prototype._parseRuleSetDeclarationAtStatement.call(this):this._tryParseMixinDeclaration()||this._tryParseRuleset(!0)||this._tryParseMixinReference()||this._parseFunction()||this._parseExtend()||e.prototype._parseRuleSetDeclaration.call(this)},t.prototype._parseKeyframeIdent=function(){return this._parseIdent([fs.Keyframe])||this._parseVariable()},t.prototype._parseKeyframeSelector=function(){return this._parseDetachedRuleSetMixin()||e.prototype._parseKeyframeSelector.call(this)},t.prototype._parseSimpleSelectorBody=function(){return this._parseSelectorCombinator()||e.prototype._parseSimpleSelectorBody.call(this)},t.prototype._parseSelector=function(e){var t=this.create(Ds),n=!1;for(e&&(n=t.addChild(this._parseCombinator()));t.addChild(this._parseSimpleSelector());){n=!0;var i=this.mark();if(t.addChild(this._parseGuard())&&this.peek(mo.CurlyL))break;this.restoreAtMark(i),t.addChild(this._parseCombinator())}return n?this.finish(t):null},t.prototype._parseSelectorCombinator=function(){if(this.peekDelim("&")){var e=this.createNode(us.SelectorCombinator);for(this.consumeToken();!this.hasWhitespace()&&(this.acceptDelim("-")||this.accept(mo.Num)||this.accept(mo.Dimension)||e.addChild(this._parseIdent())||this.acceptDelim("&")););return this.finish(e)}return null},t.prototype._parseSelectorIdent=function(){if(!this.peekInterpolatedIdent())return null;var e=this.createNode(us.SelectorInterpolation);return this._acceptInterpolatedIdent(e)?this.finish(e):null},t.prototype._parsePropertyIdentifier=function(e){void 0===e&&(e=!1);var t=/^[\w-]+/;if(!this.peekInterpolatedIdent()&&!this.peekRegExp(this.token.type,t))return null;var n=this.mark(),i=this.create(ks);i.isCustomProperty=this.acceptDelim("-")&&this.acceptDelim("-");return(e?i.isCustomProperty?i.addChild(this._parseIdent()):i.addChild(this._parseRegexp(t)):i.isCustomProperty?this._acceptInterpolatedIdent(i):this._acceptInterpolatedIdent(i,t))?(e||this.hasWhitespace()||(this.acceptDelim("+"),this.hasWhitespace()||this.acceptIdent("_")),this.finish(i)):(this.restoreAtMark(n),null)},t.prototype.peekInterpolatedIdent=function(){return this.peek(mo.Ident)||this.peekDelim("@")||this.peekDelim("$")||this.peekDelim("-")},t.prototype._acceptInterpolatedIdent=function(e,t){for(var n=this,i=!1,r=function(){var e=n.mark();return n.acceptDelim("-")&&(n.hasWhitespace()||n.acceptDelim("-"),n.hasWhitespace())?(n.restoreAtMark(e),null):n._parseInterpolation()},o=t?function(){return n.acceptRegexp(t)}:function(){return n.accept(mo.Ident)};(o()||e.addChild(this._parseInterpolation()||this.try(r)))&&(i=!0,!this.hasWhitespace()););return i},t.prototype._parseInterpolation=function(){var e=this.mark();if(this.peekDelim("@")||this.peekDelim("$")){var t=this.createNode(us.Interpolation);return this.consumeToken(),this.hasWhitespace()||!this.accept(mo.CurlyL)?(this.restoreAtMark(e),null):t.addChild(this._parseIdent())?this.accept(mo.CurlyR)?this.finish(t):this.finish(t,Pl.RightCurlyExpected):this.finish(t,Pl.IdentifierExpected)}return null},t.prototype._tryParseMixinDeclaration=function(){var e=this.mark(),t=this.create(Da);if(!t.setIdentifier(this._parseMixinDeclarationIdentifier())||!this.accept(mo.ParenthesisL))return this.restoreAtMark(e),null;if(t.getParameters().addChild(this._parseMixinParameter()))for(;(this.accept(mo.Comma)||this.accept(mo.SemiColon))&&!this.peek(mo.ParenthesisR);)t.getParameters().addChild(this._parseMixinParameter())||this.markError(t,Pl.IdentifierExpected,[],[mo.ParenthesisR]);return this.accept(mo.ParenthesisR)?(t.setGuard(this._parseGuard()),this.peek(mo.CurlyL)?this._parseBody(t,this._parseMixInBodyDeclaration.bind(this)):(this.restoreAtMark(e),null)):(this.restoreAtMark(e),null)},t.prototype._parseMixInBodyDeclaration=function(){return this._parseFontFace()||this._parseRuleSetDeclaration()},t.prototype._parseMixinDeclarationIdentifier=function(){var e;if(this.peekDelim("#")||this.peekDelim(".")){if(e=this.create(ks),this.consumeToken(),this.hasWhitespace()||!e.addChild(this._parseIdent()))return null}else{if(!this.peek(mo.Hash))return null;e=this.create(ks),this.consumeToken()}return e.referenceTypes=[fs.Mixin],this.finish(e)},t.prototype._parsePseudo=function(){if(!this.peek(mo.Colon))return null;var t=this.mark(),n=this.create(Ea);return this.consumeToken(),this.acceptIdent("extend")?this._completeExtends(n):(this.restoreAtMark(t),e.prototype._parsePseudo.call(this))},t.prototype._parseExtend=function(){if(!this.peekDelim("&"))return null;var e=this.mark(),t=this.create(Ea);return this.consumeToken(),!this.hasWhitespace()&&this.accept(mo.Colon)&&this.acceptIdent("extend")?this._completeExtends(t):(this.restoreAtMark(e),null)},t.prototype._completeExtends=function(e){if(!this.accept(mo.ParenthesisL))return this.finish(e,Pl.LeftParenthesisExpected);var t=e.getSelectors();if(!t.addChild(this._parseSelector(!0)))return this.finish(e,Pl.SelectorExpected);for(;this.accept(mo.Comma);)if(!t.addChild(this._parseSelector(!0)))return this.finish(e,Pl.SelectorExpected);return this.accept(mo.ParenthesisR)?this.finish(e):this.finish(e,Pl.RightParenthesisExpected)},t.prototype._parseDetachedRuleSetMixin=function(){if(!this.peek(mo.AtKeyword))return null;var e=this.mark(),t=this.create(Fa);return!t.addChild(this._parseVariable(!0))||!this.hasWhitespace()&&this.accept(mo.ParenthesisL)?this.accept(mo.ParenthesisR)?this.finish(t):this.finish(t,Pl.RightParenthesisExpected):(this.restoreAtMark(e),null)},t.prototype._tryParseMixinReference=function(e){void 0===e&&(e=!0);for(var t=this.mark(),n=this.create(Fa),i=this._parseMixinDeclarationIdentifier();i;){this.acceptDelim(">");var r=this._parseMixinDeclarationIdentifier();if(!r)break;n.getNamespaces().addChild(i),i=r}if(!n.setIdentifier(i))return this.restoreAtMark(t),null;var o=!1;if(this.accept(mo.ParenthesisL)){if(o=!0,n.getArguments().addChild(this._parseMixinArgument()))for(;(this.accept(mo.Comma)||this.accept(mo.SemiColon))&&!this.peek(mo.ParenthesisR);)if(!n.getArguments().addChild(this._parseMixinArgument()))return this.finish(n,Pl.ExpressionExpected);if(!this.accept(mo.ParenthesisR))return this.finish(n,Pl.RightParenthesisExpected);i.referenceTypes=[fs.Mixin]}else i.referenceTypes=[fs.Mixin,fs.Rule];return this.peek(mo.BracketL)?e||this._addLookupChildren(n):n.addChild(this._parsePrio()),o||this.peek(mo.SemiColon)||this.peek(mo.CurlyR)||this.peek(mo.EOF)?this.finish(n):(this.restoreAtMark(t),null)},t.prototype._parseMixinArgument=function(){var e=this.create(Ws),t=this.mark(),n=this._parseVariable();return n&&(this.accept(mo.Colon)?e.setIdentifier(n):this.restoreAtMark(t)),e.setValue(this._parseDetachedRuleSet()||this._parseExpr(!0))?this.finish(e):(this.restoreAtMark(t),null)},t.prototype._parseMixinParameter=function(){var e=this.create(Os);if(this.peekKeyword("@rest")){var t=this.create(Ss);return this.consumeToken(),this.accept(Zp)?(e.setIdentifier(this.finish(t)),this.finish(e)):this.finish(e,Pl.DotExpected,[],[mo.Comma,mo.ParenthesisR])}if(this.peek(Zp)){var n=this.create(Ss);return this.consumeToken(),e.setIdentifier(this.finish(n)),this.finish(e)}var i=!1;return e.setIdentifier(this._parseVariable())&&(this.accept(mo.Colon),i=!0),e.setDefaultValue(this._parseDetachedRuleSet()||this._parseExpr(!0))||i?this.finish(e):null},t.prototype._parseGuard=function(){if(!this.peekIdent("when"))return null;var e=this.create(Ma);if(this.consumeToken(),e.isNegated=this.acceptIdent("not"),!e.getConditions().addChild(this._parseGuardCondition()))return this.finish(e,Pl.ConditionExpected);for(;this.acceptIdent("and")||this.accept(mo.Comma);)if(!e.getConditions().addChild(this._parseGuardCondition()))return this.finish(e,Pl.ConditionExpected);return this.finish(e)},t.prototype._parseGuardCondition=function(){if(!this.peek(mo.ParenthesisL))return null;var e=this.create(za);return this.consumeToken(),e.addChild(this._parseExpr()),this.accept(mo.ParenthesisR)?this.finish(e):this.finish(e,Pl.RightParenthesisExpected)},t.prototype._parseFunction=function(){var e=this.mark(),t=this.create(Ps);if(!t.setIdentifier(this._parseFunctionIdentifier()))return null;if(this.hasWhitespace()||!this.accept(mo.ParenthesisL))return this.restoreAtMark(e),null;if(t.getArguments().addChild(this._parseMixinArgument()))for(;(this.accept(mo.Comma)||this.accept(mo.SemiColon))&&!this.peek(mo.ParenthesisR);)if(!t.getArguments().addChild(this._parseMixinArgument()))return this.finish(t,Pl.ExpressionExpected);return this.accept(mo.ParenthesisR)?this.finish(t):this.finish(t,Pl.RightParenthesisExpected)},t.prototype._parseFunctionIdentifier=function(){if(this.peekDelim("%")){var t=this.create(ks);return t.referenceTypes=[fs.Function],this.consumeToken(),this.finish(t)}return e.prototype._parseFunctionIdentifier.call(this)},t.prototype._parseURLArgument=function(){var t=this.mark(),n=e.prototype._parseURLArgument.call(this);if(!n||!this.peek(mo.ParenthesisR)){this.restoreAtMark(t);var i=this.create(Ss);return i.addChild(this._parseBinaryExpr()),this.finish(i)}return n},t}(Lh),iu=function(){var e=function(t,n){return(e=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)Object.prototype.hasOwnProperty.call(t,n)&&(e[n]=t[n])})(t,n)};return function(t,n){if("function"!==typeof n&&null!==n)throw new TypeError("Class extends value "+String(n)+" is not a constructor or null");function i(){this.constructor=t}e(t,n),t.prototype=null===n?Object.create(n):(i.prototype=n.prototype,new i)}}(),ru=Wa(),ou=function(e){function t(t,n){return e.call(this,"@",t,n)||this}return iu(t,e),t.prototype.createFunctionProposals=function(e,t,n,i){for(var r=0,o=e;r 50%"),example:"percentage(@number);",type:"percentage"},{name:"round",description:ru("less.builtin.round","rounds a number to a number of places"),example:"round(number, [places: 0]);"},{name:"sqrt",description:ru("less.builtin.sqrt","calculates square root of a number"),example:"sqrt(number);"},{name:"sin",description:ru("less.builtin.sin","sine function"),example:"sin(number);"},{name:"tan",description:ru("less.builtin.tan","tangent function"),example:"tan(number);"},{name:"atan",description:ru("less.builtin.atan","arctangent - inverse of tangent function"),example:"atan(number);"},{name:"pi",description:ru("less.builtin.pi","returns pi"),example:"pi();"},{name:"pow",description:ru("less.builtin.pow","first argument raised to the power of the second argument"),example:"pow(@base, @exponent);"},{name:"mod",description:ru("less.builtin.mod","first argument modulus second argument"),example:"mod(number, number);"},{name:"min",description:ru("less.builtin.min","returns the lowest of one or more values"),example:"min(@x, @y);"},{name:"max",description:ru("less.builtin.max","returns the lowest of one or more values"),example:"max(@x, @y);"}],t.colorProposals=[{name:"argb",example:"argb(@color);",description:ru("less.builtin.argb","creates a #AARRGGBB")},{name:"hsl",example:"hsl(@hue, @saturation, @lightness);",description:ru("less.builtin.hsl","creates a color")},{name:"hsla",example:"hsla(@hue, @saturation, @lightness, @alpha);",description:ru("less.builtin.hsla","creates a color")},{name:"hsv",example:"hsv(@hue, @saturation, @value);",description:ru("less.builtin.hsv","creates a color")},{name:"hsva",example:"hsva(@hue, @saturation, @value, @alpha);",description:ru("less.builtin.hsva","creates a color")},{name:"hue",example:"hue(@color);",description:ru("less.builtin.hue","returns the `hue` channel of `@color` in the HSL space")},{name:"saturation",example:"saturation(@color);",description:ru("less.builtin.saturation","returns the `saturation` channel of `@color` in the HSL space")},{name:"lightness",example:"lightness(@color);",description:ru("less.builtin.lightness","returns the `lightness` channel of `@color` in the HSL space")},{name:"hsvhue",example:"hsvhue(@color);",description:ru("less.builtin.hsvhue","returns the `hue` channel of `@color` in the HSV space")},{name:"hsvsaturation",example:"hsvsaturation(@color);",description:ru("less.builtin.hsvsaturation","returns the `saturation` channel of `@color` in the HSV space")},{name:"hsvvalue",example:"hsvvalue(@color);",description:ru("less.builtin.hsvvalue","returns the `value` channel of `@color` in the HSV space")},{name:"red",example:"red(@color);",description:ru("less.builtin.red","returns the `red` channel of `@color`")},{name:"green",example:"green(@color);",description:ru("less.builtin.green","returns the `green` channel of `@color`")},{name:"blue",example:"blue(@color);",description:ru("less.builtin.blue","returns the `blue` channel of `@color`")},{name:"alpha",example:"alpha(@color);",description:ru("less.builtin.alpha","returns the `alpha` channel of `@color`")},{name:"luma",example:"luma(@color);",description:ru("less.builtin.luma","returns the `luma` value (perceptual brightness) of `@color`")},{name:"saturate",example:"saturate(@color, 10%);",description:ru("less.builtin.saturate","return `@color` 10% points more saturated")},{name:"desaturate",example:"desaturate(@color, 10%);",description:ru("less.builtin.desaturate","return `@color` 10% points less saturated")},{name:"lighten",example:"lighten(@color, 10%);",description:ru("less.builtin.lighten","return `@color` 10% points lighter")},{name:"darken",example:"darken(@color, 10%);",description:ru("less.builtin.darken","return `@color` 10% points darker")},{name:"fadein",example:"fadein(@color, 10%);",description:ru("less.builtin.fadein","return `@color` 10% points less transparent")},{name:"fadeout",example:"fadeout(@color, 10%);",description:ru("less.builtin.fadeout","return `@color` 10% points more transparent")},{name:"fade",example:"fade(@color, 50%);",description:ru("less.builtin.fade","return `@color` with 50% transparency")},{name:"spin",example:"spin(@color, 10);",description:ru("less.builtin.spin","return `@color` with a 10 degree larger in hue")},{name:"mix",example:"mix(@color1, @color2, [@weight: 50%]);",description:ru("less.builtin.mix","return a mix of `@color1` and `@color2`")},{name:"greyscale",example:"greyscale(@color);",description:ru("less.builtin.greyscale","returns a grey, 100% desaturated color")},{name:"contrast",example:"contrast(@color1, [@darkcolor: black], [@lightcolor: white], [@threshold: 43%]);",description:ru("less.builtin.contrast","return `@darkcolor` if `@color1 is> 43% luma` otherwise return `@lightcolor`, see notes")},{name:"multiply",example:"multiply(@color1, @color2);"},{name:"screen",example:"screen(@color1, @color2);"},{name:"overlay",example:"overlay(@color1, @color2);"},{name:"softlight",example:"softlight(@color1, @color2);"},{name:"hardlight",example:"hardlight(@color1, @color2);"},{name:"difference",example:"difference(@color1, @color2);"},{name:"exclusion",example:"exclusion(@color1, @color2);"},{name:"average",example:"average(@color1, @color2);"},{name:"negation",example:"negation(@color1, @color2);"}],t}(pd);function su(e,t){var n=function(e){function t(t){return e.positionAt(t.offset).line}function n(t){return e.positionAt(t.offset+t.len).line}function i(){switch(e.languageId){case"scss":return new zp;case"less":return new eu;default:return new ls}}function r(e,i){var r=t(e),o=n(e);return r!==o?{startLine:r,endLine:o,kind:i}:null}var o=[],s=[],a=i();a.ignoreComment=!1,a.setSource(e.getText());var l=a.scan(),c=null,h=function(){switch(l.type){case mo.CurlyL:case Np:s.push({line:t(l),type:"brace",isStart:!0});break;case mo.CurlyR:if(0!==s.length){if(!(p=au(s,"brace")))break;var i=n(l);"brace"===p.type&&(c&&n(c)!==i&&i--,p.line!==i&&o.push({startLine:p.line,endLine:i,kind:void 0}))}break;case mo.Comment:var h=function(e){return"#region"===e?{line:t(l),type:"comment",isStart:!0}:{line:n(l),type:"comment",isStart:!1}},d=function(t){var n=t.text.match(/^\s*\/\*\s*(#region|#endregion)\b\s*(.*?)\s*\*\//);if(n)return h(n[1]);if("scss"===e.languageId||"less"===e.languageId){var i=t.text.match(/^\s*\/\/\s*(#region|#endregion)\b\s*(.*?)\s*/);if(i)return h(i[1])}return null}(l);if(d)if(d.isStart)s.push(d);else{var p;if(!(p=au(s,"comment")))break;"comment"===p.type&&p.line!==d.line&&o.push({startLine:p.line,endLine:d.line,kind:"region"})}else{var u=r(l,"comment");u&&o.push(u)}}c=l,l=a.scan()};for(;l.type!==mo.EOF;)h();return o}(e);return function(e,t){var n=t&&t.rangeLimit||Number.MAX_VALUE,i=e.sort((function(e,t){var n=e.startLine-t.startLine;return 0===n&&(n=e.endLine-t.endLine),n})),r=[],o=-1;return i.forEach((function(e){e.startLine=0;n--)if(e[n].type===t&&e[n].isStart)return e.splice(n,1)[0];return null}!function(){var e=[,,function(e){function t(e){this.__parent=e,this.__character_count=0,this.__indent_count=-1,this.__alignment_count=0,this.__wrap_point_index=0,this.__wrap_point_character_count=0,this.__wrap_point_indent_count=-1,this.__wrap_point_alignment_count=0,this.__items=[]}function n(e,t){this.__cache=[""],this.__indent_size=e.indent_size,this.__indent_string=e.indent_char,e.indent_with_tabs||(this.__indent_string=new Array(e.indent_size+1).join(e.indent_char)),t=t||"",e.indent_level>0&&(t=new Array(e.indent_level+1).join(this.__indent_string)),this.__base_string=t,this.__base_string_length=t.length}function i(e,i){this.__indent_cache=new n(e,i),this.raw=!1,this._end_with_newline=e.end_with_newline,this.indent_size=e.indent_size,this.wrap_line_length=e.wrap_line_length,this.indent_empty_lines=e.indent_empty_lines,this.__lines=[],this.previous_line=null,this.current_line=null,this.next_line=new t(this),this.space_before_token=!1,this.non_breaking_space=!1,this.previous_token_wrapped=!1,this.__add_outputline()}t.prototype.clone_empty=function(){var e=new t(this.__parent);return e.set_indent(this.__indent_count,this.__alignment_count),e},t.prototype.item=function(e){return e<0?this.__items[this.__items.length+e]:this.__items[e]},t.prototype.has_match=function(e){for(var t=this.__items.length-1;t>=0;t--)if(this.__items[t].match(e))return!0;return!1},t.prototype.set_indent=function(e,t){this.is_empty()&&(this.__indent_count=e||0,this.__alignment_count=t||0,this.__character_count=this.__parent.get_indent_size(this.__indent_count,this.__alignment_count))},t.prototype._set_wrap_point=function(){this.__parent.wrap_line_length&&(this.__wrap_point_index=this.__items.length,this.__wrap_point_character_count=this.__character_count,this.__wrap_point_indent_count=this.__parent.next_line.__indent_count,this.__wrap_point_alignment_count=this.__parent.next_line.__alignment_count)},t.prototype._should_wrap=function(){return this.__wrap_point_index&&this.__character_count>this.__parent.wrap_line_length&&this.__wrap_point_character_count>this.__parent.next_line.__character_count},t.prototype._allow_wrap=function(){if(this._should_wrap()){this.__parent.add_new_line();var e=this.__parent.current_line;return e.set_indent(this.__wrap_point_indent_count,this.__wrap_point_alignment_count),e.__items=this.__items.slice(this.__wrap_point_index),this.__items=this.__items.slice(0,this.__wrap_point_index),e.__character_count+=this.__character_count-this.__wrap_point_character_count,this.__character_count=this.__wrap_point_character_count," "===e.__items[0]&&(e.__items.splice(0,1),e.__character_count-=1),!0}return!1},t.prototype.is_empty=function(){return 0===this.__items.length},t.prototype.last=function(){return this.is_empty()?null:this.__items[this.__items.length-1]},t.prototype.push=function(e){this.__items.push(e);var t=e.lastIndexOf("\n");-1!==t?this.__character_count=e.length-t:this.__character_count+=e.length},t.prototype.pop=function(){var e=null;return this.is_empty()||(e=this.__items.pop(),this.__character_count-=e.length),e},t.prototype._remove_indent=function(){this.__indent_count>0&&(this.__indent_count-=1,this.__character_count-=this.__parent.indent_size)},t.prototype._remove_wrap_indent=function(){this.__wrap_point_indent_count>0&&(this.__wrap_point_indent_count-=1)},t.prototype.trim=function(){for(;" "===this.last();)this.__items.pop(),this.__character_count-=1},t.prototype.toString=function(){var e="";return this.is_empty()?this.__parent.indent_empty_lines&&(e=this.__parent.get_indent_string(this.__indent_count)):(e=this.__parent.get_indent_string(this.__indent_count,this.__alignment_count),e+=this.__items.join("")),e},n.prototype.get_indent_size=function(e,t){var n=this.__base_string_length;return t=t||0,e<0&&(n=0),n+=e*this.__indent_size,n+=t},n.prototype.get_indent_string=function(e,t){var n=this.__base_string;return t=t||0,e<0&&(e=0,n=""),t+=e*this.__indent_size,this.__ensure_cache(t),n+=this.__cache[t]},n.prototype.__ensure_cache=function(e){for(;e>=this.__cache.length;)this.__add_column()},n.prototype.__add_column=function(){var e=this.__cache.length,t=0,n="";this.__indent_size&&e>=this.__indent_size&&(e-=(t=Math.floor(e/this.__indent_size))*this.__indent_size,n=new Array(t+1).join(this.__indent_string)),e&&(n+=new Array(e+1).join(" ")),this.__cache.push(n)},i.prototype.__add_outputline=function(){this.previous_line=this.current_line,this.current_line=this.next_line.clone_empty(),this.__lines.push(this.current_line)},i.prototype.get_line_number=function(){return this.__lines.length},i.prototype.get_indent_string=function(e,t){return this.__indent_cache.get_indent_string(e,t)},i.prototype.get_indent_size=function(e,t){return this.__indent_cache.get_indent_size(e,t)},i.prototype.is_empty=function(){return!this.previous_line&&this.current_line.is_empty()},i.prototype.add_new_line=function(e){return!(this.is_empty()||!e&&this.just_added_newline())&&(this.raw||this.__add_outputline(),!0)},i.prototype.get_code=function(e){this.trim(!0);var t=this.current_line.pop();t&&("\n"===t[t.length-1]&&(t=t.replace(/\n+$/g,"")),this.current_line.push(t)),this._end_with_newline&&this.__add_outputline();var n=this.__lines.join("\n");return"\n"!==e&&(n=n.replace(/[\n]/g,e)),n},i.prototype.set_wrap_point=function(){this.current_line._set_wrap_point()},i.prototype.set_indent=function(e,t){return e=e||0,t=t||0,this.next_line.set_indent(e,t),this.__lines.length>1?(this.current_line.set_indent(e,t),!0):(this.current_line.set_indent(),!1)},i.prototype.add_raw_token=function(e){for(var t=0;t1&&this.current_line.is_empty();)this.__lines.pop(),this.current_line=this.__lines[this.__lines.length-1],this.current_line.trim();this.previous_line=this.__lines.length>1?this.__lines[this.__lines.length-2]:null},i.prototype.just_added_newline=function(){return this.current_line.is_empty()},i.prototype.just_added_blankline=function(){return this.is_empty()||this.current_line.is_empty()&&this.previous_line.is_empty()},i.prototype.ensure_empty_line_above=function(e,n){for(var i=this.__lines.length-2;i>=0;){var r=this.__lines[i];if(r.is_empty())break;if(0!==r.item(0).indexOf(e)&&r.item(-1)!==n){this.__lines.splice(i+1,0,new t(this)),this.previous_line=this.__lines[this.__lines.length-2];break}i--}},e.exports.Output=i},,,,function(e){function t(e,t){this.raw_options=n(e,t),this.disabled=this._get_boolean("disabled"),this.eol=this._get_characters("eol","auto"),this.end_with_newline=this._get_boolean("end_with_newline"),this.indent_size=this._get_number("indent_size",4),this.indent_char=this._get_characters("indent_char"," "),this.indent_level=this._get_number("indent_level"),this.preserve_newlines=this._get_boolean("preserve_newlines",!0),this.max_preserve_newlines=this._get_number("max_preserve_newlines",32786),this.preserve_newlines||(this.max_preserve_newlines=0),this.indent_with_tabs=this._get_boolean("indent_with_tabs","\t"===this.indent_char),this.indent_with_tabs&&(this.indent_char="\t",1===this.indent_size&&(this.indent_size=4)),this.wrap_line_length=this._get_number("wrap_line_length",this._get_number("max_char")),this.indent_empty_lines=this._get_boolean("indent_empty_lines"),this.templating=this._get_selection_list("templating",["auto","none","django","erb","handlebars","php","smarty"],["auto"])}function n(e,t){var n,r={};for(n in e=i(e))n!==t&&(r[n]=e[n]);if(t&&e[t])for(n in e[t])r[n]=e[t][n];return r}function i(e){var t,n={};for(t in e){n[t.replace(/-/g,"_")]=e[t]}return n}t.prototype._get_array=function(e,t){var n=this.raw_options[e],i=t||[];return"object"===typeof n?null!==n&&"function"===typeof n.concat&&(i=n.concat()):"string"===typeof n&&(i=n.split(/[^a-zA-Z0-9_\/\-]+/)),i},t.prototype._get_boolean=function(e,t){var n=this.raw_options[e];return void 0===n?!!t:!!n},t.prototype._get_characters=function(e,t){var n=this.raw_options[e],i=t||"";return"string"===typeof n&&(i=n.replace(/\\r/,"\r").replace(/\\n/,"\n").replace(/\\t/,"\t")),i},t.prototype._get_number=function(e,t){var n=this.raw_options[e];t=parseInt(t,10),isNaN(t)&&(t=0);var i=parseInt(n,10);return isNaN(i)&&(i=t),i},t.prototype._get_selection=function(e,t,n){var i=this._get_selection_list(e,t,n);if(1!==i.length)throw new Error("Invalid Option Value: The option '"+e+"' can only be one of the following values:\n"+t+"\nYou passed in: '"+this.raw_options[e]+"'");return i[0]},t.prototype._get_selection_list=function(e,t,n){if(!t||0===t.length)throw new Error("Selection list cannot be empty.");if(n=n||[t[0]],!this._is_valid_selection(n,t))throw new Error("Invalid Default Value!");var i=this._get_array(e,n);if(!this._is_valid_selection(i,t))throw new Error("Invalid Option Value: The option '"+e+"' can contain only the following values:\n"+t+"\nYou passed in: '"+this.raw_options[e]+"'");return i},t.prototype._is_valid_selection=function(e,t){return e.length&&t.length&&!e.some((function(e){return-1===t.indexOf(e)}))},e.exports.Options=t,e.exports.normalizeOpts=i,e.exports.mergeOpts=n},,function(e){var t=RegExp.prototype.hasOwnProperty("sticky");function n(e){this.__input=e||"",this.__input_length=this.__input.length,this.__position=0}n.prototype.restart=function(){this.__position=0},n.prototype.back=function(){this.__position>0&&(this.__position-=1)},n.prototype.hasNext=function(){return this.__position=0&&e=0&&t=e.length&&this.__input.substring(t-e.length,t).toLowerCase()===e},e.exports.InputScanner=n},,,,,function(e){function t(e,t){e="string"===typeof e?e:e.source,t="string"===typeof t?t:t.source,this.__directives_block_pattern=new RegExp(e+/ beautify( \w+[:]\w+)+ /.source+t,"g"),this.__directive_pattern=/ (\w+)[:](\w+)/g,this.__directives_end_ignore_pattern=new RegExp(e+/\sbeautify\signore:end\s/.source+t,"g")}t.prototype.get_directives=function(e){if(!e.match(this.__directives_block_pattern))return null;var t={};this.__directive_pattern.lastIndex=0;for(var n=this.__directive_pattern.exec(e);n;)t[n[1]]=n[2],n=this.__directive_pattern.exec(e);return t},t.prototype.readIgnored=function(e){return e.readUntilAfter(this.__directives_end_ignore_pattern)},e.exports.Directives=t},,function(e,t,n){var i=n(16).Beautifier,r=n(17).Options;e.exports=function(e,t){return new i(e,t).beautify()},e.exports.defaultOptions=function(){return new r}},function(e,t,n){var i=n(17).Options,r=n(2).Output,o=n(8).InputScanner,s=new(0,n(13).Directives)(/\/\*/,/\*\//),a=/\r\n|[\r\n]/,l=/\r\n|[\r\n]/g,c=/\s/,h=/(?:\s|\n)+/g,d=/\/\*(?:[\s\S]*?)((?:\*\/)|$)/g,p=/\/\/(?:[^\n\r\u2028\u2029]*)/g;function u(e,t){this._source_text=e||"",this._options=new i(t),this._ch=null,this._input=null,this.NESTED_AT_RULE={"@page":!0,"@font-face":!0,"@keyframes":!0,"@media":!0,"@supports":!0,"@document":!0},this.CONDITIONAL_GROUP_RULE={"@media":!0,"@supports":!0,"@document":!0}}u.prototype.eatString=function(e){var t="";for(this._ch=this._input.next();this._ch;){if(t+=this._ch,"\\"===this._ch)t+=this._input.next();else if(-1!==e.indexOf(this._ch)||"\n"===this._ch)break;this._ch=this._input.next()}return t},u.prototype.eatWhitespace=function(e){for(var t=c.test(this._input.peek()),n=0;c.test(this._input.peek());)this._ch=this._input.next(),e&&"\n"===this._ch&&(0===n||n0&&this._indentLevel--},u.prototype.beautify=function(){if(this._options.disabled)return this._source_text;var e=this._source_text,t=this._options.eol;"auto"===t&&(t="\n",e&&a.test(e||"")&&(t=e.match(a)[0]));var n=(e=e.replace(l,"\n")).match(/^[\t ]*/)[0];this._output=new r(this._options,n),this._input=new o(e),this._indentLevel=0,this._nestedLevel=0,this._ch=null;for(var i,u,m=0,f=!1,g=!1,b=!1,v=!1,y=!1,w=this._ch;i=""!==this._input.read(h),u=w,this._ch=this._input.next(),"\\"===this._ch&&this._input.hasNext()&&(this._ch+=this._input.next()),w=this._ch,this._ch;)if("/"===this._ch&&"*"===this._input.peek()){this._output.add_new_line(),this._input.back();var x=this._input.read(d),S=s.get_directives(x);S&&"start"===S.ignore&&(x+=s.readIgnored(this._input)),this.print_string(x),this.eatWhitespace(!0),this._output.add_new_line()}else if("/"===this._ch&&"/"===this._input.peek())this._output.space_before_token=!0,this._input.back(),this.print_string(this._input.read(p)),this.eatWhitespace(!0);else if("@"===this._ch)if(this.preserveSingleSpace(i),"{"===this._input.peek())this.print_string(this._ch+this.eatString("}"));else{this.print_string(this._ch);var C=this._input.peekUntilAfter(/[: ,;{}()[\]\/='"]/g);C.match(/[ :]$/)&&(C=this.eatString(": ").replace(/\s$/,""),this.print_string(C),this._output.space_before_token=!0),"extend"===(C=C.replace(/\s$/,""))?v=!0:"import"===C&&(y=!0),C in this.NESTED_AT_RULE?(this._nestedLevel+=1,C in this.CONDITIONAL_GROUP_RULE&&(b=!0)):f||0!==m||-1===C.indexOf(":")||(g=!0,this.indent())}else"#"===this._ch&&"{"===this._input.peek()?(this.preserveSingleSpace(i),this.print_string(this._ch+this.eatString("}"))):"{"===this._ch?(g&&(g=!1,this.outdent()),b?(b=!1,f=this._indentLevel>=this._nestedLevel):f=this._indentLevel>=this._nestedLevel-1,this._options.newline_between_rules&&f&&this._output.previous_line&&"{"!==this._output.previous_line.item(-1)&&this._output.ensure_empty_line_above("/",","),this._output.space_before_token=!0,"expand"===this._options.brace_style?(this._output.add_new_line(),this.print_string(this._ch),this.indent(),this._output.set_indent(this._indentLevel)):(this.indent(),this.print_string(this._ch)),this.eatWhitespace(!0),this._output.add_new_line()):"}"===this._ch?(this.outdent(),this._output.add_new_line(),"{"===u&&this._output.trim(!0),y=!1,v=!1,g&&(this.outdent(),g=!1),this.print_string(this._ch),f=!1,this._nestedLevel&&this._nestedLevel--,this.eatWhitespace(!0),this._output.add_new_line(),this._options.newline_between_rules&&!this._output.just_added_blankline()&&"}"!==this._input.peek()&&this._output.add_new_line(!0)):":"===this._ch?!f&&!b||this._input.lookBack("&")||this.foundNestedPseudoClass()||this._input.lookBack("(")||v||0!==m?(this._input.lookBack(" ")&&(this._output.space_before_token=!0),":"===this._input.peek()?(this._ch=this._input.next(),this.print_string("::")):this.print_string(":")):(this.print_string(":"),g||(g=!0,this._output.space_before_token=!0,this.eatWhitespace(!0),this.indent())):'"'===this._ch||"'"===this._ch?(this.preserveSingleSpace(i),this.print_string(this._ch+this.eatString(this._ch)),this.eatWhitespace(!0)):";"===this._ch?0===m?(g&&(this.outdent(),g=!1),v=!1,y=!1,this.print_string(this._ch),this.eatWhitespace(!0),"/"!==this._input.peek()&&this._output.add_new_line()):(this.print_string(this._ch),this.eatWhitespace(!0),this._output.space_before_token=!0):"("===this._ch?this._input.lookBack("url")?(this.print_string(this._ch),this.eatWhitespace(),m++,this.indent(),this._ch=this._input.next(),")"===this._ch||'"'===this._ch||"'"===this._ch?this._input.back():this._ch&&(this.print_string(this._ch+this.eatString(")")),m&&(m--,this.outdent()))):(this.preserveSingleSpace(i),this.print_string(this._ch),this.eatWhitespace(),m++,this.indent()):")"===this._ch?(m&&(m--,this.outdent()),this.print_string(this._ch)):","===this._ch?(this.print_string(this._ch),this.eatWhitespace(!0),!this._options.selector_separator_newline||g||0!==m||y||v?this._output.space_before_token=!0:this._output.add_new_line()):">"!==this._ch&&"+"!==this._ch&&"~"!==this._ch||g||0!==m?"]"===this._ch?this.print_string(this._ch):"["===this._ch?(this.preserveSingleSpace(i),this.print_string(this._ch)):"="===this._ch?(this.eatWhitespace(),this.print_string("="),c.test(this._ch)&&(this._ch="")):"!"!==this._ch||this._input.lookBack("\\")?(this.preserveSingleSpace(i),this.print_string(this._ch)):(this.print_string(" "),this.print_string(this._ch)):this._options.space_around_combinator?(this._output.space_before_token=!0,this.print_string(this._ch),this._output.space_before_token=!0):(this.print_string(this._ch),this.eatWhitespace(),this._ch&&c.test(this._ch)&&(this._ch=""));return this._output.get_code(t)},e.exports.Beautifier=u},function(e,t,n){var i=n(6).Options;function r(e){i.call(this,e,"css"),this.selector_separator_newline=this._get_boolean("selector_separator_newline",!0),this.newline_between_rules=this._get_boolean("newline_between_rules",!0);var t=this._get_boolean("space_around_selector_separator");this.space_around_combinator=this._get_boolean("space_around_combinator")||t;var n=this._get_selection_list("brace_style",["collapse","expand","end-expand","none","preserve-inline"]);this.brace_style="collapse";for(var r=0;r0&&fu(i,c-1);)c--;0===c||mu(i,c-1)?l=c:c=0;){var n=e.charCodeAt(t);if(n===du)return!0;if(n===pu)return!1;t--}return!1}(i,l),r=h===i.length,i=i.substring(l,h),0!==l){var p=e.offsetAt(Ba.create(t.start.line,0));o=function(e,t,n){var i=t,r=0,o=n.tabSize||4;for(;i0){var f=n.insertSpaces?ps(" ",a*o):ps("\t",o);m=m.split("\n").join("\n"+f),0===t.start.character&&(m=f+m)}return[{range:t,newText:m}]}function hu(e){return e.replace(/^\s+/,"")}var du="{".charCodeAt(0),pu="}".charCodeAt(0);function uu(e,t,n){if(e&&e.hasOwnProperty(t)){var i=e[t];if(null!==i)return i}return n}function mu(e,t){return-1!=="\r\n".indexOf(e.charAt(t))}function fu(e,t){return-1!==" \t".indexOf(e.charAt(t))}var gu={version:1.1,properties:[{name:"additive-symbols",browsers:["FF33"],syntax:"[ && ]#",relevance:50,description:"@counter-style descriptor. Specifies the symbols used by the marker-construction algorithm specified by the system descriptor. Needs to be specified if the counter system is 'additive'.",restrictions:["integer","string","image","identifier"]},{name:"align-content",values:[{name:"center",description:"Lines are packed toward the center of the flex container."},{name:"flex-end",description:"Lines are packed toward the end of the flex container."},{name:"flex-start",description:"Lines are packed toward the start of the flex container."},{name:"space-around",description:"Lines are evenly distributed in the flex container, with half-size spaces on either end."},{name:"space-between",description:"Lines are evenly distributed in the flex container."},{name:"stretch",description:"Lines stretch to take up the remaining space."}],syntax:"normal | | | ? ",relevance:62,description:"Aligns a flex container\u2019s lines within the flex container when there is extra space in the cross-axis, similar to how 'justify-content' aligns individual items within the main-axis.",restrictions:["enum"]},{name:"align-items",values:[{name:"baseline",description:"If the flex item\u2019s inline axis is the same as the cross axis, this value is identical to 'flex-start'. Otherwise, it participates in baseline alignment."},{name:"center",description:"The flex item\u2019s margin box is centered in the cross axis within the line."},{name:"flex-end",description:"The cross-end margin edge of the flex item is placed flush with the cross-end edge of the line."},{name:"flex-start",description:"The cross-start margin edge of the flex item is placed flush with the cross-start edge of the line."},{name:"stretch",description:"If the cross size property of the flex item computes to auto, and neither of the cross-axis margins are auto, the flex item is stretched."}],syntax:"normal | stretch | | [ ? ]",relevance:85,description:"Aligns flex items along the cross axis of the current line of the flex container.",restrictions:["enum"]},{name:"justify-items",values:[{name:"auto"},{name:"normal"},{name:"end"},{name:"start"},{name:"flex-end",description:'"Flex items are packed toward the end of the line."'},{name:"flex-start",description:'"Flex items are packed toward the start of the line."'},{name:"self-end",description:"The item is packed flush to the edge of the alignment container of the end side of the item, in the appropriate axis."},{name:"self-start",description:"The item is packed flush to the edge of the alignment container of the start side of the item, in the appropriate axis.."},{name:"center",description:"The items are packed flush to each other toward the center of the of the alignment container."},{name:"left"},{name:"right"},{name:"baseline"},{name:"first baseline"},{name:"last baseline"},{name:"stretch",description:"If the cross size property of the flex item computes to auto, and neither of the cross-axis margins are auto, the flex item is stretched."},{name:"save"},{name:"unsave"},{name:"legacy"}],syntax:"normal | stretch | | ? [ | left | right ] | legacy | legacy && [ left | right | center ]",relevance:53,description:"Defines the default justify-self for all items of the box, giving them the default way of justifying each box along the appropriate axis",restrictions:["enum"]},{name:"justify-self",values:[{name:"auto"},{name:"normal"},{name:"end"},{name:"start"},{name:"flex-end",description:'"Flex items are packed toward the end of the line."'},{name:"flex-start",description:'"Flex items are packed toward the start of the line."'},{name:"self-end",description:"The item is packed flush to the edge of the alignment container of the end side of the item, in the appropriate axis."},{name:"self-start",description:"The item is packed flush to the edge of the alignment container of the start side of the item, in the appropriate axis.."},{name:"center",description:"The items are packed flush to each other toward the center of the of the alignment container."},{name:"left"},{name:"right"},{name:"baseline"},{name:"first baseline"},{name:"last baseline"},{name:"stretch",description:"If the cross size property of the flex item computes to auto, and neither of the cross-axis margins are auto, the flex item is stretched."},{name:"save"},{name:"unsave"}],syntax:"auto | normal | stretch | | ? [ | left | right ]",relevance:53,description:"Defines the way of justifying a box inside its container along the appropriate axis.",restrictions:["enum"]},{name:"align-self",values:[{name:"auto",description:"Computes to the value of 'align-items' on the element\u2019s parent, or 'stretch' if the element has no parent. On absolutely positioned elements, it computes to itself."},{name:"baseline",description:"If the flex item\u2019s inline axis is the same as the cross axis, this value is identical to 'flex-start'. Otherwise, it participates in baseline alignment."},{name:"center",description:"The flex item\u2019s margin box is centered in the cross axis within the line."},{name:"flex-end",description:"The cross-end margin edge of the flex item is placed flush with the cross-end edge of the line."},{name:"flex-start",description:"The cross-start margin edge of the flex item is placed flush with the cross-start edge of the line."},{name:"stretch",description:"If the cross size property of the flex item computes to auto, and neither of the cross-axis margins are auto, the flex item is stretched."}],syntax:"auto | normal | stretch | | ? ",relevance:72,description:"Allows the default alignment along the cross axis to be overridden for individual flex items.",restrictions:["enum"]},{name:"all",browsers:["E79","FF27","S9.1","C37","O24"],values:[],syntax:"initial | inherit | unset | revert",relevance:53,references:[{name:"MDN Reference",url:"https://developer.mozilla.org/docs/Web/CSS/all"}],description:"Shorthand that resets all properties except 'direction' and 'unicode-bidi'.",restrictions:["enum"]},{name:"alt",browsers:["S9"],values:[],relevance:50,references:[{name:"MDN Reference",url:"https://developer.mozilla.org/docs/Web/CSS/alt"}],description:"Provides alternative text for assistive technology to replace the generated content of a ::before or ::after element.",restrictions:["string","enum"]},{name:"animation",values:[{name:"alternate",description:"The animation cycle iterations that are odd counts are played in the normal direction, and the animation cycle iterations that are even counts are played in a reverse direction."},{name:"alternate-reverse",description:"The animation cycle iterations that are odd counts are played in the reverse direction, and the animation cycle iterations that are even counts are played in a normal direction."},{name:"backwards",description:"The beginning property value (as defined in the first @keyframes at-rule) is applied before the animation is displayed, during the period defined by 'animation-delay'."},{name:"both",description:"Both forwards and backwards fill modes are applied."},{name:"forwards",description:"The final property value (as defined in the last @keyframes at-rule) is maintained after the animation completes."},{name:"infinite",description:"Causes the animation to repeat forever."},{name:"none",description:"No animation is performed"},{name:"normal",description:"Normal playback."},{name:"reverse",description:"All iterations of the animation are played in the reverse direction from the way they were specified."}],syntax:"#",relevance:82,references:[{name:"MDN Reference",url:"https://developer.mozilla.org/docs/Web/CSS/animation"}],description:"Shorthand property combines six of the animation properties into a single property.",restrictions:["time","timing-function","enum","identifier","number"]},{name:"animation-delay",syntax:"