From 6e64a1e5b3f2863cb63fd3c4d0f2e990ae118b72 Mon Sep 17 00:00:00 2001 From: MrFruitDude Date: Tue, 10 Mar 2026 11:01:37 -0400 Subject: [PATCH 1/2] fix: handle plugin path splitting and timeline wheel scroll --- include/xstudio/utility/string_helpers.hpp | 14 ++++- src/global_store/src/global_store.cpp | 4 +- .../src/plugin_manager_actor.cpp | 2 +- src/ui/qml/studio/src/qml_setup.cpp | 4 +- src/utility/test/string_helpers_test.cpp | 24 +++++++- ui/qml/xstudio/views/timeline/XsTimeline.qml | 56 ++++++++++++++++--- 6 files changed, 88 insertions(+), 16 deletions(-) diff --git a/include/xstudio/utility/string_helpers.hpp b/include/xstudio/utility/string_helpers.hpp index fed1569aa..5480a2a8e 100644 --- a/include/xstudio/utility/string_helpers.hpp +++ b/include/xstudio/utility/string_helpers.hpp @@ -121,6 +121,18 @@ namespace utility { return elems; } + inline constexpr char path_list_separator() { +#ifdef _WIN32 + return ';'; +#else + return ':'; +#endif + } + + inline std::vector split_path_list(const std::string &s) { + return split(s, path_list_separator()); + } + // not optimal.. inline bool starts_with(const std::string &haystack, const std::string &needle) { if (haystack.size() < needle.size()) @@ -361,4 +373,4 @@ namespace utility { } // namespace utility -} // namespace xstudio \ No newline at end of file +} // namespace xstudio diff --git a/src/global_store/src/global_store.cpp b/src/global_store/src/global_store.cpp index f97ca826a..f45abf249 100644 --- a/src/global_store/src/global_store.cpp +++ b/src/global_store/src/global_store.cpp @@ -88,7 +88,7 @@ bool xstudio::global_store::load_preferences( // folders char *plugin_path = std::getenv("XSTUDIO_PLUGIN_PATH"); if (plugin_path) { - for (const auto &p : xstudio::utility::split(plugin_path, ':')) { + for (const auto &p : xstudio::utility::split_path_list(plugin_path)) { if (fs::is_directory(p + "/preferences")) preference_load_defaults(prefs, p + "/preferences"); } @@ -441,4 +441,4 @@ utility::JsonStore GlobalStoreHelper::get_existing_or_create_new_preference( JsonStoreHelper::set(v, path, async, broadcast_change); } return default_; -} \ No newline at end of file +} diff --git a/src/plugin_manager/src/plugin_manager_actor.cpp b/src/plugin_manager/src/plugin_manager_actor.cpp index ccf87c81e..09876757a 100644 --- a/src/plugin_manager/src/plugin_manager_actor.cpp +++ b/src/plugin_manager/src/plugin_manager_actor.cpp @@ -30,7 +30,7 @@ PluginManagerActor::PluginManagerActor(caf::actor_config &cfg) : caf::event_base // xstudio plugins char *plugin_path = std::getenv("XSTUDIO_PLUGIN_PATH"); if (plugin_path) { - for (const auto &p : xstudio::utility::split(plugin_path, ':')) { + for (const auto &p : xstudio::utility::split_path_list(plugin_path)) { manager_.emplace_front_path(p); } } diff --git a/src/ui/qml/studio/src/qml_setup.cpp b/src/ui/qml/studio/src/qml_setup.cpp index c8bac5c08..38497eb87 100644 --- a/src/ui/qml/studio/src/qml_setup.cpp +++ b/src/ui/qml/studio/src/qml_setup.cpp @@ -130,7 +130,7 @@ void xstudio::ui::qml::setup_xstudio_qml_emgine(QQmlEngine *engine, caf::actor_s // with plugins char *plugin_path = std::getenv("XSTUDIO_PLUGIN_PATH"); if (plugin_path) { - for (const auto &p : xstudio::utility::split(plugin_path, ':')) { + for (const auto &p : xstudio::utility::split_path_list(plugin_path)) { // note - some xSTUDIO plugins have the backend plugin component // and a Qt/QML plugin component built into the same binary. @@ -144,4 +144,4 @@ void xstudio::ui::qml::setup_xstudio_qml_emgine(QQmlEngine *engine, caf::actor_s engine->addImportPath(QStringFromStd(p + "/qml")); } } -} \ No newline at end of file +} diff --git a/src/utility/test/string_helpers_test.cpp b/src/utility/test/string_helpers_test.cpp index 9445ee85e..12662916c 100644 --- a/src/utility/test/string_helpers_test.cpp +++ b/src/utility/test/string_helpers_test.cpp @@ -52,4 +52,26 @@ TEST(StringHelpersTest, split_vector) { EXPECT_EQ(split11[0].size(), 10); EXPECT_EQ(split11[0][0], 1); EXPECT_EQ(split11[0][9], 10); -} \ No newline at end of file +} + +TEST(StringHelpersTest, path_list_separator) { +#ifdef _WIN32 + EXPECT_EQ(path_list_separator(), ';'); +#else + EXPECT_EQ(path_list_separator(), ':'); +#endif +} + +TEST(StringHelpersTest, split_path_list) { +#ifdef _WIN32 + const auto paths = split_path_list("C:\\plugins;D:\\plugins"); + ASSERT_EQ(paths.size(), 2); + EXPECT_EQ(paths[0], "C:\\plugins"); + EXPECT_EQ(paths[1], "D:\\plugins"); +#else + const auto paths = split_path_list("/tmp/plugins:/opt/plugins"); + ASSERT_EQ(paths.size(), 2); + EXPECT_EQ(paths[0], "/tmp/plugins"); + EXPECT_EQ(paths[1], "/opt/plugins"); +#endif +} diff --git a/ui/qml/xstudio/views/timeline/XsTimeline.qml b/ui/qml/xstudio/views/timeline/XsTimeline.qml index 6a5789b80..0ebcea8ed 100644 --- a/ui/qml/xstudio/views/timeline/XsTimeline.qml +++ b/ui/qml/xstudio/views/timeline/XsTimeline.qml @@ -1391,6 +1391,43 @@ Rectangle { property var initialValue: 0 property real minScaleX: 0 + function wheelDelta(pixelDelta, angleDelta) { + return pixelDelta !== 0 ? pixelDelta : angleDelta + } + + function scrollTimelineHorizontally(deltaX) { + let stackItem = list_view.itemAtIndex(0) + if( + !stackItem || + Math.abs(deltaX) < 1 || + stackItem.scrollbar.size >= 1.0 || + stackItem.scrollbar.width <= 0 + ) { + return false + } + + let positionDelta = (stackItem.scrollbar.size / stackItem.scrollbar.width) * deltaX + stackItem.jumpToPosition(stackItem.currentPosition() + positionDelta) + return true + } + + function scrollTimelineVertically(deltaY) { + if( + hovered == null || + Math.abs(deltaY) < 1 || + !["Video Track", "Audio Track", "Gap", "Clip"].includes(hovered.itemTypeRole) + ) { + return false + } + + if(["Video Track", "Audio Track"].includes(hovered.itemTypeRole)) + hovered.parentLV.flick(0, deltaY > 0 ? 500 : -500) + else if(["Gap", "Clip"].includes(hovered.itemTypeRole)) + hovered.parentLV.parentLV.flick(0, deltaY > 0 ? 500 : -500) + + return true + } + Rectangle { id: region visible: ma.isRegionSelection @@ -1600,16 +1637,18 @@ Rectangle { } onWheel: wheel => { + let deltaX = wheelDelta(wheel.pixelDelta.x, wheel.angleDelta.x) + let deltaY = wheelDelta(wheel.pixelDelta.y, wheel.angleDelta.y) // maintain position as we zoom.. if(wheel.modifiers == Qt.ShiftModifier) { // wheel.angleDelta.y always return 0 on MacOS laptops // when SHIFT is pressed and a mouse wheel is used, but in // that case the x component is updating and usable. - let deltaY = wheel.angleDelta.y == 0 ? wheel.angleDelta.x : wheel.angleDelta.y + let zoomDelta = deltaY == 0 ? deltaX : deltaY // Limit the scale to keep it within a usable range and // avoid a negative scaleY value. - if(deltaY > 1) { + if(zoomDelta > 1) { scaleY = Math.min(2.0, scaleY + 0.2) } else { scaleY = Math.max(0.6, scaleY - 0.2) @@ -1617,7 +1656,8 @@ Rectangle { wheel.accepted = true } else if(wheel.modifiers == Qt.ControlModifier) { let tmp = scaleX - if(wheel.angleDelta.y > 1) { + let zoomDelta = deltaY == 0 ? deltaX : deltaY + if(zoomDelta > 1) { tmp += 0.2 } else { tmp -= 0.2 @@ -1625,11 +1665,9 @@ Rectangle { scaleX = Math.max((list_view.width - trackHeaderWidth) / theSessionData.timelineRect([timeline_items.rootIndex]).width, tmp) list_view.itemAtIndex(0).jumpToFrame(timelinePlayhead.logicalFrame, ListView.Center) wheel.accepted = true - } else if(hovered != null && ["Video Track", "Audio Track","Gap","Clip"].includes(hovered.itemTypeRole)) { - if(["Video Track", "Audio Track"].includes(hovered.itemTypeRole)) - hovered.parentLV.flick(0, wheel.angleDelta.y > 1 ? 500 : -500) - else if(["Gap", "Clip"].includes(hovered.itemTypeRole)) - hovered.parentLV.parentLV.flick(0, wheel.angleDelta.y > 1 ? 500 : -500) + } else if(Math.abs(deltaX) > Math.abs(deltaY)) { + wheel.accepted = scrollTimelineHorizontally(deltaX) + } else if(scrollTimelineVertically(deltaY)) { wheel.accepted = true } else { wheel.accepted = false @@ -2224,4 +2262,4 @@ Rectangle { // } // } -} \ No newline at end of file +} From 961582f0bac25d276e4467986d61f08755badd80 Mon Sep 17 00:00:00 2001 From: MrFruitDude Date: Tue, 10 Mar 2026 11:01:58 -0400 Subject: [PATCH 2/2] Improve perf tooling and media/scanner throughput Signed-off-by: MrFruitDude --- CMakeLists.txt | 50 ++ CMakePresets.json | 777 ++++++++++++++++-- README.md | 31 +- .../cacheing_media_reader_actor.hpp | 81 +- .../media_reader/frame_request_queue.hpp | 34 +- include/xstudio/scanner/scanner_actor.hpp | 18 +- scripts/perf/baseline.py | 142 ++++ share/preference/core_media_reader.json | 38 +- share/preference/core_scanner.json | 18 + .../src/cacheing_media_reader_actor.cpp | 342 ++++++-- src/media_reader/src/frame_request_queue.cpp | 173 ++-- .../test/frame_request_queue_test.cpp | 143 ++++ src/scanner/src/scanner_actor.cpp | 135 +-- src/scanner/test/scanner_actor_test.cpp | 71 +- 14 files changed, 1743 insertions(+), 310 deletions(-) create mode 100644 scripts/perf/baseline.py create mode 100644 share/preference/core_scanner.json create mode 100644 src/media_reader/test/frame_request_queue_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1206e9cad..0a596c1bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,34 @@ option(BUILD_RESKIN "Build xstudio reskin binary" ON) option(OTIO_SUBMODULE "Automatically build OpenTimelineIO as a submodule" OFF) option(USE_VCPKG "Use Vcpkg for package management" OFF) option(BUILD_PYSIDE_WIDGETS "Build xstudio player as PySide widget" OFF) +option(XSTUDIO_ENABLE_COMPILER_CACHE "Use ccache or sccache when available." ON) +option(XSTUDIO_ENABLE_PERF_TARGETS "Expose perf baseline helper targets." ON) + +set( + XSTUDIO_PREFERRED_COMPILER_CACHE + "" + CACHE STRING + "Preferred compiler cache executable to use (for example 'sccache' or 'ccache').") + +if(XSTUDIO_ENABLE_COMPILER_CACHE AND + NOT CMAKE_C_COMPILER_LAUNCHER AND + NOT CMAKE_CXX_COMPILER_LAUNCHER) + if(XSTUDIO_PREFERRED_COMPILER_CACHE) + find_program(XSTUDIO_COMPILER_CACHE_PROGRAM NAMES ${XSTUDIO_PREFERRED_COMPILER_CACHE}) + else() + find_program(XSTUDIO_COMPILER_CACHE_PROGRAM NAMES sccache ccache) + endif() + + if(XSTUDIO_COMPILER_CACHE_PROGRAM) + set( + CMAKE_C_COMPILER_LAUNCHER "${XSTUDIO_COMPILER_CACHE_PROGRAM}" + CACHE STRING "C compiler launcher" FORCE) + set( + CMAKE_CXX_COMPILER_LAUNCHER "${XSTUDIO_COMPILER_CACHE_PROGRAM}" + CACHE STRING "CXX compiler launcher" FORCE) + message(STATUS "Using compiler cache launcher: ${XSTUDIO_COMPILER_CACHE_PROGRAM}") + endif() +endif() if(WIN32) set(USE_VCPKG ON) @@ -240,6 +268,28 @@ endif() add_subdirectory(src) +if(XSTUDIO_ENABLE_PERF_TARGETS) + find_package(Python COMPONENTS Interpreter QUIET) + if(Python_Interpreter_FOUND) + add_custom_target( + perf-baseline + COMMAND + "${Python_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/scripts/perf/baseline.py" + --source-dir + "${CMAKE_CURRENT_SOURCE_DIR}" + --build-dir + "${CMAKE_BINARY_DIR}" + --output + "${CMAKE_BINARY_DIR}/perf-baseline.json" + --command + "tests:${CMAKE_CTEST_COMMAND} --output-on-failure" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + USES_TERMINAL + COMMENT "Capture a JSON perf baseline for the current build tree") + endif() +endif() + if(INSTALL_XSTUDIO) # build quickpromise diff --git a/CMakePresets.json b/CMakePresets.json index 833ec3b58..6e1952e95 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,149 +1,764 @@ { - "version": 3, + "version": 5, + "cmakeMinimumRequired": { + "major": 3, + "minor": 28, + "patch": 0 + }, "configurePresets": [ - { - "name": "default", + { + "name": "base", "hidden": true, - "binaryDir": "${sourceDir}/build", + "binaryDir": "${sourceDir}/build/${presetName}", + "generator": "Ninja", "cacheVariables": { - "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/../vcpkg/scripts/buildsystems/vcpkg.cmake", - "Qt6_DIR": "/Users/tedwaine/Qt6/6.5.3/macos/lib/cmake/Qt6", - "CMAKE_INSTALL_PREFIX": "xstudio_install", - "X_VCPKG_APPLOCAL_DEPS_INSTALL": "ON", "BUILD_DOCS": "OFF", - "USE_VCPKG": "ON" + "BUILD_TESTING": "ON", + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "CMAKE_INSTALL_PREFIX": "${sourceDir}/xstudio_install/${presetName}", + "ENABLE_CLANG_FORMAT": "OFF", + "ENABLE_CLANG_TIDY": "OFF", + "OPTIMIZE_FOR_NATIVE": "OFF", + "XSTUDIO_ENABLE_COMPILER_CACHE": "ON", + "XSTUDIO_ENABLE_PERF_TARGETS": "ON" } }, - { - "name": "windows-base", - "inherits": "default", - "condition": { - "type": "equals", - "lhs": "${hostSystemName}", - "rhs": "Windows" - }, + { + "name": "release-base", "hidden": true, - "generator": "Visual Studio 17 2022" + "inherits": "base", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } }, { - "name": "WinRelease", - "inherits": ["windows-base"], + "name": "dev-base", + "hidden": true, + "inherits": "base", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" + "CMAKE_BUILD_TYPE": "Debug" } }, { - "name": "WinRelWithDebInfo", - "inherits": ["windows-base"], + "name": "ci-base", + "hidden": true, + "inherits": "base", "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "USE_SANITIZER": "address" + "CMAKE_BUILD_TYPE": "RelWithDebInfo" } }, { - "name": "WinDebug", - "hidden": true, - "inherits": ["windows-base"], + "name": "perf-base", + "hidden": true, + "inherits": "base", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo" + } + }, + { + "name": "vcpkg-base", + "hidden": true, + "inherits": "base", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "USE_SANITIZER": "address" + "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", + "OTIO_SUBMODULE": "ON", + "USE_VCPKG": "ON", + "X_VCPKG_APPLOCAL_DEPS_INSTALL": "ON" } }, - { - "name": "macos-base-arm", + { + "name": "linux-base", + "hidden": true, "condition": { "type": "equals", "lhs": "${hostSystemName}", - "rhs": "Darwin" + "rhs": "Linux" }, - "inherits": "default", - "generator": "Unix Makefiles", + "inherits": "base", "cacheVariables": { - "VCPKG_OVERLAY_TRIPLETS" : "${sourceDir}/cmake/vcpkg_triplets", - "VCPKG_TARGET_TRIPLET": "arm-osx" + "Qt6_DIR": "$env{Qt6_DIR}" } }, - { - "name": "macos-base-intel", - "inherits": "macos-base-arm", + { + "name": "linux-vcpkg-base", + "hidden": true, + "inherits": [ + "linux-base", + "vcpkg-base" + ], "cacheVariables": { - "VCPKG_TARGET_TRIPLET": "x64-osx" + "VCPKG_TARGET_TRIPLET": "x64-xstudio-linux" } }, { - "name": "MacOSRelease", - "inherits": ["macos-base-arm"], + "name": "macos-arm-base", + "hidden": true, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + }, + "inherits": "base", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" + "Qt6_DIR": "$env{Qt6_DIR}" } }, { - "name": "MacOSRelWithDebInfo", - "inherits": ["macos-base-arm"], + "name": "macos-arm-vcpkg-base", + "hidden": true, + "inherits": [ + "macos-arm-base", + "vcpkg-base" + ], "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "USE_SANITIZER": "address" + "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/cmake/vcpkg_triplets", + "VCPKG_TARGET_TRIPLET": "arm-osx" } }, { - "name": "MacOSDebug", - "inherits": ["macos-base-arm"], + "name": "macos-intel-base", + "hidden": true, + "inherits": "macos-arm-base" + }, + { + "name": "macos-intel-vcpkg-base", + "hidden": true, + "inherits": [ + "macos-intel-base", + "vcpkg-base" + ], "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "USE_SANITIZER": "address" + "VCPKG_OVERLAY_TRIPLETS": "${sourceDir}/cmake/vcpkg_triplets", + "VCPKG_TARGET_TRIPLET": "x64-osx" } }, + { + "name": "windows-base", + "hidden": true, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + }, + "inherits": "base", + "generator": "Visual Studio 17 2022" + }, + { + "name": "windows-vcpkg-base", + "hidden": true, + "inherits": [ + "windows-base", + "vcpkg-base" + ] + }, + { + "name": "LinuxDev", + "inherits": [ + "linux-base", + "dev-base" + ] + }, + { + "name": "LinuxCI", + "inherits": [ + "linux-base", + "ci-base" + ] + }, + { + "name": "LinuxPerf", + "inherits": [ + "linux-base", + "perf-base" + ] + }, + { + "name": "LinuxDevVcpkg", + "inherits": [ + "linux-vcpkg-base", + "dev-base" + ] + }, + { + "name": "LinuxCIVcpkg", + "inherits": [ + "linux-vcpkg-base", + "ci-base" + ] + }, + { + "name": "LinuxPerfVcpkg", + "inherits": [ + "linux-vcpkg-base", + "perf-base" + ] + }, + { + "name": "MacOSArmDev", + "inherits": [ + "macos-arm-base", + "dev-base" + ] + }, + { + "name": "MacOSArmCI", + "inherits": [ + "macos-arm-base", + "ci-base" + ] + }, + { + "name": "MacOSArmPerf", + "inherits": [ + "macos-arm-base", + "perf-base" + ] + }, + { + "name": "MacOSArmDevVcpkg", + "inherits": [ + "macos-arm-vcpkg-base", + "dev-base" + ] + }, + { + "name": "MacOSArmCIVcpkg", + "inherits": [ + "macos-arm-vcpkg-base", + "ci-base" + ] + }, + { + "name": "MacOSArmPerfVcpkg", + "inherits": [ + "macos-arm-vcpkg-base", + "perf-base" + ] + }, + { + "name": "MacOSIntelDev", + "inherits": [ + "macos-intel-base", + "dev-base" + ] + }, + { + "name": "MacOSIntelCI", + "inherits": [ + "macos-intel-base", + "ci-base" + ] + }, + { + "name": "MacOSIntelPerf", + "inherits": [ + "macos-intel-base", + "perf-base" + ] + }, + { + "name": "MacOSIntelDevVcpkg", + "inherits": [ + "macos-intel-vcpkg-base", + "dev-base" + ] + }, + { + "name": "MacOSIntelCIVcpkg", + "inherits": [ + "macos-intel-vcpkg-base", + "ci-base" + ] + }, + { + "name": "MacOSIntelPerfVcpkg", + "inherits": [ + "macos-intel-vcpkg-base", + "perf-base" + ] + }, + { + "name": "WinDevVcpkg", + "inherits": [ + "windows-vcpkg-base", + "dev-base" + ] + }, + { + "name": "WinCIVcpkg", + "inherits": [ + "windows-vcpkg-base", + "ci-base" + ] + }, + { + "name": "WinPerfVcpkg", + "inherits": [ + "windows-vcpkg-base", + "perf-base" + ] + }, + { + "name": "LinuxRelease", + "inherits": [ + "linux-vcpkg-base", + "release-base" + ] + }, + { + "name": "LinuxRelWithDebInfo", + "inherits": [ + "linux-vcpkg-base", + "ci-base" + ] + }, + { + "name": "LinuxDebug", + "inherits": [ + "linux-vcpkg-base", + "dev-base" + ] + }, + { + "name": "MacOSRelease", + "inherits": [ + "macos-arm-vcpkg-base", + "release-base" + ] + }, + { + "name": "MacOSRelWithDebInfo", + "inherits": [ + "macos-arm-vcpkg-base", + "ci-base" + ] + }, + { + "name": "MacOSDebug", + "inherits": [ + "macos-arm-vcpkg-base", + "dev-base" + ] + }, { "name": "MacOSIntelRelease", - "inherits": ["macos-base-intel"], - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } + "inherits": [ + "macos-intel-vcpkg-base", + "release-base" + ] }, { "name": "MacOSIntelRelWithDebInfo", - "inherits": ["macos-base-intel"], - "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "USE_SANITIZER": "address" - } + "inherits": [ + "macos-intel-vcpkg-base", + "ci-base" + ] }, { "name": "MacOSIntelDebug", - "inherits": ["macos-base-intel"], - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "USE_SANITIZER": "address" + "inherits": [ + "macos-intel-vcpkg-base", + "dev-base" + ] + }, + { + "name": "WinRelease", + "inherits": [ + "windows-vcpkg-base", + "release-base" + ] + }, + { + "name": "WinRelWithDebInfo", + "inherits": [ + "windows-vcpkg-base", + "ci-base" + ] + }, + { + "name": "WinDebug", + "inherits": [ + "windows-vcpkg-base", + "dev-base" + ] + } + ], + "buildPresets": [ + { + "name": "LinuxDev", + "configurePreset": "LinuxDev" + }, + { + "name": "LinuxCI", + "configurePreset": "LinuxCI" + }, + { + "name": "LinuxPerf", + "configurePreset": "LinuxPerf" + }, + { + "name": "LinuxDevVcpkg", + "configurePreset": "LinuxDevVcpkg" + }, + { + "name": "LinuxCIVcpkg", + "configurePreset": "LinuxCIVcpkg" + }, + { + "name": "LinuxPerfVcpkg", + "configurePreset": "LinuxPerfVcpkg" + }, + { + "name": "MacOSArmDev", + "configurePreset": "MacOSArmDev" + }, + { + "name": "MacOSArmCI", + "configurePreset": "MacOSArmCI" + }, + { + "name": "MacOSArmPerf", + "configurePreset": "MacOSArmPerf" + }, + { + "name": "MacOSArmDevVcpkg", + "configurePreset": "MacOSArmDevVcpkg" + }, + { + "name": "MacOSArmCIVcpkg", + "configurePreset": "MacOSArmCIVcpkg" + }, + { + "name": "MacOSArmPerfVcpkg", + "configurePreset": "MacOSArmPerfVcpkg" + }, + { + "name": "MacOSIntelDev", + "configurePreset": "MacOSIntelDev" + }, + { + "name": "MacOSIntelCI", + "configurePreset": "MacOSIntelCI" + }, + { + "name": "MacOSIntelPerf", + "configurePreset": "MacOSIntelPerf" + }, + { + "name": "MacOSIntelDevVcpkg", + "configurePreset": "MacOSIntelDevVcpkg" + }, + { + "name": "MacOSIntelCIVcpkg", + "configurePreset": "MacOSIntelCIVcpkg" + }, + { + "name": "MacOSIntelPerfVcpkg", + "configurePreset": "MacOSIntelPerfVcpkg" + }, + { + "name": "WinDevVcpkg", + "configurePreset": "WinDevVcpkg", + "configuration": "Debug" + }, + { + "name": "WinCIVcpkg", + "configurePreset": "WinCIVcpkg", + "configuration": "RelWithDebInfo" + }, + { + "name": "WinPerfVcpkg", + "configurePreset": "WinPerfVcpkg", + "configuration": "RelWithDebInfo" + }, + { + "name": "LinuxRelease", + "configurePreset": "LinuxRelease" + }, + { + "name": "LinuxRelWithDebInfo", + "configurePreset": "LinuxRelWithDebInfo" + }, + { + "name": "LinuxDebug", + "configurePreset": "LinuxDebug" + }, + { + "name": "MacOSRelease", + "configurePreset": "MacOSRelease" + }, + { + "name": "MacOSRelWithDebInfo", + "configurePreset": "MacOSRelWithDebInfo" + }, + { + "name": "MacOSDebug", + "configurePreset": "MacOSDebug" + }, + { + "name": "MacOSIntelRelease", + "configurePreset": "MacOSIntelRelease" + }, + { + "name": "MacOSIntelRelWithDebInfo", + "configurePreset": "MacOSIntelRelWithDebInfo" + }, + { + "name": "MacOSIntelDebug", + "configurePreset": "MacOSIntelDebug" + }, + { + "name": "WinRelease", + "configurePreset": "WinRelease", + "configuration": "Release" + }, + { + "name": "WinRelWithDebInfo", + "configurePreset": "WinRelWithDebInfo", + "configuration": "RelWithDebInfo" + }, + { + "name": "WinDebug", + "configurePreset": "WinDebug", + "configuration": "Debug" + } + ], + "testPresets": [ + { + "name": "LinuxDev", + "configurePreset": "LinuxDev", + "output": { + "outputOnFailure": true } }, - { - "name": "linux-base", - "inherits": "default", - "cacheVariables": { - "VCPKG_TARGET_TRIPLET": "x64-xstudio-linux" + { + "name": "LinuxCI", + "configurePreset": "LinuxCI", + "output": { + "outputOnFailure": true + } + }, + { + "name": "LinuxPerf", + "configurePreset": "LinuxPerf", + "output": { + "outputOnFailure": true + } + }, + { + "name": "LinuxDevVcpkg", + "configurePreset": "LinuxDevVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "LinuxCIVcpkg", + "configurePreset": "LinuxCIVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "LinuxPerfVcpkg", + "configurePreset": "LinuxPerfVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSArmDev", + "configurePreset": "MacOSArmDev", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSArmCI", + "configurePreset": "MacOSArmCI", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSArmPerf", + "configurePreset": "MacOSArmPerf", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSArmDevVcpkg", + "configurePreset": "MacOSArmDevVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSArmCIVcpkg", + "configurePreset": "MacOSArmCIVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSArmPerfVcpkg", + "configurePreset": "MacOSArmPerfVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelDev", + "configurePreset": "MacOSIntelDev", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelCI", + "configurePreset": "MacOSIntelCI", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelPerf", + "configurePreset": "MacOSIntelPerf", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelDevVcpkg", + "configurePreset": "MacOSIntelDevVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelCIVcpkg", + "configurePreset": "MacOSIntelCIVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelPerfVcpkg", + "configurePreset": "MacOSIntelPerfVcpkg", + "output": { + "outputOnFailure": true + } + }, + { + "name": "WinDevVcpkg", + "configurePreset": "WinDevVcpkg", + "configuration": "Debug", + "output": { + "outputOnFailure": true + } + }, + { + "name": "WinCIVcpkg", + "configurePreset": "WinCIVcpkg", + "configuration": "RelWithDebInfo", + "output": { + "outputOnFailure": true + } + }, + { + "name": "WinPerfVcpkg", + "configurePreset": "WinPerfVcpkg", + "configuration": "RelWithDebInfo", + "output": { + "outputOnFailure": true } }, { "name": "LinuxRelease", - "inherits": ["linux-base"], - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" + "configurePreset": "LinuxRelease", + "output": { + "outputOnFailure": true } }, { "name": "LinuxRelWithDebInfo", - "inherits": ["linux-base"], - "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "USE_SANITIZER": "address" + "configurePreset": "LinuxRelWithDebInfo", + "output": { + "outputOnFailure": true } }, { "name": "LinuxDebug", - "inherits": ["linux-base"], - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "USE_SANITIZER": "address" + "configurePreset": "LinuxDebug", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSRelease", + "configurePreset": "MacOSRelease", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSRelWithDebInfo", + "configurePreset": "MacOSRelWithDebInfo", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSDebug", + "configurePreset": "MacOSDebug", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelRelease", + "configurePreset": "MacOSIntelRelease", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelRelWithDebInfo", + "configurePreset": "MacOSIntelRelWithDebInfo", + "output": { + "outputOnFailure": true + } + }, + { + "name": "MacOSIntelDebug", + "configurePreset": "MacOSIntelDebug", + "output": { + "outputOnFailure": true + } + }, + { + "name": "WinRelease", + "configurePreset": "WinRelease", + "configuration": "Release", + "output": { + "outputOnFailure": true + } + }, + { + "name": "WinRelWithDebInfo", + "configurePreset": "WinRelWithDebInfo", + "configuration": "RelWithDebInfo", + "output": { + "outputOnFailure": true + } + }, + { + "name": "WinDebug", + "configurePreset": "WinDebug", + "configuration": "Debug", + "output": { + "outputOnFailure": true } } ] diff --git a/README.md b/README.md index e6c534006..35ec926ef 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ -# Welcome to xSTUDIO - v1.1.0 +# Welcome to xSTUDIO 1.1.0 xSTUDIO is a media playback and review application designed for professionals working in the film and TV post production industries, particularly the Visual Effects and Feature Animation sectors. xSTUDIO is focused on providing an intuitive, easy to use interface with a high performance playback engine at its core and C++ and Python APIs for pipeline integration and customisation for total flexibility. -This codebase will build version 1.0.0 (alpha) of xstudio. There are some known issues that are currently being worked on: +This codebase builds xSTUDIO `v1.1.0`. The top-level documentation has been refreshed to match the current source tree and is split between the user guide in `docs/user_docs` and the developer reference in `docs/reference`. + +There are still some known issues that are currently being worked on: * Moderate audio distortion on playback (Windows only) -* Ueser Documentation and API Documentation is badly out-of-date. * Saved sessions might not restore media correctly (Windows only) ## Building xSTUDIO -This release of xSTUDIO can be built on various Linux flavours, Microsoft Windows and MacOS. We provide comprehensive build steps here. +This release of xSTUDIO can be built on various Linux distributions, Microsoft Windows, and macOS. We provide comprehensive build steps here. ### Building xSTUDIO for Linux @@ -23,10 +24,24 @@ This release of xSTUDIO can be built on various Linux flavours, Microsoft Window * [Windows](docs/reference/build_guides/windows.md) -### Building xSTUDIO for MacOS +### Building xSTUDIO for macOS + +* [macOS](docs/reference/build_guides/macos.md) + +### Documentation + +xSTUDIO's documentation is built with Sphinx and Doxygen. The source lives under `docs/`, with end-user workflows in `docs/user_docs` and build/API reference material in `docs/reference`. The standard build keeps documentation disabled by default; enable it with `-DBUILD_DOCS=ON` if you want to generate the HTML site locally. + +## Presets and Baselines + +The repository now ships platform-specific `dev`, `ci`, and `perf` CMake presets without hardcoded local filesystem paths. Set `VCPKG_ROOT` and `Qt6_DIR` in your environment as needed, then list the available presets with: -* [MacOS](docs/reference/build_guides/macos.md) +```bash +cmake --list-presets +``` -### Documentation Note +For repeatable JSON timing output, use the perf baseline helper: -Note that the xSTUDIO user guide is built with Sphinx using the Read-The-Docs theme. The package dependencies for building the docs can be challenging to install so we instead include the fully built docs as part of xSTUDIO's repo, as well as the source for building the docs. Our build set-up by default disables the building of the docs to make life easy! +```bash +python3 scripts/perf/baseline.py --output build/perf-baseline.json --command tests:"ctest --output-on-failure" +``` diff --git a/include/xstudio/media_reader/cacheing_media_reader_actor.hpp b/include/xstudio/media_reader/cacheing_media_reader_actor.hpp index 67f272e34..2303db3fd 100644 --- a/include/xstudio/media_reader/cacheing_media_reader_actor.hpp +++ b/include/xstudio/media_reader/cacheing_media_reader_actor.hpp @@ -6,8 +6,14 @@ #include "xstudio/media/media.hpp" #include "xstudio/media_reader/media_reader.hpp" #include "xstudio/utility/chrono.hpp" +#include "xstudio/utility/json_store.hpp" #include "xstudio/utility/uuid.hpp" +#include +#include +#include +#include + namespace xstudio { namespace media_reader { @@ -25,41 +31,84 @@ namespace media_reader { const char *name() const override { return NAME.c_str(); } private: - struct ImmediateImageReqest { + struct ImmediateImageWaiter { + ImmediateImageWaiter( + const utility::Uuid &playhead_uuid, + caf::typed_response_promise &rp) + : playhead_uuid_(playhead_uuid), response_promise_(rp) {} - ImmediateImageReqest( - const media::AVFrameID mptr, caf::typed_response_promise &rp) - : mptr_(mptr), response_promise_(rp) {} + utility::Uuid playhead_uuid_; + caf::typed_response_promise response_promise_; + }; - ImmediateImageReqest(const ImmediateImageReqest &) = default; - ImmediateImageReqest() = default; + struct PendingImmediateImageRequest { + PendingImmediateImageRequest() = default; - ImmediateImageReqest &operator=(const ImmediateImageReqest &o) = default; + explicit PendingImmediateImageRequest(const media::AVFrameID &mptr) : mptr_(mptr) {} media::AVFrameID mptr_; - caf::typed_response_promise response_promise_; + std::vector waiters_; + bool queued_{false}; + bool active_{false}; + }; + + struct WorkerPoolState { + std::vector workers_; + std::vector busy_; + size_t next_dispatch_index_{0}; + size_t next_round_robin_index_{0}; + }; + + struct WorkerPreferences { + size_t urgent_worker_count_{1}; + size_t precache_worker_count_{1}; + size_t audio_worker_count_{1}; }; - void do_urgent_get_image(); + WorkerPreferences worker_preferences(const utility::JsonStore &js) const; + void spawn_worker_pool( + WorkerPoolState &pool, + const utility::Uuid &media_reader_plugin_uuid, + const utility::JsonStore &js, + size_t worker_count); + + caf::actor next_worker(WorkerPoolState &pool); + std::optional acquire_idle_worker(WorkerPoolState &pool); + void release_worker(WorkerPoolState &pool, size_t worker_index); + + void cancel_superseded_request(const utility::Uuid &playhead_uuid); + void enqueue_urgent_image_request( + const media::AVFrameID &mptr, + const utility::Uuid &playhead_uuid, + caf::typed_response_promise &rp); + void dispatch_pending_urgent_image_requests(); + void dispatch_urgent_image_request( + const media::MediaKey &key, PendingImmediateImageRequest &request, size_t worker_index); + void finish_urgent_image_request( + const media::MediaKey &key, + size_t worker_index, + const ImageBufPtr &buf, + const caf::error *err = nullptr); + caf::typed_response_promise receive_image_buffer_request( const media::AVFrameID &mptr, const utility::Uuid playhead_uuid); ImageBufPtr make_error_buffer(const caf::error &err, const media::AVFrameID &mptr); - std::map pending_get_image_requests_; - inline static const std::string NAME = "CachingMediaReaderActor"; caf::behavior behavior_; - // bool sequential_access_; caf::actor image_cache_; caf::actor audio_cache_; - bool urgent_worker_busy_ = {false}; + WorkerPoolState urgent_workers_; + WorkerPoolState precache_workers_; + WorkerPoolState audio_workers_; - caf::actor urgent_worker_; - caf::actor precache_worker_; - caf::actor audio_worker_; + std::unordered_map playhead_pending_image_requests_; + std::unordered_map + pending_get_image_requests_; + std::deque pending_get_image_order_; ImageBufPtr blank_image_; }; diff --git a/include/xstudio/media_reader/frame_request_queue.hpp b/include/xstudio/media_reader/frame_request_queue.hpp index adcf07b41..5b7532913 100644 --- a/include/xstudio/media_reader/frame_request_queue.hpp +++ b/include/xstudio/media_reader/frame_request_queue.hpp @@ -10,7 +10,9 @@ #include "xstudio/utility/uuid.hpp" #include -#include +#include +#include +#include namespace xstudio { namespace media_reader { @@ -113,7 +115,35 @@ namespace media_reader { void clear_pending_requests(const utility::Uuid &playhead_uuid); private: - std::vector> queue_; + struct RequestEntry; + using RequestHandle = std::shared_ptr; + using PlayheadRequests = std::unordered_multimap; + + struct RequestOrder { + bool operator()(const RequestHandle &a, const RequestHandle &b) const; + }; + + struct RequestEntry { + explicit RequestEntry(FrameRequest request) : request_(std::move(request)) {} + + FrameRequest request_; + std::multiset::iterator ordered_it_; + PlayheadRequests::iterator playhead_it_; + }; + + using OrderedRequests = std::multiset; + + RequestHandle emplace_request( + std::shared_ptr frame_info, + const utility::time_point &required_by, + const utility::Uuid &requesting_playhead_uuid); + void erase_request(const RequestHandle &request); + void update_required_by( + const RequestHandle &request, const utility::time_point &required_by); + + OrderedRequests ordered_requests_; + std::unordered_map requests_by_media_key_; + PlayheadRequests requests_by_playhead_; }; } // namespace media_reader diff --git a/include/xstudio/scanner/scanner_actor.hpp b/include/xstudio/scanner/scanner_actor.hpp index dae938ffb..51c17b0c6 100644 --- a/include/xstudio/scanner/scanner_actor.hpp +++ b/include/xstudio/scanner/scanner_actor.hpp @@ -3,6 +3,10 @@ #include +#include +#include +#include + namespace xstudio::scanner { class ScanHelperActor : public caf::event_based_actor { public: @@ -15,10 +19,18 @@ class ScanHelperActor : public caf::event_based_actor { const char *name() const override { return NAME.c_str(); } private: + std::pair checksum_for_path(const std::filesystem::path &path); + inline static const std::string NAME = "ScanHelperActor"; caf::behavior behavior_; - std::map> cache_; + struct ChecksumCacheEntry { + std::string checksum_; + uintmax_t size_{0}; + std::filesystem::file_time_type modified_at_{}; + }; + + std::unordered_map cache_; }; class ScannerActor : public caf::event_based_actor { @@ -33,8 +45,12 @@ class ScannerActor : public caf::event_based_actor { const char *name() const override { return NAME.c_str(); } private: + caf::actor next_helper(); + inline static const std::string NAME = "ScannerActor"; caf::behavior behavior_; + std::vector helpers_; + size_t next_helper_index_{0}; }; } // namespace xstudio::scanner diff --git a/scripts/perf/baseline.py b/scripts/perf/baseline.py new file mode 100644 index 000000000..6f1f8e274 --- /dev/null +++ b/scripts/perf/baseline.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Capture repeatable build and runtime perf baselines as JSON.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import time +from typing import List, Tuple, Union + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dir", default=".", help="Source directory to measure.") + parser.add_argument("--build-dir", default="build", help="Build directory to measure.") + parser.add_argument("--output", required=True, help="Path to the JSON output file.") + parser.add_argument("--cmake", default="cmake", help="CMake executable to use.") + parser.add_argument("--ctest", default="ctest", help="CTest executable to use.") + parser.add_argument( + "--configure-preset", help="Configure preset to run before collecting results." + ) + parser.add_argument("--build-preset", help="Build preset to run before collecting results.") + parser.add_argument("--test-preset", help="CTest preset to run before collecting results.") + parser.add_argument( + "--command", + action="append", + default=[], + help="Extra named command to run, formatted as name:command.", + ) + return parser.parse_args() + + +def run_command( + name: str, command: Union[List[str], str], cwd: Path, shell: bool = False +) -> dict: + start = time.perf_counter() + completed = subprocess.run( + command, + cwd=str(cwd), + shell=shell, + capture_output=True, + text=True, + check=False, + ) + duration = time.perf_counter() - start + + return { + "name": name, + "command": command if isinstance(command, str) else " ".join(command), + "cwd": str(cwd), + "duration_seconds": round(duration, 6), + "returncode": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + } + + +def split_named_command(value: str) -> Tuple[str, str]: + if ":" not in value: + raise argparse.ArgumentTypeError( + f"Invalid command '{value}'. Expected the format name:command." + ) + name, command = value.split(":", 1) + if not name or not command: + raise argparse.ArgumentTypeError( + f"Invalid command '{value}'. Expected the format name:command." + ) + return name, command + + +def main() -> int: + args = parse_args() + source_dir = Path(args.source_dir).resolve() + build_dir = Path(args.build_dir).resolve() + command_dir = build_dir if build_dir.exists() else source_dir + output_path = Path(args.output).resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + + results: List[dict] = [] + + if args.configure_preset: + results.append( + run_command( + "configure", + [args.cmake, "--preset", args.configure_preset], + source_dir, + ) + ) + + if args.build_preset: + results.append( + run_command( + "build", + [args.cmake, "--build", "--preset", args.build_preset], + source_dir, + ) + ) + + if args.test_preset: + results.append( + run_command( + "test", + [args.ctest, "--preset", args.test_preset, "--output-on-failure"], + source_dir, + ) + ) + + for raw_command in args.command: + name, command = split_named_command(raw_command) + results.append(run_command(name, command, command_dir, shell=True)) + + payload = { + "source_dir": str(source_dir), + "build_dir": str(build_dir), + "timestamp_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "environment": { + "cmake": args.cmake, + "ctest": args.ctest, + "python": sys.executable, + "qt6_dir": os.environ.get("Qt6_DIR", ""), + "vcpkg_root": os.environ.get("VCPKG_ROOT", ""), + }, + "commands": results, + "summary": { + "all_passed": all(result["returncode"] == 0 for result in results), + "total_duration_seconds": round( + sum(result["duration_seconds"] for result in results), 6 + ), + }, + } + + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(output_path) + return 0 if payload["summary"]["all_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/share/preference/core_media_reader.json b/share/preference/core_media_reader.json index 3d02ae54e..a6851696c 100644 --- a/share/preference/core_media_reader.json +++ b/share/preference/core_media_reader.json @@ -30,6 +30,42 @@ "datatype": "int", "context": ["APPLICATION"] }, + "max_worker_count": { + "path": "/core/media_reader/max_worker_count", + "default_value": 4, + "description": "Number of on-demand decode workers used for interactive image requests.", + "value": 4, + "minimum": 1, + "maximum": 16, + "datatype": "int", + "context": ["APPLICATION"], + "category": "Performance", + "display_name": "Interactive Workers" + }, + "precache_worker_count": { + "path": "/core/media_reader/precache_worker_count", + "default_value": 2, + "description": "Number of background decode workers used for read-ahead and precache requests.", + "value": 2, + "minimum": 1, + "maximum": 16, + "datatype": "int", + "context": ["APPLICATION"], + "category": "Performance", + "display_name": "Precache Workers" + }, + "audio_worker_count": { + "path": "/core/media_reader/audio_worker_count", + "default_value": 1, + "description": "Number of workers used for audio decode requests.", + "value": 1, + "minimum": 1, + "maximum": 8, + "datatype": "int", + "context": ["APPLICATION"], + "category": "Performance", + "display_name": "Audio Workers" + }, "timecode_from_frame": { "path": "/core/media_reader/timecode_from_frame", "default_value": true, @@ -58,4 +94,4 @@ } } } -} \ No newline at end of file +} diff --git a/share/preference/core_scanner.json b/share/preference/core_scanner.json new file mode 100644 index 000000000..ae9d72783 --- /dev/null +++ b/share/preference/core_scanner.json @@ -0,0 +1,18 @@ +{ + "core": { + "scanner": { + "max_worker_count": { + "path": "/core/scanner/max_worker_count", + "default_value": 4, + "description": "Number of filesystem scan helpers used for checksum, relink, and rescan work.", + "value": 4, + "minimum": 1, + "maximum": 16, + "datatype": "int", + "context": ["APPLICATION"], + "category": "Performance", + "display_name": "Scanner Workers" + } + } + } +} diff --git a/src/media_reader/src/cacheing_media_reader_actor.cpp b/src/media_reader/src/cacheing_media_reader_actor.cpp index 572704a2d..62e2c1e22 100644 --- a/src/media_reader/src/cacheing_media_reader_actor.cpp +++ b/src/media_reader/src/cacheing_media_reader_actor.cpp @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +#include #include +#include #include "xstudio/media_reader/cacheing_media_reader_actor.hpp" #include "xstudio/global_store/global_store.hpp" @@ -82,6 +84,89 @@ ImageBufPtr make_blank_image() { } // namespace +CachingMediaReaderActor::WorkerPreferences +CachingMediaReaderActor::worker_preferences(const utility::JsonStore &js) const { + WorkerPreferences prefs; + + const auto cpu_count = std::max(1U, std::thread::hardware_concurrency()); + prefs.urgent_worker_count_ = std::max(1, std::min(cpu_count, 4)); + prefs.precache_worker_count_ = + std::max(1, std::min(std::max(1U, cpu_count / 2), 2)); + prefs.audio_worker_count_ = 1; + + try { + prefs.urgent_worker_count_ = std::clamp( + preference_value(js, "/core/media_reader/max_worker_count"), 1, 16); + } catch (...) { + } + + try { + prefs.precache_worker_count_ = std::clamp( + preference_value(js, "/core/media_reader/precache_worker_count"), 1, 16); + } catch (...) { + } + + try { + prefs.audio_worker_count_ = std::clamp( + preference_value(js, "/core/media_reader/audio_worker_count"), 1, 8); + } catch (...) { + } + + return prefs; +} + +void CachingMediaReaderActor::spawn_worker_pool( + WorkerPoolState &pool, + const utility::Uuid &media_reader_plugin_uuid, + const utility::JsonStore &js, + size_t worker_count) { + const auto worker_total = + std::max(1, worker_count); + + auto pm = system().registry().template get(plugin_manager_registry); + scoped_actor sys{system()}; + + pool.workers_.reserve(worker_total); + pool.busy_.assign(worker_total, false); + + for (size_t worker_index = 0; worker_index < worker_total; ++worker_index) { + auto worker = request_receive( + *sys, pm, plugin_manager::spawn_plugin_atom_v, media_reader_plugin_uuid, js); + link_to(worker); + pool.workers_.emplace_back(std::move(worker)); + } +} + +caf::actor CachingMediaReaderActor::next_worker(WorkerPoolState &pool) { + if (pool.workers_.empty()) + return caf::actor{}; + + const auto worker_index = pool.next_round_robin_index_ % pool.workers_.size(); + pool.next_round_robin_index_ = (worker_index + 1) % pool.workers_.size(); + return pool.workers_[worker_index]; +} + +std::optional CachingMediaReaderActor::acquire_idle_worker(WorkerPoolState &pool) { + if (pool.workers_.empty()) + return {}; + + for (size_t offset = 0; offset < pool.busy_.size(); ++offset) { + const auto worker_index = (pool.next_dispatch_index_ + offset) % pool.busy_.size(); + if (!pool.busy_[worker_index]) { + pool.busy_[worker_index] = true; + pool.next_dispatch_index_ = (worker_index + 1) % pool.busy_.size(); + return worker_index; + } + } + + return {}; +} + +void CachingMediaReaderActor::release_worker(WorkerPoolState &pool, size_t worker_index) { + if (worker_index < pool.busy_.size()) + pool.busy_[worker_index] = false; +} + CachingMediaReaderActor::CachingMediaReaderActor( caf::actor_config &cfg, const utility::Uuid &media_reader_plugin_uuid, @@ -94,25 +179,15 @@ CachingMediaReaderActor::CachingMediaReaderActor( print_on_exit(this, "CachingMediaReaderActor"); spdlog::debug("Created CachingMediaReaderActor."); - // create plugins.. - { - auto prefs = GlobalStoreHelper(system()); - JsonStore js; - prefs.get_group(js); + auto prefs = GlobalStoreHelper(system()); + JsonStore js; + prefs.get_group(js); - auto pm = system().registry().template get(plugin_manager_registry); - scoped_actor sys{system()}; - - precache_worker_ = request_receive( - *sys, pm, plugin_manager::spawn_plugin_atom_v, media_reader_plugin_uuid, js); - link_to(precache_worker_); - urgent_worker_ = request_receive( - *sys, pm, plugin_manager::spawn_plugin_atom_v, media_reader_plugin_uuid, js); - link_to(urgent_worker_); - audio_worker_ = request_receive( - *sys, pm, plugin_manager::spawn_plugin_atom_v, media_reader_plugin_uuid, js); - link_to(audio_worker_); - } + const auto worker_counts = worker_preferences(js); + spawn_worker_pool(urgent_workers_, media_reader_plugin_uuid, js, worker_counts.urgent_worker_count_); + spawn_worker_pool( + precache_workers_, media_reader_plugin_uuid, js, worker_counts.precache_worker_count_); + spawn_worker_pool(audio_workers_, media_reader_plugin_uuid, js, worker_counts.audio_worker_count_); if (not image_cache_) image_cache_ = system().registry().template get(image_cache_registry); @@ -130,19 +205,21 @@ CachingMediaReaderActor::CachingMediaReaderActor( blank_image_ = blank; }, - [=](get_image_atom) { - // get the next pending urgent image request - if (!urgent_worker_busy_ && pending_get_image_requests_.size()) { - do_urgent_get_image(); - } - }, + [=](get_image_atom) { dispatch_pending_urgent_image_requests(); }, [=](get_image_atom, const media::AVFrameID &mptr, bool pin, const utility::Uuid &playhead_uuid, const timebase::flicks playhead_position) -> result { + (void)playhead_position; auto rp = make_response_promise(); + auto worker = next_worker(urgent_workers_); + if (!worker) { + rp.deliver(make_error(sec::runtime_error, "No urgent media reader workers")); + return rp; + } + // first, check if the image we want is cached mail(media_cache::retrieve_atom_v, mptr.key()) .request(image_cache_, infinite) @@ -152,7 +229,7 @@ CachingMediaReaderActor::CachingMediaReaderActor( rp.deliver(buf); } else { mail(get_image_atom_v, mptr) - .request(urgent_worker_, infinite) + .request(worker, infinite) .then( [=](media_reader::ImageBufPtr buf) mutable { rp.deliver(buf); @@ -200,6 +277,12 @@ CachingMediaReaderActor::CachingMediaReaderActor( bool pin, const utility::Uuid playhead_uuid) -> result { auto rp = make_response_promise(); + auto worker = next_worker(audio_workers_); + if (!worker) { + rp.deliver(make_error(sec::runtime_error, "No audio media reader workers")); + return rp; + } + // first, check if the image we want is cached mail(media_cache::retrieve_atom_v, mptr.key()) .request(audio_cache_, infinite) @@ -209,7 +292,7 @@ CachingMediaReaderActor::CachingMediaReaderActor( rp.deliver(buf); } else { mail(get_audio_atom_v, mptr) - .request(urgent_worker_, infinite) + .request(worker, infinite) .then( [=](media_reader::AudioBufPtr buf) mutable { rp.deliver(buf); @@ -244,8 +327,13 @@ CachingMediaReaderActor::CachingMediaReaderActor( // note the caller (GlobalMediaReaderActor) handles the cacheing // of this image buffer auto rp = make_response_promise(); + auto worker = next_worker(precache_workers_); + if (!worker) { + rp.deliver(make_error(sec::runtime_error, "No precache media reader workers")); + return rp; + } mail(get_image_atom_v, mptr) - .request(precache_worker_, infinite) + .request(worker, infinite) .then( [=](media_reader::ImageBufPtr buf) mutable { rp.deliver(buf); }, [=](const caf::error &err) mutable { @@ -259,72 +347,176 @@ CachingMediaReaderActor::CachingMediaReaderActor( // note the caller (GlobalMediaReaderActor) handles the cacheing // of this image buffer auto rp = make_response_promise(); + auto worker = next_worker(precache_workers_); + if (!worker) { + rp.deliver(make_error(sec::runtime_error, "No precache audio reader workers")); + return rp; + } mail(get_audio_atom_v, mptr) - .request(precache_worker_, infinite) + .request(worker, infinite) .then( [=](media_reader::AudioBufPtr buf) mutable { rp.deliver(buf); }, [=](const caf::error &err) mutable { rp.deliver(err); }); return rp; }, - [=](get_media_detail_atom atom, const caf::uri &_uri) { - return mail(atom, _uri).delegate(urgent_worker_); + [=](get_media_detail_atom atom, const caf::uri &_uri) -> result { + auto worker = next_worker(urgent_workers_); + if (!worker) + return make_error(sec::runtime_error, "No urgent media reader workers"); + return mail(atom, _uri).delegate(worker); } ); } -void CachingMediaReaderActor::do_urgent_get_image() { +void CachingMediaReaderActor::cancel_superseded_request(const utility::Uuid &playhead_uuid) { + const auto key_it = playhead_pending_image_requests_.find(playhead_uuid); + if (key_it == playhead_pending_image_requests_.end()) + return; + + const auto pending_it = pending_get_image_requests_.find(key_it->second); + playhead_pending_image_requests_.erase(key_it); + if (pending_it == pending_get_image_requests_.end()) + return; + + auto &pending_request = pending_it->second; + pending_request.waiters_.erase( + std::remove_if( + pending_request.waiters_.begin(), + pending_request.waiters_.end(), + [&playhead_uuid](const ImmediateImageWaiter &waiter) { + return waiter.playhead_uuid_ == playhead_uuid; + }), + pending_request.waiters_.end()); + if (pending_request.waiters_.empty() && !pending_request.active_) { + pending_get_image_order_.erase( + std::remove( + pending_get_image_order_.begin(), + pending_get_image_order_.end(), + pending_it->first), + pending_get_image_order_.end()); + pending_get_image_requests_.erase(pending_it); + } +} + +void CachingMediaReaderActor::enqueue_urgent_image_request( + const media::AVFrameID &mptr, + const utility::Uuid &playhead_uuid, + caf::typed_response_promise &rp) { + cancel_superseded_request(playhead_uuid); + + auto [pending_it, inserted] = + pending_get_image_requests_.try_emplace(mptr.key(), PendingImmediateImageRequest(mptr)); + auto &pending_request = pending_it->second; + if (inserted) { + pending_request.mptr_ = mptr; + } + + pending_request.waiters_.emplace_back(playhead_uuid, rp); + playhead_pending_image_requests_[playhead_uuid] = mptr.key(); + + if (!pending_request.queued_ && !pending_request.active_) { + pending_request.queued_ = true; + pending_get_image_order_.push_back(mptr.key()); + } + + dispatch_pending_urgent_image_requests(); +} + +void CachingMediaReaderActor::dispatch_pending_urgent_image_requests() { + while (true) { + const auto worker_index = acquire_idle_worker(urgent_workers_); + if (!worker_index.has_value()) + return; + + bool dispatched_request = false; + while (!pending_get_image_order_.empty()) { + const auto key = pending_get_image_order_.front(); + pending_get_image_order_.pop_front(); + + auto pending_it = pending_get_image_requests_.find(key); + if (pending_it == pending_get_image_requests_.end()) + continue; + + auto &pending_request = pending_it->second; + pending_request.queued_ = false; + if (pending_request.active_ || pending_request.waiters_.empty()) + continue; + + pending_request.active_ = true; + dispatch_urgent_image_request(key, pending_request, *worker_index); + dispatched_request = true; + break; + } - auto p = pending_get_image_requests_.begin(); - const media::AVFrameID mptr = p->second.mptr_; - auto rp = p->second.response_promise_; - auto playhead_uuid = p->first; - pending_get_image_requests_.erase(p); + if (!dispatched_request) { + release_worker(urgent_workers_, *worker_index); + return; + } + } +} - urgent_worker_busy_ = true; - mail(get_image_atom_v, mptr) - .request(urgent_worker_, infinite) +void CachingMediaReaderActor::dispatch_urgent_image_request( + const media::MediaKey &key, + PendingImmediateImageRequest &request, + size_t worker_index) { + mail(get_image_atom_v, request.mptr_) + .request(urgent_workers_.workers_[worker_index], infinite) .then( [=](media_reader::ImageBufPtr buf) mutable { - // send the image back to the playhead that requested it - rp.deliver(buf); - - // store the image in our cache - anon_mail( - media_cache::store_atom_v, - mptr.key(), - buf, - utility::clock::now(), - playhead_uuid) - .urgent() - .send(image_cache_); - - // perhaps more urgent requests are now pending - urgent_worker_busy_ = false; - anon_mail(get_image_atom_v).send(this); + finish_urgent_image_request(key, worker_index, buf); }, [=](const caf::error &err) mutable { - // make an empty image buffer that holds the error message - media_reader::ImageBufPtr buf = make_error_buffer(err, mptr); - rp.deliver(buf); - - // store the failed image in our cache so we don't - // keep trying to load it - anon_mail( - media_cache::store_atom_v, - mptr.key(), - buf, - utility::clock::now(), - playhead_uuid) - .urgent() - .send(image_cache_); - - urgent_worker_busy_ = false; - anon_mail(get_image_atom_v).send(this); + finish_urgent_image_request(key, worker_index, ImageBufPtr(), &err); }); } +void CachingMediaReaderActor::finish_urgent_image_request( + const media::MediaKey &key, + size_t worker_index, + const ImageBufPtr &buf, + const caf::error *err) { + auto pending_it = pending_get_image_requests_.find(key); + release_worker(urgent_workers_, worker_index); + if (pending_it == pending_get_image_requests_.end()) { + dispatch_pending_urgent_image_requests(); + return; + } + + auto pending_request = std::move(pending_it->second); + pending_get_image_requests_.erase(pending_it); + + ImageBufPtr result = buf; + if (err) { + result = make_error_buffer(*err, pending_request.mptr_); + } + + std::optional cache_owner; + for (auto &waiter : pending_request.waiters_) { + auto key_it = playhead_pending_image_requests_.find(waiter.playhead_uuid_); + if (key_it != playhead_pending_image_requests_.end() && key_it->second == key) { + if (!cache_owner.has_value()) + cache_owner = waiter.playhead_uuid_; + playhead_pending_image_requests_.erase(key_it); + waiter.response_promise_.deliver(result); + } + } + + if (cache_owner.has_value()) { + anon_mail( + media_cache::store_atom_v, + key, + result, + utility::clock::now(), + *cache_owner) + .urgent() + .send(image_cache_); + } + + dispatch_pending_urgent_image_requests(); +} + caf::typed_response_promise CachingMediaReaderActor::receive_image_buffer_request( const media::AVFrameID &mptr, const utility::Uuid playhead_uuid) { @@ -338,9 +530,7 @@ caf::typed_response_promise CachingMediaReaderActor::receive_image_ // send the image back to the playhead that requested it rt.deliver(buf); } else { - // image is not cached. Update the request to load the image - pending_get_image_requests_[playhead_uuid] = ImmediateImageReqest(mptr, rt); - mail(get_image_atom_v).send(this); + enqueue_urgent_image_request(mptr, playhead_uuid, rt); } }, [=](const caf::error &err) mutable { @@ -378,4 +568,4 @@ ImageBufPtr CachingMediaReaderActor::make_error_buffer( << "\": " << caf_error_string; return media_reader::ImageBufPtr(new media_reader::ImageBuffer(err_msg.str()));*/ -} \ No newline at end of file +} diff --git a/src/media_reader/src/frame_request_queue.cpp b/src/media_reader/src/frame_request_queue.cpp index 941495505..7f78908d2 100644 --- a/src/media_reader/src/frame_request_queue.cpp +++ b/src/media_reader/src/frame_request_queue.cpp @@ -2,43 +2,70 @@ #include "xstudio/media_reader/frame_request_queue.hpp" #include "xstudio/utility/helpers.hpp" +#include + using namespace xstudio::media_reader; using namespace xstudio; -void FrameRequestQueue::add_frame_request( - const media::AVFrameID &frame_info, +bool FrameRequestQueue::RequestOrder::operator()( + const RequestHandle &a, const RequestHandle &b) const { + if (a->request_.required_by_ != b->request_.required_by_) + return a->request_.required_by_ < b->request_.required_by_; + + if (a->request_.requesting_playhead_uuid_ != b->request_.requesting_playhead_uuid_) + return a->request_.requesting_playhead_uuid_ < b->request_.requesting_playhead_uuid_; + + return a->request_.requested_frame_->key() < b->request_.requested_frame_->key(); +} + +FrameRequestQueue::RequestHandle FrameRequestQueue::emplace_request( + std::shared_ptr frame_info, const utility::time_point &required_by, const utility::Uuid &requesting_playhead_uuid) { - // auto tt = utility::clock::now(); - // spdlog::warn("{}",to_string(frame_info.uri_)); + auto inserted = std::make_shared(FrameRequest( + std::move(frame_info), required_by, requesting_playhead_uuid)); - bool matches_existing_request = false; - for (auto pp = queue_.rbegin(); pp != queue_.rend(); ++pp) { + inserted->ordered_it_ = + ordered_requests_.insert(inserted); + inserted->playhead_it_ = + requests_by_playhead_.emplace(requesting_playhead_uuid, inserted); + requests_by_media_key_[inserted->request_.requested_frame_->key()] = inserted; - if ((*pp)->requested_frame_->frame() == frame_info.frame() && - (*pp)->requested_frame_->uri() == frame_info.uri()) { - if ((*pp)->required_by_ > required_by) { - (*pp)->required_by_ = required_by; - } - matches_existing_request = true; - break; - } - } + return inserted; +} + +void FrameRequestQueue::erase_request(const RequestHandle &request) { + requests_by_media_key_.erase(request->request_.requested_frame_->key()); + requests_by_playhead_.erase(request->playhead_it_); + ordered_requests_.erase(request->ordered_it_); +} - if (!matches_existing_request) { +void FrameRequestQueue::update_required_by( + const RequestHandle &request, const utility::time_point &required_by) { + if (required_by >= request->request_.required_by_) + return; + + ordered_requests_.erase(request->ordered_it_); + request->request_.required_by_ = required_by; + request->ordered_it_ = ordered_requests_.insert(request); +} + +void FrameRequestQueue::add_frame_request( + const media::AVFrameID &frame_info, + const utility::time_point &required_by, + const utility::Uuid &requesting_playhead_uuid) { - queue_.emplace_back(new FrameRequest( - std::shared_ptr(new media::AVFrameID(frame_info)), - required_by, - requesting_playhead_uuid)); + const auto existing = requests_by_media_key_.find(frame_info.key()); + if (existing != requests_by_media_key_.end()) { + update_required_by(existing->second, required_by); + return; } - std::sort( - queue_.begin(), - queue_.end(), - [](const std::shared_ptr &a, const std::shared_ptr &b) - -> bool { return a->required_by_ < b->required_by_; }); + emplace_request( + std::make_shared(frame_info), + required_by, + requesting_playhead_uuid); } void FrameRequestQueue::add_frame_requests( @@ -48,72 +75,62 @@ void FrameRequestQueue::add_frame_requests( for (const auto &p : frames_info) { const std::shared_ptr &frame_info = (p.second); const utility::time_point &when_we_want_it = p.first; - queue_.emplace_back( - new FrameRequest(frame_info, when_we_want_it, requesting_playhead_uuid)); - } - std::sort( - queue_.begin(), - queue_.end(), - [](const std::shared_ptr &a, const std::shared_ptr &b) - -> bool { return a->required_by_ < b->required_by_; }); + const auto existing = requests_by_media_key_.find(frame_info->key()); + if (existing != requests_by_media_key_.end()) { + update_required_by(existing->second, when_we_want_it); + continue; + } + + emplace_request(frame_info, when_we_want_it, requesting_playhead_uuid); + } } std::optional FrameRequestQueue::pop_request(const std::map &exclude_playheads) { - std::optional rt = {}; - - for (auto p = queue_.begin(); p != queue_.end(); p++) { - if (!exclude_playheads.count((*p)->requesting_playhead_uuid_)) { - rt = *(*p); - queue_.erase(p); - break; - } + for (auto request = ordered_requests_.begin(); request != ordered_requests_.end(); ++request) { + const auto current = *request; + if (exclude_playheads.count(current->request_.requesting_playhead_uuid_)) + continue; + + auto result = current->request_; + erase_request(current); + return result; } - return rt; + + return {}; } void FrameRequestQueue::prune_stale_frame_requests() { + if (ordered_requests_.size() <= 20) + return; auto now = utility::clock::now(); - // bool found_stale_request = false; - for (auto pp = queue_.begin(); pp != queue_.end(); ++pp) { - if (queue_.size() > 20) { //&& (*pp).required_by < now) { - auto ppp = pp; - ppp++; - if (ppp != queue_.end() && (*ppp)->required_by_ < now) { - pp = queue_.erase(pp); - } - } + auto request = ordered_requests_.begin(); + while (request != ordered_requests_.end()) { + auto next = std::next(request); + if (next == ordered_requests_.end()) + break; + + if ((*next)->request_.required_by_ >= now) + break; + + auto current = *request; + request = next; + erase_request(current); } } void FrameRequestQueue::clear_pending_requests(const utility::Uuid &playhead_uuid) { + auto [begin, end] = requests_by_playhead_.equal_range(playhead_uuid); + std::vector to_remove; + to_remove.reserve(std::distance(begin, end)); - queue_.erase( - std::remove_if( - queue_.begin(), - queue_.end(), - [&playhead_uuid](const std::shared_ptr &x) { - return x->requesting_playhead_uuid_ == playhead_uuid; // put your condition here - }), - queue_.end()); - - /*auto pp = queue_.begin(); - auto a = queue_.end(); - auto b = queue_.end(); - - while (pp != queue_.end()) { - if ((*pp)->requesting_playhead_uuid_ == playhead_uuid) { - if (a == queue_.end()) { - a = pp; - } - b = pp; - } else { - if (a != queue_.end()) { - queue_.erase(a, b); - } - pp++; - } - }*/ -} \ No newline at end of file + for (auto it = begin; it != end; ++it) { + to_remove.emplace_back(it->second); + } + + for (const auto &request : to_remove) { + erase_request(request); + } +} diff --git a/src/media_reader/test/frame_request_queue_test.cpp b/src/media_reader/test/frame_request_queue_test.cpp new file mode 100644 index 000000000..ddebfb9fa --- /dev/null +++ b/src/media_reader/test/frame_request_queue_test.cpp @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: Apache-2.0 +#include + +#include + +#include "xstudio/media_reader/frame_request_queue.hpp" + +using namespace xstudio; +using namespace xstudio::media_reader; + +namespace { + +media::AVFrameID make_frame_id(const std::string &uri, const int frame) { + auto parsed_uri = caf::make_uri(uri); + if (!parsed_uri) + throw std::runtime_error("Failed to parse test uri"); + return media::AVFrameID(*parsed_uri, frame); +} + +size_t drain_queue(FrameRequestQueue &queue) { + size_t count = 0; + while (queue.pop_request({})) { + ++count; + } + return count; +} + +} // namespace + +TEST(FrameRequestQueueTest, DeduplicatesSingleFrameAndKeepsEarliestDeadline) { + FrameRequestQueue queue; + const auto playhead = utility::Uuid::generate(); + const auto base_tp = utility::clock::now(); + + queue.add_frame_request( + make_frame_id("file:///tmp/test.0001.exr", 10), + base_tp + std::chrono::milliseconds(20), + playhead); + queue.add_frame_request( + make_frame_id("file:///tmp/test.0001.exr", 10), + base_tp + std::chrono::milliseconds(5), + playhead); + + const auto request = queue.pop_request({}); + ASSERT_TRUE(request.has_value()); + EXPECT_EQ(request->requested_frame_->frame(), 10); + EXPECT_EQ(request->requesting_playhead_uuid_, playhead); + EXPECT_EQ(request->required_by_, base_tp + std::chrono::milliseconds(5)); + EXPECT_FALSE(queue.pop_request({}).has_value()); +} + +TEST(FrameRequestQueueTest, BatchInsertDeduplicatesByMediaKey) { + FrameRequestQueue queue; + const auto playhead = utility::Uuid::generate(); + const auto base_tp = utility::clock::now(); + + media::AVFrameIDsAndTimePoints requests; + requests.emplace_back( + base_tp + std::chrono::milliseconds(30), + std::make_shared( + make_frame_id("file:///tmp/test.0001.exr", 1))); + requests.emplace_back( + base_tp + std::chrono::milliseconds(5), + std::make_shared( + make_frame_id("file:///tmp/test.0001.exr", 1))); + requests.emplace_back( + base_tp + std::chrono::milliseconds(10), + std::make_shared( + make_frame_id("file:///tmp/test.0002.exr", 2))); + + queue.add_frame_requests(requests, playhead); + + const auto first = queue.pop_request({}); + ASSERT_TRUE(first.has_value()); + EXPECT_EQ(first->requested_frame_->frame(), 1); + EXPECT_EQ(first->required_by_, base_tp + std::chrono::milliseconds(5)); + + const auto second = queue.pop_request({}); + ASSERT_TRUE(second.has_value()); + EXPECT_EQ(second->requested_frame_->frame(), 2); + EXPECT_FALSE(queue.pop_request({}).has_value()); +} + +TEST(FrameRequestQueueTest, PopRequestSkipsExcludedPlayheads) { + FrameRequestQueue queue; + const auto excluded_playhead = utility::Uuid::generate(); + const auto included_playhead = utility::Uuid::generate(); + const auto base_tp = utility::clock::now(); + + queue.add_frame_request( + make_frame_id("file:///tmp/test.0001.exr", 1), + base_tp + std::chrono::milliseconds(5), + excluded_playhead); + queue.add_frame_request( + make_frame_id("file:///tmp/test.0002.exr", 2), + base_tp + std::chrono::milliseconds(10), + included_playhead); + + const auto request = queue.pop_request({{excluded_playhead, 1}}); + ASSERT_TRUE(request.has_value()); + EXPECT_EQ(request->requested_frame_->frame(), 2); + EXPECT_EQ(request->requesting_playhead_uuid_, included_playhead); +} + +TEST(FrameRequestQueueTest, ClearPendingRequestsRemovesOnlyMatchingPlayhead) { + FrameRequestQueue queue; + const auto removed_playhead = utility::Uuid::generate(); + const auto kept_playhead = utility::Uuid::generate(); + const auto base_tp = utility::clock::now(); + + queue.add_frame_request( + make_frame_id("file:///tmp/test.0001.exr", 1), + base_tp + std::chrono::milliseconds(5), + removed_playhead); + queue.add_frame_request( + make_frame_id("file:///tmp/test.0002.exr", 2), + base_tp + std::chrono::milliseconds(10), + kept_playhead); + + queue.clear_pending_requests(removed_playhead); + + const auto request = queue.pop_request({}); + ASSERT_TRUE(request.has_value()); + EXPECT_EQ(request->requesting_playhead_uuid_, kept_playhead); + EXPECT_FALSE(queue.pop_request({}).has_value()); +} + +TEST(FrameRequestQueueTest, PruneStaleRequestsKeepsLatestStaleRequest) { + FrameRequestQueue queue; + const auto playhead = utility::Uuid::generate(); + const auto now = utility::clock::now(); + + for (int frame = 0; frame < 24; ++frame) { + queue.add_frame_request( + make_frame_id("file:///tmp/test." + std::to_string(frame) + ".exr", frame), + now - std::chrono::milliseconds(100 - frame), + playhead); + } + + queue.prune_stale_frame_requests(); + + ASSERT_EQ(drain_queue(queue), 1U); +} diff --git a/src/scanner/src/scanner_actor.cpp b/src/scanner/src/scanner_actor.cpp index aa9f6488b..6e8d5d802 100644 --- a/src/scanner/src/scanner_actor.cpp +++ b/src/scanner/src/scanner_actor.cpp @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 +#include #include #include #include @@ -6,11 +7,14 @@ #include #include #include +#include #include "xstudio/atoms.hpp" +#include "xstudio/global_store/global_store.hpp" #include "xstudio/media/media.hpp" #include "xstudio/scanner/scanner_actor.hpp" #include "xstudio/utility/helpers.hpp" +#include "xstudio/utility/json_store.hpp" #include "xstudio/utility/logging.hpp" #include "xstudio/utility/media_reference.hpp" #include "xstudio/utility/sequence.hpp" @@ -19,6 +23,8 @@ using namespace xstudio; using namespace xstudio::utility; using namespace xstudio::scanner; using namespace caf; +using namespace xstudio::global_store; +using namespace xstudio::json_store; namespace { namespace fs = std::filesystem; @@ -63,7 +69,7 @@ media::MediaStatus check_media_status(const MediaReference &mr) { } -uintmax_t get_file_size(const std::string &path) { +uintmax_t get_file_size(const fs::path &path) { uintmax_t result = 0; try { @@ -73,7 +79,15 @@ uintmax_t get_file_size(const std::string &path) { return result; } -std::string get_checksum(const std::string &path) { +fs::file_time_type get_modified_time(const fs::path &path) { + try { + return fs::last_write_time(path); + } catch (...) { + return fs::file_time_type::min(); + } +} + +std::string get_checksum(const fs::path &path) { std::array hash; // read first and last 1k.. @@ -99,7 +113,7 @@ std::string get_checksum(const std::string &path) { myfile.close(); } catch (const std::exception &err) { - spdlog::warn("{} {} {}", __PRETTY_FUNCTION__, path, err.what()); + spdlog::warn("{} {} {}", __PRETTY_FUNCTION__, path.string(), err.what()); return std::string(); } @@ -133,6 +147,25 @@ MediaReference rescan_media_reference(MediaReference mr) { } // namespace +std::pair +ScanHelperActor::checksum_for_path(const std::filesystem::path &path) { + const auto size = get_file_size(path); + if (!size) + return std::make_pair(std::string(), 0); + + const auto modified_at = get_modified_time(path); + const auto cache_key = path.string(); + const auto cached = cache_.find(cache_key); + if (cached != cache_.end() && cached->second.size_ == size && + cached->second.modified_at_ == modified_at) { + return std::make_pair(cached->second.checksum_, cached->second.size_); + } + + auto checksum = get_checksum(path); + cache_[cache_key] = ChecksumCacheEntry{checksum, size, modified_at}; + return std::make_pair(std::move(checksum), size); +} + ScanHelperActor::ScanHelperActor(caf::actor_config &cfg) : caf::event_based_actor(cfg) { behavior_.assign( [=](media::checksum_atom, @@ -142,26 +175,12 @@ ScanHelperActor::ScanHelperActor(caf::actor_config &cfg) : caf::event_based_acto if (not urlpath) return make_error(xstudio_error::error, "Invalid url"); - auto path = uri_to_posix_path(*urlpath); - - auto size = get_file_size(path); - auto checksum = std::string(); - - if (size) - checksum = get_checksum(path); - - return std::make_pair(checksum, size); + return checksum_for_path(uri_to_posix_path(*urlpath)); }, [=](media::checksum_atom, const caf::uri &uri) -> result> { - auto path = uri_to_posix_path(uri); - auto size = get_file_size(path); - auto checksum = std::string(); - - if (size) - checksum = get_checksum(path); - return std::make_pair(checksum, size); + return checksum_for_path(uri_to_posix_path(uri)); }, [=](media::rescan_atom, const MediaReference &mr) -> result { @@ -184,48 +203,34 @@ ScanHelperActor::ScanHelperActor(caf::actor_config &cfg) : caf::event_based_acto // cache any checksums we create.. auto path = uri_to_posix_path(uri); auto cpin = std::make_pair(std::get<1>(pin), std::get<2>(pin)); + const auto iter_options = fs::directory_options::skip_permission_denied; try { - for (const auto &entry : fs::recursive_directory_iterator(path)) { + for (const auto &entry : fs::recursive_directory_iterator(path, iter_options)) { try { if (fs::is_regular_file(entry.status())) { - // check we've not alredy got it in cache.. + // check we've not alredy got it in cache.. #ifdef _WIN32 const auto puri = posix_path_to_uri(entry.path().string()); #else const auto puri = posix_path_to_uri(entry.path()); #endif - if (cache_.count(puri)) { - const auto &c = cache_.at(puri); - if (c == cpin) - return puri; - } else { -#ifdef _WIN32 - auto size = get_file_size(entry.path().string()); -#else - auto size = get_file_size(entry.path()); -#endif - if (size == cpin.second) { -#ifdef _WIN32 - auto checksum = get_checksum(entry.path().string()); -#else - auto checksum = get_checksum(entry.path()); -#endif - cache_[puri] = std::make_pair(checksum, size); - if (checksum == cpin.first) - return puri; - } - } + if (get_file_size(entry.path()) != cpin.second) + continue; + + const auto [checksum, size] = checksum_for_path(entry.path()); + if (size == cpin.second && checksum == cpin.first) + return puri; } } catch (...) { } } if (loose_match) { - for (const auto &entry : fs::recursive_directory_iterator(path)) { + for (const auto &entry : fs::recursive_directory_iterator(path, iter_options)) { try { if (fs::is_regular_file(entry.status())) { - // check we've not alredy got it in cache.. + // check we've not alredy got it in cache.. #ifdef _WIN32 const auto puri = posix_path_to_uri(entry.path().string()); #else @@ -247,8 +252,22 @@ ScanHelperActor::ScanHelperActor(caf::actor_config &cfg) : caf::event_based_acto ScannerActor::ScannerActor(caf::actor_config &cfg) : caf::event_based_actor(cfg) { - auto helper = spawn(); - link_to(helper); + auto helper_count = std::max(1, std::min(std::thread::hardware_concurrency(), 4)); + try { + auto prefs = GlobalStoreHelper(system()); + JsonStore js; + prefs.get_group(js); + helper_count = + std::clamp(preference_value(js, "/core/scanner/max_worker_count"), 1, 16); + } catch (...) { + } + + helpers_.reserve(helper_count); + for (size_t helper_index = 0; helper_index < helper_count; ++helper_index) { + auto helper = spawn(); + link_to(helper); + helpers_.emplace_back(std::move(helper)); + } system().registry().put(scanner_registry, this); @@ -273,6 +292,9 @@ ScannerActor::ScannerActor(caf::actor_config &cfg) : caf::event_based_actor(cfg) [=](media::checksum_atom atom, const caf::actor &media_source, const MediaReference &mr) { + auto helper = next_helper(); + if (!helper) + return; mail(atom, mr) .request(helper, infinite) .then( @@ -285,10 +307,16 @@ ScannerActor::ScannerActor(caf::actor_config &cfg) : caf::event_based_actor(cfg) }, [=](media::rescan_atom atom, const MediaReference &mr) { + auto helper = next_helper(); + if (!helper) + return make_error(sec::runtime_error, "No scanner workers"); return mail(atom, mr).delegate(helper); }, [=](media::checksum_atom atom, const MediaReference &mr) { + auto helper = next_helper(); + if (!helper) + return make_error(sec::runtime_error, "No scanner workers"); return mail(atom, mr).delegate(helper); }, @@ -296,6 +324,9 @@ ScannerActor::ScannerActor(caf::actor_config &cfg) : caf::event_based_actor(cfg) const media::MediaSourceChecksum &pin, const caf::uri &path, const bool loose_match) { + auto helper = next_helper(); + if (!helper) + return make_error(sec::runtime_error, "No scanner workers"); return mail(atom, pin, path, loose_match).delegate(helper); }, @@ -320,6 +351,9 @@ ScannerActor::ScannerActor(caf::actor_config &cfg) : caf::event_based_actor(cfg) const media::MediaSourceChecksum &pin, const caf::uri &path, const bool loose_match) { + auto helper = next_helper(); + if (!helper) + return; mail(atom, pin, path, loose_match) .request(helper, infinite) .then( @@ -354,6 +388,15 @@ ScannerActor::ScannerActor(caf::actor_config &cfg) : caf::event_based_actor(cfg) }); } +caf::actor ScannerActor::next_helper() { + if (helpers_.empty()) + return caf::actor{}; + + const auto helper_index = next_helper_index_ % helpers_.size(); + next_helper_index_ = (helper_index + 1) % helpers_.size(); + return helpers_[helper_index]; +} + void ScannerActor::on_exit() { system().registry().erase(scanner_registry); } diff --git a/src/scanner/test/scanner_actor_test.cpp b/src/scanner/test/scanner_actor_test.cpp index 7b2d6343c..b4a6009a7 100644 --- a/src/scanner/test/scanner_actor_test.cpp +++ b/src/scanner/test/scanner_actor_test.cpp @@ -3,12 +3,81 @@ #include #include "xstudio/atoms.hpp" +#include "xstudio/scanner/scanner_actor.hpp" +#include "xstudio/utility/helpers.hpp" + +#include +#include +#include +#include using namespace xstudio; using namespace caf; +using namespace xstudio::scanner; +using namespace xstudio::utility; #include "xstudio/utility/serialise_headers.hpp" ACTOR_TEST_SETUP() -TEST(ScannerkActorTest, Test) { fixture f; } +TEST(ScannerHelperActorTest, ChecksumCacheRefreshesWhenFileChanges) { + fixture f; + auto helper = f.self->spawn(); + + const auto tmpdir = testing::TempDir(); + const auto path = std::filesystem::path(tmpdir) / "scanner_checksum_test.mov"; + + { + std::ofstream stream(path); + stream << "alpha"; + } + + auto uri = posix_path_to_uri(path.string()); + ASSERT_FALSE(uri.empty()); + + const auto first = request_receive>( + *(f.self), helper, media::checksum_atom_v, uri); + const auto second = request_receive>( + *(f.self), helper, media::checksum_atom_v, uri); + + EXPECT_EQ(first, second); + + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + { + std::ofstream stream(path); + stream << "beta-beta"; + } + + const auto third = request_receive>( + *(f.self), helper, media::checksum_atom_v, uri); + + EXPECT_NE(first.first, third.first); + EXPECT_NE(first.second, third.second); +} + +TEST(ScannerHelperActorTest, RelinkFindsMatchingFileByChecksum) { + fixture f; + auto helper = f.self->spawn(); + + const auto tmpdir = std::filesystem::path(testing::TempDir()) / "scanner_relink"; + std::filesystem::create_directories(tmpdir); + const auto file_path = tmpdir / "shot.0001.exr"; + + { + std::ofstream stream(file_path); + stream << "frame-data"; + } + + auto file_uri = posix_path_to_uri(file_path.string()); + auto dir_uri = posix_path_to_uri(tmpdir.string()); + ASSERT_FALSE(file_uri.empty()); + ASSERT_FALSE(dir_uri.empty()); + + const auto [checksum, size] = request_receive>( + *(f.self), helper, media::checksum_atom_v, file_uri); + const media::MediaSourceChecksum pin{file_path.filename().string(), checksum, size}; + + const auto relinked = + request_receive(*(f.self), helper, media::relink_atom_v, pin, dir_uri, false); + EXPECT_EQ(relinked, file_uri); +}