Add VideoCorrelation new algorithm to merge video sequences by video …

…frame data.

Add MSND algorithm implementation with scientific paper reference.
Donate info updated. Various minor fixes.

.gitignore

@@ -1,2 +1,2 @@
 /__build
-/proprietary
+*.aps

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "thirdparty/JM"]
 	path = thirdparty/JM
 	url = https://github.com/slavanap/JM.git
-[submodule "Sub3D"]
-	path = proprietary
-	url = p:/sub3d
 [submodule "thirdparty/libudfread"]
 	path = thirdparty/libudfread
 	url = git://git.videolan.org/libudfread.git

AvsTools/CMakeLists.txt

@@ -10,6 +10,7 @@ set(HEADERS
 	Filter.RestoreAlpha.hpp
 	Filter.SequentialToSeekable.hpp
 	Filter.SetLogger.hpp
+	Filter.VideoCorrelation.hpp
 	Tools.AviSynth.hpp
 	Tools.AviSynth.Frame.hpp
 	Tools.Motion.hpp
@@ -25,19 +26,14 @@ set(SOURCES
 	Filter.RestoreAlpha.cpp
 	Filter.SequentialToSeekable.cpp
 	Filter.SetLogger.cpp
+	Filter.VideoCorrelation.cpp
 	Tools.AviSynth.cpp
 	Tools.AviSynth.Frame.cpp
 	Tools.Motion.cpp
 	Tools.Pipe.cpp
 	Tools.WinApi.cpp
 )
 
-if (EXISTS "${CMAKE_SOURCE_DIR}/proprietary/Tools.Motion.Proprietary.hpp")
-	add_definitions(-D MOTION_PROPRIETARY)
-	include_directories(${CMAKE_CURRENT_SOURCE_DIR} "${CMAKE_SOURCE_DIR}/proprietary")
-	list(APPEND HEADERS "${CMAKE_SOURCE_DIR}/proprietary/Tools.Motion.Proprietary.hpp")
-endif()
-
 set_precompiled_header(stdafx.h stdafx.cpp)
 
 add_library(${PROJECT_NAME} STATIC ${HEADERS} ${SOURCES})

AvsTools/Filter.SequentialToSeekable.hpp

@@ -30,7 +30,25 @@ namespace Filter {
 	public:
 		SequentialToSeekable(IScriptEnvironment* env, const char* command);
 		PVideoFrame WINAPI GetFrame(int n, IScriptEnvironment* env);
-
+
+		bool WINAPI GetParity(int n) override {
+			return clip->GetParity(n);
+		}
+
+		void WINAPI GetAudio(void* buf, __int64 start, __int64 count, IScriptEnvironment* env) override {
+			return clip->GetAudio(buf, start, count, env);
+		}
+
+#if defined __AVISYNTH_H__
+		void WINAPI SetCacheHints(int cachehints, int frame_range) override {
+			clip->SetCacheHints(cachehints, frame_range);
+		}
+#elif defined __AVISYNTH_6_H__
+		int WINAPI SetCacheHints(int cachehints, int frame_range) override {
+			return clip->SetCacheHints(cachehints, frame_range);
+		}
+#endif
+
 		static AvsParams CreateParams;
 		static AVSValue __cdecl Create(AVSValue args, void* user_data, IScriptEnvironment* env);
 	private:

AvsTools/Filter.VideoCorrelation.cpp

@@ -0,0 +1,215 @@
+#include "stdafx.h"
+#include <iterator>
+#include <fstream>
+
+#include "Tools.AviSynth.Frame.hpp"
+#include "Filter.VideoCorrelation.hpp"
+
+namespace Filter {
+
+	// API & Tools
+
+	std::ostream& operator<<(std::ostream& out, const VideoCorrelation::Signature& vs) {
+		out << vs.size() << std::endl;
+		for (auto it = vs.begin(); it != vs.end(); ++it)
+			out << *it << " ";
+		return out;
+	}
+
+	std::istream& operator>>(std::istream& in, VideoCorrelation::Signature& vs) {
+		size_t temp;
+		if (!(in >> temp))
+			goto error;
+		vs.resize(temp);
+		for (size_t i = 0; i < temp; ++i) {
+			if (!(in >> vs[i]))
+				goto error;
+		}
+		return in;
+	error:
+		throw std::runtime_error("Invalid array in file");
+	}
+
+	AvsParams VideoCorrelation::CreateParams = "[input]c[overlay]c[shift]i[threshold]i[preprocess_input]s";
+	AVSValue VideoCorrelation::Create(AVSValue args, void* user_data, IScriptEnvironment* env) {
+		PClip input = args[0].AsClip();
+		PClip overlay = args[1].AsClip();
+		int shift = args[2].AsInt(-1);
+		int threshold = args[3].AsInt(0);
+		const char* preprocess = args[4].AsString(nullptr);
+		try {
+			Signature sig;
+			if (preprocess != nullptr) {
+				std::ifstream f(preprocess);
+				if (!f.good())
+					throw std::runtime_error("Can't open file");
+				f >> sig;
+			}
+			return new VideoCorrelation(env, input, overlay, shift, sig, threshold);
+		}
+		catch (std::exception& e) {
+			env->ThrowError(e.what());
+			return AVSValue();
+		}
+	}
+
+	AvsParams VideoCorrelation::PreprocessParams = "c[output]s";
+	AVSValue VideoCorrelation::Preprocess(AVSValue args, void* user_data, IScriptEnvironment* env) {
+		try {
+			std::ofstream f(args[1].AsString());
+			if (!f.good())
+				throw std::runtime_error("Can't create file");
+			auto sig = Preprocess(env, args[0].AsClip());
+			f << sig;
+			f.close();
+			return AVSValue();
+		}
+		catch (std::exception& e) {
+			env->ThrowError("%s", e.what());
+			return AVSValue();
+		}
+	}
+
+	AvsParams VideoCorrelation::GetShiftParams = "[overlay]c[input_sig]s";
+	AVSValue VideoCorrelation::GetShift(AVSValue args, void* user_data, IScriptEnvironment* env) {
+		try {
+			std::ifstream f(args[1].AsString());
+			if (!f.good())
+				throw std::runtime_error("Can't open file");
+			Signature input_sig;
+			f >> input_sig;
+			return GetShift(env, args[0].AsClip(), input_sig);
+		}
+		catch (std::exception& e) {
+			env->ThrowError("%s", e.what());
+			return AVSValue();
+		}
+	}
+
+
+
+	// Algorithms
+
+	using Tools::AviSynth::Frame;
+
+	constexpr size_t MaxSigValue = (size_t)((Frame::ColorCount - 1) * (Frame::ColorCount - 1) * 3) * 100;
+
+	VideoCorrelation::Signature::value_type VideoCorrelation::ComputeForFrame(const Frame& frame) {
+		uint64_t result = 0;
+		for (int j = 0; j < frame.height(); ++j) {
+			const Frame::Pixel *row = frame.read_row(j);
+			for (int k = 0; k < frame.width(); ++k) {
+				const Frame::Pixel &px = row[k];
+				result += (uint32_t)px.r*px.r + (uint32_t)px.g*px.g + (uint32_t)px.b*px.b;
+			}
+		}
+		result = result * 100 / (frame.width() * frame.height());
+		return static_cast<SigValue>(result);
+	}
+
+	VideoCorrelation::Signature VideoCorrelation::Preprocess(IScriptEnvironment* env, PClip clip) {
+		clip = Tools::AviSynth::ConvertToRGB32(env, clip);
+		int len = clip->GetVideoInfo().num_frames;
+		Signature result;
+		result.reserve(len);
+		for (int i = 0; i < len; ++i)
+			result.push_back(ComputeForFrame(Frame(env, clip, i)));
+		return result;
+	}
+
+	int VideoCorrelation::GetShift(IScriptEnvironment* env, PClip overlay, const Signature& input_sig) {
+		if ((int)input_sig.size() <= overlay->GetVideoInfo().num_frames)
+			throw std::exception("Overlay must me shorter than input");
+		Signature overlay_sig = Preprocess(env, overlay);
+		Correlation<SigValue> c(overlay_sig, MaxSigValue);
+		for (SigValue v : input_sig)
+			if (c.Next(v))
+				break;
+		return c.GetMatch();
+	}
+
+
+
+	// Overlay implementation
+
+	constexpr int OverlaySigShiftFrames = 1200;
+	constexpr int OverlaySigLength = 1000;
+
+	VideoCorrelation::VideoCorrelation(IScriptEnvironment* env, PClip input, PClip overlay,
+		int shift, const Signature& input_sig, Signature::value_type threshold)
+		:
+		GenericVideoFilter(Tools::AviSynth::ConvertToRGB32(env, input)),
+		m_overlay(Tools::AviSynth::ConvertToRGB32(env, overlay)),
+		m_shift(shift)
+	{
+		m_overlay_vi = m_overlay->GetVideoInfo();
+
+		if (vi.width != m_overlay_vi.width || vi.height != m_overlay_vi.height || vi.pixel_type != m_overlay_vi.pixel_type)
+			env->ThrowError("Overlay and input properties must be equal");
+
+		if (vi.num_frames <= m_overlay_vi.num_frames)
+			env->ThrowError("Overlay must be shorter than input");
+
+		int overlay_sig_len = std::min(m_overlay_vi.num_frames, OverlaySigLength);
+		m_overlay_sig_shift = std::min(m_overlay_vi.num_frames - overlay_sig_len, OverlaySigShiftFrames);
+		if (m_overlay_sig_shift < OverlaySigShiftFrames)
+			fprintf(stderr, "WARNING: overlay is too short. Consider increasing its length to improve matching quality\n");
+
+		if (m_shift < 0) {
+			for (int i = 0; i < m_overlay_sig_shift; ++i)
+				m_overlay->GetFrame(i, env);
+			Signature overlay_sig;
+			overlay_sig.reserve(overlay_sig_len);
+			for (int i = m_overlay_sig_shift; i < m_overlay_sig_shift + overlay_sig_len; ++i)
+				overlay_sig.push_back(ComputeForFrame(Frame(env, m_overlay, i)));
+			m_correlation = std::make_unique<Correlation<SigValue>>(overlay_sig, MaxSigValue, threshold);
+			if (!input_sig.empty()) {
+				for (SigValue v : input_sig) {
+					m_correlation->Next(v);
+				}
+				if (m_correlation->IsMatchFound()) {
+					m_shift = m_correlation->GetMatch() - m_overlay_sig_shift;
+				}
+				else {
+					m_shift = vi.num_frames;
+					fprintf(stderr, "WARNING: no match found with specified threshold\n");
+				}
+			}
+			for (int i = 0; i < m_overlay_sig_shift; ++i)
+				m_overlay->GetFrame(i, env);
+		}
+	}
+
+	PVideoFrame VideoCorrelation::GetFrame(int n, IScriptEnvironment* env) {
+		if (m_shift < 0) {
+			if (m_correlation->Next(ComputeForFrame(Frame(env, child, n))))
+				m_shift = m_correlation->GetMatch() - m_overlay_sig_shift;
+		}
+		if (0 <= m_shift && m_shift < vi.num_frames) {
+			if (m_shift <= n && n < m_shift + m_overlay_vi.num_frames)
+				return m_overlay->GetFrame(n - m_shift, env);
+		}
+		return child->GetFrame(n, env);
+	}
+
+	void VideoCorrelation::GetAudio(void* buf, __int64 start, __int64 count, IScriptEnvironment* env) {
+		if (0 <= m_shift && m_shift < vi.num_frames) {
+			auto audio_start = vi.AudioSamplesFromFrames(m_shift);
+			auto audio_end = vi.AudioSamplesFromFrames(m_shift + m_overlay_vi.num_frames);
+			if (audio_start <= start && start < audio_end - count) {
+				m_overlay->GetAudio(buf, start - audio_start, count, env);
+				return;
+			}
+		}
+		child->GetAudio(buf, start, count, env);
+	}
+
+	bool VideoCorrelation::GetParity(int n) {
+		if (0 <= m_shift && m_shift < vi.num_frames) {
+			if (m_shift <= n && n < m_shift + m_overlay_vi.num_frames)
+				return m_overlay->GetParity(n - m_shift);
+		}
+		return child->GetParity(n);
+	}
+
+}