Merge pull request #63 from sz3/stdin

cli update: read input filenames from stdin

.github/workflows/ci.yml

@@ -45,6 +45,10 @@ jobs:
       working-directory: ${{runner.workspace}}/build
       run: make test CTEST_OUTPUT_ON_FAILURE=TRUE
 
+    - name: Usage test
+      working-directory: ${{runner.workspace}}/libcimbar/test/py
+      run: python3 -m unittest
+
   cppcheck:
     runs-on: ubuntu-18.04
 

README.md

@@ -77,12 +77,17 @@ Encode:
 * large input files may fill up your disk with pngs!
 
 ```
-./cimbar --encode -i inputfile.txt -o outputprefix -f
+./cimbar --encode -i inputfile.txt -o outputprefix
 ```
 
 Decode (extracts file into output directory):
 ```
-./cimbar outputprefix*.png -o /tmp -f
+./cimbar outputprefix*.png -o /tmp
+```
+
+Decode a series of encoded images from stdin:
+```
+echo outputprefix*.png | ./cimbar -o /tmp
 ```
 
 Encode and animate to window:

src/exe/cimbar/CMakeLists.txt

@@ -20,6 +20,7 @@ target_link_libraries(cimbar
 	wirehair
 	zstd
 	${OPENCV_LIBS}
+	${CPPFILESYSTEM}
 )
 
 add_custom_command(

src/exe/cimbar/cimbar.cpp

@@ -12,18 +12,118 @@
 
 #include "cxxopts/cxxopts.hpp"
 
+#include <cstdio>
+#include <experimental/filesystem>
 #include <functional>
 #include <iostream>
 #include <string>
 #include <vector>
 using std::string;
 using std::vector;
 
-int decode(const vector<string>& infiles, const std::function<int(cv::UMat, bool, bool)>& decode, bool no_deskew, bool undistort, int preprocess, bool color_correct)
+namespace {
+	class stdinerator
+	{
+	public:
+		stdinerator(bool done=false)
+			: _done(done)
+		{
+			read_once();
+		}
+
+		void mark_done()
+		{
+			_done = true;
+		}
+
+		void read_once()
+		{
+			if (_done)
+				return;
+
+			if (::feof(::stdin))
+			{
+				mark_done();
+				return;
+			}
+
+			std::getline(std::cin, _current);
+		}
+
+		std::string operator*() const
+		{
+			return _current;
+		}
+
+		stdinerator& operator++()
+		{
+			read_once();
+			return *this;
+		}
+
+		bool operator==(const stdinerator& rhs) const
+		{
+			return _done and rhs._done;
+		}
+
+		bool operator!=(const stdinerator& rhs) const
+		{
+			return !operator==(rhs);
+		}
+
+		static stdinerator begin()
+		{
+			return stdinerator();
+		}
+
+		static stdinerator end()
+		{
+			return stdinerator(true);
+		}
+
+	protected:
+		bool _done = false;
+		std::string _current;
+	};
+
+	struct StdinLineReader
+	{
+		stdinerator begin() const
+		{
+			return stdinerator::begin();
+		}
+
+		stdinerator end() const
+		{
+			return stdinerator::end();
+		}
+	};
+}
+
+template <typename FilenameIterable>
+int encode(const FilenameIterable& infiles, const std::string& outpath, int ecc, int color_bits, int compression_level, bool no_fountain)
+{
+	Encoder en(ecc, cimbar::Config::symbol_bits(), color_bits);
+	for (const string& f : infiles)
+	{
+		if (f.empty())
+			continue;
+		if (no_fountain)
+			en.encode(f, outpath);
+		else
+			en.encode_fountain(f, outpath, compression_level);
+	}
+	return 0;
+}
+
+template <typename FilenameIterable>
+int decode(const FilenameIterable& infiles, const std::function<int(cv::UMat, bool, bool)>& decode, bool no_deskew, bool undistort, int preprocess, bool color_correct)
 {
 	int err = 0;
 	for (const string& inf : infiles)
 	{
+		if (inf.empty())
+			continue;
 		bool shouldPreprocess = (preprocess == 1);
 		cv::UMat img = cv::imread(inf).getUMat(cv::ACCESS_RW);
 		cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
@@ -73,51 +173,55 @@ int main(int argc, char** argv)
 	unsigned compressionLevel = cimbar::Config::compression_level();
 	unsigned ecc = cimbar::Config::ecc_bytes();
 	options.add_options()
-	    ("i,in", "Encoded pngs/jpgs/etc (for decode), or file to encode", cxxopts::value<vector<string>>())
-	    ("o,out", "Output file prefix (encoding) or directory (decoding).", cxxopts::value<string>())
-	    ("c,color-bits", "Color bits. [0-3]", cxxopts::value<int>()->default_value(turbo::str::str(colorBits)))
-	    ("e,ecc", "ECC level", cxxopts::value<unsigned>()->default_value(turbo::str::str(ecc)))
-	    ("f,fountain", "Attempt fountain encode/decode", cxxopts::value<bool>())
-	    ("z,compression", "Compression level. 0 == no compression.", cxxopts::value<int>()->default_value(turbo::str::str(compressionLevel)))
-	    ("color-correct", "Toggle decoding color correction. 1 == on. 0 == off.", cxxopts::value<int>()->default_value("1"))
-	    ("encode", "Run the encoder!", cxxopts::value<bool>())
-	    ("no-deskew", "Skip the deskew step -- treat input image as already extracted.", cxxopts::value<bool>())
-	    ("undistort", "Attempt undistort step -- useful if image distortion is significant.", cxxopts::value<bool>())
-	    ("preprocess", "Run sharpen filter on the input image. 1 == on. 0 == off. -1 == guess.", cxxopts::value<int>()->default_value("-1"))
-	    ("h,help", "Print usage")
+		("i,in", "Encoded pngs/jpgs/etc (for decode), or file to encode", cxxopts::value<vector<string>>())
+		("o,out", "Output file prefix (encoding) or directory (decoding).", cxxopts::value<string>())
+		("c,color-bits", "Color bits. [0-3]", cxxopts::value<int>()->default_value(turbo::str::str(colorBits)))
+		("e,ecc", "ECC level", cxxopts::value<unsigned>()->default_value(turbo::str::str(ecc)))
+		("z,compression", "Compression level. 0 == no compression.", cxxopts::value<int>()->default_value(turbo::str::str(compressionLevel)))
+		("color-correct", "Toggle decoding color correction. 1 == on. 0 == off.", cxxopts::value<int>()->default_value("1"))
+		("encode", "Run the encoder!", cxxopts::value<bool>())
+		("no-deskew", "Skip the deskew step -- treat input image as already extracted.", cxxopts::value<bool>())
+		("no-fountain", "Disable fountain encode/decode. Will also disable compression.", cxxopts::value<bool>())
+		("undistort", "Attempt undistort step -- useful if image distortion is significant.", cxxopts::value<bool>())
+		("preprocess", "Run sharpen filter on the input image. 1 == on. 0 == off. -1 == guess.", cxxopts::value<int>()->default_value("-1"))
+		("h,help", "Print usage")
 	;
 	options.show_positional_help();
 	options.parse_positional({"in"});
 	options.positional_help("<in...>");
 
 	auto result = options.parse(argc, argv);
-	if (result.count("help") or !result.count("out") or !result.count("in"))
+	if (result.count("help"))
 	{
-	  std::cout << options.help() << std::endl;
-	  exit(0);
+		std::cerr << options.help() << std::endl;
+		exit(0);
 	}
 
-	vector<string> infiles = result["in"].as<vector<string>>();
-	string outpath = result["out"].as<string>();
+	string outpath = std::experimental::filesystem::current_path();
+	if (result.count("out"))
+		outpath = result["out"].as<string>();
+	std::cerr << "Output files will appear in " << outpath << std::endl;
 
-	bool encode = result.count("encode");
-	bool fountain = result.count("fountain");
+	bool useStdin = !result.count("in");
+	vector<string> infiles;
+	if (useStdin)
+		std::cerr << "Enter input filenames:" << std::endl;
+	else
+		infiles = result["in"].as<vector<string>>();
+
+	bool encodeFlag = result.count("encode");
+	bool no_fountain = result.count("no-fountain");
 
 	colorBits = std::min(3, result["color-bits"].as<int>());
 	compressionLevel = result["compression"].as<int>();
 	ecc = result["ecc"].as<unsigned>();
 
-	if (encode)
+	if (encodeFlag)
 	{
-		Encoder en(ecc, cimbar::Config::symbol_bits(), colorBits);
-		for (const string& f : infiles)
-		{
-			if (fountain)
-				en.encode_fountain(f, outpath, compressionLevel);
-			else
-				en.encode(f, outpath);
-		}
-		return 0;
+		if (useStdin)
+			return encode(StdinLineReader(), outpath, ecc, colorBits, compressionLevel, no_fountain);
+		else
+			return encode(infiles, outpath, ecc, colorBits, compressionLevel, no_fountain);
 	}
 
 	// else, decode
@@ -128,24 +232,32 @@ int main(int argc, char** argv)
 
 	Decoder d(ecc, colorBits);
 
-	if (fountain)
+	if (no_fountain)
 	{
-		unsigned chunkSize = cimbar::Config::fountain_chunk_size(ecc);
-		if (compressionLevel <= 0)
-		{
-			fountain_decoder_sink<std::ofstream> sink(outpath, chunkSize);
-			return decode(infiles, fountain_decode_fun(sink, d), no_deskew, undistort, preprocess, color_correct);
-		}
+		// simpler encoding, just the basics + ECC. No compression, fountain codes, etc.
+		std::ofstream f(outpath);
+		std::function<int(cv::UMat,bool,bool)> fun = [&f, &d] (cv::UMat m, bool pre, bool cc) {
+			return d.decode(m, f, pre, cc);
+		};
+		if (useStdin)
+			return decode(StdinLineReader(), fun, no_deskew, undistort, preprocess, color_correct);
+		else
+			return decode(infiles, fun, no_deskew, undistort, preprocess, color_correct);
+	}
 
-		// else -- default case, all bells and whistles
-		fountain_decoder_sink<cimbar::zstd_decompressor<std::ofstream>> sink(outpath, chunkSize);
+	// else, the good stuff
+	unsigned chunkSize = cimbar::Config::fountain_chunk_size(ecc);
+	if (compressionLevel <= 0)
+	{
+		fountain_decoder_sink<std::ofstream> sink(outpath, chunkSize, true);
 		return decode(infiles, fountain_decode_fun(sink, d), no_deskew, undistort, preprocess, color_correct);
 	}
 
-	// else
-	std::ofstream f(outpath);
-	std::function<int(cv::UMat,bool,bool)> fun = [&f, &d] (cv::UMat m, bool pre, bool cc) {
-		return d.decode(m, f, pre, cc);
-	};
-	return decode(infiles, fun, no_deskew, undistort, preprocess, color_correct);
+	// else -- default case, all bells and whistles
+	fountain_decoder_sink<cimbar::zstd_decompressor<std::ofstream>> sink(outpath, chunkSize, true);
+
+	if (useStdin)
+		return decode(StdinLineReader(), fountain_decode_fun(sink, d), no_deskew, undistort, preprocess, color_correct);
+	else
+		return decode(infiles, fountain_decode_fun(sink, d), no_deskew, undistort, preprocess, color_correct);
 }

src/lib/fountain/fountain_decoder_sink.h

@@ -5,6 +5,7 @@
 #include "FountainMetadata.h"
 #include "serialize/format.h"
 
+#include <cstdio>
 #include <set>
 #include <string>
 #include <unordered_map>
@@ -15,9 +16,10 @@ template <typename OUTSTREAM>
 class fountain_decoder_sink
 {
 public:
-	fountain_decoder_sink(std::string data_dir, unsigned chunk_size)
-	    : _dataDir(data_dir)
-	    , _chunkSize(chunk_size)
+	fountain_decoder_sink(std::string data_dir, unsigned chunk_size, bool log_writes=false)
+		: _dataDir(data_dir)
+		, _chunkSize(chunk_size)
+		, _logWrites(log_writes)
 	{
 	}
 
@@ -36,6 +38,8 @@ class fountain_decoder_sink
 		std::string file_path = fmt::format("{}/{}", _dataDir, get_filename(md));
 		OUTSTREAM f(file_path);
 		f.write((char*)data.data(), data.size());
+		if (_logWrites)
+			printf("%s\n", file_path.c_str());
 		return true;
 	}
 
@@ -143,4 +147,5 @@ class fountain_decoder_sink
 	std::unordered_map<uint8_t, fountain_decoder_stream> _streams;
 	// track the uint32_t combo of (encode_id,size) to avoid redundant work
 	std::set<uint32_t> _done;
+	bool _logWrites;
 };

test/py/helpers.py

@@ -0,0 +1,16 @@
+from os.path import join, realpath, dirname
+from tempfile import TemporaryDirectory
+
+CIMBAR_SRC = realpath(join(dirname(realpath(__file__)), '..', '..'))
+BIN_DIR = join(CIMBAR_SRC, 'dist', 'bin')
+
+
+class TestDirMixin():
+    def setUp(self):
+        self.working_dir = TemporaryDirectory()
+        super().setUp()
+
+    def tearDown(self):
+        super().tearDown()
+        with self.working_dir:
+            pass