diff --git a/src/API/Attachment.vala b/src/API/Attachment.vala index a8e1bf082..9b0e19761 100644 --- a/src/API/Attachment.vala +++ b/src/API/Attachment.vala @@ -4,7 +4,8 @@ public class Tuba.API.Attachment : Entity, Widgetizable { public string kind { get; set; default = "unknown"; } public string url { get; set; } public string? description { get; set; } - public string? t_preview_url { get; set; } + public string? blurhash { get; set; default=null; } + private string? t_preview_url { get; set; } public string? preview_url { set { this.t_preview_url = value; } get { return (this.t_preview_url == null || this.t_preview_url == "") ? url : t_preview_url; } diff --git a/src/API/Status/PreviewCard.vala b/src/API/Status/PreviewCard.vala index 2f7a0321e..0422e2bba 100644 --- a/src/API/Status/PreviewCard.vala +++ b/src/API/Status/PreviewCard.vala @@ -98,6 +98,7 @@ public class Tuba.API.PreviewCard : Entity, Widgetizable { public string provider_name { get; set; default=""; } public string provider_url { get; set; default=""; } public string? image { get; set; default=null; } + public string? blurhash { get; set; default=null; } public Gee.ArrayList? history { get; set; default = null; } public CardSpecialType card_special_type { get { diff --git a/src/Application.vala b/src/Application.vala index e61b3aaca..ff01fbe5e 100644 --- a/src/Application.vala +++ b/src/Application.vala @@ -15,6 +15,7 @@ namespace Tuba { public static EntityCache entity_cache; public static ImageCache image_cache; + public static BlurhashCache blurhash_cache; public static GLib.Regex bookwyrm_regex; public static GLib.Regex custom_emoji_regex; @@ -130,6 +131,9 @@ namespace Tuba { image_cache = new ImageCache () { maintenance_secs = 60 * 5 }; + blurhash_cache = new BlurhashCache () { + maintenance_secs = 30 + }; accounts = new SecretAccountStore (); accounts.init (); diff --git a/src/Services/Cache/AbstractCache.vala b/src/Services/Cache/AbstractCache.vala index 45225c1b9..afdac4479 100644 --- a/src/Services/Cache/AbstractCache.vala +++ b/src/Services/Cache/AbstractCache.vala @@ -85,7 +85,7 @@ public class Tuba.AbstractCache : Object { return items.has_key (get_key (id)); } - protected string insert (string id, owned Object obj) { + protected virtual string insert (string id, owned Object obj) { var key = get_key (id); debug (@"Inserting: $key"); items.@set (key, (owned) obj); diff --git a/src/Services/Cache/BlurhashCache.vala b/src/Services/Cache/BlurhashCache.vala new file mode 100644 index 000000000..ac2a0c579 --- /dev/null +++ b/src/Services/Cache/BlurhashCache.vala @@ -0,0 +1,18 @@ +public class Tuba.BlurhashCache : AbstractCache { + public Gdk.Paintable? lookup_or_decode (string? blurhash) { + if (blurhash == null) return null; + + var key = get_key (blurhash); + if (contains (key)) return lookup (key) as Gdk.Paintable?; + + var pixbuf = Tuba.Blurhash.blurhash_to_pixbuf (blurhash, 32, 32); + if (pixbuf != null) { + var paintable = Gdk.Texture.for_pixbuf (pixbuf); + insert (blurhash, paintable); + + return paintable; + } + + return null; + } +} diff --git a/src/Services/Cache/meson.build b/src/Services/Cache/meson.build index 231d5eac3..af4ad4f86 100644 --- a/src/Services/Cache/meson.build +++ b/src/Services/Cache/meson.build @@ -1,5 +1,6 @@ sources += files( 'AbstractCache.vala', + 'BlurhashCache.vala', 'EntityCache.vala', 'ImageCache.vala', ) diff --git a/src/Utils/Blurhash.vala b/src/Utils/Blurhash.vala new file mode 100644 index 000000000..b9c442f9a --- /dev/null +++ b/src/Utils/Blurhash.vala @@ -0,0 +1,192 @@ +// Blurhash decoding in pure Vala inspired by +// https://github.com/woltapp/blurhash and https://github.com/mad-gooze/fast-blurhash/ +class Tuba.Blurhash { + struct AverageColor { + int r; + int g; + int b; + } + + struct ColorSRGB { + float r; + float g; + float b; + } + + public class Base83 { + const char[] CHARACTERS = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '#', '$', '%', '*', '+', ',', '-', '.', ':', ';', '=', '?', '@', '[', ']', '^', '_', '{', '|', '}', '~' + }; + + // Unused, but works + // public static string encode (int value, int length) { + // StringBuilder res = new StringBuilder (); + + // for (int i = 1; i <= length; i++) { + // int digit = (int) (value / Math.pow (83, length - i) % 83); + // res.append_c (CHARACTERS[digit]); + // } + + // return res.str; + // } + + public static int decode (string value) { + int res = 0; + + for (int i = 0; i < value.length; i++) { + char character = value[i]; + + int index = -1; + for (int j = 0; j < 83; j++) { + if (CHARACTERS[j] == character) { + index = j; + break; + } + } + if (index == -1) return 0; + + res = res * 83 + index; + } + + return res; + } + } + + // Decodes a Base83 string partially from `start` to `end`. + // WARNING: sanitize start and end manually, this is only used + // here and only on valid blurhashes. + private static int decode_partial (string str, int start, int end) { + if (start > end || end >= str.length) return 0; + return Base83.decode (str.slice (start, end)); + } + + private static int linear_to_srgb (float value) { + float v = value.clamp (0f, 1f); + if (v <= 0.0031308) return (int) (v * 12.92f * 255 + 0.5); + + return (int) ((1.055 * Math.powf (v, 1 / 2.4f) - 0.055) * 255 + 0.5); + } + + private static float srgb_to_linear (int value) { + float v = value / 255f; + if (v <= 0.04045) return v / 12.92f; + + return Math.powf ((v + 0.055f) / 1.055f, 2.4f); + } + + private static float sign_pow (float value, float exp) { + return Math.copysignf (Math.powf (Math.fabsf (value), exp), value); + } + + public static bool is_valid_blurhash (string blurhash, out int size_flag, out int num_x, out int num_y, out int size) { + size_flag = 0; + num_y = 0; + num_x = 0; + size = 0; + + int hash_length = blurhash.length; + if (hash_length < 6) return false; + + size_flag = decode_partial (blurhash, 0, 1); + num_y = (int) Math.floorf (size_flag / 9) + 1; + num_x = (size_flag % 9) + 1; + size = num_x * num_y; + + if (hash_length != 4 + 2 * size) return false; + return true; + } + + private static AverageColor get_blurhash_average_color (string blurhash) { + int val = decode_partial (blurhash, 2, 6); + return { val >> 16, (val >> 8) & 255, val & 255 }; + } + + private static ColorSRGB decode_ac (int value, float maximum_value) { + int quant_r = (int)Math.floorf (value / (19 * 19)); + int quant_g = (int)Math.floorf (value / 19) % 19; + int quant_b = (int)value % 19; + + return ColorSRGB () { + r = sign_pow (((float)quant_r - 9) / 9, 2.0f) * maximum_value, + g = sign_pow (((float)quant_g - 9) / 9, 2.0f) * maximum_value, + b = sign_pow (((float)quant_b - 9) / 9, 2.0f) * maximum_value + }; + } + + public static uint8[]? decode_to_data (string blurhash, int width, int height, int punch = 1, bool has_alpha = true) { + int bytes_per_row = width * (has_alpha ? 4 : 3); + uint8[] res = new uint8[bytes_per_row * height]; + + int size_flag; + int num_y; + int num_x; + int size; + + if (!is_valid_blurhash (blurhash, out size_flag, out num_x, out num_y, out size)) return null; + if (punch < 1) punch = 1; + + float maximum_value = ((float)(decode_partial (blurhash, 1, 2) + 1)) / 166; + float[] colors = new float[size * 3]; + + AverageColor average_color = get_blurhash_average_color (blurhash); + colors[0] = srgb_to_linear (average_color.r); + colors[1] = srgb_to_linear (average_color.g); + colors[2] = srgb_to_linear (average_color.b); + + for (int i = 1; i < size; i++) { + int value = decode_partial (blurhash, 4 + i * 2, 6 + i * 2); + + ColorSRGB color = decode_ac (value, maximum_value); + colors[i * 3] = color.r; + colors[i * 3 + 1] = color.g; + colors[i * 3 + 2] = color.b; + } + + for (int y = 0; y < height; y++) { + float yh = (float) (Math.PI * y) / height; + for (int x = 0; x < width; x++) { + float r = 0; + float g = 0; + float b = 0; + float xw = (float) (Math.PI * x) / width; + + for (int j = 0; j < num_y; j++) { + float basis_y = Math.cosf (yh * j); + for (int i = 0; i < num_x; i++) { + float basis = Math.cosf (xw * i) * basis_y; + + int color_index = (i + j * num_x) * 3; + r += colors[color_index] * basis; + g += colors[color_index + 1] * basis; + b += colors[color_index + 2] * basis; + } + } + + int pixel_index = 4 * x + y * bytes_per_row; + res[pixel_index] = (uint8) linear_to_srgb (r); + res[pixel_index + 1] = (uint8) linear_to_srgb (g); + res[pixel_index + 2] = (uint8) linear_to_srgb (b); + + if (has_alpha) + res[pixel_index + 3] = (uint8) 255; + } + } + + return res; + } + + public static Gdk.Pixbuf? blurhash_to_pixbuf (string blurhash, int width, int height) { + uint8[]? data = decode_to_data (blurhash, width, height); + if (data == null) return null; + + return new Gdk.Pixbuf.from_data ( + data, + Gdk.Colorspace.RGB, + true, + 8, + width, + height, + 4 * height + ); + } +} diff --git a/src/Utils/meson.build b/src/Utils/meson.build index 3dc8ad191..1973b9d9b 100644 --- a/src/Utils/meson.build +++ b/src/Utils/meson.build @@ -1,4 +1,5 @@ sources += files( + 'Blurhash.vala', 'Celebrate.vala', 'DateTime.vala', 'Host.vala', diff --git a/src/Widgets/Attachment/Image.vala b/src/Widgets/Attachment/Image.vala index a51f07782..59b89e6fe 100644 --- a/src/Widgets/Attachment/Image.vala +++ b/src/Widgets/Attachment/Image.vala @@ -49,6 +49,7 @@ public class Tuba.Widgets.Attachment.Image : Widgets.Attachment.Item { protected override void on_rebind () { base.on_rebind (); pic.alternative_text = entity == null ? null : entity.description; + image_cache.request_paintable (entity.preview_url, on_cache_response); if (media_kind in VIDEO_TYPES) { @@ -72,8 +73,11 @@ public class Tuba.Widgets.Attachment.Image : Widgets.Attachment.Item { } protected virtual void on_cache_response (bool is_loaded, owned Gdk.Paintable? data) { - if (is_loaded) + if (is_loaded) { pic.paintable = data; + } else { + pic.paintable = blurhash_cache.lookup_or_decode (entity.blurhash); + } } public signal void spoiler_revealed (); diff --git a/src/Widgets/PreviewCard.vala b/src/Widgets/PreviewCard.vala index a3e74e7b2..dc1a33f50 100644 --- a/src/Widgets/PreviewCard.vala +++ b/src/Widgets/PreviewCard.vala @@ -26,8 +26,11 @@ public class Tuba.Widgets.PreviewCard : Gtk.Button { }; image_cache.request_paintable (card_obj.image, (is_loaded, paintable) => { - if (is_loaded) + if (is_loaded) { image.paintable = paintable; + } else { + image.paintable = blurhash_cache.lookup_or_decode (card_obj.blurhash); + } }); if (is_video) { diff --git a/tests/Blurhash.test.vala b/tests/Blurhash.test.vala new file mode 100644 index 000000000..de0e0dea6 --- /dev/null +++ b/tests/Blurhash.test.vala @@ -0,0 +1,126 @@ +struct TestBase83Decode { + public string encoded; + public int decoded; +} + +struct TestBlurhashValidity { + public string blurhash; + public bool valid; +} + +struct TestBlurhashRatio { + public string blurhash; + public int x; + public int y; +} + +struct TestBlurhashData { + public string blurhash; + public uint8 data_1; + public uint8 data_2; + public uint8 data_3; + public uint8 data_4; +} + +const TestBase83Decode[] BASE83_DECODE_TESTS = { + { "tuba", 31837176 }, + { "0m0R1", 27448018 }, + { "P7btDt@Pap!szZkkEoSnK5e%cg!QC4", 0 }, + { "VPmm5Ft%!9tG5hC7J^vwHToZoFJVKLDgY78kE%dPaiLB^^rv^P9f6UwR*p@c!UB", 0 }, + { "L00000fQfQfQfQfQfQfQfQfQfQfQ", -718366762 }, + { "LGF5]+Yk^6#M@-5c,1J5@[or[Q6.", 934061677 }, + { "L6PZfSjE.AyE_3t7t7R**0o#DgR4", -1746869106 }, + { "LKO2:N%2Tw=w]~RBVZRi};RPxuwH", -1074644314 }, + { "LEHLk~WB2yk8pyo0adR*.7kCMdnj", 1224798277 } +}; + +const TestBlurhashValidity[] BLURHASH_VALIDITY_TESTS = { + { "invalidblurhash", false }, + { "tuba", false }, + { "6chars", false }, + { "L00000fQfQfQfQfQfQfQfQfQfQfQ", true }, + { "LGF5]+Yk^6#M@-5c,1J5@[or[Q6.", true }, + { "L6PZfSjE.AyE_3t7t7R**0o#DgR4", true }, + { "LKO2:N%2Tw=w]~RBVZRi};RPxuwH", true }, + { "LEHLk~WB2yk8pyo0adR*.7kCMdnj", true }, + { "LEHLk~WB2yk8pyo0adR*.7kCMdnj.", false } +}; + +const TestBlurhashRatio[] BLURHASH_RATIO_TESTS = { + { "L00000fQfQfQfQfQfQfQfQfQfQfQ", 4, 3 }, + { "LGF5]+Yk^6#M@-5c,1J5@[or[Q6.", 4, 3 }, + { "L6PZfSjE.AyE_3t7t7R**0o#DgR4", 4, 3 }, + { "LKO2:N%2Tw=w]~RBVZRi};RPxuwH", 4, 3 }, + { "LEHLk~WB2yk8pyo0adR*.7kCMdnj", 4, 3 }, + { "oHF5]+Yk^6#M9wKS@-5b,1J5O[V=@[or[k6.O[TL};FxngOZE3NgjMFxS#OtcXnzj]OYNeR:JCs9", 6, 6 }, + { "o6PZfSi_.AyE8^m+_3t7t7R*WBs,*0o#DgR4.Tt,_3R*D%xt%MIpMcV@%itSI9R5Iot7-:IoM{%L", 6, 6 }, + { "oKN]Rv%2Tw=wR6cE]~RBVZRip0W9};RPxuwH%3s8tLOtxZ%gixtQI.ENa0NZIVt6%1j^M_bcRPX9", 6, 6 }, + { "oEHLk~WB2yk8$Nxupyo0adR*=ss:.7kCMdnjx]S2S#M|%1%2ENRiSis.slNHW:WBogaekBW;ofo0", 6, 6 } +}; + +const TestBlurhashData[] BLURHASH_TO_DATA_TESTS = { + { "invalid", 0, 0, 0, 0 }, + { "L00000fQfQfQfQfQfQfQfQfQfQfQ", 6, 6, 6, 255 }, + { "LGF5]+Yk^6#M@-5c,1J5@[or[Q6.", 173, 129, 188, 255 }, + { "L6PZfSjE.AyE_3t7t7R**0o#DgR4", 230, 228, 225, 255 }, + { "LKO2:N%2Tw=w]~RBVZRi};RPxuwH", 243, 194, 173, 255 }, + { "LEHLk~WB2yk8pyo0adR*.7kCMdnj", 159, 175, 181, 255 } +}; + +public void test_base83_decode () { + foreach (var test_base83_decode in BASE83_DECODE_TESTS) { + var res = Tuba.Blurhash.Base83.decode (test_base83_decode.encoded); + + assert_cmpint (res, CompareOperator.EQ, test_base83_decode.decoded); + } +} + +public void test_blurhash_validity () { + foreach (var test_blurhash_validity in BLURHASH_VALIDITY_TESTS) { + var res = Tuba.Blurhash.is_valid_blurhash (test_blurhash_validity.blurhash, null, null, null, null); + + if (test_blurhash_validity.valid) { + assert_true (res); + } else { + assert_false (res); + } + } +} + +public void test_blurhash_ratio () { + foreach (var test_blurhash_ratio in BLURHASH_RATIO_TESTS) { + var res_x = 0; + var res_y = 0; + var res = Tuba.Blurhash.is_valid_blurhash (test_blurhash_ratio.blurhash, null, out res_x, out res_y, null); + + assert_true (res); + assert_cmpint (res_x, CompareOperator.EQ, test_blurhash_ratio.x); + assert_cmpint (res_y, CompareOperator.EQ, test_blurhash_ratio.y); + } +} + +public void test_blurhash_data () { + foreach (var test_blurhash_data in BLURHASH_TO_DATA_TESTS) { + var res = Tuba.Blurhash.decode_to_data (test_blurhash_data.blurhash, 10, 10); + + if (test_blurhash_data.blurhash == "invalid") { + assert (res == null); + } else { + assert (res != null); + assert_cmpuint (res[8], CompareOperator.EQ, test_blurhash_data.data_1); + assert_cmpuint (res[9], CompareOperator.EQ, test_blurhash_data.data_2); + assert_cmpuint (res[10], CompareOperator.EQ, test_blurhash_data.data_3); + assert_cmpuint (res[11], CompareOperator.EQ, test_blurhash_data.data_4); + } + } +} + +public int main (string[] args) { + Test.init (ref args); + + Test.add_func ("/test_base83_decode", test_base83_decode); + Test.add_func ("/test_blurhash_validity", test_blurhash_validity); + Test.add_func ("/test_blurhash_ratio", test_blurhash_ratio); + Test.add_func ("/test_blurhash_data", test_blurhash_data); + return Test.run (); +} diff --git a/tests/meson.build b/tests/meson.build index 3b081815d..ffcb62257 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -1,4 +1,7 @@ unit_tests = { + 'Blurhash': files ( + meson.project_source_root() + '/src/Utils/Blurhash.vala' + ), 'Celebrate': files ( meson.project_source_root() + '/src/Utils/Celebrate.vala' ),