From 1ee3c649304a8717a5ea690235c94675b02f0715 Mon Sep 17 00:00:00 2001 From: Nexarian Date: Sat, 16 Dec 2023 12:49:56 -0500 Subject: [PATCH] Initial rfx progressive integration - Mostly base functions and utilities necessary to enable RFX Progressive - Add more EGFX work & mode flags. - Update encoder. - Does not yet include caps determination to enable RFX progressive (yet). - Update protocol constants --- common/Makefile.am | 2 +- common/ms-rdpbcgr.h | 32 +++++++++----- common/xrdp_client_info.h | 8 ++++ common/xrdp_constants.h | 46 +++++++++++++++++++++ libxrdp/xrdp_sec.c | 25 ++++++++--- xrdp/xrdp_encoder.c | 87 ++++++++++++++++++++++++++++++--------- xrdp/xrdp_encoder.h | 9 ++++ xrdp/xrdp_mm.c | 8 +++- xrdp/xrdp_painter.c | 71 ++++++++++++++++++++------------ xrdp/xrdp_types.h | 10 ++++- xrdp/xrdp_wm.c | 17 ++++++-- 11 files changed, 245 insertions(+), 70 deletions(-) diff --git a/common/Makefile.am b/common/Makefile.am index 141addc8f6..baed9f0d54 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -71,6 +71,6 @@ libcommon_la_SOURCES = \ $(PIXMAN_SOURCES) libcommon_la_LIBADD = \ - -lpthread \ + -lpthread -lrt \ $(OPENSSL_LIBS) \ $(DLOPEN_LIBS) diff --git a/common/ms-rdpbcgr.h b/common/ms-rdpbcgr.h index 517b777ce8..6e39053543 100644 --- a/common/ms-rdpbcgr.h +++ b/common/ms-rdpbcgr.h @@ -52,19 +52,29 @@ /* TS_UD_HEADER: type ((2.2.1.3.1) */ /* TODO: to be renamed */ -#define SEC_TAG_CLI_INFO 0xc001 /* CS_CORE? */ -#define SEC_TAG_CLI_CRYPT 0xc002 /* CS_SECURITY? */ -#define SEC_TAG_CLI_CHANNELS 0xc003 /* CS_CHANNELS? */ -#define SEC_TAG_CLI_4 0xc004 /* CS_CLUSTER? */ -#define SEC_TAG_CLI_MONITOR 0xc005 /* CS_MONITOR */ -#define SEC_TAG_CLI_MONITOR_EX 0xc008 /* CS_MONITOR_EX */ +#define SEC_TAG_CLI_INFO 0xc001 /* CS_CORE? */ +#define SEC_TAG_CLI_CRYPT 0xc002 /* CS_SECURITY? */ +#define SEC_TAG_CLI_CHANNELS 0xc003 /* CS_CHANNELS? */ +#define SEC_TAG_CLI_4 0xc004 /* CS_CLUSTER? */ +#define SEC_TAG_CLI_MONITOR 0xc005 /* CS_MONITOR */ +#define SEC_TAG_CLI_MONITOR_EX 0xc008 /* CS_MONITOR_EX */ /* Client Core Data: colorDepth, postBeta2ColorDepth (2.2.1.3.2) */ -#define RNS_UD_COLOR_4BPP 0xCA00 -#define RNS_UD_COLOR_8BPP 0xCA01 -#define RNS_UD_COLOR_16BPP_555 0xCA02 -#define RNS_UD_COLOR_16BPP_565 0xCA03 -#define RNS_UD_COLOR_24BPP 0xCA04 +#define RNS_UD_COLOR_4BPP 0xCA00 +#define RNS_UD_COLOR_8BPP 0xCA01 +#define RNS_UD_COLOR_16BPP_555 0xCA02 +#define RNS_UD_COLOR_16BPP_565 0xCA03 +#define RNS_UD_COLOR_24BPP 0xCA04 + +/* Client Core Data: supportedColorDepths (2.2.1.3.2) */ +#define RNS_UD_24BPP_SUPPORT 0x0001 +#define RNS_UD_16BPP_SUPPORT 0x0002 +#define RNS_UD_15BPP_SUPPORT 0x0004 +#define RNS_UD_32BPP_SUPPORT 0x0008 + +/* Client Core Data: earlyCapabilityFlags (2.2.1.3.2) */ +#define RNS_UD_CS_WANT_32BPP_SESSION 0x0002 +#define RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL 0x0100 /* Client Core Data: connectionType (2.2.1.3.2) */ #define CONNECTION_TYPE_MODEM 0x01 diff --git a/common/xrdp_client_info.h b/common/xrdp_client_info.h index f2d91ad77b..a588f16c56 100644 --- a/common/xrdp_client_info.h +++ b/common/xrdp_client_info.h @@ -215,6 +215,14 @@ struct xrdp_client_info unsigned int session_physical_height; /* in mm */ int large_pointer_support_flags; + int gfx; +}; + +enum xrdp_encoder_flags +{ + NONE = 0, + ENCODE_COMPLETE = 1 << 0, + GFX_PROGRESSIVE_RFX = 1 << 1 }; /* yyyymmdd of last incompatible change to xrdp_client_info */ diff --git a/common/xrdp_constants.h b/common/xrdp_constants.h index 231d494ab9..9ab9d17901 100644 --- a/common/xrdp_constants.h +++ b/common/xrdp_constants.h @@ -291,4 +291,50 @@ #define XR_RDP_SCAN_LSHIFT 42 #define XR_RDP_SCAN_ALT 56 +// Since we're not guaranteed to have pixman, copy these directives. +#define XRDP_PIXMAN_TYPE_ARGB 2 +#define XRDP_PIXMAN_TYPE_ABGR 3 +#define XRDP_PIXMAN_FORMAT(bpp,type,a,r,g,b) (((bpp) << 24) | \ + ((type) << 16) | \ + ((a) << 12) | \ + ((r) << 8) | \ + ((g) << 4) | \ + ((b))) + +#define XRDP_a8b8g8r8 \ + XRDP_PIXMAN_FORMAT(32, XRDP_PIXMAN_TYPE_ABGR, 8, 8, 8, 8) + +#define XRDP_a8r8g8b8 \ + XRDP_PIXMAN_FORMAT(32, XRDP_PIXMAN_TYPE_ARGB, 8, 8, 8, 8) + +#define XRDP_r5g6b5 \ + XRDP_PIXMAN_FORMAT(16, XRDP_PIXMAN_TYPE_ARGB, 0, 5, 6, 5) + +#define XRDP_a1r5g5b5 \ + XRDP_PIXMAN_FORMAT(16, XRDP_PIXMAN_TYPE_ARGB, 1, 5, 5, 5) + +#define XRDP_r3g3b2 \ + XRDP_PIXMAN_FORMAT(8, XRDP_PIXMAN_TYPE_ARGB, 0, 3, 3, 2) + +// The last used constant in pixman is 63, so use 64+ +#define XRDP_nv12 \ + XRDP_PIXMAN_FORMAT(12, 64, 0, 0, 0, 0) + +#define XRDP_i420 \ + XRDP_PIXMAN_FORMAT(12, 65, 0, 0, 0, 0) + +#define XRDP_nv12_709fr \ + XRDP_PIXMAN_FORMAT(12, 66, 0, 0, 0, 0) + +#define XRDP_yuv444_709fr \ + XRDP_PIXMAN_FORMAT(32, 67, 0, 0, 0, 0) + +// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpegfx/8131c1bc-1af8-4907-a05a-f72f4581160f +#define XRDP_yuv444_v1_stream_709fr \ + XRDP_PIXMAN_FORMAT(32, 68, 0, 0, 0, 0) + +// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpegfx/781406c3-5e24-4f2b-b6ff-42b76bf64f6d +#define XRDP_yuv444_v2_stream_709fr \ + XRDP_PIXMAN_FORMAT(32, 69, 0, 0, 0, 0) + #endif diff --git a/libxrdp/xrdp_sec.c b/libxrdp/xrdp_sec.c index 20665dfc4f..6c2057aeb1 100644 --- a/libxrdp/xrdp_sec.c +++ b/libxrdp/xrdp_sec.c @@ -2114,10 +2114,14 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s) in_uint16_le(s, supportedColorDepths); LOG_DEVEL(LOG_LEVEL_TRACE, "Received [MS-RDPBCGR] TS_UD_CS_CORE " " supportedColorDepths %s", - supportedColorDepths == 0x0001 ? "RNS_UD_24BPP_SUPPORT" : - supportedColorDepths == 0x0002 ? "RNS_UD_16BPP_SUPPORT" : - supportedColorDepths == 0x0004 ? "RNS_UD_15BPP_SUPPORT" : - supportedColorDepths == 0x0008 ? "RNS_UD_32BPP_SUPPORT" : + supportedColorDepths == RNS_UD_24BPP_SUPPORT + ? "RNS_UD_24BPP_SUPPORT" : + supportedColorDepths == RNS_UD_16BPP_SUPPORT + ? "RNS_UD_16BPP_SUPPORT" : + supportedColorDepths == RNS_UD_15BPP_SUPPORT + ? "RNS_UD_15BPP_SUPPORT" : + supportedColorDepths == RNS_UD_32BPP_SUPPORT + ? "RNS_UD_32BPP_SUPPORT" : "unknown"); if (!s_check_rem(s, 2)) @@ -2129,11 +2133,20 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s) LOG_DEVEL(LOG_LEVEL_TRACE, "Received [MS-RDPBCGR] TS_UD_CS_CORE " " earlyCapabilityFlags 0x%4.4x", earlyCapabilityFlags); - if ((earlyCapabilityFlags & 0x0002) && (supportedColorDepths & 0x0008)) + if ((earlyCapabilityFlags & RNS_UD_CS_WANT_32BPP_SESSION) + && (supportedColorDepths & RNS_UD_32BPP_SUPPORT)) { client_info->bpp = 32; } - + if (earlyCapabilityFlags & RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL) + { + LOG(LOG_LEVEL_INFO, "client supports gfx protocol"); + self->rdp_layer->client_info.gfx = 1; + } + else + { + LOG_DEVEL(LOG_LEVEL_INFO, "client DOES NOT support gfx"); + } if (!s_check_rem(s, 64)) { return 0; diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index 03203cd791..8c3d9d0b17 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -32,10 +32,17 @@ #include "rfxcodec_encode.h" #endif - - #define XRDP_SURCMD_PREFIX_BYTES 256 +#ifdef XRDP_RFXCODEC +/* LH3 LL3, HH3 HL3, HL2 LH2, LH1 HH2, HH1 HL1 todo check this */ +static const unsigned char g_rfx_quantization_values[] = +{ + 0x66, 0x66, 0x77, 0x88, 0x98, + 0x76, 0x77, 0x88, 0x98, 0xA9 +}; +#endif + /*****************************************************************************/ static int process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc); @@ -77,16 +84,24 @@ xrdp_encoder_create(struct xrdp_mm *mm) client_info = mm->wm->client_info; + /* RemoteFX 7.1 requires LAN but GFX does not */ if (client_info->mcs_connection_type != CONNECTION_TYPE_LAN) { - return 0; + if ((mm->egfx_flags & (XRDP_EGFX_H264 | XRDP_EGFX_RFX_PRO)) == 0) + { + return 0; + } } if (client_info->bpp < 24) { return 0; } - self = (struct xrdp_encoder *)g_malloc(sizeof(struct xrdp_encoder), 1); + self = g_new0(struct xrdp_encoder, 1); + if (self == NULL) + { + return NULL; + } self->mm = mm; if (client_info->jpeg_codec_id != 0) @@ -96,12 +111,29 @@ xrdp_encoder_create(struct xrdp_mm *mm) self->in_codec_mode = 1; self->codec_quality = client_info->jpeg_prop[0]; client_info->capture_code = 0; - client_info->capture_format = - /* XRDP_a8b8g8r8 */ - (32 << 24) | (3 << 16) | (8 << 12) | (8 << 8) | (8 << 4) | 8; + client_info->capture_format = XRDP_a8b8g8r8; self->process_enc = process_enc_jpg; } #ifdef XRDP_RFXCODEC + else if (mm->egfx_flags & XRDP_EGFX_RFX_PRO) + { + LOG(LOG_LEVEL_INFO, + "xrdp_encoder_create: starting gfx rfx pro codec session"); + self->in_codec_mode = 1; + client_info->capture_code = 2; + self->process_enc = process_enc_rfx; + self->gfx = 1; + self->quants = (const char *) g_rfx_quantization_values; + self->num_quants = 2; + self->quant_idx_y = 0; + self->quant_idx_u = 1; + self->quant_idx_v = 1; + self->codec_handle = rfxcodec_encode_create( + mm->wm->screen->width, + mm->wm->screen->height, + RFX_FORMAT_YUV, + RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1); + } else if (client_info->rfx_codec_id != 0) { LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_create: starting rfx codec session"); @@ -120,9 +152,7 @@ xrdp_encoder_create(struct xrdp_mm *mm) self->codec_id = client_info->h264_codec_id; self->in_codec_mode = 1; client_info->capture_code = 3; - client_info->capture_format = - /* XRDP_nv12 */ - (12 << 24) | (64 << 16) | (0 << 12) | (0 << 8) | (0 << 4) | 0; + client_info->capture_format = XRDP_nv12; self->process_enc = process_enc_h264; } else @@ -131,7 +161,9 @@ xrdp_encoder_create(struct xrdp_mm *mm) return 0; } - LOG_DEVEL(LOG_LEVEL_INFO, "init_xrdp_encoder: initializing encoder codec_id %d", self->codec_id); + LOG_DEVEL(LOG_LEVEL_INFO, + "init_xrdp_encoder: initializing encoder codec_id %d", + self->codec_id); /* setup required FIFOs */ self->fifo_to_proc = fifo_create(xrdp_enc_data_destructor); @@ -146,8 +178,17 @@ xrdp_encoder_create(struct xrdp_mm *mm) self->xrdp_encoder_event_processed = g_create_wait_obj(buf); g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_term", pid); self->xrdp_encoder_term = g_create_wait_obj(buf); - self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15; - self->frames_in_flight = client_info->max_unacknowledged_frame_count; + if (client_info->gfx) + { + // Magic numbers... Why? + self->frames_in_flight = 2; + self->max_compressed_bytes = 3145728; + } + else + { + self->frames_in_flight = client_info->max_unacknowledged_frame_count; + self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15; + } /* make sure frames_in_flight is at least 1 */ self->frames_in_flight = MAX(self->frames_in_flight, 1); @@ -354,9 +395,9 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) tiles[index].y = y; tiles[index].cx = cx; tiles[index].cy = cy; - tiles[index].quant_y = 0; - tiles[index].quant_cb = 0; - tiles[index].quant_cr = 0; + tiles[index].quant_y = self->quant_idx_y; + tiles[index].quant_cb = self->quant_idx_u; + tiles[index].quant_cr = self->quant_idx_v; } count = enc->num_drects; @@ -376,9 +417,11 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) tiles_written = rfxcodec_encode(self->codec_handle, out_data + XRDP_SURCMD_PREFIX_BYTES, &out_data_bytes, enc->data, - enc->width, enc->height, enc->width * 4, + enc->width, enc->height, + enc->width * 4, rfxrects, enc->num_drects, - tiles, tiles_left, 0, 0); + tiles, enc->num_crects, + self->quants, self->num_quants); } } @@ -399,7 +442,11 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) enc_done->enc = enc; enc_done->cx = self->mm->wm->screen->width; enc_done->cy = self->mm->wm->screen->height; - + if (self->gfx) + { + enc_done->flags = (enum xrdp_encoder_flags) + ((int)enc_done->flags | GFX_PROGRESSIVE_RFX); + } enc_done->continuation = all_tiles_written > 0; if (tiles_written > 0) { @@ -429,7 +476,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) static int process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) { - LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_x264:"); + LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_h264: dummy func"); return 0; } diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h index 59bebaa865..a5c32a58a2 100644 --- a/xrdp/xrdp_encoder.h +++ b/xrdp/xrdp_encoder.h @@ -3,6 +3,7 @@ #define _XRDP_ENCODER_H #include "arch.h" +#include "xrdp_client_info.h" struct fifo; struct xrdp_enc_data; @@ -27,6 +28,13 @@ struct xrdp_encoder int frame_id_server; /* last frame id received from Xorg */ int frame_id_server_sent; int frames_in_flight; + int gfx; + int gfx_ack_off; + const char *quants; + int num_quants; + int quant_idx_y; + int quant_idx_u; + int quant_idx_v; }; /* used when scheduling tasks in xrdp_encoder.c */ @@ -63,6 +71,7 @@ struct xrdp_enc_data_done int y; int cx; int cy; + enum xrdp_encoder_flags flags; }; typedef struct xrdp_enc_data_done XRDP_ENC_DATA_DONE; diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index ccade62ae7..8769a61d97 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -79,7 +79,13 @@ xrdp_mm_create(struct xrdp_wm *owner) self->wm->client_info->rfx_codec_id, self->wm->client_info->h264_codec_id); - self->encoder = xrdp_encoder_create(self); + if ((self->wm->client_info->gfx == 0) && + ((self->wm->client_info->h264_codec_id != 0) || + (self->wm->client_info->jpeg_codec_id != 0) || + (self->wm->client_info->rfx_codec_id != 0))) + { + self->encoder = xrdp_encoder_create(self); + } return self; } diff --git a/xrdp/xrdp_painter.c b/xrdp/xrdp_painter.c index 50ab371af6..fce0d3780d 100644 --- a/xrdp/xrdp_painter.c +++ b/xrdp/xrdp_painter.c @@ -93,35 +93,54 @@ xrdp_painter_send_dirty(struct xrdp_painter *self) Bpp = 4; } - jndex = 0; - error = xrdp_region_get_rect(self->dirty_region, jndex, &rect); - while (error == 0) - { - cx = rect.right - rect.left; - cy = rect.bottom - rect.top; - ldata = (char *)g_malloc(cx * cy * Bpp, 0); - if (ldata == 0) + if (self->session->client_info->gfx) + { + if (self->wm->screen_dirty_region == NULL) { - return 1; + self->wm->screen_dirty_region = xrdp_region_create(self->wm); } - src = self->wm->screen->data; - src += self->wm->screen->line_size * rect.top; - src += rect.left * Bpp; - dst = ldata; - for (index = 0; index < cy; index++) + jndex = 0; + error = xrdp_region_get_rect(self->dirty_region, jndex, &rect); + while (error == 0) { - g_memcpy(dst, src, cx * Bpp); - src += self->wm->screen->line_size; - dst += cx * Bpp; + xrdp_region_add_rect(self->wm->screen_dirty_region, &rect); + jndex++; + error = xrdp_region_get_rect(self->dirty_region, jndex, &rect); } - LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_send_dirty: x %d y %d cx %d cy %d", - rect.left, rect.top, cx, cy); - libxrdp_send_bitmap(self->session, cx, cy, bpp, - ldata, rect.left, rect.top, cx, cy); - g_free(ldata); - - jndex++; + } + else + { + jndex = 0; error = xrdp_region_get_rect(self->dirty_region, jndex, &rect); + while (error == 0) + { + cx = rect.right - rect.left; + cy = rect.bottom - rect.top; + ldata = (char *)g_malloc(cx * cy * Bpp, 0); + if (ldata == 0) + { + return 1; + } + src = self->wm->screen->data; + src += self->wm->screen->line_size * rect.top; + src += rect.left * Bpp; + dst = ldata; + for (index = 0; index < cy; index++) + { + g_memcpy(dst, src, cx * Bpp); + src += self->wm->screen->line_size; + dst += cx * Bpp; + } + LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_send_dirty:" + " x %d y %d cx %d cy %d", + rect.left, rect.top, cx, cy); + libxrdp_send_bitmap(self->session, cx, cy, bpp, + ldata, rect.left, rect.top, cx, cy); + g_free(ldata); + + jndex++; + error = xrdp_region_get_rect(self->dirty_region, jndex, &rect); + } } xrdp_region_delete(self->dirty_region); @@ -145,8 +164,8 @@ xrdp_painter_create(struct xrdp_wm *wm, struct xrdp_session *session) self->rop = 0xcc; /* copy will use 0xcc */ self->clip_children = 1; - - if (self->session->client_info->no_orders_supported) + if (self->session->client_info->no_orders_supported || + self->session->client_info->gfx) { #if defined(XRDP_PAINTER) if (painter_create(&(self->painter)) != PT_ERROR_NONE) diff --git a/xrdp/xrdp_types.h b/xrdp/xrdp_types.h index 5f8a730027..d0bce05706 100644 --- a/xrdp/xrdp_types.h +++ b/xrdp/xrdp_types.h @@ -374,6 +374,13 @@ enum display_resize_state "unknown" \ ) +enum xrdp_egfx_flags +{ + XRDP_EGFX_NONE = 0, + XRDP_EGFX_H264 = 1, + XRDP_EGFX_RFX_PRO = 2 +}; + struct xrdp_mm { struct xrdp_wm *wm; /* owner */ @@ -410,7 +417,8 @@ struct xrdp_mm int dynamic_monitor_chanid; struct xrdp_egfx *egfx; int egfx_up; - + int egfx_flags; + int gfx_delay_autologin; /* Resize on-the-fly control */ struct display_control_monitor_layout_data *resize_data; struct list *resize_queue; diff --git a/xrdp/xrdp_wm.c b/xrdp/xrdp_wm.c index 8a34b03393..8d0a48c317 100644 --- a/xrdp/xrdp_wm.c +++ b/xrdp/xrdp_wm.c @@ -767,10 +767,19 @@ xrdp_wm_init(struct xrdp_wm *self) list_add_strdup(self->mm->login_values, r); } - /* - * Skip the login box and go straight to the connection phase - */ - xrdp_wm_set_login_state(self, WMLS_START_CONNECT); + if (self->session->client_info->gfx && !self->mm->egfx_up) + { + /* gfx session but have not recieved caps advertise yet, + set flag so we will connect to backend later */ + self->mm->gfx_delay_autologin = 1; + } + else + { + /* + * Skip the login box and go straight to the connection phase + */ + xrdp_wm_set_login_state(self, WMLS_START_CONNECT); + } } else {