Initial rfx progressive integration

- Mostly base functions and utilities necessary to enable RFX
  Progressive
- Add more EGFX work & mode flags.
- Update encoder.
- Does not yet include caps determination to enable RFX progressive
  (yet).

common/Makefile.am

@@ -71,6 +71,6 @@ libcommon_la_SOURCES = \
   $(PIXMAN_SOURCES)
 
 libcommon_la_LIBADD = \
-  -lpthread \
+  -lpthread -lrt \
   $(OPENSSL_LIBS) \
   $(DLOPEN_LIBS)

common/xrdp_client_info.h

@@ -215,6 +215,14 @@ struct xrdp_client_info
     unsigned int session_physical_height; /* in mm */
 
     int large_pointer_support_flags;
+    int gfx;
+};
+
+enum xrdp_encoder_flags
+{
+    NONE                                   = 0,
+    ENCODE_COMPLETE                        = 1 << 0,
+    GFX_PROGRESSIVE_RFX                    = 1 << 1
 };
 
 /* yyyymmdd of last incompatible change to xrdp_client_info */

common/xrdp_constants.h

@@ -291,4 +291,50 @@
 #define XR_RDP_SCAN_LSHIFT 42
 #define XR_RDP_SCAN_ALT    56
 
+// Since we're not guaranteed to have pixman, copy these directives.
+#define XRDP_PIXMAN_TYPE_ARGB   2
+#define XRDP_PIXMAN_TYPE_ABGR   3
+#define XRDP_PIXMAN_FORMAT(bpp,type,a,r,g,b)    (((bpp) << 24) |  \
+        ((type) << 16) | \
+        ((a) << 12) |    \
+        ((r) << 8) |     \
+        ((g) << 4) |     \
+        ((b)))
+
+#define XRDP_a8b8g8r8 \
+    XRDP_PIXMAN_FORMAT(32, XRDP_PIXMAN_TYPE_ABGR, 8, 8, 8, 8)
+
+#define XRDP_a8r8g8b8 \
+    XRDP_PIXMAN_FORMAT(32, XRDP_PIXMAN_TYPE_ARGB, 8, 8, 8, 8)
+
+#define XRDP_r5g6b5 \
+    XRDP_PIXMAN_FORMAT(16, XRDP_PIXMAN_TYPE_ARGB, 0, 5, 6, 5)
+
+#define XRDP_a1r5g5b5 \
+    XRDP_PIXMAN_FORMAT(16, XRDP_PIXMAN_TYPE_ARGB, 1, 5, 5, 5)
+
+#define XRDP_r3g3b2 \
+    XRDP_PIXMAN_FORMAT(8, XRDP_PIXMAN_TYPE_ARGB, 0, 3, 3, 2)
+
+// The last used constant in pixman is 63, so use 64+
+#define XRDP_nv12 \
+    XRDP_PIXMAN_FORMAT(12, 64, 0, 0, 0, 0)
+
+#define XRDP_i420 \
+    XRDP_PIXMAN_FORMAT(12, 65, 0, 0, 0, 0)
+
+#define XRDP_nv12_709fr \
+    XRDP_PIXMAN_FORMAT(12, 66, 0, 0, 0, 0)
+
+#define XRDP_yuv444_709fr \
+    XRDP_PIXMAN_FORMAT(32, 67, 0, 0, 0, 0)
+
+// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpegfx/8131c1bc-1af8-4907-a05a-f72f4581160f
+#define XRDP_yuv444_v1_stream_709fr \
+    XRDP_PIXMAN_FORMAT(32, 68, 0, 0, 0, 0)
+
+// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpegfx/781406c3-5e24-4f2b-b6ff-42b76bf64f6d
+#define XRDP_yuv444_v2_stream_709fr \
+    XRDP_PIXMAN_FORMAT(32, 69, 0, 0, 0, 0)
+
 #endif

libxrdp/xrdp_sec.c

@@ -2133,7 +2133,15 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s)
     {
         client_info->bpp = 32;
     }
-
+    if (earlyCapabilityFlags & 0x100) /* RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL */
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client supports gfx");
+        self->rdp_layer->client_info.gfx = 1;
+    }
+    else
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client DOES NOT support gfx");
+    }
     if (!s_check_rem(s, 64))
     {
         return 0;

xrdp/xrdp_encoder.c

@@ -32,10 +32,17 @@
 #include "rfxcodec_encode.h"
 #endif
 
-
-
 #define XRDP_SURCMD_PREFIX_BYTES 256
 
+#ifdef XRDP_RFXCODEC
+/* LH3 LL3, HH3 HL3, HL2 LH2, LH1 HH2, HH1 HL1 todo check this */
+static const unsigned char g_rfx_quantization_values[] =
+{
+    0x66, 0x66, 0x77, 0x88, 0x98,
+    0x76, 0x77, 0x88, 0x98, 0xA9
+};
+#endif
+
 /*****************************************************************************/
 static int
 process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
@@ -77,16 +84,24 @@ xrdp_encoder_create(struct xrdp_mm *mm)
 
     client_info = mm->wm->client_info;
 
+    /* RemoteFX 7.1 requires LAN but GFX does not */
     if (client_info->mcs_connection_type != CONNECTION_TYPE_LAN)
     {
-        return 0;
+        if ((mm->egfx_flags & (XRDP_EGFX_H264 | XRDP_EGFX_RFX_PRO)) == 0)
+        {
+            return 0;
+        }
     }
     if (client_info->bpp < 24)
     {
         return 0;
     }
 
-    self = (struct xrdp_encoder *)g_malloc(sizeof(struct xrdp_encoder), 1);
+    self = g_new0(struct xrdp_encoder, 1);
+    if (self == NULL)
+    {
+        return NULL;
+    }
     self->mm = mm;
 
     if (client_info->jpeg_codec_id != 0)
@@ -96,12 +111,29 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         self->in_codec_mode = 1;
         self->codec_quality = client_info->jpeg_prop[0];
         client_info->capture_code = 0;
-        client_info->capture_format =
-            /* XRDP_a8b8g8r8 */
-            (32 << 24) | (3 << 16) | (8 << 12) | (8 << 8) | (8 << 4) | 8;
+        client_info->capture_format = XRDP_a8b8g8r8;
         self->process_enc = process_enc_jpg;
     }
 #ifdef XRDP_RFXCODEC
+    else if (mm->egfx_flags & XRDP_EGFX_RFX_PRO)
+    {
+        LOG(LOG_LEVEL_INFO,
+            "xrdp_encoder_create: starting gfx rfx pro codec session");
+        self->in_codec_mode = 1;
+        client_info->capture_code = 2;
+        self->process_enc = process_enc_rfx;
+        self->gfx = 1;
+        self->quants = (const char *) g_rfx_quantization_values;
+        self->num_quants = 2;
+        self->quant_idx_y = 0;
+        self->quant_idx_u = 1;
+        self->quant_idx_v = 1;
+        self->codec_handle = rfxcodec_encode_create(
+                                 mm->wm->screen->width,
+                                 mm->wm->screen->height,
+                                 RFX_FORMAT_YUV,
+                                 RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1);
+    }
     else if (client_info->rfx_codec_id != 0)
     {
         LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_create: starting rfx codec session");
@@ -120,9 +152,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         self->codec_id = client_info->h264_codec_id;
         self->in_codec_mode = 1;
         client_info->capture_code = 3;
-        client_info->capture_format =
-            /* XRDP_nv12 */
-            (12 << 24) | (64 << 16) | (0 << 12) | (0 << 8) | (0 << 4) | 0;
+        client_info->capture_format = XRDP_nv12;
         self->process_enc = process_enc_h264;
     }
     else
@@ -131,7 +161,9 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         return 0;
     }
 
-    LOG_DEVEL(LOG_LEVEL_INFO, "init_xrdp_encoder: initializing encoder codec_id %d", self->codec_id);
+    LOG_DEVEL(LOG_LEVEL_INFO,
+              "init_xrdp_encoder: initializing encoder codec_id %d",
+              self->codec_id);
 
     /* setup required FIFOs */
     self->fifo_to_proc = fifo_create(xrdp_enc_data_destructor);
@@ -146,8 +178,17 @@ xrdp_encoder_create(struct xrdp_mm *mm)
     self->xrdp_encoder_event_processed = g_create_wait_obj(buf);
     g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_term", pid);
     self->xrdp_encoder_term = g_create_wait_obj(buf);
-    self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15;
-    self->frames_in_flight = client_info->max_unacknowledged_frame_count;
+    if (client_info->gfx)
+    {
+        // Magic numbers... Why?
+        self->frames_in_flight = 2;
+        self->max_compressed_bytes = 3145728;
+    }
+    else
+    {
+        self->frames_in_flight = client_info->max_unacknowledged_frame_count;
+        self->max_compressed_bytes = client_info->max_fastpath_frag_bytes & ~15;
+    }
     /* make sure frames_in_flight is at least 1 */
     self->frames_in_flight = MAX(self->frames_in_flight, 1);
 
@@ -354,9 +395,9 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                     tiles[index].y = y;
                     tiles[index].cx = cx;
                     tiles[index].cy = cy;
-                    tiles[index].quant_y = 0;
-                    tiles[index].quant_cb = 0;
-                    tiles[index].quant_cr = 0;
+                    tiles[index].quant_y = self->quant_idx_y;
+                    tiles[index].quant_cb = self->quant_idx_u;
+                    tiles[index].quant_cr = self->quant_idx_v;
                 }
 
                 count = enc->num_drects;
@@ -376,9 +417,11 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
                 tiles_written = rfxcodec_encode(self->codec_handle,
                                                 out_data + XRDP_SURCMD_PREFIX_BYTES,
                                                 &out_data_bytes, enc->data,
-                                                enc->width, enc->height, enc->width * 4,
+                                                enc->width, enc->height,
+                                                enc->width * 4,
                                                 rfxrects, enc->num_drects,
-                                                tiles, tiles_left, 0, 0);
+                                                tiles, enc->num_crects,
+                                                self->quants, self->num_quants);
             }
         }
 
@@ -399,7 +442,11 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
         enc_done->enc = enc;
         enc_done->cx = self->mm->wm->screen->width;
         enc_done->cy = self->mm->wm->screen->height;
-
+        if (self->gfx)
+        {
+            enc_done->flags = (enum xrdp_encoder_flags)
+                              ((int)enc_done->flags | GFX_PROGRESSIVE_RFX);
+        }
         enc_done->continuation = all_tiles_written > 0;
         if (tiles_written > 0)
         {
@@ -429,7 +476,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 static int
 process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 {
-    LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_x264:");
+    LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_h264: dummy func");
     return 0;
 }
 

xrdp/xrdp_encoder.h

@@ -3,6 +3,7 @@
 #define _XRDP_ENCODER_H
 
 #include "arch.h"
+#include "xrdp_client_info.h"
 struct fifo;
 
 struct xrdp_enc_data;
@@ -27,6 +28,13 @@ struct xrdp_encoder
     int frame_id_server; /* last frame id received from Xorg */
     int frame_id_server_sent;
     int frames_in_flight;
+    int gfx;
+    int gfx_ack_off;
+    const char *quants;
+    int num_quants;
+    int quant_idx_y;
+    int quant_idx_u;
+    int quant_idx_v;
 };
 
 /* used when scheduling tasks in xrdp_encoder.c */
@@ -63,6 +71,7 @@ struct xrdp_enc_data_done
     int y;
     int cx;
     int cy;
+    enum xrdp_encoder_flags flags;
 };
 
 typedef struct xrdp_enc_data_done XRDP_ENC_DATA_DONE;

xrdp/xrdp_mm.c

@@ -79,7 +79,13 @@ xrdp_mm_create(struct xrdp_wm *owner)
               self->wm->client_info->rfx_codec_id,
               self->wm->client_info->h264_codec_id);
 
-    self->encoder = xrdp_encoder_create(self);
+    if ((self->wm->client_info->gfx == 0) &&
+            ((self->wm->client_info->h264_codec_id != 0) ||
+             (self->wm->client_info->jpeg_codec_id != 0) ||
+             (self->wm->client_info->rfx_codec_id != 0)))
+    {
+        self->encoder = xrdp_encoder_create(self);
+    }
 
     return self;
 }

xrdp/xrdp_painter.c

@@ -93,35 +93,54 @@ xrdp_painter_send_dirty(struct xrdp_painter *self)
         Bpp = 4;
     }
 
-    jndex = 0;
-    error = xrdp_region_get_rect(self->dirty_region, jndex, &rect);
-    while (error == 0)
-    {
-        cx = rect.right - rect.left;
-        cy = rect.bottom - rect.top;
-        ldata = (char *)g_malloc(cx * cy * Bpp, 0);
-        if (ldata == 0)
+    if (self->session->client_info->gfx)
+    {
+        if (self->wm->screen_dirty_region == NULL)
         {
-            return 1;
+            self->wm->screen_dirty_region = xrdp_region_create(self->wm);
         }
-        src = self->wm->screen->data;
-        src += self->wm->screen->line_size * rect.top;
-        src += rect.left * Bpp;
-        dst = ldata;
-        for (index = 0; index < cy; index++)
+        jndex = 0;
+        error = xrdp_region_get_rect(self->dirty_region, jndex, &rect);
+        while (error == 0)
         {
-            g_memcpy(dst, src, cx * Bpp);
-            src += self->wm->screen->line_size;
-            dst += cx * Bpp;
+            xrdp_region_add_rect(self->wm->screen_dirty_region, &rect);
+            jndex++;
+            error = xrdp_region_get_rect(self->dirty_region, jndex, &rect);
         }
-        LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_send_dirty: x %d y %d cx %d cy %d",
-                  rect.left, rect.top, cx, cy);
-        libxrdp_send_bitmap(self->session, cx, cy, bpp,
-                            ldata, rect.left, rect.top, cx, cy);
-        g_free(ldata);
-
-        jndex++;
+    }
+    else
+    {
+        jndex = 0;
         error = xrdp_region_get_rect(self->dirty_region, jndex, &rect);
+        while (error == 0)
+        {
+            cx = rect.right - rect.left;
+            cy = rect.bottom - rect.top;
+            ldata = (char *)g_malloc(cx * cy * Bpp, 0);
+            if (ldata == 0)
+            {
+                return 1;
+            }
+            src = self->wm->screen->data;
+            src += self->wm->screen->line_size * rect.top;
+            src += rect.left * Bpp;
+            dst = ldata;
+            for (index = 0; index < cy; index++)
+            {
+                g_memcpy(dst, src, cx * Bpp);
+                src += self->wm->screen->line_size;
+                dst += cx * Bpp;
+            }
+            LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_painter_send_dirty:"
+                      " x %d y %d cx %d cy %d",
+                      rect.left, rect.top, cx, cy);
+            libxrdp_send_bitmap(self->session, cx, cy, bpp,
+                                ldata, rect.left, rect.top, cx, cy);
+            g_free(ldata);
+
+            jndex++;
+            error = xrdp_region_get_rect(self->dirty_region, jndex, &rect);
+        }
     }
 
     xrdp_region_delete(self->dirty_region);
@@ -145,8 +164,8 @@ xrdp_painter_create(struct xrdp_wm *wm, struct xrdp_session *session)
     self->rop = 0xcc; /* copy will use 0xcc */
     self->clip_children = 1;
 
-
-    if (self->session->client_info->no_orders_supported)
+    if (self->session->client_info->no_orders_supported ||
+            self->session->client_info->gfx)
     {
 #if defined(XRDP_PAINTER)
         if (painter_create(&(self->painter)) != PT_ERROR_NONE)