fix(PlayCookie) PLAY_TOKEN cookie rejected because userprofile exceed…

…s 4096 chars (datahub-project#5114)

build.gradle

@@ -103,12 +103,15 @@ project.ext.externalDependency = [
     'parseqTest': 'com.linkedin.parseq:parseq:3.0.7:test',
     'parquet': 'org.apache.parquet:parquet-avro:1.12.0',
     'picocli': 'info.picocli:picocli:4.5.0',
-    'playCache': 'com.typesafe.play:play-cache_2.11:2.6.18',
-    'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.11:2.0.8',
-    'playDocs': 'com.typesafe.play:play-docs_2.11:2.6.18',
-    'playGuice': 'com.typesafe.play:play-guice_2.11:2.6.18',
-    'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.11:2.6.18',
-    'playTest': 'com.typesafe.play:play-test_2.11:2.6.18',
+    'playCache': 'com.typesafe.play:play-cache_2.12:2.7.6',
+    'playEhcache': 'com.typesafe.play:play-ehcache_2.12:2.7.6',
+    'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.0.8',
+    'playDocs': 'com.typesafe.play:play-docs_2.12:2.7.6',
+    'playGuice': 'com.typesafe.play:play-guice_2.12:2.7.6',
+    'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.12:2.7.6',
+    'playAkkaHttpServer': 'com.typesafe.play:play-akka-http-server_2.12:2.7.6',
+    'playServer': 'com.typesafe.play:play-server_2.12:2.7.6',
+    'playTest': 'com.typesafe.play:play-test_2.12:2.7.6',
     'pac4j': 'org.pac4j:pac4j-oidc:3.6.0',
     'playPac4j': 'org.pac4j:play-pac4j_2.11:7.0.1',
     'postgresql': 'org.postgresql:postgresql:42.3.3',

datahub-frontend/app/auth/AuthModule.java

@@ -21,6 +21,7 @@
 import org.pac4j.core.context.session.SessionStore;
 import org.pac4j.play.LogoutController;
 import org.pac4j.play.http.PlayHttpActionAdapter;
+import org.pac4j.play.store.PlayCacheSessionStore;
 import org.pac4j.play.store.PlayCookieSessionStore;
 import org.pac4j.play.store.PlaySessionStore;
 import org.pac4j.play.store.ShiroAesDataEncrypter;
@@ -33,6 +34,7 @@
 import auth.sso.SsoConfigs;
 import auth.sso.SsoManager;
 import controllers.SsoCallbackController;
+import play.cache.SyncCacheApi;
 import utils.ConfigUtil;
 
 import static auth.AuthUtils.*;
@@ -52,6 +54,8 @@ public class AuthModule extends AbstractModule {
      * We hash this value (SHA1), then take the first 16 bytes as the AES key.
      */
     private static final String PAC4J_AES_KEY_BASE_CONF = "play.http.secret.key";
+    private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
+
     private final com.typesafe.config.Config _configs;
 
     public AuthModule(final Environment environment, final com.typesafe.config.Config configs) {
@@ -60,22 +64,38 @@ public AuthModule(final Environment environment, final com.typesafe.config.Confi
 
     @Override
     protected void configure() {
-        PlayCookieSessionStore playCacheCookieStore;
-        try {
-            // To generate a valid encryption key from an input value, we first
-            // hash the input to generate a fixed-length string. Then, we convert
-            // it to hex and slice the first 16 bytes, because AES key length must strictly
-            // have a specific length.
-            final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF);
-            final String aesKeyHash = DigestUtils.sha1Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8));
-            final String aesEncryptionKey = aesKeyHash.substring(0, 16);
-            playCacheCookieStore = new PlayCookieSessionStore(
-                new ShiroAesDataEncrypter(aesEncryptionKey));
-        } catch (Exception e) {
-            throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e);
+        /**
+         * In Pac4J, you are given the option to store the profiles of authenticated users in either
+         * (i) PlayCacheSessionStore - saves your data in the Play cache or
+         * (ii) PlayCookieSessionStore saves your data in the Play session cookie
+         * However there is problem (https://github.com/datahub-project/datahub/issues/4448) observed when storing the Pac4j profile in cookie.
+         * Whenever the profile returned by Pac4j is greater than 4096 characters, the response will be rejected by the browser.
+         * Default to PlayCacheCookieStore so that datahub-frontend container remains as a stateless service
+         */
+        String sessionStoreProvider = _configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF);
+
+        if (sessionStoreProvider.equals("PlayCacheSessionStore")) {
+            final PlayCacheSessionStore playCacheSessionStore = new PlayCacheSessionStore(getProvider(SyncCacheApi.class));
+            bind(SessionStore.class).toInstance(playCacheSessionStore);
+            bind(PlaySessionStore.class).toInstance(playCacheSessionStore);
+        } else {
+            PlayCookieSessionStore playCacheCookieStore;
+            try {
+                // To generate a valid encryption key from an input value, we first
+                // hash the input to generate a fixed-length string. Then, we convert
+                // it to hex and slice the first 16 bytes, because AES key length must strictly
+                // have a specific length.
+                final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF);
+                final String aesKeyHash = DigestUtils.sha1Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8));
+                final String aesEncryptionKey = aesKeyHash.substring(0, 16);
+                playCacheCookieStore = new PlayCookieSessionStore(
+                        new ShiroAesDataEncrypter(aesEncryptionKey));
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e);
+            }
+            bind(SessionStore.class).toInstance(playCacheCookieStore);
+            bind(PlaySessionStore.class).toInstance(playCacheCookieStore);
         }
-        bind(SessionStore.class).toInstance(playCacheCookieStore);
-        bind(PlaySessionStore.class).toInstance(playCacheCookieStore);
 
         try {
             bind(SsoCallbackController.class).toConstructor(SsoCallbackController.class.getConstructor(

datahub-frontend/conf/application.conf

@@ -26,6 +26,24 @@ play.modules.disabled += "play.api.mvc.CookiesModule"
 play.modules.enabled += "play.api.mvc.LegacyCookiesModule"
 play.modules.enabled += "auth.AuthModule"
 
+# Legacy Configuration to avoid code changes, update to modern approaches eventually
+play.allowHttpContext = true
+play.allowGlobalApplication = true
+
+# We override the Akka server provider to allow setting the max header count to a higher value
+# This is useful while using proxies like Envoy that result in the frontend server rejecting GMS
+# responses as there's more than the max of 64 allowed headers
+play.server.provider = server.CustomAkkaHttpServerProvider
+play.http.server.akka.max-header-count = 64
+play.http.server.akka.max-header-count = ${?DATAHUB_AKKA_MAX_HEADER_COUNT}
+play.server.akka.max-header-value-length = 8k
+play.server.akka.max-header-value-length = ${?DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH}
+
+# pac4j configuration
+# default to PlayCookieSessionStore to keep datahub-frontend's statelessness
+pac4j.sessionStore.provider= "PlayCookieSessionStore"
+pac4j.sessionStore.provider= ${?PAC4J_SESSIONSTORE_PROVIDER}
+
 # Database configuration
 # ~~~~~
 # You can declare as many datasources as you want.
@@ -210,4 +228,4 @@ metadataService.auth.enabled=${?METADATA_SERVICE_AUTH_ENABLED}
 systemClientId = __datahub_system # Change this to something random.
 systemClientSecret = JohnSnowKnowsNothing # Along with this.
 systemClientId=${?DATAHUB_SYSTEM_CLIENT_ID}
-systemClientSecret=${?DATAHUB_SYSTEM_CLIENT_SECRET}
+systemClientSecret=${?DATAHUB_SYSTEM_CLIENT_SECRET}

datahub-frontend/play.gradle

@@ -38,12 +38,21 @@ dependencies {
   play externalDependency.jerseyCore
   play externalDependency.jerseyGuava
 
-  play externalDependency.pac4j
-  play externalDependency.playPac4j
-  play externalDependency.shiroCore
-  play externalDependency.playCache
-  play externalDependency.playWs
-  play externalDependency.kafkaClients
+  implementation(externalDependency.pac4j) {
+    exclude group: "net.minidev", module: "json-smart"
+    exclude group: "com.nimbusds", module: "nimbus-jose-jwt"
+  }
+  implementation "com.nimbusds:nimbus-jose-jwt:7.9"
+  implementation externalDependency.jsonSmart
+  implementation externalDependency.playPac4j
+  implementation externalDependency.shiroCore
+  implementation externalDependency.playCache
+  implementation externalDependency.playEhcache
+  implementation externalDependency.playWs
+  implementation externalDependency.playServer
+  implementation externalDependency.playAkkaHttpServer
+  implementation externalDependency.kafkaClients
+  implementation externalDependency.awsMskIamAuth
 
   playTest externalDependency.mockito
   playTest externalDependency.playTest

docs/debugging.md

@@ -204,4 +204,19 @@ You'll need to ingest some metadata of the following form to see it inside the D
   },
   "proposedDelta": null
 }
-```
+```
+
+## I've configured OIDC, but I cannot login. I get continuously redirected. What do I do? 
+
+Sorry to hear that!
+
+This phenomena may be due to the size of a Cookie DataHub uses to authenticate its users. If it's too large ( > 4096), then you'll see this behavior. The cookie embeds an encoded version of the information returned by your OIDC Identity Provider - if they return a lot of information, this can be the root cause.
+
+One solution is to use Play Cache to persist this session information for a user. This means the attributes about the user (and their session info) will be stored in an in-memory store in the `datahub-frontend` service, instead of a browser-side cookie.
+
+To configure the Play Cache session store, you can set the env variable "PAC4J_SESSIONSTORE_PROVIDER" as "PlayCacheSessionStore" for the `datahub-frontend` container. 
+
+Do note that there are downsides to using the Play Cache. Specifically, it will make `datahub-frontend` a stateful server. If you have multiple instances of `datahub-frontend` deployed, you'll need to ensure that the same user is deterministically routed to the same service container (since the sessions are stored in memory). If you're using a single instance of `datahub-frontend` (the default), then things should "just work". 
+
+For more details, please refer to https://github.com/datahub-project/datahub/pull/5114
+