Remote connectivity to S/4 Cloud is failing from CAP

[ejazsyed]

Describe the bug
Hi Team, i am trying to consume business partner in S/4HANA cloud from CAP application . However i am getting the below error.

Error Message from logger

   2024-10-08T11:29:17.98+0000 [CELL/0] OUT Cell ff9adc8f-659f-4ec8-9e30-868c238e2faa successfully destroyed container for instance 472a8642-c41c-4ee2-4d71-4c99
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT << received sap.s4.beh.businesspartner.v1.BusinessPartner.Changed.v1 { BusinessPartner: '1000100' }
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT BusinessPartnerID 1000100
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"env-destination-accessor","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"Attempting to retrieve destination from environment variable.","msg":"Attempting to retrieve destination from environment variable.","timestamp":"2024-10-08T11:30:51.868Z","written_at":"2024-10-08T11:30:51.868Z","written_ts":1728387051868}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"env-destination-accessor","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"No environment variable set.","msg":"No environment variable set.","timestamp":"2024-10-08T11:30:51.868Z","written_at":"2024-10-08T11:30:51.868Z","written_ts":1728387051868}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"jwt","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","msg":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","timestamp":"2024-10-08T11:30:51.868Z","written_at":"2024-10-08T11:30:51.868Z","written_ts":1728387051868}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"jwt","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","msg":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","timestamp":"2024-10-08T11:30:51.868Z","written_at":"2024-10-08T11:30:51.868Z","written_ts":1728387051868}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"jwt","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"JWT user_id is: undefined.","msg":"JWT user_id is: undefined.","timestamp":"2024-10-08T11:30:51.868Z","written_at":"2024-10-08T11:30:51.868Z","written_ts":1728387051868}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"jwt","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","msg":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","timestamp":"2024-10-08T11:30:51.868Z","written_at":"2024-10-08T11:30:51.868Z","written_ts":1728387051868}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"register-destination","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"Could not retrieve 'incidents-api-access' from registered destinations.","msg":"Could not retrieve 'incidents-api-access' from registered destinations.","timestamp":"2024-10-08T11:30:51.869Z","written_at":"2024-10-08T11:30:51.869Z","written_ts":1728387051869}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"destination-accessor-vcap","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"Attempting to retrieve destination from service binding.","msg":"Attempting to retrieve destination from service binding.","timestamp":"2024-10-08T11:30:51.869Z","written_at":"2024-10-08T11:30:51.869Z","written_ts":1728387051869}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"jwt","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","msg":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","timestamp":"2024-10-08T11:30:51.869Z","written_at":"2024-10-08T11:30:51.869Z","written_ts":1728387051869}
   2024-10-08T11:30:51.86+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"jwt","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","msg":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","timestamp":"2024-10-08T11:30:51.869Z","written_at":"2024-10-08T11:30:51.869Z","written_ts":1728387051869}
   2024-10-08T11:30:51.87+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"destination-accessor-vcap","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"Could not retrieve destination from service binding. If you are not using SAP Extension Factory, this information probably does not concern you. Could not find service with name: 'incidents-api-access'.","msg":"Could not retrieve destination from service binding. If you are not using SAP Extension Factory, this information probably does not concern you. Could not find service with name: 'incidents-api-access'.","timestamp":"2024-10-08T11:30:51.870Z","written_at":"2024-10-08T11:30:51.870Z","written_ts":1728387051870}
   2024-10-08T11:30:51.87+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"destination-accessor-service","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"Attempting to retrieve destination from destination service.","msg":"Attempting to retrieve destination from destination service.","timestamp":"2024-10-08T11:30:51.870Z","written_at":"2024-10-08T11:30:51.870Z","written_ts":1728387051870}
   2024-10-08T11:30:51.87+0000 [APP/PROC/WEB/0] OUT {"custom_fields":{"messageContext":"jwt","package":"connectivity"},"level":"debug","logger":"sap-cloud-sdk-logger","message":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","msg":"JWT zid is: undefined, app_tid is: 3525ffdc-60f2-4d05-a036-d70428d549dd.","timestamp":"2024-10-08T11:30:51.870Z","written_at":"2024-10-08T11:30:51.870Z","written_ts":1728387051870}
   2024-10-08T11:30:52.12+0000 [APP/PROC/WEB/0] ERR {"level":"warn","logger":"remote","correlation_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","tenant_id":"3525ffdc-60f2-4d05-a036-d70428d549dd","host":"referenceapps-s4eventpub-n6e7jhsl-dev-remote-srv.cfapps.eu10-004.hana.ondemand.com","user_agent":"Go-http-client/2.0","content_length":"29","request_size_b":"29","accept_encoding":"gzip","authorization":"***","b3":"6586dd72b78e366e4420b4fe2aa01fd8-8e8b6a95e939c04e-0","ce_id":"17928f26-b16f-1eef-a1ad-189928a54940","ce_kafkaoffset":"5","ce_kafkapartition":"1","ce_kafkatopic":"beb-st-system.f3d69622e9baddf16ed7803f07ee2f35","ce_sapconsumertenant":"SA0176014160","ce_sapemtenantid":"3525ffdc-60f2-4d05-a036-d70428d549dd","ce_source":"/default/sap.s4.beh/730431274","ce_specversion":"1.0","ce_time":"2024-10-08T11:30:51Z","ce_type":"sap.s4.beh.businesspartner.v1.BusinessPartner.Changed.v1","content_type":"application/json","x_b3_sampled":"0","x_b3_spanid":"8e8b6a95e939c04e","x_b3_traceid":"6586dd72b78e366e4420b4fe2aa01fd8","x_cf_applicationid":"3401667f-aef0-4e3d-91c9-389eb7d509c5","x_cf_instanceid":"47bf6ee3-0091-457d-7737-fc86","x_cf_instanceindex":"0","x_forwarded_for":"3.76.26.241, 10.0.136.1","x_forwarded_proto":"https","x_request_start":"1728387051721","x_scp_request_id":"22ee7a5c-5c43-4521-bc3d-dcda6d6628f1-670517EB-37D2073","x_vcap_request_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","request_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","x_correlation_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","timestamp":"2024-10-08T11:30:52.122Z","msg":"Error during request to remote service: Failed to load destination.","stacktrace":["Error: Error during request to remote service: Failed to load destination.","at run (/home/vcap/app/node_modules/@sap/cds/libx/_runtime/remote/utils/client.js:238:31)","at process.processTicksAndRejections (node:internal/process/task_queues:95:5)","at async RemoteService.on_handler (/home/vcap/app/node_modules/@sap/cds/libx/_runtime/remote/Service.js:274:20)","at async next (/home/vcap/app/node_modules/@sap/cds/lib/srv/srv-dispatch.js:69:17)","at async RemoteService.handle (/home/vcap/app/node_modules/@sap/cds/lib/srv/srv-dispatch.js:67:10)","at async RemoteService.handle (/home/vcap/app/node_modules/@sap/cds/libx/_runtime/remote/Service.js:286:16)","at async ProcessorService.onBusinessPartnerChanged (/home/vcap/app/srv/services.js:57:20)","at async EventBroker.<anonymous> (/home/vcap/app/srv/services.js:34:110)","at async Promise.all (index 0)","at async EventBroker.handle (/home/vcap/app/node_modules/@sap/cds/lib/srv/srv-dispatch.js:66:21)"],"statusCode":502,"reason":{"message":"Error during request to remote service: Failed to load destination. Caused by: Could not fetch JWT bearer token for service of type xsuaa: HTTP response from https://s4eventpub-n6e7jhsl.authentication.eu10.hana.ondemand.com/oauth/token was 401: {\"error\":\"invalid_token\",\"error_description\":\"Some parties were not in the token audience: 3b78c4df-e21c-43a6-8775-799b42bd8e0c\"}.","request":{"method":"GET","url":"/sap/opu/odata/sap/API_BUSINESS_PARTNER/A_BusinessPartner?$top=10","headers":{"accept":"application/json,text/plain","x-correlation-id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"}},"correlationId":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"},"layer":"cds","component_type":"application","container_id":"10.0.201.6","component_id":"3401667f-aef0-4e3d-91c9-389eb7d509c5","component_name":"remote-srv","component_instance":0,"source_instance":0,"organization_name":"referenceapps_s4eventpub-n6e7jhsl","organization_id":"6e524a27-7a88-4b8d-b75e-7490f14ee023","space_name":"dev","space_id":"7e8116be-4b42-4dbb-8351-5a59adf6dcdc","#cf":{"string":[{"k":"reason","v":{"message":"Error during request to remote service: Failed to load destination. Caused by: Could not fetch JWT bearer token for service of type xsuaa: HTTP response from https://s4eventpub-n6e7jhsl.authentication.eu10.hana.ondemand.com/oauth/token was 401: {\"error\":\"invalid_token\",\"error_description\":\"Some parties were not in the token audience: 3b78c4df-e21c-43a6-8775-799b42bd8e0c\"}.","request":{"method":"GET","url":"/sap/opu/odata/sap/API_BUSINESS_PARTNER/A_BusinessPartner?$top=10","headers":{"accept":"application/json,text/plain","x-correlation-id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"}},"correlationId":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"},"i":3}]},"type":"log"}
   2024-10-08T11:30:52.12+0000 [APP/PROC/WEB/0] ERR {"level":"error","logger":"event-broker","correlation_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","tenant_id":"3525ffdc-60f2-4d05-a036-d70428d549dd","host":"referenceapps-s4eventpub-n6e7jhsl-dev-remote-srv.cfapps.eu10-004.hana.ondemand.com","user_agent":"Go-http-client/2.0","content_length":"29","request_size_b":"29","accept_encoding":"gzip","authorization":"***","b3":"6586dd72b78e366e4420b4fe2aa01fd8-8e8b6a95e939c04e-0","ce_id":"17928f26-b16f-1eef-a1ad-189928a54940","ce_kafkaoffset":"5","ce_kafkapartition":"1","ce_kafkatopic":"beb-st-system.f3d69622e9baddf16ed7803f07ee2f35","ce_sapconsumertenant":"SA0176014160","ce_sapemtenantid":"3525ffdc-60f2-4d05-a036-d70428d549dd","ce_source":"/default/sap.s4.beh/730431274","ce_specversion":"1.0","ce_time":"2024-10-08T11:30:51Z","ce_type":"sap.s4.beh.businesspartner.v1.BusinessPartner.Changed.v1","content_type":"application/json","x_b3_sampled":"0","x_b3_spanid":"8e8b6a95e939c04e","x_b3_traceid":"6586dd72b78e366e4420b4fe2aa01fd8","x_cf_applicationid":"3401667f-aef0-4e3d-91c9-389eb7d509c5","x_cf_instanceid":"47bf6ee3-0091-457d-7737-fc86","x_cf_instanceindex":"0","x_forwarded_for":"3.76.26.241, 10.0.136.1","x_forwarded_proto":"https","x_request_start":"1728387051721","x_scp_request_id":"22ee7a5c-5c43-4521-bc3d-dcda6d6628f1-670517EB-37D2073","x_vcap_request_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","request_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","x_correlation_id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8","timestamp":"2024-10-08T11:30:52.123Z","msg":"ERROR during inbound event processing: Error during request to remote service: Failed to load destination.","stacktrace":["Error: Error during request to remote service: Failed to load destination.","at run (/home/vcap/app/node_modules/@sap/cds/libx/_runtime/remote/utils/client.js:238:31)","at process.processTicksAndRejections (node:internal/process/task_queues:95:5)","at async RemoteService.on_handler (/home/vcap/app/node_modules/@sap/cds/libx/_runtime/remote/Service.js:274:20)","at async next (/home/vcap/app/node_modules/@sap/cds/lib/srv/srv-dispatch.js:69:17)","at async RemoteService.handle (/home/vcap/app/node_modules/@sap/cds/lib/srv/srv-dispatch.js:67:10)","at async RemoteService.handle (/home/vcap/app/node_modules/@sap/cds/libx/_runtime/remote/Service.js:286:16)","at async ProcessorService.onBusinessPartnerChanged (/home/vcap/app/srv/services.js:57:20)","at async EventBroker.<anonymous> (/home/vcap/app/srv/services.js:34:110)","at async Promise.all (index 0)","at async EventBroker.handle (/home/vcap/app/node_modules/@sap/cds/lib/srv/srv-dispatch.js:66:21)"],"statusCode":502,"reason":{"message":"Error during request to remote service: Failed to load destination. Caused by: Could not fetch JWT bearer token for service of type xsuaa: HTTP response from https://s4eventpub-n6e7jhsl.authentication.eu10.hana.ondemand.com/oauth/token was 401: {\"error\":\"invalid_token\",\"error_description\":\"Some parties were not in the token audience: 3b78c4df-e21c-43a6-8775-799b42bd8e0c\"}.","request":{"method":"GET","url":"/sap/opu/odata/sap/API_BUSINESS_PARTNER/A_BusinessPartner?$top=10","headers":{"accept":"application/json,text/plain","x-correlation-id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"}},"correlationId":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"},"layer":"cds","component_type":"application","container_id":"10.0.201.6","component_id":"3401667f-aef0-4e3d-91c9-389eb7d509c5","component_name":"remote-srv","component_instance":0,"source_instance":0,"organization_name":"referenceapps_s4eventpub-n6e7jhsl","organization_id":"6e524a27-7a88-4b8d-b75e-7490f14ee023","space_name":"dev","space_id":"7e8116be-4b42-4dbb-8351-5a59adf6dcdc","#cf":{"string":[{"k":"reason","v":{"message":"Error during request to remote service: Failed to load destination. Caused by: Could not fetch JWT bearer token for service of type xsuaa: HTTP response from https://s4eventpub-n6e7jhsl.authentication.eu10.hana.ondemand.com/oauth/token was 401: {\"error\":\"invalid_token\",\"error_description\":\"Some parties were not in the token audience: 3b78c4df-e21c-43a6-8775-799b42bd8e0c\"}.","request":{"method":"GET","url":"/sap/opu/odata/sap/API_BUSINESS_PARTNER/A_BusinessPartner?$top=10","headers":{"accept":"application/json,text/plain","x-correlation-id":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"}},"correlationId":"6586dd72-b78e-366e-4420-b4fe2aa01fd8"},"i":3}]},"type":"log"}

Points:

1. looks like JWT zid is: undefined.
2. Also didnt understand why the destination incidents-api-access is not able fetchdstination service when i am calling inside messaging handler (messaging.on). Although same code is working outside of messaging handler.

To Reproduce
Deploy the CAP application. Send the message from SAP S/4 Cloud

Expected behavior
CAP should return business partner details from S/4HANA as soon message is consumed

Used Versions:

@cap-js/asyncapi: 1.0.2
@cap-js/db-service: 1.13.0
@cap-js/openapi: 1.0.6
@sap/cds: 8.3.0
@sap/cds-compiler: 5.3.0
@sap/cds-dk: 8.3.0
@sap/cds-dk (global): 8.2.3
@sap/cds-fiori: 1.2.7
@sap/cds-foss: 5.0.1
@sap/cds-mtxs: 2.2.0
@sap/eslint-plugin-cds: 3.1.0
Node.js: v20.12.0
home: /home/user/projects/remote/node_modules/@sap/cds

"@sap-cloud-sdk/connectivity": "^3.22.0",
"@sap-cloud-sdk/http-client": "^3.22.0",
"@sap-cloud-sdk/resilience": "^3.22.0",
"@sap-cloud-sdk/util": "^3.22.0",

Code Examples

class ProcessorService extends cds.ApplicationService {
  /** Registering custom event handlers */
  async init() {
   
/* This is working fine 
    const bupa1 = await cds.connect.to('API_BUSINESS_PARTNER');
    const { A_BusinessPartner } = bupa1.entities;
    const results = await bupa1.run(SELECT(A_BusinessPartner).limit(10));
    console.log("<<result ",results)
*/

    this.messaging = await cds.connect.to('messaging');
    this.messaging.on('sap.s4.beh.businesspartner.v1.BusinessPartner.Changed.v1', async ({ event, data }) => await this.onBusinessPartnerChanged(event, data))

    return super.init();
  }


  async onBusinessPartnerChanged(event, data){
    const { Customers } = this.entities;
    const { BusinessPartner, BusinessPartnerAddress } = this.remoteService.entities;
    //If Business Partner exists in Cache, then update
    console.log('<< received', event, data)
    const Id = data.BusinessPartner;
    console.log("BusinessPartnerID", Id);

/* This is failing 

*/
    const bupa = await cds.connect.to('API_BUSINESS_PARTNER');
    const { A_BusinessPartner } = bupa.entities;
    const result = await bupa.run(SELECT(A_BusinessPartner).limit(10));
    console.log("<<result ",result)