WSO2 3.2.0 : Analytics and Dashboard, Cannot borrow client on SSL port 7712 + Required field 'message' was not present

Question

I installed WSO2 3.2.0 recently :

OS : Debian 9
WSO2 API Manager 3.2.0 from the official repository
WSO2 Analytics 3.2.0 from the zip archive
Everything running on the same machine with the same domain name
Valid certificate for this domain name, let's say wso2.mydomain.com

I tried to install Analytics following the documentation, but I think there is a communication issue between analytics worker and API Manager. When I try to go on my dashboard here, all I got is a blue screen.

Here are my warns and errors in API Manager logs :

tail -f /usr/lib/wso2/wso2am/3.2.0/repository/logs/wso2carbon.log

WARN {org.wso2.carbon.databridge.agent.endpoint.DataEndpointGroup} - No receiver is reachable at reconnection, will try to reconnect every 30 sec

ERROR {org.wso2.carbon.databridge.agent.endpoint.DataEndpointConnectionWorker} - Error while trying to connect to the endpoint. Cannot borrow client for ssl://wso2.mydomain.com:7712. org.wso2.carbon.databridge.agent.exception.DataEndpointLoginException: Cannot borrow client for ssl://wso2.mydomain.com:7712.

Caused by: org.wso2.carbon.databridge.agent.exception.DataEndpointLoginException: Error while trying to login to the data receiver.

Caused by: org.apache.thrift.transport.TTransportException

Here are my warns and errors in Analytics logs :

tail -f /usr/lib/wso2/wso2an/3.2.0/wso2/worker/logs/carbon.log

ERROR {org.apache.thrift.server.TThreadPoolServer} - Thrift error occurred during processing of message. org.apache.thrift.protocol.TProtocolException: Required field 'message' was not present! Struct: ThriftAuthenticationException(message:null)

WARN {org.apache.thrift.transport.TIOStreamTransport} - Error closing output stream. java.net.SocketException: Connection or outbound has closed

tail -f /usr/lib/wso2/wso2an/3.2.0/wso2/dashboard/logs/carbon.log

WARN {org.wso2.msf4j.internal.MSF4JHttpConnectorListener} - Error in http connector listener : 'Remote client closed the connection before initiating outbound response'

Here is my full API Manager deployment config at /usr/lib/wso2/wso2am/3.2.0/repository/conf/deployment.toml :

[server]
hostname = "wso2.mydomain.com"
node_ip = "127.0.0.1"
#offset=0
mode = "single" #single or ha
base_path = "${carbon.protocol}://${carbon.host}:${carbon.management.port}"
#discard_empty_caches = false
server_role = "default"

[super_admin]
username = "admin"
password = "mypassword"
create_admin_account = true

[user_store]
type = "database_unique_id"

[database.apim_db]
type = "h2"
url = "jdbc:h2:./repository/database/WSO2AM_DB;AUTO_SERVER=TRUE;DB_CLOSE_ON_EXIT=FALSE"
username = "wso2carbon"
password = "wso2carbon"

[database.shared_db]
type = "h2"
url = "jdbc:h2:./repository/database/WSO2SHARED_DB;DB_CLOSE_ON_EXIT=FALSE"
username = "wso2carbon"
password = "wso2carbon"

[keystore.tls]
file_name =  "wso2carbon.jks"
type =  "JKS"
password =  "wso2carbon"
alias =  "owacustom"
key_password =  "wso2carbon"

[keystore.primary]
file_name =  "wso2carbon.jks"
type =  "JKS"
password =  "wso2carbon"
alias =  "owacustom"
key_password =  "wso2carbon"

[keystore.internal]
file_name =  "wso2carbon.jks"
type =  "JKS"
password =  "wso2carbon"
alias =  "owacustom"
key_password =  "wso2carbon"

[[apim.gateway.environment]]
name = "Production and Sandbox"
type = "hybrid"
display_in_api_console = true
description = "This is a hybrid gateway that handles both production and sandbox token traffic."
show_as_token_endpoint_url = true
service_url = "https://wso2.mydomain.com:9443/services/"
username= "${admin.username}"
password= "${admin.password}"
ws_endpoint = "ws://wso2.mydomain.com:9099"
wss_endpoint = "wss://wso2.mydomain.com:8099"
http_endpoint = "http://wso2.mydomain.com:${http.nio.port}"
https_endpoint = "https://wso2.mydomain.com:${https.nio.port}"

#[apim.cache.gateway_token]
#enable = true
#expiry_time = "900s"

#[apim.cache.resource]
#enable = true
#expiry_time = "900s"

#[apim.cache.km_token]
#enable = false
#expiry_time = "15m"

#[apim.cache.recent_apis]
#enable = false

#[apim.cache.scopes]
#enable = true

#[apim.cache.publisher_roles]
#enable = true

#[apim.cache.jwt_claim]
#enable = true
#expiry_time = "15m"

#[apim.cache.tags]
#expiry_time = "2m"

[apim.analytics]
enable = true
store_api_url = "https://wso2.mydomain.com:7444"
username = "$ref{super_admin.username}"
password = "$ref{super_admin.password}"
#event_publisher_type = "default"
#event_publisher_impl = "org.wso2.carbon.apimgt.usage.publisher.APIMgtUsageDataBridgeDataPublisher"
#publish_response_size = true

#[[apim.analytics.url_group]]
#analytics_url =["tcp://wso2.mydomain.com:7611"]
#analytics_auth_url =["ssl://wso2.mydomain.com:7711"]
#type = "loadbalance"

[[apim.analytics.url_group]]
analytics_url =["tcp://wso2.mydomain.com:7612"]
analytics_auth_url =["ssl://wso2.mydomain.com:7712"]
#type = "failover"

[apim.key_manager]
service_url = "https://wso2.mydomain.com:9443/services/"
username = "$ref{super_admin.username}"
password = "$ref{super_admin.password}"
#pool.init_idle_capacity = 50
#pool.max_idle = 100
#key_validation_handler_type = "default"
#key_validation_handler_type = "custom"
#key_validation_handler_impl = "org.wso2.carbon.apimgt.keymgt.handlers.DefaultKeyValidationHandler"

[apim.idp]
server_url = "https://wso2.mydomain.com:9443"
authorize_endpoint = "https://wso2.mydomain.com:9443/oauth2/authorize"
oidc_logout_endpoint = "https://wso2.mydomain.com:9443/oidc/logout"
oidc_check_session_endpoint = "https://wso2.mydomain.com:9443/oidc/checksession"

#[apim.jwt]
#enable = true
#encoding = "base64" # base64,base64url
#generator_impl = "org.wso2.carbon.apimgt.keymgt.token.JWTGenerator"
#claim_dialect = "http://wso2.org/claims"
#convert_dialect = false
#header = "X-JWT-Assertion"
#signing_algorithm = "SHA256withRSA"
#enable_user_claims = true
#claims_extractor_impl = "org.wso2.carbon.apimgt.impl.token.ExtendedDefaultClaimsRetriever"

[apim.oauth_config]
#enable_outbound_auth_header = false
#auth_header = "Authorization"
revoke_endpoint = "https://wso2.mydomain.com:${https.nio.port}/revoke"
#enable_token_encryption = false
#enable_token_hashing = false

[apim.devportal]
url = "https://wso2.mydomain.com:9443/devportal"
#enable_application_sharing = false
#if application_sharing_type, application_sharing_impl both defined priority goes to application_sharing_impl
#application_sharing_type = "default" #changed type, saml, default #todo: check the new config for rest api
#application_sharing_impl = "org.wso2.carbon.apimgt.impl.SAMLGroupIDExtractorImpl"
#display_multiple_versions = false
#display_deprecated_apis = false
#enable_comments = true
#enable_ratings = true
#enable_forum = true
#enable_anonymous_mode=true

[apim.cors]
allow_origins = "*"
allow_methods = ["GET","PUT","POST","DELETE","PATCH","OPTIONS"]
allow_headers = ["authorization","Access-Control-Allow-Origin","Content-Type","SOAPAction","apikey", "testKey"]
allow_credentials = false

[apim.throttling]
#enable_data_publishing = true
#enable_policy_deploy = true
#enable_blacklist_condition = true
#enable_persistence = true
throttle_decision_endpoints = ["tcp://wso2.mydomain.com:5672","tcp://wso2.mydomain.com:5672"]

#[apim.throttling.blacklist_condition]
#start_delay = "5m"
#period = "1h"

#[apim.throttling.jms]
#start_delay = "5m"

#[apim.throttling.event_sync]
#hostName = "0.0.0.0"
#port = 11224

#[apim.throttling.event_management]
#hostName = "0.0.0.0"
#port = 10005

#[[apim.throttling.url_group]]
#traffic_manager_urls = ["tcp://wso2.mydomain.com:9611"]
#traffic_manager_auth_urls = ["ssl://wso2.mydomain.com:9711"]
#type = "loadbalance"

[[apim.throttling.url_group]]
traffic_manager_urls = ["tcp://wso2.mydomain.com.com:9611"]
traffic_manager_auth_urls = ["ssl://wso2.mydomain.com:9711"]
#type = "failover"

[apim.workflow]
#enable = false
service_url = "https://wso2.mydomain.com:9445/bpmn"
#username = "$ref{super_admin.username}"
#password = "$ref{super_admin.password}"
callback_endpoint = "https://wso2.mydomain.com:9443/api/am/admin/v0.17/workflows/update-workflow-status"
token_endpoint = "https://wso2.mydomain.com:${https.nio.port}/token"
client_registration_endpoint = "https://wso2.mydomain.com:9443/client-registration/v0.17/register"
#client_registration_username = "$ref{super_admin.username}"
#client_registration_password = "$ref{super_admin.password}"

#data bridge config
#[transport.receiver]
#type = "binary"
#worker_threads = 10
#session_timeout = "30m"
#keystore.file_name = "$ref{keystore.tls.file_name}"
#keystore.password = "$ref{keystore.tls.password}"
#tcp_port = 9611
#ssl_port = 9711
#ssl_receiver_thread_pool_size = 100
#tcp_receiver_thread_pool_size = 100
#ssl_enabled_protocols = ["TLSv1","TLSv1.1","TLSv1.2"]
#ciphers = ["SSL_RSA_WITH_RC4_128_MD5","SSL_RSA_WITH_RC4_128_SHA"]

[apim.notification]
#from_address = "APIM.com"
#username = "APIM"
#password = "APIM+123"
hostname = "wso2.mydomain.com"
#port = 3025
#enable_start_tls = false
#enable_authentication = true

apim.token.revocation]
#notifier_impl = "org.wso2.carbon.apimgt.keymgt.events.TokenRevocationNotifierImpl"
#enable_realtime_notifier = true
#realtime_notifier.ttl = 5000
#enable_persistent_notifier = true
persistent_notifier.hostname = "https://wso2.mydomain.com:2379/v2/keys/jti/"
#persistent_notifier.ttl = 5000
#persistent_notifier.username = "root"
#persistent_notifier.password = "root"

[[event_handler]]
name="userPostSelfRegistration"
subscriptions=["POST_ADD_USER"]

[service_provider]
sp_name_regex = "^[\\sa-zA-Z0-9._-]*$"

[database.local]
url = "jdbc:h2:./repository/database/WSO2CARBON_DB;DB_CLOSE_ON_EXIT=FALSE"

[[event_listener]]
id = "token_revocation"
type = "org.wso2.carbon.identity.core.handler.AbstractIdentityHandler"
name = "org.wso2.is.notification.ApimOauthEventInterceptor"
order = 1
[event_listener.properties]
notification_endpoint = "https://wso2.mydomain.com:9443/internal/data/v1/notify"
username = "${admin.username}"
password = "${admin.password}"
'header.X-WSO2-KEY-MANAGER' = "default"

# Remove "WSO2 Carbon Server" from server value in header response
[transport.http.properties]
server="wso2.mydomain.com"

# Intermediate compatibility (recommended) ciphers list see https://wiki.mozilla.org/Security/Server_Side_TLS
[transport.https.sslHostConfig.properties]
ciphers="TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, ECDHE-ECDSA-AES128-GCM-SHA256, ECDHE-RSA-AES128-GCM-SHA256, ECDHE-ECDSA-AES256-GCM-SHA384, ECDHE-RSA-AES256-GCM-SHA384, ECDHE-ECDSA-CHACHA20-POLY1305, ECDHE-RSA-CHACHA20-POLY1305, DHE-RSA-AES128-GCM-SHA256, DHE-RSA-AES256-GCM-SHA384"

Here is the lines I added to the API Manager boostrap file on /usr/lib/wso2/wso2am/3.2.0/bin/wso2server.sh :

-Djdk.tls.ephemeralDHKeySize=2048 \
-Djdk.tls.rejectClientInitiatedRenegotiation=true \

I just modified the super admin password in my worker deployment file at /usr/lib/wso2/wso2an/3.2.0/conf/worker/deployment.yaml :

# Authentication configuration
auth.configs:
  type: 'local'        # Type of the IdP client used
  userManager:
    adminRole: admin   # Admin role which is granted all permissions
    userStore:         # User store
      users:
       -
         user:
           username: admin
           password: bXlwYXNzd29yZA==
           roles: 1
      roles:
       -
         role:
           id: 1
           displayName: admin

  # Configuration to enable apim alerts
#analytics.solutions:
#  APIM-alerts.enabled: true

And I replaced some localhost values by my domain name + modified the super admin password in my dashboard deployment file at /usr/lib/wso2/wso2an/3.2.0/conf/dashboard/deployment.yaml :

## Authentication configuration
auth.configs:
  type: apim
  ssoEnabled: true
  properties:
    adminScope: apim_analytics:admin_carbon.super
    allScopes: apim_analytics:admin openid apim:api_view apim:subscribe apim_analytics:monitoring_dashboard:own apim_analytics:monitoring_dashboard:edit apim_analytics:monitoring_dashboard:view apim_analytics:business_analytics:own apim_analytics:business_analytics:edit apim_analytics:business_analytics:view apim_analytics:api_analytics:own apim_analytics:api_analytics:edit apim_analytics:api_analytics:view apim_analytics:application_analytics:own apim_analytics:application_analytics:edit
    adminUsername: admin
    adminPassword: mypassword
    kmDcrUrl: https://wso2.mydomain.com:9443/client-registration/v0.17/register
    kmTokenUrlForRedirection: https://wso2.mydomain.com:9443/oauth2
    kmTokenUrl: https://wso2.mydomain.com:9443/oauth2
    kmUsername: admin
    kmPassword: admin
    portalAppContext: analytics-dashboard
    businessRulesAppContext : business-rules
    cacheTimeout: 30
    baseUrl: https://wso2.mydomain.com:9643
    grantType: authorization_code
    publisherUrl: https://wso2.mydomain.com:9443
    devPortalUrl: https://wso2.mydomain.com:9443
    externalLogoutUrl: https://wso2.mydomain.com:9443/oidc/logout

Also, my client-truststore.jks and wso2carbon.jks files are the same for both products :

/usr/lib/wso2/wso2am/3.2.0/repository/resources/security
/usr/lib/wso2/wso2an/3.2.0/resources/security

Am I missing something? Thanks by advance!

No there is an error, I quoted it at the beginning of my post. I'll edit my question to separate API Manager and Analytics errors. — Johnson
can you disable analytics first and check whether the API manager is running without any errors? — chashikajw
Sure. It gives me no error but some warns : WARN {org.apache.tomcat.util.net.SSLUtilBase} - jsseUtil.trustedCertNotValid WARN {org.wso2.carbon.apimgt.jms.listener.utils.JMSUtils} - Cannot locate destination : keyManager / throttleData / notification / cacheInvalidation / tokenRevocation — Johnson

chashikajw chashikajw · Accepted Answer · 2020-11-23T15:07:01

can you check whether wso2am-analytics-3.2.0/conf/dashboard/deployment.yaml---> AM_DB -> jdbcUrl path was correctly configured to your wso2 API manager am_db. Maybe your path is incorrect and should change it according to your path.

WSO2 3.2.0 : Analytics and Dashboard, Cannot borrow client on SSL port 7712 + Required field 'message' was not present

1 Answers