Cosmos DB: The request failed because the client was unable to establish connections to 1 endpoints across 1 regions

0
votes

Below exception logged in the application insights for HTTP Trigger Azure function.

The request failed because the client was unable to establish connections to 1 endpoints across 1 regions. The client CPU was overloaded during the attempted request

"outerId":"0","message":"Response status code does not indicate success: ServiceUnavailable (503); Substatus: 0; ActivityId: 35d16569-41da-42ed-bc5c-8f9a5c49078e; Reason: (The request failed because the client was unable to establish connections to 1 endpoints across 1 regions. The client CPU was overloaded during the attempted request.\r\nActivityId: 35d16569-41da-42ed-bc5c-8f9a5c49078e, Please see CosmosDiagnostics, Windows/10.0.14393 cosmos-netstandard-sdk/3.9.0);","parsedStack":[{"assembly":"Microsoft.Azure.Cosmos.Client, Version=3.9.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.Cosmos.ResponseMessage.EnsureSuccessStatusCode","level":0,"line":0},{"assembly":"Microsoft.Azure.Cosmos.Client, Version=3.9.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.Cosmos.ReadFeedResponse1.CreateResponse","level":1,"line":0},{"assembly":"Microsoft.Azure.Cosmos.Client, Version=3.9.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.Cosmos.FeedIteratorCore1+d__6.MoveNext","level":2,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":3,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":4,"line":0},{"assembly":"FleetHub.Notifications.Domain, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Domain.Repositories.CosmosDBRepository1+<GetItemsAsync>d__10.MoveNext","level":5,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":6,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":7,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":8,"line":0},{"assembly":"FleetHub.Notifications.Queries, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Queries.Handlers.RetrieveNotificationQueryHandler+<GetNotifications>d__4.MoveNext","level":9,"line":41,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Queries\\Handlers\\RetrieveNotificationQueryHandler.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":10,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":11,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":12,"line":0},{"assembly":"FleetHub.Notifications.Queries, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Queries.Handlers.RetrieveNotificationQueryHandler+<GetNotificationSummary>d__5.MoveNext","level":13,"line":54,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Queries\\Handlers\\RetrieveNotificationQueryHandler.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":14,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":15,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":16,"line":0},{"assembly":"FleetHub.Notifications.Queries, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Queries.Handlers.RetrieveNotificationQueryHandler+<HandleAsync>d__3.MoveNext","level":17,"line":28,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Queries\\Handlers\\RetrieveNotificationQueryHandler.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":18,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":19,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":20,"line":0},{"assembly":"FleetHub.Notifications.Azure.Functions, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Azure.Functions.Notifications.RetrieveNotifications+<Run>d__0.MoveNext","level":21,"line":39,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Azure.Functions\\Notifications\\RetrieveNotifications.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":22,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":23,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":24,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter1.GetResult","level":25,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionInvoker2+<InvokeAsync>d__10.MoveNext","level":26,"line":52,"fileName":"C:\\projects\\azure-webjobs-sdk-rqm4t\\src\\Microsoft.Azure.WebJobs.Host\\Executors\\FunctionInvoker.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":27,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":28,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":29,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter1.GetResult","level":30,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+FunctionInvocationFilterInvoker+d__10.MoveNext","level":31,"line":1116,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":32,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+FunctionInvocationFilterInvoker+d__10.MoveNext","level":33,"line":1154,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":34,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":35,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":36,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter`1.GetResult","level":37,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__27.MoveNext","level":38,"line":547,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":39,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":40,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":41,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.GetResult","level":42,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__26.MoveNext","level":43,"line":492,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":44,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":45,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":46,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.GetResult","level":47,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__25.MoveNext","level":48,"line":438,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":49,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":50,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":51,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.GetResult","level":52,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__19.MoveNext","level":53,"line":245,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"}],"type":"Microsoft.Azure.Cosmos.CosmosException","id":"23479096"}]

Azure Function:

public static class RetrieveNotifications
    {
        [FunctionName(nameof(RetrieveNotifications))]
        public static async Task<IActionResult> Run(
            [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
            [Inject] ILoggingService loggingService,
            [Inject] IQueryValidator queryValidator,
            [Inject] IQueryHandler<RetrieveNotificationQuery, NotificationSummaryResponse> queryHandler)
        {
            var logger = new Logger(loggingService);

            try
            {
                logger.Info($"Retrieve Notifications request received on {nameof(RetrieveNotifications)}");

                IActionResult actionResult = null;

                string requestBody = await new StreamReader(req.Body).ReadToEndAsync();

                var query = requestBody.AsPoco<RetrieveNotificationQuery>();

                if (queryValidator.Validate<RetrieveNotificationQuery>(req, query, new RetrieveNotificationQueryValidator(), logger, ref actionResult))
                {
                    var response = await queryHandler.HandleAsync(query, logger);
                    actionResult = new OkObjectResult(response);
                }
                return actionResult;
            }
            catch (Exception ex)
            {
                logger.Error($"Exception while processing {nameof(RetrieveNotifications)}", ex,
                nameof(RetrieveNotifications));

                throw;
            }
        }
    }
1
azure-functionsazure-cosmosdb

1 Answers

1
votes

Your exception contains the source of the connectivity problem:

The client CPU was overloaded during the attempted request

This means that CPU was starved and could not handle TCP requests within the expected timeout interval. It is within the official documentation causes: https://docs.microsoft.com/azure/cosmos-db/troubleshoot-dot-net-sdk-request-timeout#high-cpu-utilization

Check your instance metrics and see if you need to scale the size of the instances to handle the load of operations you are doing. I see you are injecting references into the Function, I guess these are Singleton instances of the CosmosClient (if not, they should be).