0
votes

Below exception logged in the application insights for HTTP Trigger Azure function.

The request failed because the client was unable to establish connections to 1 endpoints across 1 regions. The client CPU was overloaded during the attempted request

"outerId":"0","message":"Response status code does not indicate success: ServiceUnavailable (503); Substatus: 0; ActivityId: 35d16569-41da-42ed-bc5c-8f9a5c49078e; Reason: (The request failed because the client was unable to establish connections to 1 endpoints across 1 regions. The client CPU was overloaded during the attempted request.\r\nActivityId: 35d16569-41da-42ed-bc5c-8f9a5c49078e, Please see CosmosDiagnostics, Windows/10.0.14393 cosmos-netstandard-sdk/3.9.0);","parsedStack":[{"assembly":"Microsoft.Azure.Cosmos.Client, Version=3.9.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.Cosmos.ResponseMessage.EnsureSuccessStatusCode","level":0,"line":0},{"assembly":"Microsoft.Azure.Cosmos.Client, Version=3.9.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.Cosmos.ReadFeedResponse1.CreateResponse","level":1,"line":0},{"assembly":"Microsoft.Azure.Cosmos.Client, Version=3.9.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.Cosmos.FeedIteratorCore1+d__6.MoveNext","level":2,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":3,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":4,"line":0},{"assembly":"FleetHub.Notifications.Domain, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Domain.Repositories.CosmosDBRepository1+<GetItemsAsync>d__10.MoveNext","level":5,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":6,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":7,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":8,"line":0},{"assembly":"FleetHub.Notifications.Queries, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Queries.Handlers.RetrieveNotificationQueryHandler+<GetNotifications>d__4.MoveNext","level":9,"line":41,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Queries\\Handlers\\RetrieveNotificationQueryHandler.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":10,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":11,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":12,"line":0},{"assembly":"FleetHub.Notifications.Queries, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Queries.Handlers.RetrieveNotificationQueryHandler+<GetNotificationSummary>d__5.MoveNext","level":13,"line":54,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Queries\\Handlers\\RetrieveNotificationQueryHandler.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":14,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":15,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":16,"line":0},{"assembly":"FleetHub.Notifications.Queries, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Queries.Handlers.RetrieveNotificationQueryHandler+<HandleAsync>d__3.MoveNext","level":17,"line":28,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Queries\\Handlers\\RetrieveNotificationQueryHandler.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":18,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":19,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":20,"line":0},{"assembly":"FleetHub.Notifications.Azure.Functions, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null","method":"FleetHub.Notifications.Azure.Functions.Notifications.RetrieveNotifications+<Run>d__0.MoveNext","level":21,"line":39,"fileName":"D:\\a\\144\\s\\FleetHub.Notifications.Azure.Functions\\Notifications\\RetrieveNotifications.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":22,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":23,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":24,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter1.GetResult","level":25,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionInvoker2+<InvokeAsync>d__10.MoveNext","level":26,"line":52,"fileName":"C:\\projects\\azure-webjobs-sdk-rqm4t\\src\\Microsoft.Azure.WebJobs.Host\\Executors\\FunctionInvoker.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":27,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":28,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":29,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter1.GetResult","level":30,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+FunctionInvocationFilterInvoker+d__10.MoveNext","level":31,"line":1116,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":32,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+FunctionInvocationFilterInvoker+d__10.MoveNext","level":33,"line":1154,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":34,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":35,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":36,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter`1.GetResult","level":37,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__27.MoveNext","level":38,"line":547,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":39,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":40,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":41,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.GetResult","level":42,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__26.MoveNext","level":43,"line":492,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":44,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":45,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":46,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.GetResult","level":47,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__25.MoveNext","level":48,"line":438,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw","level":49,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess","level":50,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification","level":51,"line":0},{"assembly":"System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e","method":"System.Runtime.CompilerServices.TaskAwaiter.GetResult","level":52,"line":0},{"assembly":"Microsoft.Azure.WebJobs.Host, Version=3.0.23.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35","method":"Microsoft.Azure.WebJobs.Host.Executors.FunctionExecutor+d__19.MoveNext","level":53,"line":245,"fileName":"C:\projects\azure-webjobs-sdk-rqm4t\src\Microsoft.Azure.WebJobs.Host\Executors\FunctionExecutor.cs"}],"type":"Microsoft.Azure.Cosmos.CosmosException","id":"23479096"}]

Azure Function:

public static class RetrieveNotifications
    {
        [FunctionName(nameof(RetrieveNotifications))]
        public static async Task<IActionResult> Run(
            [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
            [Inject] ILoggingService loggingService,
            [Inject] IQueryValidator queryValidator,
            [Inject] IQueryHandler<RetrieveNotificationQuery, NotificationSummaryResponse> queryHandler)
        {
            var logger = new Logger(loggingService);

            try
            {
                logger.Info($"Retrieve Notifications request received on {nameof(RetrieveNotifications)}");

                IActionResult actionResult = null;

                string requestBody = await new StreamReader(req.Body).ReadToEndAsync();

                var query = requestBody.AsPoco<RetrieveNotificationQuery>();

                if (queryValidator.Validate<RetrieveNotificationQuery>(req, query, new RetrieveNotificationQueryValidator(), logger, ref actionResult))
                {
                    var response = await queryHandler.HandleAsync(query, logger);
                    actionResult = new OkObjectResult(response);
                }
                return actionResult;
            }
            catch (Exception ex)
            {
                logger.Error($"Exception while processing {nameof(RetrieveNotifications)}", ex,
                nameof(RetrieveNotifications));

                throw;
            }
        }
    }
1

1 Answers

1
votes

Your exception contains the source of the connectivity problem:

The client CPU was overloaded during the attempted request

This means that CPU was starved and could not handle TCP requests within the expected timeout interval. It is within the official documentation causes: https://docs.microsoft.com/azure/cosmos-db/troubleshoot-dot-net-sdk-request-timeout#high-cpu-utilization

Check your instance metrics and see if you need to scale the size of the instances to handle the load of operations you are doing. I see you are injecting references into the Function, I guess these are Singleton instances of the CosmosClient (if not, they should be).