DialogFlow/Actions: Allow Google Assistant user to create Event in Google Calendar from Actions App

Question

Target/Summary: I have an Actions App developed in Google DialogFlow and I want the user be able to create Google Calendar Event using the App (from Google Assistant). In other words, authenticate the user to Allow my app to use his Calendar to create Events.

What is done:

Since Google Actions doesn't allow use of Google Auth/Token endpoints, I opted to use http://www.auth0.com.

Created an account (used my Google account) on auth0.com, created an Application and setup the following values using their management panel (Domain, CliendId and ClientSecret generated by auth0):

Created OAuth Client Id on Google Cloud Console Credentials page:

Configured Actions Account Linking like so:

Went back to auth0.com > Connections > Social > Enabled Google:

Checked the "Sign in required" in DialogFlow > Integrations > Google Assistant:

Wrote log on the first line of my DialogFlow Webhook method to log the following Response:

  {
  "originalRequest":{
     "source":"google",
     "version":"2",
     "data":{
        "isInSandbox":true,
        "surface":{
           "capabilities":[
              {
                 "name":"actions.capability.AUDIO_OUTPUT"
              },
              {
                 "name":"actions.capability.WEB_BROWSER"
              },
              {
                 "name":"actions.capability.MEDIA_RESPONSE_AUDIO"
              },
              {
                 "name":"actions.capability.SCREEN_OUTPUT"
              }
           ]
        },
        "inputs":[
           {
              "rawInputs":[
                 {
                    "query":"test",
                    "inputType":"KEYBOARD"
                 }
              ],
              "arguments":[
                 {
                    "rawText":"test",
                    "textValue":"test",
                    "name":"text"
                 }
              ],
              "intent":"actions.intent.TEXT"
           }
        ],
        "user":{
           "lastSeen":"2018-05-03T11:40:57Z",
           "accessToken":"4CfRs-Lt5lWVQuyOYODvf1xxxxxxx",
           "locale":"en-US",
           "userId":"15229245xxxxx"
        },
        "conversation":{
           "conversationId":"15253476xxxxx",
           "type":"ACTIVE",
           "conversationToken":"[\"authentication\",\"wh_patient-details\"]"
        },
        "availableSurfaces":[
           {
              "capabilities":[
                 {
                    "name":"actions.capability.AUDIO_OUTPUT"
                 },
                 {
                    "name":"actions.capability.SCREEN_OUTPUT"
                 }
              ]
           }
        ]
     }
  },
  "id":"1d6ed865-0803-49ca-bbac-xxxx",
  "timestamp":"2018-05-03T11:42:22.835Z",
  "lang":"en-us",
  "result":{
     "source":"agent",
     "resolvedQuery":"test",
     "speech":"",
     "action":"v00.xxxxx",
     "actionIncomplete":false,
     "parameters":{
        "CallEnum":"Test"
     },
     "contexts":[
        {
           "name":"authentication",
           "parameters":{
              "CallEnum":"Test",
              "CallEnum.original":""
           },
           "lifespan":1
        },
        {
           "name":"actions_capability_screen_output",
           "parameters":{
              "CallEnum":"Test",
              "CallEnum.original":""
           },
           "lifespan":0
        },
        {
           "name":"actions_capability_audio_output",
           "parameters":{
              "CallEnum":"Test",
              "CallEnum.original":""
           },
           "lifespan":0
        },
        {
           "name":"wh_patient-details",
           "parameters":{
              "patientId":0,
              "CallEnum":"Test",
              "fallbackLifespan":0,
              "providerId":0,
              "practiceId":0,
              "CallEnum.original":"",
              "fullDob":"01 January, 0001"
           },
           "lifespan":199
        },
        {
           "name":"google_assistant_input_type_keyboard",
           "parameters":{
              "CallEnum":"Test",
              "CallEnum.original":""
           },
           "lifespan":0
        },
        {
           "name":"actions_capability_web_browser",
           "parameters":{
              "CallEnum":"Test",
              "CallEnum.original":""
           },
           "lifespan":0
        },
        {
           "name":"actions_capability_media_response_audio",
           "parameters":{
              "CallEnum":"Test",
              "CallEnum.original":""
           },
           "lifespan":0
        }
     ],
     "metadata":{
        "intentName":"v00xxxx",
        "isResponseToSlotfilling":false,
        "intentId":"c7bd9113-d5b4-4312-8851-xxxxxxx",
        "webhookUsed":"true",
        "webhookForSlotFillingUsed":"false",
        "nluResponseTime":556
     },
     "fulfillment":{
        "speech":"Test",
        "messages":[
           {
              "type":0,
              "speech":"Test"
           }
        ]
     },
     "score":0.8399999737739563
  },
  "status":{
     "code":200,
     "errorType":"success"
  },
  "sessionId":"152534xxxxxxx",
  "isStackdriverLoggingEnabled":false

}

where the relevant section is:

"user":{
           "lastSeen":"2018-05-03T11:40:57Z",
           "accessToken":"4CfRs-Lt5lWVQuyOYODvf1xxxxxxx",
           "locale":"en-US",
           "userId":"15229245xxxxx"
        }

From this post: See answer by @Prisoner

The auth token (which you have issued, because you're the OAuth server) will be sent in the JSON object at originalRequest.data.user.accessToken.

So I used the authorization token from above in the code below:

string clientId = "361385932727-ksg6jgjxxxxxSNIP";
string clientSecret = "rc2K1UUyntxxxxxxSNIP";
string accessToken = jsonObject.SelectToken("originalRequest.data.user.accessToken");
string userId = jsonObject.SelectToken("originalRequest.data.user.userId");

IAuthorizationCodeFlow flow =
        new GoogleAuthorizationCodeFlow(new GoogleAuthorizationCodeFlow.Initializer
        {
            ClientSecrets = new ClientSecrets
            {
                ClientId = clientId,
                ClientSecret = clientSecret
            },
            Scopes = new[] { CalendarService.Scope.Calendar }
        });

TokenResponse token = flow.ExchangeCodeForTokenAsync(userId, accessToken, 
    "https://oauth-redirect.googleusercontent.com/r/xxxxxxxxx", CancellationToken.None).Result;

UserCredential credential = new UserCredential(flow, userId, new TokenResponse { AccessToken = token.AccessToken });
CalendarService service = new CalendarService(new BaseClientService.Initializer()
{
    HttpClientInitializer = credential,
    ApplicationName = "Test Auth0",
});

var list = service.CalendarList.List().Execute().Items;

And the exception:

Error:"invalid_grant", Description:"Malformed auth code.", Uri:""
Stacktrace:    at Google.Apis.Auth.OAuth2.Requests.TokenRequestExtenstions.<ExecuteAsync>d__0.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at Google.Apis.Auth.OAuth2.Flows.AuthorizationCodeFlow.<FetchTokenAsync>d__35.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at Google.Apis.Auth.OAuth2.Flows.AuthorizationCodeFlow.<FetchTokenAsync>d__35.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at Google.Apis.Auth.OAuth2.Flows.AuthorizationCodeFlow.<ExchangeCodeForTokenAsync>d__30.MoveNext()

And when I change the ClientId/ClientSecret in the code above to the one from auth0.com, the exception is:

Error:"invalid_client", Description:"The OAuth client was not found.", Uri:""
Stacktrace:    at Google.Apis.Auth.OAuth2.Requests.TokenRequestExtenstions.<ExecuteAsync>d__0.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at Google.Apis.Auth.OAuth2.Flows.AuthorizationCodeFlow.<FetchTokenAsync>d__35.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at Google.Apis.Auth.OAuth2.Flows.AuthorizationCodeFlow.<FetchTokenAsync>d__35.MoveNext()
--- End of stack trace from previous location where exception was thrown ---
   at System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw()
   at System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   at Google.Apis.Auth.OAuth2.Flows.AuthorizationCodeFlow.<ExchangeCodeForTokenAsync>d__30.MoveNext()

What am I missing here? Can somebody help.

Without giving us the access token or refresh token can you update your original question with some additional info: how are you getting the tokens in the first place? Are you requesting offline access? I don't see where you've defined accessToken in the source fragment you've given. How are you getting the userId and how are you associating it with the tokens? — Prisoner
for MVC you should be using GoogleWebAuthorizationBroker.Authorize the code you are using now is for installed applications — DaImTo
question where exactly are you suppling the refresh token I don't see anything about that in your code — DaImTo
Are you calling this function each time you call your webhook? — Nick Felker
I have updated the original question with more information. If you want more, please do let me know. — Abdullah

Abdullah Abdullah · Accepted Answer · 2018-05-09T06:16:21

Here are the "missing" steps after which I got this thing working. I am sure there must be alternate ways of doing this but I will suffice with this until I get some feedback here.

Note: Google Actions Console UI has changed so the screens in the original question maybe different (but are complete).

Make sure the Calendar API is enabled in the Google Cloud Console (API Library) against the project you have selected.

Go back to https://manage.auth0.com > APIs. Edit the listed System API > Machine to Machine Applications. Authorize your listed Application and select/tick the following two scopes:

read:users
read:user_idp_tokens

Go to Google Actions Console > Account Linking (see point no. 4) and check if you have set the following scopes:

Change your code by:
a. Call https://[domain].auth0.com/userinfo with the Authorization token from originalRequest.data.user.accessToken (see point 7 and 8). Successful response should give you the userId
b. Post on https://[domain].auth0.com/oauth/token with request body containing your client_id, client_secret, audience, grant_type. Successful response should give you a new access_token.
c. Change the Authorization token to the newly acquired access_token from point 11.b and make a call to https://[domain].auth0.com/api/v2/users/[userId] where the userId is the one you have from point 11.a. Successful response should give you a "Google" access_token and userId (under identities).
d. Change the authorization token in the header with the one from point 11.c. This is the token you use to call the Google APIs. For example, for Calendars, call https://www.googleapis.com/calendar/v3/users/me/calendarList. You will get the required response.

Here is the code (I have reused the variables):

string responseText = string.Empty;
string clientId = "DCuPWHknmv_k2xxxxxxxxxxxxxxxxx";     //Auth0 ClientId
string clientSecret = "7G3xlreFIULPZ9OtwlOxCX99xxxxxxxxxxxxxxxxxxx";    //Auth0 ClientSecret
string accessToken = jsonObject.SelectToken("originalRequest.data.user.accessToken").ToString();

try
{
    using (var httpClient = new HttpClient())
    {
        httpClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + accessToken);
        var url = "https://xxx.auth0.com/userinfo";
        responseText = httpClient.GetStringAsync(url).Result;

        JObject jsonUserInfo = JObject.Parse(responseText);
        string userId = jsonUserInfo.SelectToken("sub").ToString();

        url = "https://xxx.auth0.com/oauth/token";
        var content = new FormUrlEncodedContent(new[]
        {
            new KeyValuePair<string, string>("client_id", clientId),
            new KeyValuePair<string, string>("client_secret", clientSecret),
            new KeyValuePair<string, string>("audience", "https://[domain].auth0.com/api/v2/"),
            new KeyValuePair<string, string>("grant_type", "client_credentials")
        });

        var postResult = httpClient.PostAsync(url, content).Result;
        jsonUserInfo = JObject.Parse(postResult.Content.ReadAsStringAsync().Result);
        accessToken = jsonUserInfo.SelectToken("access_token").ToString();

        httpClient.DefaultRequestHeaders.Remove("Authorization");
        httpClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + accessToken);

        url = "https://xxx.auth0.com/api/v2/users/" + userId;
        jsonUserInfo = JObject.Parse(httpClient.GetStringAsync(url).Result);
        accessToken = jsonUserInfo.SelectToken("identities[0].access_token").ToString();
        userId = jsonUserInfo.SelectToken("identities[0].user_id").ToString();

        httpClient.DefaultRequestHeaders.Remove("Authorization");
        httpClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + accessToken);

        url = "https://www.googleapis.com/calendar/v3/users/me/calendarList";
        responseText = httpClient.GetStringAsync(url).Result;
    }
}

DialogFlow/Actions: Allow Google Assistant user to create Event in Google Calendar from Actions App

1 Answers