Google Apps Script UrlFetchApp and Ajax Data

Question

I am trying to fetch data behind a login in Google Apps script. My code below works and fetches both the login and data page (after logging in) without issues.

function fetch() {
  var loginurl = "https://somedomain.com/login";
  var dataurl = "https://somedomain.com/orders#status=0&p=1&per=100";
  var params = {
    "method": "post",
    "payload": {
      "UserName": "myusername",
      "Password": "mypassword",
      "action": "Log In",
      "testcookie": 1
    },
    "followRedirects": false
  };

  var response = UrlFetchApp.fetch(loginurl, params);

  if (response.getResponseCode() == 302) {
    var headers = response.getAllHeaders();
    var cookies = typeof headers['Set-Cookie'] == 'string' ? [ headers['Set-Cookie'] ] : headers['Set-Cookie'];
    for (var i = 0; i < cookies.length; i++) {
      cookies[i] = cookies[i].split(';')[0];
    };
    params = {"method": "get","headers": {"Cookie": cookies.join(';')}};
    response = UrlFetchApp.fetch(dataurl, params);
    Logger.log(response.getContentText());
  } else {
    Logger.log(getResponseCode());
  }
}

My problem is the fetch response for the data page does not include the actual data I need since it is not fully loaded by the website (ajax) when the response is sent back. I can even see the "ajaxLoading" div is visible in the ContentText of the response.

Is there any kind of work-around for this?

inkt inkt · Accepted Answer · 2018-02-16T22:03:56

Solved: The dataurl in my example above was taking my parameters and posting to a totally different url, and i was able to call that url directly for the data I was after.

Google Apps Script UrlFetchApp and Ajax Data

1 Answers