Apache Avro library failed to parse file nodejs

Question

I'm trying to upload a Avro file from google staorage to google big query using nodejs

var gcloud = require('gcloud')({
  keyFilename: '../config/keyfile.json',
  projectId: 'my-project-id'
});

var request = require('request');

var bigquery = gcloud.bigquery();


var dataset = bigquery.dataset('my-data-set');
var table = dataset.table('my-table');



  var metadata = {

   sourceFormat: 'AVRO'
  };

   var url = 'https://storage.googleapis.com/' + 'my-bucket' + '/' + 'yob1900.avro';

    request.get(url)
      .pipe(table.createWriteStream(metadata))
      .on('complete', function (job) {
        // Wait up to 60 seconds for job to complete
        pollJobUntilDone(job, 60000, 0, function (err, metadata) {
          if (err) {

            return callback(err);
          }
          console.log('job completed', metadata);

        });
      });

function pollJobUntilDone (job, timeout, timeWaited, callback) {
  job.getMetadata(function (err, metadata) {
    if (err) {
      return callback(err);
    }
    if (timeWaited > timeout) {
      return callback(new Error('Timed out waiting for job to complete'));
    }
    if (metadata.status && (metadata.status.state === 'RUNNING' ||
      metadata.status.state === 'PENDING')) {
      setTimeout(function () {
        console.log('working...');
        pollJobUntilDone(job, timeout, timeWaited + 5000, callback);
      }, 5000);
    } else {
      callback(null, metadata);
    }
  });
}

i receive this error

'The Apache Avro library failed to parse file file-00000000.'

(the 'yob1900' file was downloaded from bigquery documentation https://cloud.google.com/bigquery/loading-data#loading_json_files) when I try doing this through the webUI I success. Any suggestions?

I don't know nodejs but it seems the avro file is not passed to BigQuery directly. The error message says file name "file-00000000" instead of "yob1900.avro". Some kind of conversion is done by nodejs. — Hua Zhang

dina dina · Accepted Answer · 2016-06-13T19:57:39

0

votes

Avro formats weren't soported by gcloud. Issue was fixed "gcloud": "^0.36.0"

Apache Avro library failed to parse file nodejs

1 Answers