0
votes

My Function is triggered by a cloud storage event and will load files into a BigQuery table, my issue is that we recieved some .zip files with the same name and the function is attempting to load these files as well and this is causing some issues with the table. I need to make the code only process files that are .csv. Below is the code I have so far:

exports.ToBigQuery = (event, callback) => {
  const file = event.data;
  const context = event.context;

  const BigQuery = require('@google-cloud/bigquery');
  const Storage = require('@google-cloud/storage');

  const projectId = "gas-ddr";
  const datasetId = "gas_ddr";
  const bucketName = file.bucket;
  const filename = file.name;

  const dashOffset = filename.indexOf('-');
  const tableId = filename.substring(0, dashOffset);

  console.log(`Load ${filename} into ${tableId}.`);

 // Instantiates clients
  const bigquery = new BigQuery({
    projectId: projectId,
  });

  const storage = Storage({
    projectId: projectId,
  });

  const metadata = {
  allowJaggedRows: true,
  skipLeadingRows: 1

 };

  let job;

  // Loads data from a Google Cloud Storage file into the table
  bigquery
    .dataset(datasetId)
    .table(tableId)
    .load(storage.bucket(bucketName).file(filename),metadata)
    .then(results => {
      job = results[0];
      console.log(`Job ${job.id} started.`);

      // Wait for the job to finish
      return job;
    })
    .then(metadata => {
      // Check the job's status for errors
      const errors = metadata.status.errors;
      if (errors && errors.length > 0) {
        throw errors;
      }
    })
    .then(() => {
      console.log(`Job ${job.id} completed.`);
    })
    .catch(err => {
      console.error('ERROR:', err);
    });

  callback();
};
1

1 Answers

3
votes

This is simply a javascript related question. You can simply extract the extension part of a filename and process files accordingly:

function getExtension(filename) {
    var parts = filename.split('.');
    return parts[parts.length - 1];
}


if (getExtension(filename) == "csv") {
  // Loads data from a Google Cloud Storage file into the table
  bigquery
     .dataset(datasetId)
  ...
}