0
votes

Hello im currently trying to establish daily data transfers from Google Cloud Storage to Big Query tables. Theses tables are just meant to store raw data (JSON files) and i unnest them later with scheduled queries. I have no issue when i create the table manually but im getting the error " Error while reading data, error message: JSON table encountered too many errors, giving up. Rows: 1; errors: 1" when i launch a test transfer (even if I previously deleted the data in the table).

Here is an example of the raw data table/JSON schema:

{
  "fields": [{
    "name": "data",
    "type": "RECORD",
    "mode": "REPEATED",
    "schema": {
      "fields": [{
        "name": "relationships",
        "type": "RECORD",
        "mode": "NULLABLE",
        "schema": {
          "fields": [{
            "name": "createdBy",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }, {
            "name": "validationWorkflow",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "REPEATED",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }, {
            "name": "projects",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "REPEATED",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }, {
            "name": "expensesReport",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }, {
            "name": "agency",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }, {
            "name": "files",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "STRING",
                "mode": "REPEATED"
              }]
            }
          }, {
            "name": "resource",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }, {
            "name": "validations",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "REPEATED",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }, {
            "name": "orders",
            "type": "RECORD",
            "mode": "NULLABLE",
            "schema": {
              "fields": [{
                "name": "data",
                "type": "RECORD",
                "mode": "REPEATED",
                "schema": {
                  "fields": [{
                    "name": "type",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }]
            }
          }]
        }
      }, {
        "name": "attributes",
        "type": "RECORD",
        "mode": "NULLABLE",
        "schema": {
          "fields": [{
            "name": "plannedTimes",
            "type": "STRING",
            "mode": "REPEATED"
          }, {
            "name": "state",
            "type": "STRING",
            "mode": "NULLABLE"
          }, {
            "name": "exceptionalTimes",
            "type": "STRING",
            "mode": "REPEATED"
          }, {
            "name": "closed",
            "type": "BOOLEAN",
            "mode": "NULLABLE"
          }, {
            "name": "informationComments",
            "type": "STRING",
            "mode": "NULLABLE"
          }, {
            "name": "regularTimes",
            "type": "RECORD",
            "mode": "REPEATED",
            "schema": {
              "fields": [{
                "name": "batch",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "title",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }, {
                "name": "workUnitType",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "name",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "activityType",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "reference",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }, {
                "name": "project",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "reference",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }, {
                "name": "row",
                "type": "INTEGER",
                "mode": "NULLABLE"
              }, {
                "name": "delivery",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "endDate",
                    "type": "DATE",
                    "mode": "NULLABLE"
                  }, {
                    "name": "startDate",
                    "type": "DATE",
                    "mode": "NULLABLE"
                  }, {
                    "name": "title",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "id",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }, {
                "name": "startDate",
                "type": "DATE",
                "mode": "NULLABLE"
              }, {
                "name": "duration",
                "type": "FLOAT",
                "mode": "NULLABLE"
              }, {
                "name": "id",
                "type": "INTEGER",
                "mode": "NULLABLE"
              }]
            }
          }, {
            "name": "updateDate",
            "type": "TIMESTAMP",
            "mode": "NULLABLE"
          }, {
            "name": "creationDate",
            "type": "TIMESTAMP",
            "mode": "NULLABLE"
          }, {
            "name": "absencesTimes",
            "type": "RECORD",
            "mode": "REPEATED",
            "schema": {
              "fields": [{
                "name": "workUnitType",
                "type": "RECORD",
                "mode": "NULLABLE",
                "schema": {
                  "fields": [{
                    "name": "name",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "activityType",
                    "type": "STRING",
                    "mode": "NULLABLE"
                  }, {
                    "name": "reference",
                    "type": "INTEGER",
                    "mode": "NULLABLE"
                  }]
                }
              }, {
                "name": "duration",
                "type": "FLOAT",
                "mode": "NULLABLE"
              }, {
                "name": "startDate",
                "type": "DATE",
                "mode": "NULLABLE"
              }]
            }
          }, {
            "name": "term",
            "type": "STRING",
            "mode": "NULLABLE"
          }]
        }
      }, {
        "name": "type",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "id",
        "type": "INTEGER",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "date",
    "type": "DATE",
    "mode": "NULLABLE"
  }]
}

I know that BQ works better with JSON newline delimited format but this table must contains raw data even if its just 1 row in the end. The weirdest thing is that the transfer works for some files with similar schemas.

What should I do to make these transfers work ?

Thank you for your help

So you are trying to use Data Transfer Service to load json files from GCS to a table in BigQuery that has this specific schema that you shared, right? - JMA
Yes exactly like a daily routine. - malevil667