So I'm a little bit new to big query and gcp in general.
I'm trying to send data to big query, and when it does, it's pulling my production credentials when it should be using my dev credentials.
application-local.yml
spring:
config:
name: local
jpa:
show-sql: true
generate-ddl: true
hibernate:
ddl-auto: update
database-platform: org.hibernate.dialect.PostgreSQLDialect
datasource:
platform: postgres
url: jdbc:postgresql://localhost:5432/db_name
username: username
password: TestPass
driverClassName: org.postgresql.Driver
profiles:
active: local
application:
name: user-api
cloud:
gcp:
bigquery:
dataset-name: app-dev
project-id: app-dev
credentials:
location: classpath:google-dev.json
sql:
enabled: false
credentials:
encoded-key: [b64 credentials]
project-id: app-dev
config:
project-id: app-dev
starting logs showing it picking up the dev project
2021-05-05 14:09:23.683 INFO 37046 --- [ main] o.s.c.g.a.c.GcpContextAutoConfiguration : The default project ID is app-dev
2021-05-05 14:09:23.707 INFO 37046 --- [ main] o.s.c.g.core.DefaultCredentialsProvider : Default credentials provider for service account [email protected]
2021-05-05 14:09:23.707 INFO 37046 --- [ main] o.s.c.g.core.DefaultCredentialsProvider : Scopes in use by default credentials: [https://www.googleapis.com/auth/pubsub, https://www.googleapis.com/auth/spanner.admin, https://www.googleapis.com/auth/spanner.data, https://www.googleapis.com/auth/datastore, https://www.googleapis.com/auth/sqlservice.admin, https://www.googleapis.com/auth/devstorage.read_only, https://www.googleapis.com/auth/devstorage.read_write, https://www.googleapis.com/auth/cloudruntimeconfig, https://www.googleapis.com/auth/trace.append, https://www.googleapis.com/auth/cloud-platform, https://www.googleapis.com/auth/cloud-vision, https://www.googleapis.com/auth/bigquery, https://www.googleapis.com/auth/monitoring.write]
When it actually tries to run big query inserts
2021-05-05 14:09:27.900 ERROR 37046 --- [ask-scheduler-1] o.s.integration.handler.LoggingHandler : com.google.cloud.bigquery.BigQueryException: Not found: Dataset app-prod:app-dev
at com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc.translate(HttpBigQueryRpc.java:115)
at com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc.insertAll(HttpBigQueryRpc.java:494)
at com.google.cloud.bigquery.BigQueryImpl.insertAll(BigQueryImpl.java:1036)
at ca.app.api.domain.repository.datalake.DataLakeInterface$DefaultImpls.save(DataLakeInterface.kt:27)
at ca.app.api.domain.repository.datalake.CycleEventDataLakeRepository.save(CycleEventDataLakeRepository.kt:12)
at ca.app.api.tasks.DataLakeSyncTasks.syncCycleEvents(DataLakeSyncTasks.kt:99)
at ca.app.api.tasks.DataLakeSyncTasks.syncDataLake(DataLakeSyncTasks.kt:71)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:84)
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base/java.lang.Thread.run(Thread.java:832)
Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 404 Not Found
POST https://www.googleapis.com/bigquery/v2/projects/app-prod/datasets/app-dev/tables/cycle_event_table/insertAll?prettyPrint=false
{
"code" : 404,
"errors" : [ {
"domain" : "global",
"message" : "Not found: Dataset app-prod:app-dev",
"reason" : "notFound"
} ],
"message" : "Not found: Dataset app-prod:app-dev",
"status" : "NOT_FOUND"
}
I have no clue where it's picking up app-prod. The only place app-prod is used is application-prod.yml which only has the prod profile. The auth file and the base64 key all only contain app-dev.
