BigQuery plugin defaults to dataset project for temporary bucket
Description
If I configure the BigQuery source plugin to read from a public dataset but omit the "Temporary Bucket Name" field, then I get an error indicating that it is trying to create the temporary GCS bucket in the dataset project (logs below).
Specifying a value for this field resolves the issue.
2019-07-03 00:23:24,999 - ERROR [MapReduceRunner-phase-1:i.c.c.i.a.r.ProgramControllerServiceAdapter@97] - MapReduce Program 'phase-1' failed.
java.io.IOException: Unable to create Cloud Storage bucket '8f2a1aef-4032-4cc8-8ad0-0cb3b4cd27bb' in the same location ('US') as BigQuery dataset 'campaign_tutorial'. Please use a bucket that is in the same location as the dataset.
at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createBucket(BigQueryUtil.java:196) ~[na:na]
at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createResources(BigQueryUtil.java:149) ~[na:na]
at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:131) ~[na:na]
at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:69) ~[na:na]
at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.lambda$prepareRun$0(WrappedBatchSource.java:51) ~[na:na]
at io.cdap.cdap.etl.common.plugin.Caller$1.call(Caller.java:30) ~[na:na]
at io.cdap.cdap.etl.common.plugin.StageLoggingCaller.call(StageLoggingCaller.java:40) ~[na:na]
at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:50) ~[na:na]
at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:36) ~[na:na]
at io.cdap.cdap.etl.common.submit.SubmitterPlugin.lambda$prepareRun$2(SubmitterPlugin.java:71) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext$2.run(AbstractContext.java:551) ~[na:na]
at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.finishExecute(Transactions.java:224) ~[na:na]
at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.execute(Transactions.java:211) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:546) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:534) ~[na:na]
at io.cdap.cdap.etl.common.submit.SubmitterPlugin.prepareRun(SubmitterPlugin.java:69) ~[na:na]
at io.cdap.cdap.etl.batch.PipelinePhasePreparer.prepare(PipelinePhasePreparer.java:111) ~[na:na]
at io.cdap.cdap.etl.batch.mapreduce.MapReducePreparer.prepare(MapReducePreparer.java:97) ~[na:na]
at io.cdap.cdap.etl.batch.mapreduce.ETLMapReduce.initialize(ETLMapReduce.java:192) ~[na:na]
at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:109) ~[na:na]
at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:32) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:182) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:177) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.lambda$initializeProgram$1(AbstractContext.java:640) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:600) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.initializeProgram(AbstractContext.java:637) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.beforeSubmit(MapReduceRuntimeService.java:547) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:226) ~[na:na]
at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$2$1.run(MapReduceRuntimeService.java:450) [na:na]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212]
Caused by: com.google.cloud.storage.StorageException: 108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638.
at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:220) ~[na:na]
at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:263) ~[na:na]
at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:112) ~[na:na]
at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:109) ~[na:na]
at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:89) ~[na:na]
at com.google.cloud.RetryHelper.run(RetryHelper.java:74) ~[na:na]
at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:51) ~[na:na]
at com.google.cloud.storage.StorageImpl.create(StorageImpl.java:108) ~[na:na]
at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createBucket(BigQueryUtil.java:190) ~[na:na]
... 30 common frames omitted
Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
{
"code" : 403,
"errors" : [ {
"domain" : "global",
"message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638.",
"reason" : "forbidden"
} ],
"message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638."
}
at com.google.api.client.googleapis.json.GoogleJsonResponseException.from(GoogleJsonResponseException.java:146) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:113) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:40) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest$1.interceptResponse(AbstractGoogleClientRequest.java:321) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1065) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:469) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:260) ~[na:na]
... 37 common frames omitted
2019-07-03 00:23:25,213 - ERROR [MapReduceRunner-phase-1:i.c.c.i.a.r.ProgramControllerServiceAdapter@98] - MapReduce program 'phase-1' failed with error: 403 Forbidden
{
"code" : 403,
"errors" : [ {
"domain" : "global",
"message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638.",
"reason" : "forbidden"
} ],
"message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638."
}. Please check the system logs for more details.
com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
{
"code" : 403,
"errors" : [ {
"domain" : "global",
"message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638.",
"reason" : "forbidden"
} ],
"message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638."
}
at com.google.api.client.googleapis.json.GoogleJsonResponseException.from(GoogleJsonResponseException.java:146) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:113) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:40) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest$1.interceptResponse(AbstractGoogleClientRequest.java:321) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1065) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:469) ~[bigquery-connector-0.10.11-hadoop2.jar:na]
at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:260) ~[na:na]
at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:112) ~[na:na]
at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:109) ~[na:na]
at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:89) ~[na:na]
at com.google.cloud.RetryHelper.run(RetryHelper.java:74) ~[na:na]
at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:51) ~[na:na]
at com.google.cloud.storage.StorageImpl.create(StorageImpl.java:108) ~[na:na]
at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createBucket(BigQueryUtil.java:190) ~[na:na]
at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createResources(BigQueryUtil.java:149) ~[na:na]
at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:131) ~[na:na]
at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:69) ~[na:na]
at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.lambda$prepareRun$0(WrappedBatchSource.java:51) ~[na:na]
at io.cdap.cdap.etl.common.plugin.Caller$1.call(Caller.java:30) ~[na:na]
at io.cdap.cdap.etl.common.plugin.StageLoggingCaller.call(StageLoggingCaller.java:40) ~[na:na]
at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:50) ~[na:na]
at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:36) ~[na:na]
at io.cdap.cdap.etl.common.submit.SubmitterPlugin.lambda$prepareRun$2(SubmitterPlugin.java:71) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext$2.run(AbstractContext.java:551) ~[na:na]
at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.finishExecute(Transactions.java:224) ~[na:na]
at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.execute(Transactions.java:211) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:546) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:534) ~[na:na]
at io.cdap.cdap.etl.common.submit.SubmitterPlugin.prepareRun(SubmitterPlugin.java:69) ~[na:na]
at io.cdap.cdap.etl.batch.PipelinePhasePreparer.prepare(PipelinePhasePreparer.java:111) ~[na:na]
at io.cdap.cdap.etl.batch.mapreduce.MapReducePreparer.prepare(MapReducePreparer.java:97) ~[na:na]
at io.cdap.cdap.etl.batch.mapreduce.ETLMapReduce.initialize(ETLMapReduce.java:192) ~[na:na]
at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:109) ~[na:na]
at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:32) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:182) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:177) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.lambda$initializeProgram$1(AbstractContext.java:640) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:600) ~[na:na]
at io.cdap.cdap.internal.app.runtime.AbstractContext.initializeProgram(AbstractContext.java:637) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.beforeSubmit(MapReduceRuntimeService.java:547) ~[na:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:226) ~[na:na]
at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na]
at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$2$1.run(MapReduceRuntimeService.java:450) [na:na]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212]
Release Notes
Fixed a bug in the BigQuery source that cause automatic bucket creation to fail if the dataset is in a different project.
If I configure the BigQuery source plugin to read from a public dataset but omit the "Temporary Bucket Name" field, then I get an error indicating that it is trying to create the temporary GCS bucket in the dataset project (logs below).
Specifying a value for this field resolves the issue.
2019-07-03 00:23:24,999 - ERROR [MapReduceRunner-phase-1:i.c.c.i.a.r.ProgramControllerServiceAdapter@97] - MapReduce Program 'phase-1' failed. java.io.IOException: Unable to create Cloud Storage bucket '8f2a1aef-4032-4cc8-8ad0-0cb3b4cd27bb' in the same location ('US') as BigQuery dataset 'campaign_tutorial'. Please use a bucket that is in the same location as the dataset. at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createBucket(BigQueryUtil.java:196) ~[na:na] at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createResources(BigQueryUtil.java:149) ~[na:na] at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:131) ~[na:na] at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:69) ~[na:na] at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.lambda$prepareRun$0(WrappedBatchSource.java:51) ~[na:na] at io.cdap.cdap.etl.common.plugin.Caller$1.call(Caller.java:30) ~[na:na] at io.cdap.cdap.etl.common.plugin.StageLoggingCaller.call(StageLoggingCaller.java:40) ~[na:na] at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:50) ~[na:na] at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:36) ~[na:na] at io.cdap.cdap.etl.common.submit.SubmitterPlugin.lambda$prepareRun$2(SubmitterPlugin.java:71) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext$2.run(AbstractContext.java:551) ~[na:na] at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.finishExecute(Transactions.java:224) ~[na:na] at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.execute(Transactions.java:211) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:546) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:534) ~[na:na] at io.cdap.cdap.etl.common.submit.SubmitterPlugin.prepareRun(SubmitterPlugin.java:69) ~[na:na] at io.cdap.cdap.etl.batch.PipelinePhasePreparer.prepare(PipelinePhasePreparer.java:111) ~[na:na] at io.cdap.cdap.etl.batch.mapreduce.MapReducePreparer.prepare(MapReducePreparer.java:97) ~[na:na] at io.cdap.cdap.etl.batch.mapreduce.ETLMapReduce.initialize(ETLMapReduce.java:192) ~[na:na] at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:109) ~[na:na] at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:32) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:182) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:177) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.lambda$initializeProgram$1(AbstractContext.java:640) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:600) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.initializeProgram(AbstractContext.java:637) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.beforeSubmit(MapReduceRuntimeService.java:547) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:226) ~[na:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$2$1.run(MapReduceRuntimeService.java:450) [na:na] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212] Caused by: com.google.cloud.storage.StorageException: 108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638. at com.google.cloud.storage.spi.v1.HttpStorageRpc.translate(HttpStorageRpc.java:220) ~[na:na] at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:263) ~[na:na] at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:112) ~[na:na] at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:109) ~[na:na] at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:89) ~[na:na] at com.google.cloud.RetryHelper.run(RetryHelper.java:74) ~[na:na] at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:51) ~[na:na] at com.google.cloud.storage.StorageImpl.create(StorageImpl.java:108) ~[na:na] at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createBucket(BigQueryUtil.java:190) ~[na:na] ... 30 common frames omitted Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden { "code" : 403, "errors" : [ { "domain" : "global", "message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638.", "reason" : "forbidden" } ], "message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638." } at com.google.api.client.googleapis.json.GoogleJsonResponseException.from(GoogleJsonResponseException.java:146) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:113) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:40) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest$1.interceptResponse(AbstractGoogleClientRequest.java:321) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1065) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:469) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:260) ~[na:na] ... 37 common frames omitted 2019-07-03 00:23:25,213 - ERROR [MapReduceRunner-phase-1:i.c.c.i.a.r.ProgramControllerServiceAdapter@98] - MapReduce program 'phase-1' failed with error: 403 Forbidden { "code" : 403, "errors" : [ { "domain" : "global", "message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638.", "reason" : "forbidden" } ], "message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638." }. Please check the system logs for more details. com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden { "code" : 403, "errors" : [ { "domain" : "global", "message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638.", "reason" : "forbidden" } ], "message" : "108604513871-compute@developer.gserviceaccount.com does not have storage.buckets.create access to project 888472635638." } at com.google.api.client.googleapis.json.GoogleJsonResponseException.from(GoogleJsonResponseException.java:146) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:113) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest.newExceptionOnError(AbstractGoogleJsonClientRequest.java:40) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest$1.interceptResponse(AbstractGoogleClientRequest.java:321) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1065) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:469) ~[bigquery-connector-0.10.11-hadoop2.jar:na] at com.google.cloud.storage.spi.v1.HttpStorageRpc.create(HttpStorageRpc.java:260) ~[na:na] at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:112) ~[na:na] at com.google.cloud.storage.StorageImpl$2.call(StorageImpl.java:109) ~[na:na] at com.google.api.gax.retrying.DirectRetryingExecutor.submit(DirectRetryingExecutor.java:89) ~[na:na] at com.google.cloud.RetryHelper.run(RetryHelper.java:74) ~[na:na] at com.google.cloud.RetryHelper.runWithRetries(RetryHelper.java:51) ~[na:na] at com.google.cloud.storage.StorageImpl.create(StorageImpl.java:108) ~[na:na] at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createBucket(BigQueryUtil.java:190) ~[na:na] at io.cdap.plugin.gcp.bigquery.util.BigQueryUtil.createResources(BigQueryUtil.java:149) ~[na:na] at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:131) ~[na:na] at io.cdap.plugin.gcp.bigquery.source.BigQuerySource.prepareRun(BigQuerySource.java:69) ~[na:na] at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.lambda$prepareRun$0(WrappedBatchSource.java:51) ~[na:na] at io.cdap.cdap.etl.common.plugin.Caller$1.call(Caller.java:30) ~[na:na] at io.cdap.cdap.etl.common.plugin.StageLoggingCaller.call(StageLoggingCaller.java:40) ~[na:na] at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:50) ~[na:na] at io.cdap.cdap.etl.common.plugin.WrappedBatchSource.prepareRun(WrappedBatchSource.java:36) ~[na:na] at io.cdap.cdap.etl.common.submit.SubmitterPlugin.lambda$prepareRun$2(SubmitterPlugin.java:71) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext$2.run(AbstractContext.java:551) ~[na:na] at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.finishExecute(Transactions.java:224) ~[na:na] at io.cdap.cdap.data2.transaction.Transactions$CacheBasedTransactional.execute(Transactions.java:211) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:546) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:534) ~[na:na] at io.cdap.cdap.etl.common.submit.SubmitterPlugin.prepareRun(SubmitterPlugin.java:69) ~[na:na] at io.cdap.cdap.etl.batch.PipelinePhasePreparer.prepare(PipelinePhasePreparer.java:111) ~[na:na] at io.cdap.cdap.etl.batch.mapreduce.MapReducePreparer.prepare(MapReducePreparer.java:97) ~[na:na] at io.cdap.cdap.etl.batch.mapreduce.ETLMapReduce.initialize(ETLMapReduce.java:192) ~[na:na] at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:109) ~[na:na] at io.cdap.cdap.api.mapreduce.AbstractMapReduce.initialize(AbstractMapReduce.java:32) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:182) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$1.initialize(MapReduceRuntimeService.java:177) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.lambda$initializeProgram$1(AbstractContext.java:640) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.execute(AbstractContext.java:600) ~[na:na] at io.cdap.cdap.internal.app.runtime.AbstractContext.initializeProgram(AbstractContext.java:637) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.beforeSubmit(MapReduceRuntimeService.java:547) ~[na:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService.startUp(MapReduceRuntimeService.java:226) ~[na:na] at com.google.common.util.concurrent.AbstractExecutionThreadService$1$1.run(AbstractExecutionThreadService.java:47) ~[com.google.guava.guava-13.0.1.jar:na] at io.cdap.cdap.internal.app.runtime.batch.MapReduceRuntimeService$2$1.run(MapReduceRuntimeService.java:450) [na:na] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_212]