diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java index 96a10deb3..841a790ee 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java @@ -105,7 +105,7 @@ public HttpBigQueryRpc(BigQueryOptions options) { this.options = options; bigquery = new Bigquery.Builder(transport, new GsonFactory(), initializer) - .setRootUrl(options.getHost()) + .setRootUrl(options.getResolvedApiaryHost("bigquery")) .setApplicationName(options.getApplicationName()) .build(); } @@ -114,9 +114,16 @@ private static BigQueryException translate(IOException exception) { return new BigQueryException(exception); } + private void validateRPC() throws BigQueryException, IOException { + if (!this.options.hasValidUniverseDomain()) { + throw new BigQueryException(BigQueryException.UNKNOWN_CODE, "Invalid universe domain"); + } + } + @Override public Dataset getDataset(String projectId, String datasetId, Map options) { try { + validateRPC(); return bigquery .datasets() .get(projectId, datasetId) @@ -135,6 +142,7 @@ public Dataset getDataset(String projectId, String datasetId, Map opt @Override public Tuple> listDatasets(String projectId, Map options) { try { + validateRPC(); DatasetList datasetsList = bigquery .datasets() @@ -159,6 +167,7 @@ public Tuple> listDatasets(String projectId, Map options) { try { + validateRPC(); return bigquery .datasets() .insert(dataset.getDatasetReference().getProjectId(), dataset) @@ -173,6 +182,7 @@ public Dataset create(Dataset dataset, Map options) { @Override public Table create(Table table, Map options) { try { + validateRPC(); // unset the type, as it is output only table.setType(null); TableReference reference = table.getTableReference(); @@ -190,6 +200,7 @@ public Table create(Table table, Map options) { @Override public Routine create(Routine routine, Map options) { try { + validateRPC(); RoutineReference reference = routine.getRoutineReference(); return bigquery .routines() @@ -205,6 +216,7 @@ public Routine create(Routine routine, Map options) { @Override public Job create(Job job, Map options) { try { + validateRPC(); String projectId = job.getJobReference() != null ? job.getJobReference().getProjectId() @@ -223,6 +235,7 @@ public Job create(Job job, Map options) { @Override public Job createJobForQuery(Job job) { try { + validateRPC(); String projectId = job.getJobReference() != null ? job.getJobReference().getProjectId() @@ -236,6 +249,7 @@ public Job createJobForQuery(Job job) { @Override public boolean deleteDataset(String projectId, String datasetId, Map options) { try { + validateRPC(); bigquery .datasets() .delete(projectId, datasetId) @@ -255,6 +269,7 @@ public boolean deleteDataset(String projectId, String datasetId, Map @Override public Dataset patch(Dataset dataset, Map options) { try { + validateRPC(); DatasetReference reference = dataset.getDatasetReference(); return bigquery .datasets() @@ -270,6 +285,7 @@ public Dataset patch(Dataset dataset, Map options) { @Override public Table patch(Table table, Map options) { try { + validateRPC(); // unset the type, as it is output only table.setType(null); TableReference reference = table.getTableReference(); @@ -289,6 +305,7 @@ public Table patch(Table table, Map options) { public Table getTable( String projectId, String datasetId, String tableId, Map options) { try { + validateRPC(); return bigquery .tables() .get(projectId, datasetId, tableId) @@ -316,6 +333,7 @@ private String getTableMetadataOption(Map options) { public Tuple> listTables( String projectId, String datasetId, Map options) { try { + validateRPC(); TableList tableList = bigquery .tables() @@ -351,6 +369,7 @@ public Table apply(TableList.Tables tablePb) { @Override public boolean deleteTable(String projectId, String datasetId, String tableId) { try { + validateRPC(); bigquery.tables().delete(projectId, datasetId, tableId).execute(); return true; } catch (IOException ex) { @@ -365,6 +384,7 @@ public boolean deleteTable(String projectId, String datasetId, String tableId) { @Override public Model patch(Model model, Map options) { try { + validateRPC(); // unset the type, as it is output only ModelReference reference = model.getModelReference(); return bigquery @@ -382,6 +402,7 @@ public Model patch(Model model, Map options) { public Model getModel( String projectId, String datasetId, String modelId, Map options) { try { + validateRPC(); return bigquery .models() .get(projectId, datasetId, modelId) @@ -401,6 +422,7 @@ public Model getModel( public Tuple> listModels( String projectId, String datasetId, Map options) { try { + validateRPC(); ListModelsResponse modelList = bigquery .models() @@ -420,6 +442,7 @@ public Tuple> listModels( @Override public boolean deleteModel(String projectId, String datasetId, String modelId) { try { + validateRPC(); bigquery.models().delete(projectId, datasetId, modelId).execute(); return true; } catch (IOException ex) { @@ -434,6 +457,7 @@ public boolean deleteModel(String projectId, String datasetId, String modelId) { @Override public Routine update(Routine routine, Map options) { try { + validateRPC(); RoutineReference reference = routine.getRoutineReference(); return bigquery .routines() @@ -451,6 +475,7 @@ public Routine update(Routine routine, Map options) { public Routine getRoutine( String projectId, String datasetId, String routineId, Map options) { try { + validateRPC(); return bigquery .routines() .get(projectId, datasetId, routineId) @@ -470,6 +495,7 @@ public Routine getRoutine( public Tuple> listRoutines( String projectId, String datasetId, Map options) { try { + validateRPC(); ListRoutinesResponse routineList = bigquery .routines() @@ -491,6 +517,7 @@ public Tuple> listRoutines( @Override public boolean deleteRoutine(String projectId, String datasetId, String routineId) { try { + validateRPC(); bigquery.routines().delete(projectId, datasetId, routineId).execute(); return true; } catch (IOException ex) { @@ -506,6 +533,7 @@ public boolean deleteRoutine(String projectId, String datasetId, String routineI public TableDataInsertAllResponse insertAll( String projectId, String datasetId, String tableId, TableDataInsertAllRequest request) { try { + validateRPC(); return bigquery .tabledata() .insertAll(projectId, datasetId, tableId, request) @@ -520,6 +548,7 @@ public TableDataInsertAllResponse insertAll( public TableDataList listTableData( String projectId, String datasetId, String tableId, Map options) { try { + validateRPC(); return bigquery .tabledata() .list(projectId, datasetId, tableId) @@ -544,6 +573,7 @@ public TableDataList listTableDataWithRowLimit( Integer maxResultPerPage, String pageToken) { try { + validateRPC(); return bigquery .tabledata() .list(projectId, datasetId, tableId) @@ -559,6 +589,7 @@ public TableDataList listTableDataWithRowLimit( @Override public Job getJob(String projectId, String jobId, String location, Map options) { try { + validateRPC(); return bigquery .jobs() .get(projectId, jobId) @@ -578,6 +609,7 @@ public Job getJob(String projectId, String jobId, String location, Map> listJobs(String projectId, Map options) { try { + validateRPC(); Bigquery.Jobs.List request = bigquery .jobs() @@ -650,6 +683,7 @@ public Job apply(JobList.Jobs jobPb) { @Override public boolean cancel(String projectId, String jobId, String location) { try { + validateRPC(); bigquery .jobs() .cancel(projectId, jobId) @@ -669,6 +703,7 @@ public boolean cancel(String projectId, String jobId, String location) { @Override public boolean deleteJob(String projectId, String jobName, String location) { try { + validateRPC(); bigquery .jobs() .delete(projectId, jobName) @@ -685,6 +720,7 @@ public boolean deleteJob(String projectId, String jobName, String location) { public GetQueryResultsResponse getQueryResults( String projectId, String jobId, String location, Map options) { try { + validateRPC(); return bigquery .jobs() .getQueryResults(projectId, jobId) @@ -707,6 +743,7 @@ public GetQueryResultsResponse getQueryResults( public GetQueryResultsResponse getQueryResultsWithRowLimit( String projectId, String jobId, String location, Integer maxResultPerPage, Long timeoutMs) { try { + validateRPC(); return bigquery .jobs() .getQueryResults(projectId, jobId) @@ -723,6 +760,7 @@ public GetQueryResultsResponse getQueryResultsWithRowLimit( @Override public QueryResponse queryRpc(String projectId, QueryRequest content) { try { + validateRPC(); return bigquery.jobs().query(projectId, content).execute(); } catch (IOException ex) { throw translate(ex); @@ -732,7 +770,7 @@ public QueryResponse queryRpc(String projectId, QueryRequest content) { @Override public String open(Job loadJob) { try { - String builder = options.getHost(); + String builder = options.getResolvedApiaryHost("bigquery"); if (!builder.endsWith("/")) { builder += "/"; } @@ -807,6 +845,7 @@ public Job write( @Override public Policy getIamPolicy(String resourceId, Map options) { try { + validateRPC(); GetIamPolicyRequest policyRequest = new GetIamPolicyRequest(); if (null != Option.REQUESTED_POLICY_VERSION.getLong(options)) { policyRequest = @@ -828,6 +867,7 @@ public Policy getIamPolicy(String resourceId, Map options) { @Override public Policy setIamPolicy(String resourceId, Policy policy, Map options) { try { + validateRPC(); SetIamPolicyRequest policyRequest = new SetIamPolicyRequest().setPolicy(policy); return bigquery .tables() @@ -843,6 +883,7 @@ public Policy setIamPolicy(String resourceId, Policy policy, Map opti public TestIamPermissionsResponse testIamPermissions( String resourceId, List permissions, Map options) { try { + validateRPC(); TestIamPermissionsRequest permissionsRequest = new TestIamPermissionsRequest().setPermissions(permissions); return bigquery diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index d8d673360..ba30ea0d4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -54,6 +54,7 @@ import com.google.cloud.bigquery.BigQueryDryRunResult; import com.google.cloud.bigquery.BigQueryError; import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.BigQueryResult; import com.google.cloud.bigquery.BigQuerySQLException; import com.google.cloud.bigquery.CloneDefinition; @@ -149,6 +150,7 @@ import com.google.common.io.BaseEncoding; import com.google.common.util.concurrent.ListenableFuture; import com.google.gson.JsonObject; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -773,6 +775,65 @@ public class ITBigQueryTest { private static final String PUBLIC_PROJECT = "bigquery-public-data"; private static final String PUBLIC_DATASET = "census_bureau_international"; + private static final String FAKE_JSON_CRED_WITH_GOOGLE_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"googleapis.com\"\n" + + "}"; + private static final String FAKE_JSON_CRED_WITH_INVALID_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"fake.domain\"\n" + + "}"; + private static BigQuery bigquery; private static Storage storage; @@ -6323,4 +6384,77 @@ public void testStatelessQueriesWithLocation() throws Exception { bigQuery.delete(dataset.getDatasetId(), DatasetDeleteOption.deleteContents()); } } + + @Test + public void testUniverseDomainWithInvalidUniverseDomain() { + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + BigQueryOptions bigQueryOptions = + bigqueryHelper + .getOptions() + .toBuilder() + .setCredentials(loadCredentials(FAKE_JSON_CRED_WITH_GOOGLE_DOMAIN)) + .setUniverseDomain("invalid.domain") + .build(); + BigQuery bigQuery = bigQueryOptions.getService(); + + try { + // Use list dataset to send RPC to invalid domain. + bigQuery.listDatasets("bigquery-public-data"); + fail("RPCs to invalid universe domain should fail"); + } catch (BigQueryException e) { + assertNotNull(e.getMessage()); + assertThat((e.getMessage().contains("Invalid universe domain"))).isTrue(); + } + } + + @Test + public void testInvalidUniverseDomainWithMismatchCredentials() { + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + BigQueryOptions bigQueryOptions = + bigqueryHelper + .getOptions() + .toBuilder() + .setCredentials(loadCredentials(FAKE_JSON_CRED_WITH_INVALID_DOMAIN)) + .build(); + BigQuery bigQuery = bigQueryOptions.getService(); + + try { + // Use list dataset to send RPC to invalid domain. + bigQuery.listDatasets("bigquery-public-data"); + fail("RPCs to invalid universe domain should fail"); + } catch (BigQueryException e) { + assertNotNull(e.getMessage()); + assertThat((e.getMessage().contains("Invalid universe domain"))).isTrue(); + } + } + + @Test + public void testUniverseDomainWithMatchingDomain() { + // Test a valid domain using the default credentials and Google default universe domain. + RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); + BigQueryOptions bigQueryOptions = + bigqueryHelper.getOptions().toBuilder().setUniverseDomain("googleapis.com").build(); + BigQuery bigQuery = bigQueryOptions.getService(); + + // Verify that all is well by listing a dataset. + Page datasets = bigQuery.listDatasets("bigquery-public-data"); + Iterator iterator = datasets.iterateAll().iterator(); + Set datasetNames = new HashSet<>(); + while (iterator.hasNext()) { + datasetNames.add(iterator.next().getDatasetId().getDataset()); + } + for (String type : PUBLIC_DATASETS) { + assertTrue(datasetNames.contains(type)); + } + } + + static GoogleCredentials loadCredentials(String credentialFile) { + try { + InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes()); + return GoogleCredentials.fromStream(keyStream); + } catch (IOException e) { + fail("Couldn't create fake JSON credentials."); + } + return null; + } }