Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit fed28c7

Browse files
Praful Makanistephaniewang526
Praful Makani
and
stephaniewang526
authoredOct 6, 2020
docs(samples): add query external bigtable using temp table (#763)
* docs(samples): add query external bigtable using temp table * docs(samples): address feedback * nit * docs(samples): address comments * update projectId Co-authored-by: Stephanie Wang <stephaniewang526@users.noreply.github.com>
1 parent 85ee12f commit fed28c7

File tree

5 files changed

+322
-0
lines changed

5 files changed

+322
-0
lines changed
 

‎samples/install-without-bom/pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,12 @@
6060
<version>1.31.0</version>
6161
</dependency>
6262
<!-- Test dependencies -->
63+
<dependency>
64+
<groupId>com.google.cloud</groupId>
65+
<artifactId>google-cloud-bigtable</artifactId>
66+
<version>1.15.0</version>
67+
<scope>test</scope>
68+
</dependency>
6369
<dependency>
6470
<groupId>junit</groupId>
6571
<artifactId>junit</artifactId>

‎samples/snapshot/pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@
5858
<version>1.31.0</version>
5959
</dependency>
6060
<!-- Test dependencies -->
61+
<dependency>
62+
<groupId>com.google.cloud</groupId>
63+
<artifactId>google-cloud-bigtable</artifactId>
64+
<version>1.15.0</version>
65+
<scope>test</scope>
66+
</dependency>
6167
<dependency>
6268
<groupId>junit</groupId>
6369
<artifactId>junit</artifactId>

‎samples/snippets/pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@
7373
<!-- [END bigquery-enduser-installed-packages] -->
7474

7575
<!-- Test dependencies -->
76+
<dependency>
77+
<groupId>com.google.cloud</groupId>
78+
<artifactId>google-cloud-bigtable</artifactId>
79+
<version>1.15.0</version>
80+
<scope>test</scope>
81+
</dependency>
7682
<dependency>
7783
<groupId>junit</groupId>
7884
<artifactId>junit</artifactId>

‎samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtableTemp.java

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquery;
18+
19+
// [START bigquery_query_external_bigtable_temp]
20+
import com.google.cloud.bigquery.BigQuery;
21+
import com.google.cloud.bigquery.BigQueryException;
22+
import com.google.cloud.bigquery.BigQueryOptions;
23+
import com.google.cloud.bigquery.BigtableColumn;
24+
import com.google.cloud.bigquery.BigtableColumnFamily;
25+
import com.google.cloud.bigquery.BigtableOptions;
26+
import com.google.cloud.bigquery.ExternalTableDefinition;
27+
import com.google.cloud.bigquery.QueryJobConfiguration;
28+
import com.google.cloud.bigquery.TableResult;
29+
import com.google.common.collect.ImmutableList;
30+
import org.apache.commons.codec.binary.Base64;
31+
32+
// Sample to queries an external bigtable data source using a temporary table
33+
public class QueryExternalBigtableTemp {
34+
35+
public static void main(String[] args) {
36+
// TODO(developer): Replace these variables before running the sample.
37+
String projectId = "MY_PROJECT_ID";
38+
String bigtableInstanceId = "MY_INSTANCE_ID";
39+
String bigtableTableName = "MY_BIGTABLE_NAME";
40+
String bigqueryTableName = "MY_TABLE_NAME";
41+
String sourceUri =
42+
String.format(
43+
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
44+
projectId, bigtableInstanceId, bigtableTableName);
45+
String query = String.format("SELECT * FROM %s ", bigqueryTableName);
46+
queryExternalBigtableTemp(bigqueryTableName, sourceUri, query);
47+
}
48+
49+
public static void queryExternalBigtableTemp(String tableName, String sourceUri, String query) {
50+
try {
51+
// Initialize client that will be used to send requests. This client only needs to be created
52+
// once, and can be reused for multiple requests.
53+
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();
54+
55+
BigtableColumnFamily.Builder statsSummary = BigtableColumnFamily.newBuilder();
56+
57+
// Configuring Columns
58+
BigtableColumn connectedCell =
59+
BigtableColumn.newBuilder()
60+
.setQualifierEncoded(Base64.encodeBase64String("connected_cell".getBytes()))
61+
.setFieldName("connected_cell")
62+
.setType("STRING")
63+
.setEncoding("TEXT")
64+
.build();
65+
BigtableColumn connectedWifi =
66+
BigtableColumn.newBuilder()
67+
.setQualifierEncoded(Base64.encodeBase64String("connected_wifi".getBytes()))
68+
.setFieldName("connected_wifi")
69+
.setType("STRING")
70+
.setEncoding("TEXT")
71+
.build();
72+
BigtableColumn osBuild =
73+
BigtableColumn.newBuilder()
74+
.setQualifierEncoded(Base64.encodeBase64String("os_build".getBytes()))
75+
.setFieldName("os_build")
76+
.setType("STRING")
77+
.setEncoding("TEXT")
78+
.build();
79+
80+
// Configuring column family and columns
81+
statsSummary
82+
.setColumns(ImmutableList.of(connectedCell, connectedWifi, osBuild))
83+
.setFamilyID("stats_summary")
84+
.setOnlyReadLatest(true)
85+
.setEncoding("TEXT")
86+
.setType("STRING")
87+
.build();
88+
89+
// Configuring BigtableOptions is optional.
90+
BigtableOptions options =
91+
BigtableOptions.newBuilder()
92+
.setIgnoreUnspecifiedColumnFamilies(true)
93+
.setReadRowkeyAsString(true)
94+
.setColumnFamilies(ImmutableList.of(statsSummary.build()))
95+
.build();
96+
97+
// Configure the external data source and query job.
98+
ExternalTableDefinition externalTable =
99+
ExternalTableDefinition.newBuilder(sourceUri, options).build();
100+
QueryJobConfiguration queryConfig =
101+
QueryJobConfiguration.newBuilder(query)
102+
.addTableDefinition(tableName, externalTable)
103+
.build();
104+
105+
// Example query
106+
TableResult results = bigquery.query(queryConfig);
107+
108+
results
109+
.iterateAll()
110+
.forEach(row -> row.forEach(val -> System.out.printf("%s,", val.toString())));
111+
112+
System.out.println("Query on external temporary table performed successfully.");
113+
} catch (BigQueryException | InterruptedException e) {
114+
System.out.println("Query not performed \n" + e.toString());
115+
}
116+
}
117+
}
118+
// [END bigquery_query_external_bigtable_temp]

‎samples/snippets/src/test/java/com/example/bigquery/QueryExternalBigtableTempIT.java

+186
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquery;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static junit.framework.TestCase.assertNotNull;
21+
22+
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
23+
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
24+
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
25+
import com.google.cloud.bigtable.data.v2.models.BulkMutation;
26+
import com.google.cloud.bigtable.data.v2.models.Mutation;
27+
import com.google.protobuf.ByteString;
28+
import java.io.ByteArrayOutputStream;
29+
import java.io.IOException;
30+
import java.io.PrintStream;
31+
import java.util.UUID;
32+
import java.util.logging.Level;
33+
import java.util.logging.Logger;
34+
import org.junit.After;
35+
import org.junit.Before;
36+
import org.junit.BeforeClass;
37+
import org.junit.Test;
38+
39+
public class QueryExternalBigtableTempIT {
40+
41+
private final Logger log = Logger.getLogger(this.getClass().getName());
42+
private static final String ID = UUID.randomUUID().toString().substring(0, 8);
43+
private static final String TABLE_ID = "bigquery-samples-test" + ID;
44+
private static final String COLUMN_FAMILY_NAME = "stats_summary";
45+
private static final long TIMESTAMP = System.currentTimeMillis() * 1000;
46+
private static final String CONNECTED_CELL = "connected_cell";
47+
private static final String CONNECTED_WIFI = "connected_wifi";
48+
private static final String OS_BUILD = "os_build";
49+
private ByteArrayOutputStream bout;
50+
private PrintStream out;
51+
private PrintStream originalPrintStream;
52+
53+
private static final String INSTANCE = requireEnvVar("BIGTABLE_TESTING_INSTANCE");
54+
private static final String PROJECT = requireEnvVar("SAMPLES_TESTING_PROJECT");
55+
56+
private static String requireEnvVar(String varName) {
57+
String value = System.getenv(varName);
58+
assertNotNull(
59+
"Environment variable " + varName + " is required to perform these tests.",
60+
System.getenv(varName));
61+
return value;
62+
}
63+
64+
@BeforeClass
65+
public static void checkRequirements() {
66+
requireEnvVar("GOOGLE_CLOUD_PROJECT");
67+
requireEnvVar("BIGTABLE_TESTING_INSTANCE");
68+
}
69+
70+
@Before
71+
public void setUp() throws IOException {
72+
bout = new ByteArrayOutputStream();
73+
out = new PrintStream(bout);
74+
originalPrintStream = System.out;
75+
System.setOut(out);
76+
77+
// create a temporary bigtable table.
78+
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
79+
CreateTableRequest createTableRequest =
80+
CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_NAME);
81+
client.createTable(createTableRequest);
82+
}
83+
// inserting temporary rows.
84+
try (BigtableDataClient client = BigtableDataClient.create(PROJECT, INSTANCE)) {
85+
BulkMutation bulkMutation =
86+
BulkMutation.create(TABLE_ID)
87+
.add(
88+
"phone#4c410523#20190501",
89+
Mutation.create()
90+
.setCell(
91+
COLUMN_FAMILY_NAME,
92+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
93+
TIMESTAMP,
94+
1)
95+
.setCell(
96+
COLUMN_FAMILY_NAME,
97+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
98+
TIMESTAMP,
99+
1)
100+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.003"))
101+
.add(
102+
"phone#4c410523#20190502",
103+
Mutation.create()
104+
.setCell(
105+
COLUMN_FAMILY_NAME,
106+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
107+
TIMESTAMP,
108+
1)
109+
.setCell(
110+
COLUMN_FAMILY_NAME,
111+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
112+
TIMESTAMP,
113+
1)
114+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.004"))
115+
.add(
116+
"phone#4c410523#20190505",
117+
Mutation.create()
118+
.setCell(
119+
COLUMN_FAMILY_NAME,
120+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
121+
TIMESTAMP,
122+
0)
123+
.setCell(
124+
COLUMN_FAMILY_NAME,
125+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
126+
TIMESTAMP,
127+
1)
128+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"))
129+
.add(
130+
"phone#5c10102#20190501",
131+
Mutation.create()
132+
.setCell(
133+
COLUMN_FAMILY_NAME,
134+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
135+
TIMESTAMP,
136+
1)
137+
.setCell(
138+
COLUMN_FAMILY_NAME,
139+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
140+
TIMESTAMP,
141+
1)
142+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190401.002"))
143+
.add(
144+
"phone#5c10102#20190502",
145+
Mutation.create()
146+
.setCell(
147+
COLUMN_FAMILY_NAME,
148+
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
149+
TIMESTAMP,
150+
1)
151+
.setCell(
152+
COLUMN_FAMILY_NAME,
153+
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
154+
TIMESTAMP,
155+
0)
156+
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"));
157+
158+
client.bulkMutateRows(bulkMutation);
159+
}
160+
}
161+
162+
@After
163+
public void tearDown() throws IOException {
164+
// Clean up
165+
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
166+
client.deleteTable(TABLE_ID);
167+
}
168+
// restores print statements in the original method
169+
System.out.flush();
170+
System.setOut(originalPrintStream);
171+
log.log(Level.INFO, bout.toString());
172+
}
173+
174+
@Test
175+
public void testQueryExternalBigtableTemp() {
176+
String tableName = "EXTERNAL_TEMP_TABLE_FROM_BIGTABLE_TEST_" + ID;
177+
String query = String.format("SELECT * FROM %s ", tableName);
178+
String sourceUri =
179+
String.format(
180+
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
181+
PROJECT, INSTANCE, TABLE_ID);
182+
QueryExternalBigtableTemp.queryExternalBigtableTemp(tableName, sourceUri, query);
183+
assertThat(bout.toString())
184+
.contains("Query on external temporary table performed successfully.");
185+
}
186+
}

0 commit comments

Comments
 (0)
Failed to load comments.