-
Notifications
You must be signed in to change notification settings - Fork 301
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
fix: load table from json #1248
Changes from 37 commits
46c54eb
c237d2b
1975087
1d06dcd
74efb3a
92915d7
04654c0
aa1eba7
7a6b5c4
f21b6cf
d6cf378
8f4d034
8d735e0
6afe300
66d4581
9a5d71f
dc9838d
1d64518
d883af3
0535084
68511b2
b3faa95
3ca4dc8
e1f9187
1c3d6a8
287a282
0fb771e
efdd87b
51e79b8
04c9fc8
6f8bcc1
ebe3348
57b371a
28f728e
e83d3bf
0d075b3
81d5345
2dc5795
5c3e2b2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -832,8 +832,18 @@ def test_load_table_from_json_basic_use(self): | |
) | ||
|
||
json_rows = [ | ||
{"name": "John", "age": 18, "birthday": "2001-10-15", "is_awesome": False}, | ||
{"name": "Chuck", "age": 79, "birthday": "1940-03-10", "is_awesome": True}, | ||
{ | ||
"name": "John", | ||
"age": "18", | ||
"birthday": "2001-10-15", | ||
"is_awesome": False, | ||
}, | ||
{ | ||
"name": "Chuck", | ||
"age": "79", | ||
"birthday": "1940-03-10", | ||
"is_awesome": True, | ||
}, | ||
] | ||
|
||
dataset_id = _make_dataset_id("bq_system_test") | ||
|
@@ -859,6 +869,64 @@ def test_load_table_from_json_basic_use(self): | |
self.assertEqual(tuple(table.schema), table_schema) | ||
self.assertEqual(table.num_rows, 2) | ||
|
||
def test_load_table_from_json_table_exists(self): | ||
table_schema = (bigquery.SchemaField("age", "STRING", mode="REQUIRED"),) | ||
json_rows = [ | ||
{ | ||
"age": "18", | ||
}, | ||
{ | ||
"age": "79", | ||
}, | ||
] | ||
|
||
dataset_id = _make_dataset_id("bq_system_test") | ||
self.temp_dataset(dataset_id) | ||
table_id = "{}.{}.load_table_from_json_bug_check_with_schema".format( | ||
Config.CLIENT.project, dataset_id | ||
) | ||
|
||
# create an empty table | ||
table = helpers.retry_403(Config.CLIENT.create_table)(Table(table_id)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The test name and comment say this table should be created with a schema, but I don't see the schema here. |
||
self.to_delete.insert(0, table) | ||
|
||
job_config = bigquery.LoadJobConfig(schema=table_schema) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need to provide schema here? |
||
load_job = Config.CLIENT.load_table_from_json( | ||
json_rows, table_id, job_config=job_config | ||
) | ||
load_job.result() | ||
|
||
table = Config.CLIENT.get_table(table) | ||
fetched = self._fetch_single_page(table) | ||
row_tuples = [r.values() for r in fetched] | ||
assert type(table.schema[0].field_type) is str | ||
abecerrilsalas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
assert type(row_tuples[0][0]) is str | ||
|
||
def test_load_table_from_json_bug_table_not_exists(self): | ||
table_schema = (bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"),) | ||
json_rows = [ | ||
{ | ||
"age": "18", | ||
}, | ||
{ | ||
"age": "79", | ||
}, | ||
] | ||
dataset_id = _make_dataset_id("bq_system_test") | ||
self.temp_dataset(dataset_id) | ||
table_id = "{}.{}.load_table_from_json_bug_check".format( | ||
Config.CLIENT.project, dataset_id | ||
) | ||
|
||
job_config = bigquery.LoadJobConfig() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add table as destination so it will be created during the load job |
||
job_config.autodetect = True | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm not following this portion. What is the |
||
load_job = Config.CLIENT.load_table_from_json(json_rows, table_id) | ||
load_job.result() | ||
|
||
table = Config.CLIENT.get_table(table_id) | ||
self.assertTrue(table.schema) | ||
self.assertEqual(tuple(table.schema), table_schema) | ||
|
||
def test_load_table_from_json_schema_autodetect(self): | ||
json_rows = [ | ||
{"name": "John", "age": 18, "birthday": "2001-10-15", "is_awesome": False}, | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why are we calling
_table_arg_to_table_ref()
here?The very first line of
get_table()
calls that function.