-
Notifications
You must be signed in to change notification settings - Fork 0
/
bq_flow.yaml
75 lines (75 loc) · 2.3 KB
/
bq_flow.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
- init:
assign:
- project_id: PROJECT_ID
- dataset_id: DATASET_ID
- create_slots_query: ${"CALL `" +project_id+ "." +dataset_id+ ".get_slots`()"}
- train_model_query: ${"CALL `" +project_id+ "." +dataset_id+ ".train_model`()"}
- drop_slots_query: ${"CALL `" +project_id+ "." +dataset_id+ ".drop_slots`()"}
- create_slots:
call: googleapis.bigquery.v2.jobs.insert
args:
projectId: ${project_id}
body:
configuration:
query:
query: ${create_slots_query}
useLegacySql: false
result: create_job_response
- log_create_job_id:
call: sys.log
args:
text: ${"Creating slots. Job id " + create_job_response.jobReference.jobId}
- wait_for_slots:
call: sys.sleep
args:
seconds: 120
- train_model:
call: googleapis.bigquery.v2.jobs.insert
args:
projectId: ${project_id}
body:
configuration:
query:
query: ${train_model_query}
useLegacySql: false
priority: BATCH
result: train_job_response
- get_train_job_info:
call: googleapis.bigquery.v2.jobs.get
args:
projectId: ${project_id}
jobId: ${train_job_response.jobReference.jobId}
result: train_job_info
- check_train_job_state:
switch:
- condition: ${train_job_info.status.state != "DONE"}
steps:
- log_train_status:
call: sys.log
args:
text: Still training
severity: INFO
- wait_for_training:
call: sys.sleep
args:
seconds: 60
next: get_train_job_info
next: drop_slots
- drop_slots:
call: googleapis.bigquery.v2.jobs.insert
args:
projectId: ${project_id}
body:
configuration:
query:
query: ${drop_slots_query}
useLegacySql: false
result: drop_job_response
- get_drop_job_info:
call: googleapis.bigquery.v2.jobs.get
args:
projectId: ${project_id}
jobId: ${drop_job_response.jobReference.jobId}
result: drop_job_info
- the_end:
return: ${"Slots dropped? " + drop_job_info.status.state}