-
Notifications
You must be signed in to change notification settings - Fork 7
/
main.tf
114 lines (98 loc) · 3.21 KB
/
main.tf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
terraform {
required_providers {
google = {
source = "hashicorp/google"
version = "4.51.0"
}
}
}
provider "google" {
# Credentials only needs to be set if you do not have the GOOGLE_APPLICATION_CREDENTIALS set
# credentials =
credentials = file(var.credentials)
project = var.project
region = var.region
}
resource "google_storage_bucket" "data-lake-bucket_new" {
name = var.datalake-bucket
location = var.location
# Optional, but recommended settings:
storage_class = "STANDARD"
uniform_bucket_level_access = true
force_destroy = true
lifecycle_rule {
condition {
age = 1
}
action {
type = "AbortIncompleteMultipartUpload"
}
}
}
resource "google_storage_bucket_object" "load_files" {
for_each = fileset("${path.module}/01-data-cleaned", "**/*")
name = each.value
bucket = google_storage_bucket.data-lake-bucket_new.name
source = "${path.module}/01-data-cleaned/${each.value}"
}
resource "google_bigquery_dataset" "dataset" {
dataset_id = var.bq_dataset_name
project = var.project
location = var.location
delete_contents_on_destroy = true
}
data "archive_file" "function_code" {
type = "zip"
source_dir = "${path.module}/function_code"
output_path = "${path.module}/function_code.zip"
}
resource "google_storage_bucket_object" "function_code" {
name = "function_code.zip"
bucket = google_storage_bucket.data-lake-bucket_new.name
source = data.archive_file.function_code.output_path
}
resource "google_cloudfunctions_function" "function" {
name = "streaming"
description = "Function to load data from GCS to BigQuery"
available_memory_mb = 256
source_archive_bucket = google_storage_bucket.data-lake-bucket_new.name
source_archive_object = google_storage_bucket_object.function_code.name
entry_point = "streaming"
runtime = "python310"
environment_variables = {
PROJECT_ID = var.project
BQ_DATASET = "datawarehouse_ye_go_cs_419123"
BQ_TABLE = "reviews_unified"
}
event_trigger {
event_type = "google.storage.object.finalize"
resource = google_storage_bucket.data-lake-bucket_new.name
}
}
/*
data "archive_file" "function_code" {
type = "zip"
source_dir = "${path.module}/function_code"
output_path = "${path.module}/function_code.zip"
}
resource "google_storage_bucket_object" "function_code" {
name = "function_code.zip"
bucket = google_storage_bucket.data-lake-bucket_new.name
source = data.archive_file.function_code.output_path
}
resource "google_cloudfunctions_function" "function" {
name = "streaming"
description = "Function to load data from GCS to BigQuery"
available_memory_mb = 256
source_archive_bucket = google_storage_bucket.data-lake-bucket_new.name
source_archive_object = google_storage_bucket_object.function_code.name
trigger_http = true
entry_point = "streaming"
runtime = "python310"
environment_variables = {
PROJECT_ID = var.project
BQ_DATASET = "your_bigquery_dataset"
BQ_TABLE = "your_bigquery_table"
}
}
*/