-
Notifications
You must be signed in to change notification settings - Fork 1
/
get_sheetsdoc.js
159 lines (148 loc) · 5.55 KB
/
get_sheetsdoc.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
// import regeneratorRuntime from "regenerator-runtime";
import { asyncForEach } from "./util";
export async function get_sheetsdoc(sheets_doc_key) {
let result = {};
try {
await gapi.client.init({
apiKey: process.env.GOOGLE_API_KEY,
});
let sheets_arr = await gapi.client.request({
path: `https://sheets.googleapis.com/v4/spreadsheets/${sheets_doc_key}?&fields=sheets.properties`,
});
await asyncForEach(sheets_arr.result.sheets, async (a_sheet) => {
let single_sheet_title = a_sheet["properties"]["title"];
let enc_single_sheet_title = encodeURIComponent(single_sheet_title);
// console.log(single_sheet_title);
let sheet_data = await gapi.client.request({
path: `https://sheets.googleapis.com/v4/spreadsheets/${sheets_doc_key}/values/${enc_single_sheet_title}`,
});
// console.log("---");
// console.log(single_sheet_title);
// console.log(sheet_data);
result[single_sheet_title] = sheet_data.result.values
? objectify(sheet_data.result.values)
: [];
// console.log(result[single_sheet_title]);
});
return result;
} catch (e) {
console.log(e);
return false;
}
}
function objectify(values_arr) {
try {
let col_headers = values_arr.shift();
return values_arr.map((row) => {
let obj = {};
col_headers.forEach((head, i) => {
let v = row[i];
// this seems to be the way drive shaft handles this
obj[head] = v === "" || v === undefined ? null : v;
});
return obj;
});
} catch (e) {
console.log(e);
// console.log("values_arr");
// console.log(values_arr);
return [];
}
}
// similar to above but gets csvs and an array
export async function get_sheetsdoc_csvs(sheets_doc_key) {
let result = [];
try {
await gapi.client.init({
apiKey: process.env.GOOGLE_API_KEY,
});
let sheets_arr = await gapi.client.request({
path: `https://sheets.googleapis.com/v4/spreadsheets/${sheets_doc_key}?&fields=sheets.properties`,
});
await asyncForEach(sheets_arr.result.sheets, async (a_sheet) => {
let single_sheet_title = a_sheet["properties"]["title"];
let enc_single_sheet_title = encodeURIComponent(single_sheet_title);
// console.log("title", single_sheet_title);
let sheet_id = a_sheet["properties"]["sheetId"].toString();
// https://stackoverflow.com/a/33727897/83859
let url = `https://docs.google.com/spreadsheets/d/${sheets_doc_key}/gviz/tq?tqx=out:csv&sheet=${enc_single_sheet_title}`;
let resp = await fetch(url, { mode: "cors" });
let text = await resp.text();
// console.log("\n\n");
// console.log(sheet_id);
// console.log(text.slice(0, 50));
// emoji in s3 file metadat are a problem
// this strips them, but is imperfect
// moved this down, can't do it above the request, we query based on this!
single_sheet_title = single_sheet_title.replace(/\W/g, " ");
result.push({ sheet_id, text, single_sheet_title, sheets_doc_key });
// result[sheet_id] = text;
// console.log(text);
});
} catch (e) {
console.log(e);
return false;
}
return result;
// https://docs.google.com/spreadsheets/d/{key}/gviz/tq?tqx=out:csv&sheet={sheet_name}
}
// a newer test, more complicated tabsheet titles, escaping, also just for dev
// gapi.load("client", async () => {
// // let result = await get_sheetsdoc(
// let result = await get_sheetsdoc_csvs(
// "1xa0iLqYKz8x9Yc_rfhtmSOJQ2EGgeUVjvV4A8LsIaxY"
// );
// console.log(result);
// });
// JUST FOR DEV
// uncomment this to run the test above
// need to load gapi first
// gapi.load("client", () => {
// test_get_sheetsdoc();
// });
// just for dev !
import fixture from "../fixtures/raai_index_2019.json";
// hacky fix for the encoding issue
fixture["Leader's List 2019 vs 2017"] =
fixture["Leader\\x27s List 2019 vs 2017"];
delete fixture["Leader\\x27s List 2019 vs 2017"];
// usefull http://jsondiff.com/
// Note, this make shift test only gets 7/11 of the tables for the test document,
// which seems to be character encoding issues (as in the fixture key above), exacerbated by a complicated test case
export async function test_get_sheetsdoc() {
console.log("running transform test");
console.log(" against fixture:");
console.log(fixture);
// let fixture_string = JSON.stringify(fixture);
// console.log(fixture_string.split("\x27").length);
// console.log(fixture_string.split(`\x26`).length);
// return;
// Zach Ref Copy of RAAI live data template 2019
// https://docs.google.com/spreadsheets/d/1aySa6njMLlXT39FHm5ikHCxoxHF-HY0JF76ERzTxm88/edit#gid=2140363911
let result = await get_sheetsdoc(
"1aySa6njMLlXT39FHm5ikHCxoxHF-HY0JF76ERzTxm88"
);
let keys = Object.keys(fixture);
let score = 0;
console.log("using fixture keys");
// console.log('using results keys')
// let keys = Object.keys(result);
for (let k of keys) {
console.log(`Checking ${k}`);
let they_match = JSON.stringify(fixture[k]) === JSON.stringify(result[k]);
if (!result[k]) {
console.log("mismatch! not found at all");
} else if (they_match) {
console.log("✅ THEY MATCH");
score += 1;
} else {
console.log(`fixture is ${JSON.stringify(fixture[k]).length} c long `);
console.log(`result is ${JSON.stringify(result[k]).length} c long `);
console.log(`fixture is ${fixture[k].length} rows long `);
console.log(`result is ${result[k].length} rows long `);
console.log("NO MATCH");
}
console.log(" ");
}
console.log(`Score: ${score} (/${keys.length})`);
}