-
Notifications
You must be signed in to change notification settings - Fork 0
/
gdrivesync.py
276 lines (244 loc) · 9.58 KB
/
gdrivesync.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
'''
gdrivesync
Sync files in local directory to a Google Drive directory.
'''
__version__ = '0.1.0'
import httplib2
import json
import fnmatch
import sys
import os
import getopt
import logging
from apiclient.discovery import build
from apiclient.http import MediaIoBaseUpload
from oauth2client.client import OAuth2WebServerFlow, OAuth2Credentials
logging.basicConfig(level=logging.INFO)
class GoogleCredentials(object):
def __init__(self, config_file):
self.config_file = config_file
self.config = {}
def _load_credentials(self):
self.config = json.loads(open(self.config_file).read())
def _save_credentials(self):
fd = open(self.config_file, 'w')
fd.write(json.dumps(self.config))
def _get_credentials(self):
OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'
REDIRECT_URI = 'http://localhost'
flow = OAuth2WebServerFlow(self.config['client_id'],
self.config['client_secret'],
OAUTH_SCOPE,
REDIRECT_URI)
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
credentials = flow.step2_exchange(code)
return credentials
def get_service(self):
self._load_credentials()
if not self.config.get('credentials'):
logging.info("Credentials not found, begin the Oauth process.")
credentials = self._get_credentials()
self.config['credentials'] = credentials.to_json()
self._save_credentials()
else:
credentials_json = self.config.get('credentials')
credentials = OAuth2Credentials.from_json(credentials_json)
# Create an httplib2.Http object and authorize it with our credentials
http = httplib2.Http()
http = credentials.authorize(http)
drive_service = build('drive', 'v2', http=http)
return drive_service
def match_any(name, patterns):
'''
Check if the name match any patterns.
'''
for pattern in patterns:
if fnmatch.fnmatch(name, pattern):
return True
return False
def walk_files(root_path='.', patterns_include=None, patterns_exclude=None):
'''
A wrapper of os.walk(). If patterns_include is provided, only files matched
any of patterns_include are returned; if patterns_exclude is provided,
files matched any of patterns_exclude are removed from the results.
The function will first evaluate patterns_include, then patterns_exclude.
'''
result = []
if not root_path.endswith('/'):
root_path += '/'
for path, __, names in os.walk(root_path):
prefix = path[len(root_path):]
files = []
for name in names:
fullname = (prefix + '/' + name) if prefix else name
if patterns_include and\
(not match_any(fullname, patterns_include)):
continue
if patterns_exclude and match_any(fullname, patterns_exclude):
continue
files.append(name)
if files:
result.append([prefix, files])
return result
class GDriveSync(object):
def __init__(self, service, target):
'''
:param service: The service of get_service by GoogleCredentials.
:param target: The target folder to sync.
'''
self.service = service
self.target = target
def get_config(self):
if not self.target.endswith('/'):
self.target += '/'
content = open('%s.gdrivesync' % self.target).read()
self.config = json.loads(content)
def get_directory(self, title, parent_id):
'''
Get the file object with title and parent_id.
If not found, a None object will be returned.
'''
q = 'title = "%s" and "%s" in parents' % (title, parent_id)
q += ' and trashed = false'
q += ' and mimeType = "application/vnd.google-apps.folder"'
r = self.service.files().list(q=q).execute() # use the first matched
if r['items']:
r = r['items'][0]
else:
r = None
return r
def create_folder(self, title, parent_id):
'''
Create a folder with title under a parnet folder with parent_id.
'''
params = {'title': title,
'parents': [{'id': parent_id}],
'mimeType': 'application/vnd.google-apps.folder'}
r = self.service.files().insert(body=params).execute()
return r
def recursive_path(self, path, root='root', ensure_path=False):
'''
Recursivly locate the path.
:param path: The spcified path,
heading and tailing '/' will be ignored.
:param root: The parent folder id of path, default 'root'.
:param ensure_path: If True, the not found folder will be created.
'''
directories = [i for i in path.strip().split('/') if i]
parent_id = root
r = None
for directory in directories:
r = self.get_directory(directory, parent_id)
if r:
parent_id = r['id']
elif ensure_path:
r = self.create_folder(directory, parent_id)
parent_id = r['id']
else:
raise Exception("Folder %s not found." % directory)
if r:
return r
else:
return None
def insert_file(self, root_path, path, title, parent_id):
'''
Insert a file to google drive.
'''
if not root_path.endswith('/'):
root_path += '/'
filename = root_path + path + '/' + title
logging.info("Inserting %s to folder %s." % (filename, parent_id))
media_body = MediaIoBaseUpload(open(filename),
'text/plain',
resumable=True)
body = {
'title': title,
'description': title,
'mimeType': 'text/plain',
'parents': [{'id': parent_id}],
'indexableText.text': open(filename).read()}
file = self.service.files().insert(body=body,
media_body=media_body).execute()
return file
def update_file(self, file_obj, root_path, path, title, parent_id):
'''
Update a file to google drive.
'''
if not root_path.endswith('/'):
root_path += '/'
filename = root_path + path + '/' + title
logging.info("Updating %s to folder %s." % (filename, parent_id))
media_body = MediaIoBaseUpload(open(filename),
'text/plain',
resumable=True)
body = {'title': title,
'description': title,
'mimeType': 'text/plain',
'indexableText.text': open(filename).read()}
file = self.service.files().update(body=body,
media_body=media_body,
fileId=file_obj['id'],
newRevision=True).execute()
return file
def get_file(self, title, parent_id):
'''
Get the file object named title in folder parent_id.
'''
q = 'title = "%s" and "%s" in parents' % (title, parent_id)
q += ' and trashed = false'
q += ' and mimeType != "application/vnd.google-apps.folder"'
r = self.service.files().list(q=q).execute()
if r['items']:
r = r['items'][0]
else:
r = None
return r
def upload_files(self, root_path, files, root_id):
'''
Upload files located in root_path to folder root_id on
Google Drive.
'''
for prefix, names in files:
if prefix:
path_obj = self.recursive_path(prefix,
root=root_id,
ensure_path=True)
parent_id = path_obj['id']
else:
parent_id = root_id
for name in names:
file_obj = self.get_file(name, parent_id)
if file_obj:
self.update_file(file_obj,
root_path,
prefix,
name,
parent_id)
else:
self.insert_file(root_path, prefix, name, parent_id)
def run(self):
self.get_config()
files = walk_files(root_path=self.target,
patterns_include=self.config.get('include'),
patterns_exclude=self.config.get('exclude'))
remote_id = self.config.get('remote_id')
if remote_id is None:
if self.config.get('remote_dir'):
path_obj = self.recursive_path(self.config.get('remote_dir'))
if path_obj:
remote_id = path_obj['id']
if remote_id is None:
raise Exception("Remote path or id not existed.")
self.upload_files(self.target, files, remote_id)
def main():
opts, args = getopt.gnu_getopt(sys.argv[1:], 'c:')
opts = dict(opts)
config_file = 'config.json' if not '-c' in opts else opts['-c']
gc = GoogleCredentials(config_file)
service = gc.get_service()
for arg in args:
GDriveSync(service, arg).run()
if __name__ == '__main__':
main()