Skip to content

Commit

Permalink
fix(history storage): clean up old history for a task upon its run
Browse files Browse the repository at this point in the history
as a temporary solution to issue #41
  • Loading branch information
pistonsky committed Jun 26, 2016
1 parent ba5ce65 commit 2c4dfba
Showing 1 changed file with 8 additions and 3 deletions.
11 changes: 8 additions & 3 deletions lib/fusion_history_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,19 @@ def prepare_task_run(self, task_run):
task_path = self.get_task_run_path(task_run)
if not os.path.isdir(task_path):
os.makedirs(task_path)
self.remove_old_day_dirs(task_run)
self.remove_old_history_for_task_id(task_run.task_id, days_limit=30)

def remove_old_day_dirs(self, task_run, days_limit=30):
day_dirs = glob.glob(self.storage_dir + '/' + task_run.task_id + '/*')
def remove_old_history_for_task_id(self, task_id, days_limit=30):
day_dirs = glob.glob(self.storage_dir + '/' + task_id + '/*')
if len(day_dirs) > days_limit:
day_dirs.sort()
last_day = day_dirs[0]
shutil.rmtree(last_day)
self.sqlite_connection.execute('''

This comment has been minimized.

Copy link
@pistonsky

pistonsky Jun 26, 2016

Author Contributor

Shouldn't do it too often. Maybe keep track of last cleanup in some class variable, and run the following DELETE only when days_limit (or, say, days_limit/2) has passed since we last cleaned up.

delete from task_runs
where task_id = '%s' and started_at < datetime('now', '-%d days');
''' % (task_id, days_limit))
self.sqlite_connection.commit()

def select_to_dict(self, sql, columns=COLUMNS):
cursor = self.sqlite_connection.execute(sql)
Expand Down

0 comments on commit 2c4dfba

Please sign in to comment.