forked from simon987/od-database
-
Notifications
You must be signed in to change notification settings - Fork 0
/
callbacks.py
73 lines (46 loc) · 2.29 KB
/
callbacks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
from tasks import Task, TaskResult
from reddit_bot import RedditBot
import praw
from search.search import SearchEngine
import json
class PostCrawlCallback:
def __init__(self, task: Task):
self.task = task
if self.task.callback_args:
self.task.callback_args = json.loads(self.task.callback_args)
def run(self, task_result: TaskResult, search: SearchEngine):
raise NotImplementedError
class PostCrawlCallbackFactory:
@staticmethod
def get_callback(task: Task):
if task.callback_type == "reddit_post":
return RedditPostCallback(task)
elif task.callback_type == "reddit_comment":
return RedditCommentCallback(task)
elif task.callback_type == "discord":
return DiscordCallback(task)
class RedditCallback(PostCrawlCallback):
def __init__(self, task: Task):
super().__init__(task)
reddit = praw.Reddit('opendirectories-bot',
user_agent='github.com/simon987/od-database (by /u/Hexahedr_n)')
self.reddit_bot = RedditBot("crawled.txt", reddit)
def run(self, task_result: TaskResult, search: SearchEngine):
raise NotImplementedError
class RedditPostCallback(RedditCallback):
def run(self, task_result: TaskResult, search: SearchEngine):
print("Reddit post callback for task " + str(self.task))
class RedditCommentCallback(RedditCallback):
def run(self, task_result: TaskResult, search: SearchEngine):
comment_id = self.task.callback_args["comment_id"]
print("Editing comment comment " + comment_id)
search.refresh() # Make sure the newly indexed documents are available before commenting
stats = search.get_stats(self.task.website_id)
message = self.reddit_bot.get_comment(stats, self.task.website_id,
message="There you go! This website was crawled in `" +
str(int(task_result.end_time - task_result.start_time)) + "s`")
print(message)
self.reddit_bot.edit(self.reddit_bot.reddit.comment(comment_id), message)
class DiscordCallback(PostCrawlCallback):
def run(self, task_result: TaskResult, search: SearchEngine):
print("Discord callback for task " + str(self.task))