diff --git a/config/plugins.d/alerts.conf b/config/plugins.d/alerts.conf new file mode 100644 index 0000000..06d2298 --- /dev/null +++ b/config/plugins.d/alerts.conf @@ -0,0 +1,11 @@ +### Reports Plugin Configuration for Webcampak + +[alerts] + +### If enabled, load a plugin named `example` either from the Python module +### `webcampak.cli.plugins.example` or from the file path +### `/var/lib/webcampak/plugins/example.py` +enable_plugin = true + +### Additional plugin configuration settings +###foo = bar diff --git a/webcampak/cli/plugins/alerts.py b/webcampak/cli/plugins/alerts.py new file mode 100644 index 0000000..9b6d8e0 --- /dev/null +++ b/webcampak/cli/plugins/alerts.py @@ -0,0 +1,63 @@ +"""Example Plugin for Webcampak.""" + +from cement.core.controller import CementBaseController, expose +from cement.core import handler, hook + +from webcampak.core.wpakAlertsCapture import alertsCapture + +def alerts_plugin_hook(app): + # do something with the ``app`` object here. + pass + +class ExamplePluginController(CementBaseController): + class Meta: + # name that the controller is displayed at command line + label = 'alerts' + + # text displayed next to the label in ``--help`` output + description = 'Trigger some alerts based on specific events or checks' + + # stack this controller on-top of ``base`` (or any other controller) + stacked_on = 'base' + + # determines whether the controller is nested, or embedded + stacked_type = 'nested' + + # these arguments are only going to display under + # ``$ webcampak alerts --help`` + arguments = [ + ( + ['-s', '--sourceid'], + dict( + help='Run the alert only for the specified source', + action='store', + ) + ) + ] + + @expose(hide=True) + def default(self): + self.app.log.info("Please indicate which command to run") + + @expose(help="Alert if a capture is running late based on source schedule") + def capture(self): + self.app.log.info("Starting Capture Alert", __file__) + if self.app.pargs.config_dir != None: + config_dir = self.app.pargs.config_dir + else: + config_dir = self.app.config.get('webcampak', 'config_dir') + + try: + start = alertsCapture(self.app.log, self.app.config, config_dir, self.app.pargs.sourceid) + start.run() + except Exception: + self.app.log.fatal("Ooops! Something went terribly wrong, stack trace below:", exc_info=True) + raise + + +def load(app): + # register the plugin class.. this only happens if the plugin is enabled + handler.register(ExamplePluginController) + + # register a hook (function) to run after arguments are parsed. + hook.register('post_argument_parsing', alerts_plugin_hook) diff --git a/webcampak/cli/plugins/reports.py b/webcampak/cli/plugins/reports.py index 86a24d7..36db2c0 100644 --- a/webcampak/cli/plugins/reports.py +++ b/webcampak/cli/plugins/reports.py @@ -25,15 +25,6 @@ class Meta: # these arguments are only going to display under # ``$ webcampak reports --help`` - arguments = [ - ( - ['-t', '--thread'], - dict( - help='Start/Stop a specific XFer job thread', - action='store', - ) - ) - ] @expose(hide=True) def default(self): @@ -41,7 +32,7 @@ def default(self): @expose(help="Daily reports for all sources") def daily(self): - self.app.log.info("Starting XFer Dispatch", __file__) + self.app.log.info("Starting Daily report for all sources", __file__) if self.app.pargs.config_dir != None: config_dir = self.app.pargs.config_dir else: diff --git a/webcampak/cli/plugins/stats.py b/webcampak/cli/plugins/stats.py index c5a94ea..136bcad 100644 --- a/webcampak/cli/plugins/stats.py +++ b/webcampak/cli/plugins/stats.py @@ -35,6 +35,13 @@ class Meta: action='store', ) ) + , ( + ['--full'], + dict( + help='Run consolidation on all system logs', + action='store_true', + ) + ) ] @expose(hide=True) @@ -65,7 +72,7 @@ def consolidate(self): try: consolidate = statsConsolidate(self.app.log, self.app.config, config_dir) - consolidate.run() + consolidate.run(self.app.pargs.full) except Exception: self.app.log.fatal("Ooops! Something went terribly wrong, stack trace below:", exc_info=True) raise diff --git a/webcampak/core/wpakAlertsCapture.py b/webcampak/core/wpakAlertsCapture.py new file mode 100644 index 0000000..01872e5 --- /dev/null +++ b/webcampak/core/wpakAlertsCapture.py @@ -0,0 +1,488 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright 2010-2016 Eurotechnia (support@webcampak.com) +# This file is part of the Webcampak project. +# Webcampak is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +# Webcampak is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License along with Webcampak. +# If not, see http://www.gnu.org/licenses/ + +import os +import time +import gettext +import json +from datetime import tzinfo, timedelta, datetime +import pytz +from dateutil import tz +import dateutil.parser +from tabulate import tabulate +import socket + +from wpakConfigObj import Config +from wpakConfigCache import configCache +from wpakTimeUtils import timeUtils +from wpakSourcesUtils import sourcesUtils +from wpakFileUtils import fileUtils +from wpakDbUtils import dbUtils +from wpakEmailObj import emailObj +from wpakAlertsObj import alertObj +from wpakAlertsEmails import alertsEmails + +class alertsCapture(object): + """ This class is used to verify if pictures are properly captured and not running late + + Args: + log: A class, the logging interface + appConfig: A class, the app config interface + config_dir: A string, filesystem location of the configuration directory + sourceId: Source ID of the source to capture + + Attributes: + tbc + """ + + def __init__(self, log, appConfig, config_dir, sourceId): + self.log = log + self.appConfig = appConfig + self.config_dir = config_dir + self.sourceId = sourceId + + self.configPaths = Config(self.log, self.config_dir + 'param_paths.yml') + self.dirEtc = self.configPaths.getConfig('parameters')['dir_etc'] + self.dirConfig = self.configPaths.getConfig('parameters')['dir_config'] + self.dirBin = self.configPaths.getConfig('parameters')['dir_bin'] + self.dirSources = self.configPaths.getConfig('parameters')['dir_sources'] + self.dirSourceLive = self.configPaths.getConfig('parameters')['dir_source_live'] + self.dirSourceCapture = self.configPaths.getConfig('parameters')['dir_source_capture'] + self.dirLocale = self.configPaths.getConfig('parameters')['dir_locale'] + self.dirLocaleMessage = self.configPaths.getConfig('parameters')['dir_locale_message'] + self.dirLocaleEmails = self.configPaths.getConfig('parameters')['dir_locale_emails'] + self.dirStats = self.configPaths.getConfig('parameters')['dir_stats'] + self.dirCache = self.configPaths.getConfig('parameters')['dir_cache'] + self.dirEmails = self.configPaths.getConfig('parameters')['dir_emails'] + self.dirResources = self.configPaths.getConfig('parameters')['dir_resources'] + self.dirLogs = self.configPaths.getConfig('parameters')['dir_logs'] + self.dirXferQueue = self.configPaths.getConfig('parameters')['dir_xfer'] + 'queued/' + + self.setupLog() + + self.configGeneral = Config(self.log, self.dirConfig + 'config-general.cfg') + self.initGetText(self.dirLocale, self.configGeneral.getConfig('cfgsystemlang'), + self.configGeneral.getConfig('cfggettextdomain')) + + self.timeUtils = timeUtils(self) + self.sourcesUtils = sourcesUtils(self) + self.dbUtils = dbUtils(self) + self.configCache = configCache(self) + self.fileUtils = fileUtils(self) + self.alertsEmails = alertsEmails(self) + + + def setupLog(self): + """ Setup logging to file""" + reportsLogs = self.dirLogs + "alerts/" + if not os.path.exists(reportsLogs): + os.makedirs(reportsLogs) + logFilename = reportsLogs + "alert.log" + self.appConfig.set(self.log._meta.config_section, 'file', logFilename) + self.appConfig.set(self.log._meta.config_section, 'rotate', True) + self.appConfig.set(self.log._meta.config_section, 'max_bytes', 512000) + self.appConfig.set(self.log._meta.config_section, 'max_files', 10) + self.log._setup_file_log() + + def initGetText(self, dirLocale, cfgsystemlang, cfggettextdomain): + """ Initialize Gettext with the corresponding translation domain + + Args: + dirLocale: A string, directory location of the file + cfgsystemlang: A string, webcampak-level language configuration parameter from config-general.cfg + cfggettextdomain: A string, webcampak-level gettext domain configuration parameter from config-general.cfg + + Returns: + None + """ + self.log.debug("alertsCapture.initGetText(): Start") + try: + t = gettext.translation(cfggettextdomain, dirLocale, [cfgsystemlang], fallback=True) + _ = t.ugettext + t.install() + self.log.info("alertsCapture.initGetText(): " + _( + "Initialized gettext with Domain: %(cfggettextdomain)s - Language: %(cfgsystemlang)s - Path: %(dirLocale)s") + % {'cfggettextdomain': cfggettextdomain, 'cfgsystemlang': cfgsystemlang, + 'dirLocale': dirLocale}) + except: + self.log.error("No translation file available") + + def run(self): + """ Initiate daily reports creation for all sources """ + self.log.info("alertsCapture.run(): " + _("Initiate alerts capture")) + + if self.sourceId != None: + sourceAlerts = [self.sourceId] + else: + sourceAlerts = self.sourcesUtils.getActiveSourcesIds() + + for currentSource in sourceAlerts: + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Processing source: %(currentSource)s") % {'currentSource': str(currentSource)}) + configSource = self.configCache.loadSourceConfig("source", self.dirEtc + 'config-source' + str(currentSource) + '.cfg', currentSource) + if configSource.getConfig('cfgemailerroractivate') == "yes" and (configSource.getConfig('cfgemailalerttime') == "yes" or configSource.getConfig('cfgemailalertscheduleslot') == "yes"): + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Email alerts are enabled for this source") % {'currentSource': str(currentSource)}) + + currentTime = self.timeUtils.getCurrentSourceTime(configSource) + latestPicture = self.sourcesUtils.getLatestPicture(currentSource) + lastCaptureTime = self.timeUtils.getTimeFromFilename(latestPicture, configSource) + secondsDiff = int((currentTime-lastCaptureTime).total_seconds()) + + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Last picture: %(latestPicture)s") % {'currentSource': str(currentSource), 'latestPicture': str(latestPicture)}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Current Source Time: %(currentTime)s") % {'currentSource': str(currentSource), 'currentTime': str(currentTime.isoformat())}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Last Capture Time: %(lastCaptureTime)s") % {'currentSource': str(currentSource), 'lastCaptureTime': str(lastCaptureTime.isoformat())}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Seconds since last capture: %(secondsDiff)s") % {'currentSource': str(currentSource), 'secondsDiff': str(secondsDiff)}) + + alertsFile = self.dirSources + "source" + str(currentSource) + "/resources/alerts/" + currentTime.strftime("%Y%m%d") + ".jsonl"; + lastAlertFile = self.dirSources + "source" + str(currentSource) + "/resources/alerts/last-alert.json"; + lastEmailFile = self.dirSources + "source" + str(currentSource) + "/resources/alerts/last-email.json"; + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Alerts Log file: %(alertsFile)s") % {'currentSource': str(currentSource), 'alertsFile': alertsFile}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Last Alert file: %(alertsFile)s") % {'currentSource': str(currentSource), 'alertsFile': lastAlertFile}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Last Email file: %(alertsFile)s") % {'currentSource': str(currentSource), 'alertsFile': lastEmailFile}) + + lastAlert = alertObj(self.log, lastAlertFile) + lastAlert.loadAlertFile() + + lastEmail = alertObj(self.log, lastEmailFile) + lastEmail.loadAlertFile() + + currentAlert = alertObj(self.log, alertsFile) + currentAlert.setAlertValue("sourceid", currentSource) + currentAlert.setAlertValue("currentTime", currentTime.isoformat()) + currentAlert.setAlertValue("lastPicture", latestPicture) + currentAlert.setAlertValue("lastCaptureTime", lastCaptureTime.isoformat()) + currentAlert.setAlertValue("nextCaptureTime", None) + currentAlert.setAlertValue("missedCapture", None) + currentAlert.setAlertValue("secondsSinceLastCapture", secondsDiff) + currentAlert.setAlertValue("missedCapturesSinceLastEmail", None) + currentAlert.setAlertValue("email", None) + + if lastEmail.getAlert() != {} and lastEmail.getAlertValue("lastCaptureTime") == currentAlert.getAlertValue("lastCaptureTime"): + lastEmailTime = dateutil.parser.parse(lastEmail.getAlertValue("currentTime")) + secondsSinceLastEmail = int((currentTime-lastEmailTime).total_seconds()) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Seconds since last email: %(secondsSinceLastEmail)s") % {'currentSource': str(currentSource), 'secondsSinceLastEmail': str(secondsSinceLastEmail)}) + else: + secondsSinceLastEmail = None + currentAlert.setAlertValue("secondsSinceLastEmail", secondsSinceLastEmail) + + + # Determine alert state based on time since last capture + timeAlertStatus = None + if configSource.getConfig('cfgemailalerttime') == "yes": + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Time based alert enabled") % {'currentSource': str(currentSource)}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Time based: cfgemailalerttime: %(cfgemailalerttime)s") % {'currentSource': str(currentSource), 'cfgemailalerttime': str(configSource.getConfig('cfgemailalerttime'))}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Time based: cfgemailalerttimefailure: %(cfgemailalerttimefailure)s") % {'currentSource': str(currentSource), 'cfgemailalerttimefailure': str(configSource.getConfig('cfgemailalerttimefailure'))}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Time based: cfgemailalerttimereminder: %(cfgemailalerttimereminder)s") % {'currentSource': str(currentSource), 'cfgemailalerttimereminder': str(configSource.getConfig('cfgemailalerttimereminder'))}) + + # Determine if source id in an error state + minutesDiff = int(secondsDiff / 60) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Time based: Minutes since last capture: %(minutesDiff)s") % {'currentSource': str(currentSource), 'minutesDiff': str(minutesDiff)}) + if minutesDiff >= int(configSource.getConfig('cfgemailalerttimefailure')): + timeAlertStatus = "ERROR" + if secondsSinceLastEmail != None and secondsSinceLastEmail >= (int(configSource.getConfig('cfgemailalerttimereminder'))*60): + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Requesting to send a reminder email based on time since last email") % {'currentSource': str(currentSource)}) + currentAlert.setAlertValue("email", True) + currentAlert.setAlertValue("emailType", "REMINDER") + else: + timeAlertStatus = "GOOD" + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Time based alert Status: %(timeAlertStatus)s") % {'currentSource': str(currentSource), 'timeAlertStatus': timeAlertStatus}) + + # Determine alert state based on per-configured alert schedule + scheduleAlertStatus = None + if configSource.getConfig('cfgemailalertscheduleslot') == "yes": + sourceSchedule = self.getSourceSchedule(currentSource) + if sourceSchedule != {}: + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - An alert schedule is available for the source") % {'currentSource': str(currentSource)}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Schedule slot based alert enabled") % {'currentSource': str(currentSource)}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Schedule slot based: cfgemailalertscheduleslot: %(cfgemailalertscheduleslot)s") % {'currentSource': str(currentSource), 'cfgemailalertscheduleslot': str(configSource.getConfig('cfgemailalertscheduleslot'))}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Schedule slot based: cfgemailalertscheduleslotfailure: %(cfgemailalertscheduleslotfailure)s") % {'currentSource': str(currentSource), 'cfgemailalertscheduleslotfailure': str(configSource.getConfig('cfgemailalertscheduleslotfailure'))}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Schedule slot based: cfgemailalertscheduleslotreminder: %(cfgemailalertscheduleslotreminder)s") % {'currentSource': str(currentSource), 'cfgemailalertscheduleslotreminder': str(configSource.getConfig('cfgemailalertscheduleslotreminder'))}) + + if configSource.getConfig('cfgemailalertscheduleslotgrace') != "" and int(configSource.getConfig('cfgemailalertscheduleslotgrace')) > 0: + #Offset current date by the grace period, this offset is used to include the time taken by the picture to arrive into the source (for example if uploaded from a remote webcampak) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Grace Period: Substracting a grace period of %(gracePeriod)s minutes from current date") % {'currentSource': str(currentSource), 'gracePeriod': configSource.getConfig('cfgemailalertscheduleslotgrace')}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Grace Period: Orignial Time %(currentTime)s") % {'currentSource': str(currentSource), 'currentTime': currentTime.isoformat()}) + currentTime = currentTime - timedelta(minutes=int(configSource.getConfig('cfgemailalertscheduleslotgrace'))) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Grace Period: Updated Time %(currentTime)s") % {'currentSource': str(currentSource), 'currentTime': currentTime.isoformat()}) + # If the latest picture has been captured after current time, look for an older picture as later captured pictures should be taken in consideration in subsequent script execution + if int(lastCaptureTime.strftime("%Y%m%d%H%M%S")) > int(currentTime.strftime("%Y%m%d%H%M%S")): + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Latest picture captured after the updated time with grace period. Looking for an older picture") % {'currentSource': str(currentSource)}) + latestPicture = self.sourcesUtils.getLatestPicture(currentSource, currentTime) + lastCaptureTime = self.timeUtils.getTimeFromFilename(latestPicture, configSource) + + missedCapture = self.getCountMissedSlots(currentTime, lastCaptureTime, sourceSchedule) + nextCaptureTime = self.getNextCaptureSlot(currentTime, sourceSchedule, configSource) + currentAlert.setAlertValue("missedCapture", missedCapture) + currentAlert.setAlertValue("nextCaptureTime", nextCaptureTime.isoformat()) + + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Total Missed Captures: %(missedCapture)s") % {'currentSource': str(currentSource), 'missedCapture': missedCapture}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Next Planned Capture Time: %(nextCaptureTime)s") % {'currentSource': str(currentSource), 'nextCaptureTime': str(nextCaptureTime.isoformat())}) + + cfgemailalertscheduleslotfailure = int(configSource.getConfig('cfgemailalertscheduleslotfailure')) + if missedCapture == 0: + scheduleAlertStatus = "GOOD" + elif missedCapture > 0 and int(cfgemailalertscheduleslotfailure) > missedCapture: + scheduleAlertStatus = "LATE" + elif missedCapture >= int(cfgemailalertscheduleslotfailure): + scheduleAlertStatus = "ERROR" + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Schedule slot Alert Status: %(scheduleAlertStatus)s") % {'currentSource': str(currentSource), 'scheduleAlertStatus': str(scheduleAlertStatus)}) + + # Calculate number of missed capture since last email + if lastEmail.getAlert() != {} and scheduleAlertStatus != "GOOD" and lastEmail.getAlertValue("lastCaptureTime") == currentAlert.getAlertValue("lastCaptureTime"): + lastEmailMissedCaptures = lastEmail.getAlertValue("missedCapture") + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Last Email Missed Captures: %(lastEmailMissedCaptures)s") % {'currentSource': str(currentSource), 'lastEmailMissedCaptures': str(lastEmailMissedCaptures)}) + + missedCaptureSinceLastEmail = missedCapture - lastEmailMissedCaptures + currentAlert.setAlertValue("missedCapturesSinceLastEmail", missedCaptureSinceLastEmail) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Missed Capture since last email: %(missedCaptureSinceLastEmail)s") % {'currentSource': str(currentSource), 'missedCaptureSinceLastEmail': str(missedCaptureSinceLastEmail)}) + + if missedCaptureSinceLastEmail >= int(configSource.getConfig('cfgemailalertscheduleslotreminder')): + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Requesting to send a reminder email based on number of missed captures since last email") % {'currentSource': str(currentSource)}) + currentAlert.setAlertValue("email", True) + currentAlert.setAlertValue("emailType", "REMINDER") + + else: + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Alert Schedule is empty for the source") % {'currentSource': str(currentSource)}) + + # Determine overall Alert Status + if (timeAlertStatus == "ERROR" or scheduleAlertStatus == "ERROR"): + alertStatus = "ERROR" + elif scheduleAlertStatus == "LATE": + alertStatus = "LATE" + else: + alertStatus = "GOOD" + + if lastAlert.getAlertValue("status") == "ERROR" and alertStatus == "GOOD": + alertStatus = "RECOVER" + + currentAlert.setAlertValue("status", alertStatus) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Overeall alert status: %(alertStatus)s") % {'currentSource': str(currentSource), 'alertStatus': str(alertStatus)}) + + lastAlertMissedCapture = lastAlert.getAlertValue("missedCapture") + lastAlertMinutesDiff = lastAlert.getAlertValue("minutesDiff") + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: cfgemailalwaysnotify: %(cfgemailalwaysnotify)s") % {'currentSource': str(currentSource), 'cfgemailalwaysnotify': configSource.getConfig('cfgemailalwaysnotify')}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: cfgemailalerttime: %(cfgemailalerttime)s") % {'currentSource': str(currentSource), 'cfgemailalerttime': configSource.getConfig('cfgemailalwaysnotify')}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: lastAlertMinutesDiff: %(lastAlertMinutesDiff)s") % {'currentSource': str(currentSource), 'lastAlertMinutesDiff': str(lastAlertMinutesDiff)}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: cfgemailalerttimefailure: %(cfgemailalerttimefailure)s") % {'currentSource': str(currentSource), 'cfgemailalerttimefailure': configSource.getConfig('cfgemailalerttimefailure')}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: cfgemailalertscheduleslot: %(cfgemailalertscheduleslot)s") % {'currentSource': str(currentSource), 'cfgemailalertscheduleslot': configSource.getConfig('cfgemailalertscheduleslot')}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: lastAlertMissedCapture: %(lastAlertMissedCapture)s") % {'currentSource': str(currentSource), 'lastAlertMissedCapture': str(lastAlertMissedCapture)}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: cfgemailalertscheduleslotfailure: %(cfgemailalertscheduleslotfailure)s") % {'currentSource': str(currentSource), 'cfgemailalertscheduleslotfailure': configSource.getConfig('cfgemailalertscheduleslotfailure')}) + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Validator for email alerts: lastAlert Status: %(lastAlertStatus)s") % {'currentSource': str(currentSource), 'lastAlertStatus': lastAlert.getAlertValue("status")}) + + # This section is used to determine if an email alert or a reminder should be sent + if alertStatus == "RECOVER": + if configSource.getConfig('cfgemailalwaysnotify') == "yes": + currentAlert.setAlertValue("email", True) + currentAlert.setAlertValue("emailType", "RECOVER") + elif configSource.getConfig('cfgemailalwaysnotify') == "no" and configSource.getConfig('cfgemailalerttime') == "yes" and (lastAlertMinutesDiff == None or int(lastAlertMinutesDiff) >= int(configSource.getConfig('cfgemailalerttimefailure'))): + currentAlert.setAlertValue("email", True) + currentAlert.setAlertValue("emailType", "RECOVER") + elif configSource.getConfig('cfgemailalwaysnotify') == "no" and configSource.getConfig('cfgemailalertscheduleslot') == "yes" and (lastAlertMissedCapture == None or int(lastAlertMissedCapture) >= int(cfgemailalertscheduleslotfailure)): + currentAlert.setAlertValue("email", True) + currentAlert.setAlertValue("emailType", "RECOVER") + elif alertStatus == "ERROR" and (lastAlert.getAlertValue("status") == "GOOD" or lastAlert.getAlertValue("status") == "LATE" or lastAlert.getAlertValue("status") == "RECOVER" or lastEmail.getAlert() == {}): + currentAlert.setAlertValue("email", True) + currentAlert.setAlertValue("emailType", "NEW") + + if currentAlert.getAlertValue("email") == True: + self.log.info("alertsCapture.run(): " + _("Source: %(currentSource)s - Requesting an email alert to be sent for the source") % {'currentSource': str(currentSource)}) + currentAlert.writeAlertFile(lastEmailFile) + if currentAlert.getAlertValue("emailType") == "RECOVER": + self.alertsEmails.sendCaptureSuccess(currentAlert) + else: + self.alertsEmails.sendCaptureError(currentAlert) + + currentAlert.archiveAlertFile() + currentAlert.writeAlertFile(lastAlertFile) + else: + self.log.info("alertsCapture.run(): " + _("Schedule based email alerts disabled for the source")) + self.log.info("alertsCapture.run(): " + _("---------")) + + + def getNextCaptureSlot(self, currentTime, sourceSchedule, configSource): + """ Calculates the next expected capture slot based on calendar """ + self.log.debug("alertsCapture.getNextCaptureSlot(): " + _("Start")) + + sourceTimeDayOfWeek = currentTime.strftime("%w") + if sourceTimeDayOfWeek == 0: # Sunday is 7, not 0 + sourceTimeDayOfWeek = 7 + sourceTimeHour = currentTime.strftime("%H") + sourceTimeMinute = currentTime.strftime("%M") + sourceTimeWeek = currentTime.strftime("%W") + sourceTimeYear = currentTime.strftime("%Y") + sourceTargetWeek = sourceTimeWeek + sourceTime = int(str(sourceTimeDayOfWeek) + str(sourceTimeHour) + str(sourceTimeMinute)) + + nextScanTime = None + for scanTime in sorted(sourceSchedule): + if sourceSchedule[scanTime] == "Y": + self.log.debug("alertsCapture.getNextCaptureSlot(): " + _("Scanning Day: %(scanDay)s Hour: %(scanHour)s minute: %(scanMinute)s - Status: %(slotActive)s") % {'scanDay': str(scanTime)[0], 'scanHour': str(scanTime)[1:3], 'scanMinute': str(scanTime)[3:6], 'slotActive': sourceSchedule[scanTime]}) + if scanTime >= sourceTime: + nextScanTime = scanTime + break + if nextScanTime == None: + sourceTargetWeek = sourceTimeWeek + 1 + for scanTime in sorted(sourceSchedule): + if sourceSchedule[scanTime] == "Y": + self.log.debug("alertsCapture.getNextCaptureSlot(): " + _("Scanning Day: %(scanDay)s Hour: %(scanHour)s minute: %(scanMinute)s - Status: %(slotActive)s") % {'scanDay': str(scanTime)[0], 'scanHour': str(scanTime)[1:3], 'scanMinute': str(scanTime)[3:6], 'slotActive': sourceSchedule[scanTime]}) + nextScanTime = scanTime + break + + self.log.info("alertsCapture.getNextCaptureSlot(): " + _("Next Capture slot: %(nextScanTime)s") % {'nextScanTime': nextScanTime}) + + # Build next capture date + targetDayOfWeek = int(str(scanTime)[0]) + if (targetDayOfWeek == 7): + sourceTargetWeek = int(sourceTargetWeek) + 1 + targetDayOfWeek = 0 + if sourceTargetWeek == 53: + sourceTimeYear = int(sourceTimeYear) + 1 + sourceTargetWeek = 0 + + nextCaptureTime = datetime.strptime(str(sourceTimeYear) + "-" + str(sourceTargetWeek) + "-" + str(targetDayOfWeek) + "-" + str(nextScanTime)[1:3] + "-" + str(nextScanTime)[3:6], "%Y-%W-%w-%H-%M") + + if configSource.getConfig('cfgcapturetimezone') != "": # Update the timezone from UTC to the source's timezone + self.log.info("alertsCapture.getNextCaptureSlot(): " + _("Source timezone is: %(sourceTimezone)s") % {'sourceTimezone': configSource.getConfig('cfgcapturetimezone')}) + sourceTimezone = tz.gettz(configSource.getConfig('cfgcapturetimezone')) + nextCaptureTime = nextCaptureTime.replace(tzinfo=sourceTimezone) + + return nextCaptureTime + + + def getCountMissedSlots(self, currentTime, lastCaptureTime, sourceSchedule): + """ Calculate the number of missed slots between last captured picture and current date using capture schedule """ + self.log.debug("alertsCapture.getCountMissedSlots(): " + _("Start")) + + sourceTimeDayOfWeek = currentTime.strftime("%w") + if sourceTimeDayOfWeek == 0: # Sunday is 7, not 0 + sourceTimeDayOfWeek = 7 + sourceTimeHour = currentTime.strftime("%H") + sourceTimeMinute = currentTime.strftime("%M") + sourceTimeWeek = currentTime.strftime("%W") + sourceTimeYear = currentTime.strftime("%Y") + sourceTime = int(str(sourceTimeDayOfWeek) + str(sourceTimeHour) + str(sourceTimeMinute)) + + captureTimeDayOfWeek = lastCaptureTime.strftime("%w") + if captureTimeDayOfWeek == 0: # Sunday is 7, not 0 + captureTimeDayOfWeek = 7 + captureTimeHour = lastCaptureTime.strftime("%H") + captureTimeMinute = lastCaptureTime.strftime("%M") + captureTimeWeek = lastCaptureTime.strftime("%W") + captureTimeYear = lastCaptureTime.strftime("%Y") + captureTime = int(str(captureTimeDayOfWeek) + str(captureTimeHour) + str(captureTimeMinute)) + + missedCaptureRoundOne = 0 + missedCaptureRoundTwo = 0 + missedPicturesInDiffWeek = 0 + fullWeekCaptures = len(sourceSchedule) + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Analyzing source schedule")) + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Source Time: %(sourceTime)s")% {'sourceTime': sourceTime}) + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Capture Time: %(captureTime)s")% {'captureTime': captureTime}) + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Number of captures in full week: %(fullWeekCaptures)s")% {'fullWeekCaptures': fullWeekCaptures}) + if (captureTimeWeek != sourceTimeWeek): + diffWeek = ((int(sourceTimeYear)*52) + int(sourceTimeWeek)) - ((int(captureTimeYear)*52) + int(captureTimeWeek)) - 1 + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Number of week difference: %(diffWeek)s")% {'diffWeek': diffWeek}) + missedPicturesInDiffWeek = diffWeek * fullWeekCaptures + + # Scan all capture times backward, and count number of slots until it get a match between capture slot and capture time, if no match it keeps going + for scanTime in reversed(sorted(sourceSchedule)): + if sourceSchedule[scanTime] == "Y": + if scanTime == captureTime and sourceTimeWeek == captureTimeWeek: + break + if scanTime <= sourceTime: + missedCaptureRoundOne += 1 + self.log.debug("alertsCapture.getCountMissedSlots(): " + _("Scanning Day: %(scanDay)s Hour: %(scanHour)s minute: %(scanMinute)s - Status: %(slotActive)s") % {'scanDay': str(scanTime)[0], 'scanHour': str(scanTime)[1:3], 'scanMinute': str(scanTime)[3:6], 'slotActive': sourceSchedule[scanTime]}) + + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Number of missed captures in round 1: %(missedCaptureRoundOne)s")% {'missedCaptureRoundOne': missedCaptureRoundOne}) + + if sourceTimeWeek != captureTimeWeek: + # Scan all capture times backward, and count number of slots until it get a match between capture slot and capture time, if no match it keeps going + for scanTime in reversed(sorted(sourceSchedule)): + if sourceSchedule[scanTime] == "Y": + if scanTime == captureTime: + break + if scanTime >= captureTime: + missedCaptureRoundTwo += 1 + self.log.debug("alertsCapture.getCountMissedSlots(): " + _("Scanning Day: %(scanDay)s Hour: %(scanHour)s minute: %(scanMinute)s - Status: %(slotActive)s") % {'scanDay': str(scanTime)[0], 'scanHour': str(scanTime)[1:3], 'scanMinute': str(scanTime)[3:6], 'slotActive': sourceSchedule[scanTime]}) + + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Number of missed captures in round 2: %(missedCaptureRoundTwo)s")% {'missedCaptureRoundTwo': missedCaptureRoundTwo}) + + missedCapture = missedCaptureRoundOne + missedCaptureRoundTwo + missedPicturesInDiffWeek + + + """ + for scanDay in reversed(sorted(sourceSchedule)): + for scanHour in reversed(sorted(sourceSchedule[scanDay])): + for scanMinute in reversed(sorted(sourceSchedule[scanDay][scanHour])): + if sourceSchedule[scanDay][scanHour][scanMinute] == "Y": + if sourceTimeDayOfWeek == scanDay and scanHour == sourceTimeHour and scanMinute <= sourceTimeMinute: + missedCapture += 1 + + self.log.info("alertsCapture.getCountMissedSlots(): " + _("Scanning Day: %(scanDay)s Hour: %(scanHour)s minute: %(scanMinute)s - Status: %(slotActive)s") % {'scanDay': scanDay, 'scanHour': scanHour, 'scanMinute': scanMinute, 'slotActive': sourceSchedule[scanDay][scanHour][scanMinute]}) + """ + return missedCapture + + def getSourceSchedule(self, sourceId): + """ Verify if schedule exists for the source """ + self.log.debug("alertsCapture.checkScheduleActive(): " + _("Start")) + sourceScheduleFile = self.dirEtc + 'config-source' + str(sourceId) + '-schedule.json' + if os.path.isfile(sourceScheduleFile): + try: + with open(sourceScheduleFile) as sourceSchedule: + sourceScheduleObj = json.load(sourceSchedule) + sourceScheduleNum = self.convertScheduleToFlat(sourceScheduleObj) + return sourceScheduleNum + except Exception: + self.log.error("alertsCapture.getSourceSchedule(): " + _("File appears corrupted: %(sourceScheduleFile)s ") % {'sourceScheduleFile': sourceScheduleFile}) + else: + return {} + + def convertScheduleToNumericalIndex(self, sourceSchedule): + self.log.debug("alertsCapture.convertScheduleToNumericalIndex(): " + _("Start")) + self.log.info("alertsCapture.convertScheduleToNumericalIndex(): " + _("Converting object to numerical index")) + sourceScheduleNum = {} + for scanDay in sourceSchedule: + scanDayNum = int(scanDay) + sourceScheduleNum[scanDayNum] = {}; + for scanHour in sourceSchedule[scanDay]: + scanHourNum = int(scanHour) + sourceScheduleNum[scanDayNum][scanHourNum] = {}; + for scanMinute in sourceSchedule[scanDay][scanHour]: + scanMinuteNum = int(scanMinute) + sourceScheduleNum[scanDayNum][scanHourNum][scanMinuteNum] = sourceSchedule[scanDay][scanHour][scanMinute] + return sourceScheduleNum + + # As a tentative to simplify the core, return the schedule as a flat array + def convertScheduleToFlat(self, sourceSchedule): + self.log.debug("alertsCapture.convertScheduleToNumericalIndex(): " + _("Start")) + self.log.info("alertsCapture.convertScheduleToNumericalIndex(): " + _("Converting object to flat array")) + sourceScheduleFlat = {} + for scanDay in sourceSchedule: + scanDayNum = int(scanDay) + for scanHour in sourceSchedule[scanDay]: + scanHourNum = int(scanHour) + if scanHourNum < 10: + scanHourTxt = "0" + str(scanHourNum) + else: + scanHourTxt = str(scanHourNum) + for scanMinute in sourceSchedule[scanDay][scanHour]: + scanMinuteNum = int(scanMinute) + if scanMinuteNum < 10: + scanMinuteTxt = "0" + str(scanMinuteNum) + else: + scanMinuteTxt = str(scanMinuteNum) + fullKey = int(str(scanDayNum) + scanHourTxt + scanMinuteTxt) + sourceScheduleFlat[fullKey] = sourceSchedule[scanDay][scanHour][scanMinute] + return sourceScheduleFlat \ No newline at end of file diff --git a/webcampak/core/wpakAlertsEmails.py b/webcampak/core/wpakAlertsEmails.py new file mode 100644 index 0000000..f1ddd46 --- /dev/null +++ b/webcampak/core/wpakAlertsEmails.py @@ -0,0 +1,157 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright 2010-2016 Eurotechnia (support@webcampak.com) +# This file is part of the Webcampak project. +# Webcampak is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +# Webcampak is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License along with Webcampak. +# If not, see http://www.gnu.org/licenses/ + +import os +import time +import gettext +import json +from datetime import tzinfo, timedelta, datetime +import pytz +from dateutil import tz +import dateutil.parser +from tabulate import tabulate +import socket + +from wpakConfigObj import Config +from wpakConfigCache import configCache +from wpakTimeUtils import timeUtils +from wpakSourcesUtils import sourcesUtils +from wpakFileUtils import fileUtils +from wpakDbUtils import dbUtils +from wpakEmailObj import emailObj +from wpakFTPUtils import FTPUtils +from wpakAlertsObj import alertObj + +class alertsEmails(object): + """ This class contains functions used to send email to source users + + Args: + parentClass: The parent class + + Attributes: + tbc + """ + + def __init__(self, parentClass): + self.log = parentClass.log + + self.configPaths = parentClass.configPaths + self.dirSources = self.configPaths.getConfig('parameters')['dir_sources'] + self.dirLocale = self.configPaths.getConfig('parameters')['dir_locale'] + self.dirLocaleEmails = self.configPaths.getConfig('parameters')['dir_locale_emails'] + self.dirEmails = self.configPaths.getConfig('parameters')['dir_emails'] + + self.configGeneral = parentClass.configGeneral + self.dbUtils = parentClass.dbUtils + self.configCache = parentClass.configCache + self.fileUtils = parentClass.fileUtils + + def loadEmailTemplateFile(self, configSource, TemplateFilename): + """Simple function to load an email template (either subject or content)""" + self.log.debug("alertsEmails.loadEmailTemplateFile(): " + _("Start")) + + templateFile = self.dirLocale + configSource.getConfig('cfgsourcelanguage') + "/" + self.dirLocaleEmails + TemplateFilename + if os.path.isfile(templateFile) == False: + templateFile = self.dirLocale + "en_US.utf8/" + self.dirLocaleEmails + TemplateFilename + if os.path.isfile(templateFile): + self.log.info("alertsEmails.sendCaptureSuccess(): " + _("Using message subject file: %(templateFile)s") % {'templateFile': templateFile}) + templateFileContent = open(templateFile, 'r') + templateContent = templateFileContent.read() + templateFileContent.close() + return templateContent + else: + return None + + + def sendCaptureError(self, currentAlert): + """ This function queue an email to inform the user that the capture is failing + The email's content and subject is store within the locale's directory corresponding to the language configured for the source. + + Args: + currentAlert: a capture alert object + + Returns: + None + """ + self.log.debug("alertsEmails.sendCaptureSuccess(): " + _("Start")) + configSource = self.configCache.getSourceConfig("source", currentAlert.getAlertValue("sourceid")) + + emailContent = self.loadEmailTemplateFile(configSource, "alertErrorContent.txt") + if currentAlert.getAlertValue("emailType") == "REMINDER": + emailSubject = self.loadEmailTemplateFile(configSource, "alertErrorReminderSubject.txt") + else: + emailSubject = self.loadEmailTemplateFile(configSource, "alertErrorSubject.txt") + + if emailContent != None and emailSubject != None: + currentSourceTime = dateutil.parser.parse(currentAlert.getAlertValue("currentTime")) + lastCatpureTime = dateutil.parser.parse(currentAlert.getAlertValue("lastCaptureTime")) + emailSubject = emailSubject.replace("#CURRENTHOSTNAME#", socket.gethostname()) + emailSubject = emailSubject.replace("#CURRENTSOURCE#", str(currentAlert.getAlertValue("sourceid"))) + emailSubject = emailSubject.replace("#CURRENTSOURCENAME#", self.dbUtils.getSourceName(currentAlert.getAlertValue("sourceid"))) + emailSubject = emailSubject.replace("#LASTCAPTURETIME#", lastCatpureTime.strftime("%c")) + emailContent = emailContent.replace("#CURRENTSOURCETIME#", currentSourceTime.strftime("%c")) + emailContent = emailContent.replace("#LASTCAPTURETIME#", lastCatpureTime.strftime("%c")) + newEmail = emailObj(self.log, self.dirEmails, self.fileUtils) + newEmail.setFrom({'email': self.configGeneral.getConfig('cfgemailsendfrom')}) + newEmail.setTo(self.dbUtils.getSourceEmailUsers(currentAlert.getAlertValue("sourceid"))) + newEmail.setBody(emailContent) + newEmail.setSubject(emailSubject) + newEmail.writeEmailObjectFile() + else: + self.log.info("alertsEmails.sendCaptureSuccess(): " + _("Unable to find default translation files to be used")) + + def sendCaptureSuccess(self, currentAlert): + """ This function queue an email to inform the user that the capture is successful. + The email's content and subject is store within the locale's directory corresponding to the language configured for the source. + If a filename is provided, a picture can be sent along the email. + + Args: + currentAlert: a capture alert object + + Returns: + None + """ + self.log.debug("alertsEmails.sendCaptureSuccess(): " + _("Start")) + configSource = self.configCache.getSourceConfig("source", currentAlert.getAlertValue("sourceid")) + + emailContent = self.loadEmailTemplateFile(configSource, "alertWorkingContent.txt") + emailSubject = self.loadEmailTemplateFile(configSource, "alertWorkingSubject.txt") + + if emailContent != None and emailSubject != None: + emailSubject = emailSubject.replace("#CURRENTHOSTNAME#", socket.gethostname()) + emailSubject = emailSubject.replace("#CURRENTSOURCE#", str(currentAlert.getAlertValue("sourceid"))) + emailSubject = emailSubject.replace("#CURRENTSOURCENAME#", self.dbUtils.getSourceName(currentAlert.getAlertValue("sourceid"))) + currentSourceTime = dateutil.parser.parse( currentAlert.getAlertValue("currentTime")) + lastCatpureTime = dateutil.parser.parse( currentAlert.getAlertValue("lastCaptureTime")) + emailContent = emailContent.replace("#CURRENTSOURCETIME#", currentSourceTime.strftime("%c")) + emailContent = emailContent.replace("#LASTCAPTURETIME#", lastCatpureTime.strftime("%c")) + newEmail = emailObj(self.log, self.dirEmails, self.fileUtils) + newEmail.setFrom({'email': self.configGeneral.getConfig('cfgemailsendfrom')}) + newEmail.setTo(self.dbUtils.getSourceEmailUsers(currentAlert.getAlertValue("sourceid"))) + newEmail.setBody(emailContent) + newEmail.setSubject(emailSubject) + if currentAlert.getAlertValue("lastPicture") != None and int(configSource.getConfig('cfgemailsuccesspicturewidth')) > 0: + captureDirectory = currentAlert.getAlertValue("lastPicture")[:8] + dirCurrentSourcePictures = self.dirSources + 'source' + str(currentAlert.getAlertValue("sourceid")) + '/' + self.configPaths.getConfig('parameters')['dir_source_pictures'] + if os.path.isfile(dirCurrentSourcePictures + captureDirectory + "/" + currentAlert.getAlertValue("lastPicture")): + captureFilename = dirCurrentSourcePictures + captureDirectory + "/" + currentAlert.getAlertValue("lastPicture") + newEmail.addAttachment({'PATH': captureFilename, + 'WIDTH': int(configSource.getConfig('cfgemailsuccesspicturewidth')), + 'NAME': 'last-capture.jpg'}) + newEmail.writeEmailObjectFile() + else: + self.log.info("alertsEmails.sendCaptureSuccess(): " + _("Unable to find default translation files to be used")) + diff --git a/webcampak/core/wpakAlertsObj.py b/webcampak/core/wpakAlertsObj.py new file mode 100644 index 0000000..e6fbced --- /dev/null +++ b/webcampak/core/wpakAlertsObj.py @@ -0,0 +1,183 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright 2010-2016 Eurotechnia (support@webcampak.com) +# This file is part of the Webcampak project. +# Webcampak is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +# Webcampak is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License along with Webcampak. +# If not, see http://www.gnu.org/licenses/ + +import os +import json +import dateutil.parser +import jsonschema +import subprocess + +from wpakFileUtils import fileUtils + + +class alertObj(object): + """ Builds an object containing details about an alert + + Args: + log: A class, the logging interface + alertFile: A string, path to a jsonl file containing an archive of all capture objects for a specific day + + Attributes: + log: A class, the logging interface + alertFile: A string, path to a jsonl file containing an archive of all capture objects for a specific day + lastAlert: A dictionary, containing all values of the capture object + """ + + def __init__(self, log, alertFile=None): + self.log = log + self.alertFile = alertFile + + # Declare the schema used + # The schema is validate each time single values are set or the entire dictionary is loaded or set + self.schema = { + "$schema": "http://json-schema.org/draft-04/schema#" + , "title": "alertObj" + , "description": "Used to log details associated with a capture" + , "type": "object" + , "additionalProperties": False + , "properties": { + "sourceid": {"type": ["number", "null"], "description": "ID of the source"} + , "status": {"type": ["string", "null"], "description": "Status of the alert (GOOD, ERROR, LATE)"} + , "email": {"type": ["boolean", "null"], "description": "Record if an email will be sent"} + , "emailType": {"type": ["string", "null"], "description": "Type of the email (NEW, REMINDER)"} + , "currentTime": {"type": ["string", "null"], "description": "Current time for the source"} + , "lastPicture": {"type": ["string", "null"], "description": "Filename of the last picture captured"} + , "lastCaptureTime": {"type": ["string", "null"], "description": "Last time capture was done according to schedule"} + , "nextCaptureTime": {"type": ["string", "null"], "description": "Next time a capture is scheduled"} + , "secondsSinceLastCapture": {"type": ["number", "null"], "description": "Number of seconds between current source time and last schedule-based capture"} + , "missedCapture": {"type": ["number", "null"], "description": "Number of missed captures as per the calendar"} + , "incidentFile": {"type": ["string", "null"], "description": "Filename used to record the incident"} + , "missedCapturesSinceLastEmail": {"type": ["number", "null"], "description": "Number of missed captures since last email"} + , "secondsSinceLastEmail": {"type": ["number", "null"], "description": "Number of seconds since last email"} + } + } + self.initAlert() + + # Getters and Setters + def setAlertValue(self, index, value): + self.lastAlert[index] = value + jsonschema.validate(self.lastAlert, self.schema) + + def getAlertValue(self, index): + if (self.lastAlert.has_key(index)): + return self.lastAlert[index] + else: + return None + + def setAlert(self, lastAlert): + jsonschema.validate(lastAlert, self.schema) + self.lastAlert = lastAlert + + def getAlert(self): + jsonschema.validate(self.lastAlert, self.schema) + return self.lastAlert + + def setAlertFile(self, alertFile): + self.log.info("alertObj.setAlertFile(): " + _("Alert file set to: %(alertFile)s") % {'alertFile': alertFile}) + self.alertFile = alertFile + + def getAlertFile(self): + return self.alertFile + + def initAlert(self): + """Initialize the object values to 0 or None""" + self.log.debug("alertObj.initAlert(): " + _("Start")) + self.lastAlert = {} + + def getLastAlertTime(self): + """Return the last capture date from the object""" + self.log.debug("alertObj.getLastAlertTime(): " + _("Start")) + try: + lastAlertTime = dateutil.parser.parse(self.getAlertValue('captureDate')) + self.log.info("alertObj.getLastAlertTime(): " + _("Last capture time: %(lastAlertTime)s") % { + 'lastAlertTime': lastAlertTime}) + return lastAlertTime + except: + return None + + def loadAlertFile(self): + """Load the capture file into memory, if there was no previous capture, return an initialized version of the object""" + self.log.debug("alertObj.loadAlertFile(): " + _("Start")) + lastAlert = self.loadJsonFile(self.getAlertFile()) + if lastAlert != None: + self.setAlert(lastAlert) + else: + self.initAlert() + + def writeAlertFile(self, alertFile = None): + """Write the content of the object into a capture file""" + self.log.debug("alertObj.writeAlertFile(): " + _("Start")) + if alertFile == None: + alertFile = self.alertFile + self.log.info("alertObj.writeAlertFile(): " + _("Preparing to write to: %(alertFile)s") % {'alertFile': str(alertFile)}) + if self.writeJsonFile(alertFile, self.getAlert()) == True: + self.log.info("alertObj.writeAlertFile(): " + _("Successfully saved last capture file to: %(alertFile)s") % {'alertFile': str(alertFile)}) + return True + else: + self.log.error("alertObj.writeAlertFile(): " + _("Error saving last capture file")) + return False + + def archiveAlertFile(self): + """Append the content of the object into a log file containing previous captures""" + self.log.debug("alertObj.archiveAlertFile(): " + _("Start")) + if self.archiveJsonFile(self.alertFile, self.getAlert()) == True: + self.log.info("alertObj.archiveAlertFile(): " + _("Successfully archived capture file to: %(alertFile)s") % {'alertFile': str(self.alertFile)}) + return True + else: + self.log.error("alertObj.archiveAlertFile(): " + _("Error saving last capture file")) + return False + + def loadLastAlert(self): + """Load the last alert into the object""" + self.log.debug("alertObj.loadLastAlert(): " + _("Start")) + if os.path.isfile(self.alertFile): + alertJson = self.getLastLine(self.alertFile) + self.lastAlert = json.loads(alertJson) + else: + self.initAlert() + + def loadJsonFile(self, jsonFile): + """Loads the content of a JSON file""" + self.log.debug("alertObj.loadJsonFile(): " + _("Start")) + if os.path.isfile(jsonFile): + self.log.info("alertObj.loadJsonFile(): " + _("Load JSON file into memory: %(jsonFile)s") % {'jsonFile': jsonFile}) + with open(jsonFile) as threadJsonFile: + threadJson = json.load(threadJsonFile) + return threadJson + return None + + def writeJsonFile(self, jsonFile, jsonContent): + """Write the content of a dictionary to a JSON file""" + self.log.info("alertObj.writeJsonFile(): " + _("Writing to: %(jsonFile)s") % {'jsonFile': jsonFile}) + if fileUtils.CheckFilepath(jsonFile) != "": + with open(jsonFile, "w") as threadJsonFile: + threadJsonFile.write(json.dumps(jsonContent)) + return True + return False + + def archiveJsonFile(self, jsonFile, jsonContent): + """Append the content of a dictionary to a JSONL file""" + self.log.info("alertObj.archiveJsonFile(): " + _("Writing to: %(jsonFile)s") % {'jsonFile': jsonFile}) + if fileUtils.CheckFilepath(jsonFile) != "": + with open(jsonFile, "a+") as threadJsonFile: + threadJsonFile.write(json.dumps(jsonContent) + '\n') + return True + return False + + def getLastLine(self, jsonFile): + """Append the content of a dictionary to a JSONL file""" + self.log.info("alertObj.getLastLine(): " + _("Get last alert line of file: %(jsonFile)s") % {'jsonFile': jsonFile}) + return subprocess.check_output(['tail', '-1', jsonFile])[0:-1] diff --git a/webcampak/core/wpakCapture.py b/webcampak/core/wpakCapture.py index 0245e98..aa13180 100644 --- a/webcampak/core/wpakCapture.py +++ b/webcampak/core/wpakCapture.py @@ -321,79 +321,12 @@ def run(self): storedJpgSize + storedRawSize)) processedPicturesCount = processedPicturesCount + 1 - # We check if the previous capture was failing, if yes, send a capture success email - if os.path.isfile(self.dirCache + "source" + self.currentSourceId + "-errorcount"): - currentErrorCount = self.captureUtils.getCustomCounter('errorcount') - self.log.info("capture.run(): " + _( - "Process found that previous capture(s) failed, error count: %(currentErrorCount)s") % { - 'currentErrorCount': str(currentErrorCount)}) - if (currentErrorCount >= int( - self.configSource.getConfig('cfgemailalertfailure')) or self.configSource.getConfig( - 'cfgemailalwaysnotify') == "yes") and self.configSource.getConfig( - 'cfgemailerroractivate') == "yes": - self.log.info( - "capture.run(): " + _("Preparation of an email to inform that the system is back")) - self.captureEmails.sendCaptureSuccess(self.captureFilename) - else: - self.log.info("capture.run(): " + _( - "Not enough capture errors to trigger an action. Threshold: %(currentErrorThreshold)s errors") % { - 'currentErrorThreshold': str( - self.configSource.getConfig('cfgemailalertfailure'))}) - self.log.info( - "capture.run(): " + _("Deleting 'errorcount' file for source (to reset error counter)")) - os.remove(self.dirCache + "source" + self.currentSourceId + "-errorcount") - - # At this point the file is deemed valid, we can therefore delete any possible error count - if os.path.isfile(self.dirCache + "source" + self.currentSourceId + "-errorcount"): - os.remove(self.dirCache + "source" + self.currentSourceId + "-errorcount") - if os.path.isfile(self.dirCache + "source" + self.currentSourceId + "-errorcountemail"): - os.remove(self.dirCache + "source" + self.currentSourceId + "-errorcountemail") - if os.path.isfile(self.dirCache + "source" + self.currentSourceId + "-errorcountphidget"): - os.remove(self.dirCache + "source" + self.currentSourceId + "-errorcountphidget") - self.log.info("capture.run(): " + _("Capture process completed")) self.currentCaptureDetails.setCaptureValue('captureSuccess', True) else: self.log.info("capture.run(): " + _("Unable to capture picture")) self.captureUtils.generateFailedCaptureHotlink() self.currentCaptureDetails.setCaptureValue('captureSuccess', False) - previousErrorCount = self.captureUtils.getCustomCounter('errorcount') - self.log.info("capture.run(): " + _("Previous Error Count was: %(previousErrorCount)s") % { - 'previousErrorCount': previousErrorCount}) - currentErrorCount = previousErrorCount + 1 - self.captureUtils.setCustomCounter('errorcount', currentErrorCount) - - # If the system is configured to send an email in case of capture error - # It stores a counter of the number of failure since last email - # If over the reminder, will reset this counter and resend the email - if int(currentErrorCount) >= int( - self.configSource.getConfig('cfgemailalertfailure')) and self.configSource.getConfig( - 'cfgemailerroractivate') == "yes" and self.lastCaptureDetails.getLastCaptureTime() != None: - self.log.info( - "capture.run(): " + _("Last Successful capture took place at: %(lastSuccessCapture)s") % { - 'lastSuccessCapture': str(self.lastCaptureDetails.getLastCaptureTime().isoformat())}) - if os.path.isfile(self.dirCache + "source" + self.currentSourceId + "-errorcountemail"): - currentEmailCounter = self.captureUtils.getCustomCounter("errorcountemail") + 1 - self.captureUtils.setCustomCounter("errorcountemail", currentEmailCounter) - if int(self.configSource.getConfig('cfgemailalertreminder')) == int(currentEmailCounter): - self.log.info("capture.run(): " + _( - "Error counter is: %(currentEmailCounter)s (Total: %(currentErrorCount)s), sending a reminder (= %(cfgemailalertreminder)s)") % { - 'currentEmailCounter': str(currentEmailCounter), - 'currentErrorCount': str(currentErrorCount), - 'cfgemailalertreminder': self.configSource.getConfig( - 'cfgemailalertreminder')}) - self.captureUtils.setCustomCounter("errorcountemail", 0) - if self.captureUtils.getCustomCounter("errorcountemail") >= 1: - self.log.info("capture.run(): " + _( - "Error email already sent, error counter since last email: %(currentEmailCounter)s, next email at: %(cfgemailalertreminder)s") % { - 'currentEmailCounter': str(currentEmailCounter), - 'cfgemailalertreminder': self.configSource.getConfig( - 'cfgemailalertreminder')}) - else: - if self.configSource.getConfig('cfgemaildirectalert') == "yes": - self.captureEmails.sendCaptureError(currentErrorCount, - self.lastCaptureDetails.getLastCaptureTime()) - self.captureUtils.setCustomCounter("errorcountemail", "1") if self.configSource.getConfig('cfgcapturedeleteafterdays') != "0": # Purge old pictures (by day) diff --git a/webcampak/core/wpakConfigCache.py b/webcampak/core/wpakConfigCache.py new file mode 100644 index 0000000..3cca9e0 --- /dev/null +++ b/webcampak/core/wpakConfigCache.py @@ -0,0 +1,62 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# Copyright 2010-2016 Eurotechnia (support@webcampak.com) +# This file is part of the Webcampak project. +# Webcampak is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, +# or (at your option) any later version. + +# Webcampak is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License along with Webcampak. +# If not, see http://www.gnu.org/licenses/ + + +from wpakConfigObj import Config + +class configCache(object): + """ A very simple class used to cache source configuration and avoid re-reading from the same file too frequently + + Args: + log: A class, the logging interface + appConfig: A class, the app config interface + config_dir: A string, filesystem location of the configuration directory + sourceId: Source ID of the source to capture + + Attributes: + tbc + """ + + def __init__(self, parentClass): + self.log = parentClass.log + self.configCache = {} + + def loadSourceConfig(self, type, filepath, sourceId = None): + """ Load the source configuration + + Args: + type: Type of configuration file + filepath: Full path of the configuration file + sourceId: Source id of the configuration file, if None, means its config-general + + """ + self.log.debug("configCache.loadSourceConfig(): " + _("Start")) + if sourceId == None: + sourceId = 0 + + self.configCache[sourceId] = {} + self.configCache[sourceId][type] = Config(self.log, filepath) + return self.configCache[sourceId][type] + + def getSourceConfig(self, type, sourceId = None): + """Get source config previously loaded""" + self.log.debug("configCache.getSourceConfig(): " + _("Start")) + if sourceId == None: + sourceId = 0 + + return self.configCache[sourceId][type] + + diff --git a/webcampak/core/wpakConfigObj.py b/webcampak/core/wpakConfigObj.py index 074aa7e..5b9196d 100644 --- a/webcampak/core/wpakConfigObj.py +++ b/webcampak/core/wpakConfigObj.py @@ -21,7 +21,6 @@ from configobj import ConfigObj - # This class is used to set or get values from configobj functions class Config: def __init__(self, log, filePath): @@ -50,7 +49,11 @@ def getFullConfig(self): ## key: configuration key # Return: configuration value def getConfig(self, key): - return self.currentConfig[key] + if key in self.currentConfig: + return self.currentConfig[key] + else: + self.log.error("Config.getConfig(): Unable to find config key: " + str(key)) + return None # Function: setConfig # Description; Function used to set configuration settings diff --git a/webcampak/core/wpakDbUtils.py b/webcampak/core/wpakDbUtils.py index c67b415..d0682b1 100644 --- a/webcampak/core/wpakDbUtils.py +++ b/webcampak/core/wpakDbUtils.py @@ -60,8 +60,8 @@ def getSourceEmailUsers(self, sourceId): users.append({'name': firstname + ' ' + lastname, 'email': email}) return users - def getUserWithSourceAlerts(self): - self.log.debug("dbUtils.getUserWithSourceAlerts(): " + _("Start")) + def getUsersWithSourceAlerts(self): + self.log.debug("dbUtils.getUsersWithSourceAlerts(): " + _("Start")) if (self.dbConnection == None): self.openDb() @@ -93,6 +93,27 @@ def getUserWithSourceAlerts(self): self.closeDb() return users + def getUsersAlertsForSource(self, sourceId): + self.log.debug("dbUtils.getUsersAlertsForSource(): " + _("Start")) + if (self.dbConnection == None): + self.openDb() + + dbQuery = "SELECT USE.USE_ID USE_ID, USE.EMAIL EMAIL, USE.FIRSTNAME FIRSTNAME, USE.LASTNAME LASTNAME \ + FROM USERS USE \ + LEFT JOIN USERS_SOURCES USESOU ON USE.USE_ID = USESOU.USE_ID \ + LEFT JOIN SOURCES SOU ON USESOU.SOU_ID = SOU.SOU_ID \ + WHERE USESOU.ALERTS_FLAG = 'Y' AND SOU.SOU_ID = :souId\ + GROUP BY USE.USE_ID\ + ORDER BY USE.USERNAME"; + + self.dbCursor.execute(dbQuery, {'souId': sourceId}) + users = [] + for row in self.dbCursor.fetchall(): + useId, email, firstname, lastname = row + users.append({'useId': useId, 'name': firstname + ' ' + lastname, 'email': email}) + self.closeDb() + return users + def getSourcesForUser(self, useId): self.log.debug("dbUtils.getSourcesForUser(): " + _("Start")) if (self.dbConnection == None): diff --git a/webcampak/core/wpakFileUtils.py b/webcampak/core/wpakFileUtils.py index bba4153..35b0b22 100644 --- a/webcampak/core/wpakFileUtils.py +++ b/webcampak/core/wpakFileUtils.py @@ -22,13 +22,11 @@ from dateutil import tz import time - class fileUtils: def __init__(self, parentClass): self.log = parentClass.log self.config_dir = parentClass.config_dir self.configPaths = parentClass.configPaths - self.configSource = parentClass.configSource self.configGeneral = parentClass.configGeneral # self.dirEtc = self.configPaths.getConfig('parameters')['dir_etc'] diff --git a/webcampak/core/wpakReportsDaily.py b/webcampak/core/wpakReportsDaily.py index 73ed62a..b1e2116 100644 --- a/webcampak/core/wpakReportsDaily.py +++ b/webcampak/core/wpakReportsDaily.py @@ -166,7 +166,7 @@ def run(self): 'jsonReportFile': str(jsonReportFile)}) self.log.info("reportsDaily.run(): " + _("Getting ready to send reports")) - for currentUser in self.dbUtils.getUserWithSourceAlerts(): + for currentUser in self.dbUtils.getUsersWithSourceAlerts(): self.sendReportEmail(currentUser, emailReports) def sendReportEmail(self, currentUser, emailReports): diff --git a/webcampak/core/wpakSourcesUtils.py b/webcampak/core/wpakSourcesUtils.py index ce3019a..6cf0e16 100644 --- a/webcampak/core/wpakSourcesUtils.py +++ b/webcampak/core/wpakSourcesUtils.py @@ -16,7 +16,7 @@ import os from wpakConfigObj import Config - +from wpakFileUtils import fileUtils class sourcesUtils: def __init__(self, parentClass): @@ -28,6 +28,7 @@ def __init__(self, parentClass): self.dirSources = self.configPaths.getConfig('parameters')['dir_sources'] self.configGeneral = parentClass.configGeneral + self.fileUtils = fileUtils(self) def getSourcesIds(self): self.log.info("sourcesUtils.getSources(): " + _("Start")) @@ -53,3 +54,24 @@ def getActiveSourcesIds(self): 'currentSource': str(currentSource)}) activeSourcesIds.sort() return activeSourcesIds + + def getLatestPicture(self, sourceId, currentTime = None): + self.log.debug("sourcesUtils.getLatestPicture(): " + _("Start")) + self.log.info("sourcesUtils.getLatestPicture(): " + _("Scanning source: %(sourceId)s") % {'sourceId': str(sourceId)}) + if currentTime != None: + currentTime = currentTime.strftime("%Y%m%d%H%M%S") + self.log.info("sourcesUtils.getLatestPicture(): " + _("Looking for picture captured before: %(currentTime)s") % {'currentTime': str(currentTime)}) + + dirCurrentSourcePictures = self.dirSources + 'source' + str(sourceId) + '/' + self.configPaths.getConfig('parameters')['dir_source_pictures'] + for listpictdir in sorted(os.listdir(dirCurrentSourcePictures), reverse=True): + if listpictdir[:2] == "20" and os.path.isdir(dirCurrentSourcePictures + listpictdir): + for listpictfiles in sorted(os.listdir(dirCurrentSourcePictures + listpictdir), reverse=True): + if listpictfiles[:2] == "20" and self.fileUtils.CheckJpegFile(dirCurrentSourcePictures + listpictdir + "/" + listpictfiles) == True: + if currentTime == None or int(listpictfiles[:14]) < int(currentTime): + self.log.info("fileUtils.getLatestPicture(): " + _("Last Picture: %(lastScannedPicture)s") % {'lastScannedPicture': str(dirCurrentSourcePictures + listpictdir + "/" + listpictfiles)}) + return listpictfiles + break; + else: + self.log.info("fileUtils.getLatestPicture(): " + _("Picture captured more recently than specified date: %(lastScannedPicture)s") % {'lastScannedPicture': str(dirCurrentSourcePictures + listpictdir + "/" + listpictfiles)}) + + #break; \ No newline at end of file diff --git a/webcampak/core/wpakStatsConsolidate.py b/webcampak/core/wpakStatsConsolidate.py index 362d89d..841d81a 100644 --- a/webcampak/core/wpakStatsConsolidate.py +++ b/webcampak/core/wpakStatsConsolidate.py @@ -40,6 +40,7 @@ def __init__(self, log, appConfig, config_dir): self.dirSources = self.configPaths.getConfig('parameters')['dir_sources'] self.dirLogs = self.configPaths.getConfig('parameters')['dir_logs'] self.dirStats = self.configPaths.getConfig('parameters')['dir_stats'] + self.dirStatsConsolidated = self.dirStats + "consolidated/" self.setupLog() @@ -65,8 +66,10 @@ def setupLog(self): # Description: Start the threads processing process # Each thread will get started in its own thread. # Return: Nothing - def run(self): + def run(self, fullPass): self.log.info("statsConsolidate.run(): Running Stats Collection") + # fullPass indicates the system should clear previous consolidated files and run on the entire history + self.log.info("statsConsolidate.run(): fullPass: " + str(fullPass)) # The following call will convert all .txt to .jsonl files # convertTxtArchive(g, self.dirStats) @@ -74,55 +77,56 @@ def run(self): # Remove and recreate consolidated stats directory self.log.info("statsConsolidate.run(): Step 0: Deleting consolidated/ directory") - shutil.rmtree(self.dirStats + "consolidated/") - os.makedirs(self.dirStats + "consolidated/") + if os.path.exists(self.dirStatsConsolidated) and fullPass == True: + shutil.rmtree(self.dirStatsConsolidated) + if not os.path.exists(self.dirStatsConsolidated): + os.makedirs(self.dirStatsConsolidated) # 1- Convert start - with days (contains hours), month (contains days), and year (contains months) self.log.info("statsConsolidate.run(): Step 1: Crunching hours") skipCount = 0 for scanFile in sorted(os.listdir(self.dirStats), reverse=True): - if os.path.splitext(scanFile)[1].lower() == '.jsonl' and len( - os.path.splitext(scanFile)[0]) == 8 and skipCount < 2: + if os.path.splitext(scanFile)[1].lower() == '.jsonl' and len(os.path.splitext(scanFile)[0]) == 8 and skipCount < 2: self.log.info("statsConsolidate.run(): Step 1: Source File: " + self.dirStats + scanFile) - self.log.info( - "statsConsolidate.run(): Step 1: Destination File: " + self.dirStats + "consolidated/" + scanFile) - skipCount = self.checkProcessFile(scanFile, "consolidated/" + scanFile, skipCount, '23:55', 11, 16) + self.log.info("statsConsolidate.run(): Step 1: Destination File: " + self.dirStatsConsolidated + scanFile) dayStats = self.parseSourceHoursFile(scanFile) dayStats = self.crunchHourFile(dayStats) self.saveHourFile(dayStats, scanFile) + if fullPass == False: + skipCount = self.checkProcessFile(self.dirStats + scanFile, self.dirStatsConsolidated + scanFile, skipCount, '23:55', 11, 16) + self.log.info("statsConsolidate.run(): Step 1: skipCount: " + str(skipCount)) + # 2- Convert Months, using days previously converted self.log.info("statsConsolidate.run(): Step 2: Crunching days") skipCount = 0 - for scanFile in sorted(os.listdir(self.dirStats + "/consolidated"), reverse=True): + for scanFile in sorted(os.listdir(self.dirStatsConsolidated), reverse=True): # 201601.jsonl monthFile = scanFile[0:6] + '.jsonl' self.log.info("statsConsolidate.run(): Step 2: Saving to month File: " + monthFile) - if os.path.splitext(scanFile)[1].lower() == '.jsonl' and len( - os.path.splitext(scanFile)[0]) == 8 and skipCount < 2: - self.log.info( - "statsConsolidate.run(): Step 2: Source File: " + self.dirStats + "consolidated/" + scanFile) - self.log.info( - "statsConsolidate.run(): Step 2: Destination File: " + self.dirStats + "consolidated/" + monthFile) - skipCount = self.checkProcessFile("consolidated/" + scanFile, "consolidated/" + monthFile, skipCount, - '31', 8, 10) + if os.path.splitext(scanFile)[1].lower() == '.jsonl' and len(os.path.splitext(scanFile)[0]) == 8 and skipCount < 1: + self.log.info("statsConsolidate.run(): Step 2: Source File: " + self.dirStatsConsolidated + scanFile) + self.log.info("statsConsolidate.run(): Step 2: Destination File: " + self.dirStatsConsolidated + monthFile) dayStats = self.parseSourceDaysFile(scanFile) dayStats = self.crunchDayFile(dayStats) self.saveDayFile(dayStats, monthFile) + if fullPass == False: + skipCount = self.checkProcessFile(self.dirStatsConsolidated + scanFile, self.dirStatsConsolidated + monthFile, skipCount,'31', 8, 10) + self.log.info("statsConsolidate.run(): Step 2: skipCount: " + str(skipCount)) + # 3- Convert Years self.log.info("statsConsolidate.run(): Step 2: Crunching years") - for scanFile in sorted(os.listdir(self.dirStats + "/consolidated"), reverse=True): + for scanFile in sorted(os.listdir(self.dirStatsConsolidated), reverse=True): if len(os.path.splitext(scanFile)[0]) == 4: - os.remove(self.dirStats + "consolidated/" + scanFile) + os.remove(self.dirStatsConsolidated + scanFile) skipCount = 0 - for scanFile in sorted(os.listdir(self.dirStats + "/consolidated"), reverse=True): + for scanFile in sorted(os.listdir(self.dirStatsConsolidated), reverse=True): # 201601.jsonl yearFile = scanFile[0:4] + '.jsonl' - if os.path.splitext(scanFile)[1].lower() == '.jsonl' and len( - os.path.splitext(scanFile)[0]) == 6 and skipCount < 3: - # skipCount = self.checkProcessFile(g, self.dirStats + "consolidated/" + scanFile, self.dirStats + "consolidated/" + yearFile, skipCount, '31', 8, 10) + if os.path.splitext(scanFile)[1].lower() == '.jsonl' and len(os.path.splitext(scanFile)[0]) == 6 and skipCount < 3: + # skipCount = self.checkProcessFile(g, self.dirStatsConsolidated + scanFile, self.dirStatsConsolidated + yearFile, skipCount, '31', 8, 10) # print 'COUNT: ' + str(skipCount) dayStats = self.parseSourceMonthsFile(scanFile) dayStats = self.crunchDayFile(dayStats) @@ -204,7 +208,7 @@ def parseSourceDaysFile(self, scanFile): self.log.info("statsConsolidate.parseSourceDaysFile() - Processing Days file: " + scanFile) dayStats = OrderedDict() # Start with days - for line in reversed(open(self.dirStats + "consolidated/" + scanFile).readlines()): + for line in reversed(open(self.dirStatsConsolidated + scanFile).readlines()): try: currentStatsLine = json.loads(line, object_pairs_hook=OrderedDict) except Exception: @@ -232,7 +236,7 @@ def parseSourceMonthsFile(self, scanFile): self.log.info("statsConsolidate.parseSourceMonthsFile() - Processing Months file: " + scanFile) dayStats = OrderedDict() # Start with days - for line in reversed(open(self.dirStats + "consolidated/" + scanFile).readlines()): + for line in reversed(open(self.dirStatsConsolidated + scanFile).readlines()): try: currentStatsLine = json.loads(line, object_pairs_hook=OrderedDict) except Exception: @@ -294,12 +298,12 @@ def crunchDayFile(self, dayStats): def saveHourFile(self, dayStats, scanFile): self.log.info("statsConsolidate.saveHourFile() - Start") - if os.path.isfile(self.dirStats + "consolidated/" + scanFile): + if os.path.isfile(self.dirStatsConsolidated + scanFile): self.log.info( - "statsConsolidate.saveHourFile() - Json file: exists, deleting... " + self.dirStats + "consolidated/" + scanFile) - os.remove(self.dirStats + "consolidated/" + scanFile) + "statsConsolidate.saveHourFile() - Json file: exists, deleting... " + self.dirStatsConsolidated + scanFile) + os.remove(self.dirStatsConsolidated + scanFile) for dayHour in reversed(dayStats.keys()): - with open(self.dirStats + "consolidated/" + scanFile, "a") as consolidatedStatFile: + with open(self.dirStatsConsolidated + scanFile, "a") as consolidatedStatFile: consolidatedStatFile.write(json.dumps(dayStats[dayHour]) + "\n") def prependLine(self, filename, line): @@ -311,28 +315,33 @@ def prependLine(self, filename, line): # Save crunched numbers to file def saveDayFile(self, dayStats, scanFile): self.log.info("statsConsolidate.saveHourFile() - Start") - if os.path.isfile(self.dirStats + "consolidated/" + scanFile): + if os.path.isfile(self.dirStatsConsolidated + scanFile): for dayHour in reversed(dayStats.keys()): - self.prependLine(self.dirStats + "consolidated/" + scanFile, json.dumps(dayStats[dayHour]) + "\n") + self.prependLine(self.dirStatsConsolidated + scanFile, json.dumps(dayStats[dayHour]) + "\n") else: for dayHour in reversed(dayStats.keys()): - with open(self.dirStats + "consolidated/" + scanFile, "a") as consolidatedStatFile: + with open(self.dirStatsConsolidated + scanFile, "a") as consolidatedStatFile: consolidatedStatFile.write(json.dumps(dayStats[dayHour]) + "\n") # Identify if we want to process this file - # If scanned date is 23:55 and target file exists then increase count + # The system is configured to collect system stats at 5mn interval, so last recorded date of a single day should be 23:55 + # If last line of the file corresponds to the end of a day or to the end of a month and target file exists then increase count def checkProcessFile(self, sourceFile, targetFile, skipCount, searchTime, searchStart, searchEnd): # self.log.info("statsConsolidate.checkProcessFile() - Source File: " + sourceFile) # self.log.info("statsConsolidate.checkProcessFile() - Target File: " + targetFile) - for line in reversed(open(self.dirStats + sourceFile).readlines()): + for line in reversed(open(sourceFile).readlines()): try: currentStatsLine = json.loads(line, object_pairs_hook=OrderedDict) except Exception: self.log.error("statsConsolidate.checkProcessFile(): Unable to decode JSON line: " + line) break # if currentStatsLine['date'][11:16] == searchTime and os.path.isfile(targetFile): + self.log.debug("statsConsolidate.checkProcessFile(): CurrentStatsLine: " + currentStatsLine['date'][searchStart:searchEnd]) + self.log.debug("statsConsolidate.checkProcessFile(): searchTime: " + searchTime) + self.log.debug("statsConsolidate.checkProcessFile(): isfile: " + targetFile) + self.log.debug("statsConsolidate.checkProcessFile(): isfile: " + str(os.path.isfile(targetFile))) if currentStatsLine['date'][searchStart:searchEnd] == searchTime and os.path.isfile(targetFile): return skipCount + 1 else: diff --git a/webcampak/core/wpakTimeUtils.py b/webcampak/core/wpakTimeUtils.py index e569047..f1ef669 100644 --- a/webcampak/core/wpakTimeUtils.py +++ b/webcampak/core/wpakTimeUtils.py @@ -50,19 +50,17 @@ def getCurrentDateIso(self): # If capture is configured to be delayed there are two option, use script start date or capture date def getCurrentSourceTime(self, sourceConfig): self.log.debug("timeUtils.getCurrentSourceTime(): " + _("Start")) - cfgnowsource = datetime.utcnow() if sourceConfig.getConfig('cfgcapturetimezone') != "": # Update the timezone from UTC to the source's timezone - self.log.info("timeUtils.getCurrentSourceTime(): " + _("Source Timezone is: %(sourceTimezone)s") % { - 'sourceTimezone': sourceConfig.getConfig('cfgcapturetimezone')}) + self.log.info("timeUtils.getCurrentSourceTime(): " + _("Source Timezone is: %(sourceTimezone)s") % {'sourceTimezone': sourceConfig.getConfig('cfgcapturetimezone')}) sourceTimezone = tz.gettz(sourceConfig.getConfig('cfgcapturetimezone')) - cfgnowsource = cfgnowsource.replace(tzinfo=tz.gettz('UTC')) - cfgnowsource = cfgnowsource.astimezone(sourceTimezone) - self.log.info("timeUtils.getCurrentSourceTime(): " + _("Current source time: %(cfgnowsource)s") % { - 'cfgnowsource': cfgnowsource.isoformat()}) + cfgnowsource = datetime.now(sourceTimezone) + else: + cfgnowsource = datetime.utcnow() + self.log.info("timeUtils.getCurrentSourceTime(): " + _("Current source time: %(cfgnowsource)s") % {'cfgnowsource': cfgnowsource.isoformat()}) return cfgnowsource # Using a webcampak timestamp, capture the file date and time - def getTimeFromFilename(self, fileName, sourceConfig, dateFormat): + def getTimeFromFilename(self, fileName, sourceConfig, dateFormat = "YYYYMMDDHHMMSS"): self.log.debug("timeUtils.getTimeFromFilename(): " + _("Start")) self.log.info( "timeUtils.getTimeFromFilename(): " + _("Extract time from: %(fileName)s using format %(dateFormat)s") % {'fileName': fileName, 'dateFormat': dateFormat}) @@ -71,16 +69,13 @@ def getTimeFromFilename(self, fileName, sourceConfig, dateFormat): fileTime = datetime.strptime(os.path.splitext(os.path.basename(fileName))[0], "%Y%m%d_%H%M%S") else: fileTime = datetime.strptime(os.path.splitext(os.path.basename(fileName))[0], "%Y%m%d%H%M%S") - if sourceConfig.getConfig( - 'cfgcapturetimezone') != "": # Update the timezone from UTC to the source's timezone - self.log.info("timeUtils.getTimeFromFilename(): " + _("Source timezone is: %(sourceTimezone)s") % { - 'sourceTimezone': sourceConfig.getConfig('cfgcapturetimezone')}) + if sourceConfig.getConfig('cfgcapturetimezone') != "": # Update the timezone from UTC to the source's timezone + self.log.info("timeUtils.getTimeFromFilename(): " + _("Source timezone is: %(sourceTimezone)s") % {'sourceTimezone': sourceConfig.getConfig('cfgcapturetimezone')}) sourceTimezone = tz.gettz(sourceConfig.getConfig('cfgcapturetimezone')) fileTime = fileTime.replace(tzinfo=sourceTimezone) - #fileTime = fileTime.replace(tzinfo=tz.gettz('UTC')) - #fileTime = fileTime.astimezone(sourceTimezone) - self.log.info("timeUtils.getTimeFromFilename(): " + _("Picture date is: %(picDate)s") % { - 'picDate': fileTime.isoformat()}) + else: + fileTime = fileTime.replace(tzinfo=tz.gettz('UTC')) + self.log.info("timeUtils.getTimeFromFilename(): " + _("Picture date is: %(picDate)s") % {'picDate': fileTime.isoformat()}) return fileTime except: return False