fix: Handles failed events for #223. (#232)

xAPI-vle · Aug 22, 2018 · 6e33b63 · 6e33b63
1 parent a90d6e5
commit 6e33b63
Show file tree

Hide file tree

Showing 10 changed files with 240 additions and 77 deletions.
diff --git a/classes/task/emit_task.php b/classes/task/emit_task.php
@@ -31,25 +31,56 @@ public function get_name() {
         return get_string('taskemit', 'logstore_xapi');
     }
 
+    private function get_failed_events($events) {
+        $nonloadedevents = array_filter($events, function ($loadedevent) {
+            return $loadedevent['loaded'] === false;
+        });
+        $failedevents = array_map(function ($nonloadedevent) {
+            return $nonloadedevent['event'];
+        }, $nonloadedevents);
+        return $failedevents;
+    }
+
+    private function get_event_ids($loadedevents) {
+        return array_map(function ($loadedevent) {
+            return $loadedevent['event']->id;
+        }, $loadedevents);
+    }
+
+    private function extract_events($limitnum) {
+        global $DB;
+        $conditions = null;
+        $sort = '';
+        $fields = '*';
+        $limitfrom = 0;
+        $extractedevents = $DB->get_records('logstore_xapi_log', $conditions, $sort, $fields, $limitfrom, $limitnum);
+        return $extractedevents;
+    }
+
+    private function delete_processed_events($events) {
+        global $DB;
+        $eventids = $this->get_event_ids($events);
+        $DB->delete_records_list('logstore_xapi_log', 'id', $eventids);
+        mtrace("Events (".implode(', ', $eventids).") have been successfully sent to LRS.");
+    }
+
+    private function store_failed_events($events) {
+        global $DB;
+        $failedevents = $this->get_failed_events($events);
+        $DB->insert_records('logstore_xapi_failed_log', $failedevents);
+    }
+
     /**
      * Do the job.
      * Throw exceptions on errors (the job will be retried).
      */
     public function execute() {
-        global $DB;
         $manager = get_log_manager();
         $store = new store($manager);
-        $conditions = null;
-        $sort = '';
-        $fields = '*';
-        $limitfrom = 0;
-        $limitnum = $store->get_max_batch_size();
-        $extractedevents = $DB->get_records('logstore_xapi_log', $conditions, $sort, $fields, $limitfrom, $limitnum);
+
+        $extractedevents = $this->extract_events($store->get_max_batch_size());
         $loadedevents = $store->process_events($extractedevents);
-        $loadedeventids = array_map(function ($transformedevent) {
-            return $transformedevent['eventid'];
-        }, $loadedevents);
-        $DB->delete_records_list('logstore_xapi_log', 'id', $loadedeventids);
-        mtrace("Events (".implode(', ', $loadedeventids).") have been successfully sent to LRS.");
+        $this->store_failed_events($loadedevents);
+        $this->delete_processed_events($loadedevents);
     }
 }
diff --git a/db/install.xml b/db/install.xml
@@ -37,5 +37,39 @@
         <INDEX NAME="user-module" UNIQUE="false" FIELDS="userid, contextlevel, contextinstanceid, crud, edulevel, timecreated"/>
       </INDEXES>
     </TABLE>
+
+    <TABLE NAME="logstore_xapi_failed_log" COMMENT="xAPI holding table for failed events">
+      <FIELDS>
+        <FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
+        <FIELD NAME="eventname" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="component" TYPE="char" LENGTH="100" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="action" TYPE="char" LENGTH="100" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="target" TYPE="char" LENGTH="100" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="objecttable" TYPE="char" LENGTH="50" NOTNULL="false" SEQUENCE="false"/>
+        <FIELD NAME="objectid" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false"/>
+        <FIELD NAME="crud" TYPE="char" LENGTH="1" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="edulevel" TYPE="int" LENGTH="1" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="contextid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="contextlevel" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="contextinstanceid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="userid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="courseid" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false"/>
+        <FIELD NAME="relateduserid" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false"/>
+        <FIELD NAME="anonymous" TYPE="int" LENGTH="1" NOTNULL="true" DEFAULT="0" SEQUENCE="false" COMMENT="Was this event anonymous at the time of triggering?"/>
+        <FIELD NAME="other" TYPE="text" NOTNULL="false" SEQUENCE="false"/>
+        <FIELD NAME="timecreated" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
+        <FIELD NAME="origin" TYPE="char" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="cli, cron, ws, etc."/>
+        <FIELD NAME="ip" TYPE="char" LENGTH="45" NOTNULL="false" SEQUENCE="false" COMMENT="IP address"/>
+        <FIELD NAME="realuserid" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="real user id when logged-in-as"/>
+      </FIELDS>
+      <KEYS>
+        <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
+      </KEYS>
+      <INDEXES>
+        <INDEX NAME="timecreated" UNIQUE="false" FIELDS="timecreated"/>
+        <INDEX NAME="course-time" UNIQUE="false" FIELDS="courseid, anonymous, timecreated"/>
+        <INDEX NAME="user-module" UNIQUE="false" FIELDS="userid, contextlevel, contextinstanceid, crud, edulevel, timecreated"/>
+      </INDEXES>
+    </TABLE>
   </TABLES>
 </XMLDB>
diff --git a/db/upgrade.php b/db/upgrade.php
@@ -16,62 +16,68 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-function xmldb_logstore_xapi_upgrade($oldversion) {
-    global $CFG, $DB;
-
-    $dbman = $DB->get_manager();
+function create_xapi_log_table($dbman, $tablename) {
+    // Define table to be created.
+    $table = new xmldb_table($tablename);
 
-    if ($oldversion < 2015081001) {
+    // Adding fields to table.
+    $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
+    $table->add_field('eventname', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('component', XMLDB_TYPE_CHAR, '100', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('action', XMLDB_TYPE_CHAR, '100', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('target', XMLDB_TYPE_CHAR, '100', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('objecttable', XMLDB_TYPE_CHAR, '50', null, null, null, null);
+    $table->add_field('objectid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
+    $table->add_field('crud', XMLDB_TYPE_CHAR, '1', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('edulevel', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('contextid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('contextlevel', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('contextinstanceid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('userid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('courseid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
+    $table->add_field('relateduserid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
+    $table->add_field('anonymous', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, null, '0');
+    $table->add_field('other', XMLDB_TYPE_TEXT, null, null, null, null, null);
+    $table->add_field('timecreated', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
+    $table->add_field('origin', XMLDB_TYPE_CHAR, '10', null, null, null, null);
+    $table->add_field('ip', XMLDB_TYPE_CHAR, '45', null, null, null, null);
+    $table->add_field('realuserid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
 
-        // Define table logstore_xapi_log to be created.
-        $table = new xmldb_table('logstore_xapi_log');
+    // Adding keys to table.
+    $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
 
-        // Adding fields to table logstore_xapi_log.
-        $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
-        $table->add_field('eventname', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('component', XMLDB_TYPE_CHAR, '100', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('action', XMLDB_TYPE_CHAR, '100', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('target', XMLDB_TYPE_CHAR, '100', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('objecttable', XMLDB_TYPE_CHAR, '50', null, null, null, null);
-        $table->add_field('objectid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
-        $table->add_field('crud', XMLDB_TYPE_CHAR, '1', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('edulevel', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('contextid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('contextlevel', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('contextinstanceid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('userid', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('courseid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
-        $table->add_field('relateduserid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
-        $table->add_field('anonymous', XMLDB_TYPE_INTEGER, '1', null, XMLDB_NOTNULL, null, '0');
-        $table->add_field('other', XMLDB_TYPE_TEXT, null, null, null, null, null);
-        $table->add_field('timecreated', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, null);
-        $table->add_field('origin', XMLDB_TYPE_CHAR, '10', null, null, null, null);
-        $table->add_field('ip', XMLDB_TYPE_CHAR, '45', null, null, null, null);
-        $table->add_field('realuserid', XMLDB_TYPE_INTEGER, '10', null, null, null, null);
+    // Adding indexes to table.
+    $table->add_index('timecreated', XMLDB_INDEX_NOTUNIQUE, array('timecreated'));
+    $table->add_index('course-time', XMLDB_INDEX_NOTUNIQUE, array('courseid', 'anonymous', 'timecreated'));
+    $table->add_index('user-module', XMLDB_INDEX_NOTUNIQUE, array(
+        'userid',
+        'contextlevel',
+        'contextinstanceid',
+        'crud',
+        'edulevel',
+        'timecreated'
+    ));
 
-        // Adding keys to table logstore_xapi_log.
-        $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
+    // Conditionally launch create table.
+    if (!$dbman->table_exists($table)) {
+        $dbman->create_table($table);
+    }
+}
 
-        // Adding indexes to table logstore_xapi_log.
-        $table->add_index('timecreated', XMLDB_INDEX_NOTUNIQUE, array('timecreated'));
-        $table->add_index('course-time', XMLDB_INDEX_NOTUNIQUE, array('courseid', 'anonymous', 'timecreated'));
-        $table->add_index('user-module', XMLDB_INDEX_NOTUNIQUE, array(
-            'userid',
-            'contextlevel',
-            'contextinstanceid',
-            'crud',
-            'edulevel',
-            'timecreated'
-        ));
+function xmldb_logstore_xapi_upgrade($oldversion) {
+    global $DB;
 
-        // Conditionally launch create table for logstore_xapi_log.
-        if (!$dbman->table_exists($table)) {
-            $dbman->create_table($table);
-        }
+    $dbman = $DB->get_manager();
 
-        // Xapi savepoint reached.
+    if ($oldversion < 2015081001) {
+        create_xapi_log_table($dbman, 'logstore_xapi_log');
         upgrade_plugin_savepoint(true, 2015081001, 'logstore', 'xapi');
     }
 
+    if ($oldversion < 2018082100) {
+        create_xapi_log_table($dbman, 'logstore_xapi_failed_log');
+        upgrade_plugin_savepoint(true, 2018082100, 'logstore', 'xapi');
+    }
+
     return true;
 }
diff --git a/src/loader/log.php b/src/loader/log.php
@@ -18,11 +18,13 @@
 
 defined('MOODLE_INTERNAL') || die();
 
+use src\loader\utils as utils;
+
 function load(array $config, array $transformedevents) {
     $statements = array_reduce($transformedevents, function ($result, $transformedevent) {
         $eventstatements = $transformedevent['statements'];
         return array_merge($result, $eventstatements);
     }, []);
     echo(json_encode($statements, JSON_PRETTY_PRINT)."\n");
-    return $transformedevents;
+    return utils\construct_loaded_events($transformedevents, true);
 }
diff --git a/src/loader/lrs.php b/src/loader/lrs.php
@@ -17,6 +17,8 @@
 namespace src\loader\lrs;
 defined('MOODLE_INTERNAL') || die();
 
+use src\loader\utils as utils;
+
 function correct_endpoint($endpoint) {
     $endswithstatements = substr($endpoint, -11) === "/statements";
     if ($endswithstatements) {
@@ -25,17 +27,13 @@ function correct_endpoint($endpoint) {
     return rtrim($endpoint, '/');
 }
 
-function load_transormed_events_to_lrs(array $config, array $transformedevents) {
+function send_http_statements(array $config, array $statements) {
     $endpoint = $config['lrs_endpoint'];
     $username = $config['lrs_username'];
     $password = $config['lrs_password'];
 
     $url = correct_endpoint($endpoint).'/statements';
     $auth = base64_encode($username.':'.$password);
-    $statements = array_reduce($transformedevents, function ($result, $transformedevent) {
-        $eventstatements = $transformedevent['statements'];
-        return array_merge($result, $eventstatements);
-    }, []);
     $postdata = json_encode($statements);
 
     $request = curl_init();
@@ -55,10 +53,25 @@ function load_transormed_events_to_lrs(array $config, array $transformedevents)
 
     if ($responsecode !== 200) {
         throw new \Exception($responsetext);
-        return [];
     }
+}
 
-    return $transformedevents;
+function load_transormed_events_to_lrs(array $config, array $transformedevents) {
+    try {
+        $statements = array_reduce($transformedevents, function ($result, $transformedevent) {
+            $eventstatements = $transformedevent['statements'];
+            return array_merge($result, $eventstatements);
+        }, []);
+        send_http_statements($config, $statements);
+        $loadedevents = utils\construct_loaded_events($transformedevents, true);
+        return $loadedevents;
+    } catch (\Exception $e) {
+        $logerror = $config['log_error'];
+        $logerror("Failed load for event id #" . $eventobj->id . ": " .  $e->getMessage());
+        $logerror($e->getTraceAsString());
+        $loadedevents = utils\construct_loaded_events($transformedevents, false);
+        return $loadedevents;
+    }
 }
 
 function get_event_batches(array $config, array $transformedevents) {
@@ -69,11 +82,19 @@ function get_event_batches(array $config, array $transformedevents) {
     return [$transformedevents];
 }
 
-function load(array $config, array $transformedevents) {
-    $batches = get_event_batches($config, $transformedevents);
+function load(array $config, array $events) {
+    // Attempts to load events that were transformed successfully in batches.
+    $successfultransformevents = utils\filter_transformed_events($events, true);
+    $batches = get_event_batches($config, $successfultransformevents);
     $loadedevents = array_reduce($batches, function ($result, $batch) use ($config) {
         $loadedbatchevents = load_transormed_events_to_lrs($config, $batch);
         return array_merge($result, $loadedbatchevents);
     }, []);
-    return $loadedevents;
+
+    // Flags events that weren't transformed successfully as events that didn't load.
+    $failedtransformevents = utils\filter_transformed_events($events, false);
+    $nonloadedevents = utils\construct_loaded_events($failedtransformevents, false);
+
+    // Returns loaded and non-loaded events to avoid re-processing.
+    return array_merge($loadedevents, $nonloadedevents);
 }
diff --git a/src/loader/none.php b/src/loader/none.php
@@ -18,6 +18,8 @@
 
 defined('MOODLE_INTERNAL') || die();
 
+use src\loader\utils as utils;
+
 function load(array $config, array $transformedevents) {
-    return $transformedevents;
+    return utils\construct_loaded_events($transformedevents, true);
 }
diff --git a/src/loader/utils/construct_loaded_events.php b/src/loader/utils/construct_loaded_events.php
@@ -0,0 +1,31 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace src\loader\utils;
+
+defined('MOODLE_INTERNAL') || die();
+
+function construct_loaded_events(array $transformedevents, $loaded) {
+    $loadedevents = array_map(function ($transformedevent) use ($loaded) {
+        return [
+            'event' => $transformedevent['event'],
+            'statements' => $transformedevent['statements'],
+            'transformed' => $transformedevent['transformed'],
+            'loaded' => $loaded,
+        ];
+    }, $transformedevents);
+    return $loadedevents;
+}
diff --git a/src/loader/utils/filter_transformed_events.php b/src/loader/utils/filter_transformed_events.php
@@ -0,0 +1,26 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace src\loader\utils;
+
+defined('MOODLE_INTERNAL') || die();
+
+function filter_transformed_events(array $events, $transformed) {
+    $filteredevents = array_filter($events, function ($event) use ($transformed) {
+        return $event['transformed'] === $transformed;
+    });
+    return $filteredevents;
+}