Implement DynamoDB ORM (#68)

Adds the DynamoDB ORM dynamoid and switches all routes to use it. Implements the database structure from the design document

Frontend/src/pages/registrations/components/RegistrationList.jsx

@@ -2,8 +2,8 @@ import React, { useState } from 'react'
 import deleteRegistration from '../../../api/registration/delete/delete_registration'
 import getRegistrations from '../../../api/registration/get/get_registrations'
 import updateRegistration from '../../../api/registration/patch/update_registration'
-import StatusDropdown from '../../register/components/StatusDropdown'
 import styles from './list.module.scss'
+import StatusDropdown from './StatusDropdown'
 
 function RegistrationRow({
   competitorId,

Frontend/src/pages/register/components/StatusDropdown.jsx → Frontend/src/pages/registrations/components/StatusDropdown.jsx

@@ -1,7 +1,7 @@
 import React from 'react'
 
 export default function StatusDropdown({ status, setStatus }) {
-  const options = ['waiting', 'accepted']
+  const options = ['waiting', 'accepted', 'deleted']
   return (
     <select onChange={(e) => setStatus(e.target.value)} value={status}>
       {options.map((opt) => (

Gemfile

@@ -30,6 +30,7 @@ gem 'redis', '~> 4.0'
 
 # DynamoDB for storing registrations
 gem 'aws-sdk-dynamodb'
+gem 'dynamoid', '3.8.0'
 
 # SQS for adding data into a queue
 gem 'aws-sdk-sqs'

Gemfile.lock

@@ -96,6 +96,10 @@ GEM
       irb (>= 1.5.0)
       reline (>= 0.3.1)
     diff-lcs (1.5.0)
+    dynamoid (3.8.0)
+      activemodel (>= 4)
+      aws-sdk-dynamodb (~> 1.0)
+      concurrent-ruby (>= 1.0)
     erubi (1.12.0)
     globalid (1.1.0)
       activesupport (>= 5.0)
@@ -263,6 +267,7 @@ DEPENDENCIES
   aws-sdk-sqs
   bootsnap
   debug
+  dynamoid (= 3.8.0)
   hiredis
   jbuilder
   kredis

app/controllers/registration_controller.rb

@@ -5,6 +5,35 @@
 require_relative '../helpers/competitor_api'
 
 class RegistrationController < ApplicationController
+  before_action :ensure_lane_exists, only: [:create]
+
+  def ensure_lane_exists
+    user_id = params[:competitor_id]
+    competition_id = params[:competition_id]
+    @queue_url = ENV["QUEUE_URL"] || $sqs.get_queue_url(queue_name: 'registrations.fifo').queue_url
+    # TODO: Cache this call?
+    lane_created = begin
+      Registrations.find("#{competition_id}-#{user_id}")
+      true
+    rescue Dynamoid::Errors::RecordNotFound
+      false
+    end
+    unless lane_created
+      step_data = {
+        user_id: user_id,
+        competition_id: competition_id,
+        step: 'Lane Init',
+      }
+      id = SecureRandom.uuid
+      $sqs.send_message({
+                          queue_url: @queue_url,
+                          message_body: step_data.to_json,
+                          message_group_id: id,
+                          message_deduplication_id: id,
+                        })
+    end
+  end
+
   def create
     competitor_id = params[:competitor_id]
     competition_id = params[:competition_id]
@@ -18,16 +47,18 @@ def create
     id = SecureRandom.uuid
 
     step_data = {
-      competitor_id: competitor_id,
+      user_id: competitor_id,
       competition_id: competition_id,
-      event_ids: event_ids,
-      registration_status: 'waiting',
+      lane_name: 'Competing',
       step: 'Event Registration',
+      step_details: {
+        registration_status: 'waiting',
+        event_ids: event_ids,
+      },
     }
-    queue_url = ENV["QUEUE_URL"] || @sqs.get_queue_url(queue_name: 'registrations.fifo').queue_url
 
     $sqs.send_message({
-                        queue_url: queue_url,
+                        queue_url: @queue_url,
                         message_body: step_data.to_json,
                         message_group_id: id,
                         message_deduplication_id: id,
@@ -37,68 +68,56 @@ def create
   end
 
   def update
-    competitor_id = params[:competitor_id]
+    user_id = params[:competitor_id]
     competition_id = params[:competition_id]
     status = params[:status]
 
-    unless validate_request(competitor_id, competition_id, status)
+    unless validate_request(user_id, competition_id, status)
       Metrics.registration_validation_errors_counter.increment
       return render json: { status: 'User cannot register, wrong format' }, status: :forbidden
     end
-
-    # Specify the key attributes for the item to be updated
-    key = {
-      'competitor_id' => competitor_id,
-      'competition_id' => competition_id,
-    }
-
-    # Set the expression for the update operation
-    update_expression = 'set registration_status = :s'
-    expression_attribute_values = {
-      ':s' => status,
-    }
-
     begin
-      # Update the item in the table
-      $dynamodb.update_item({
-                              table_name: 'Registrations',
-                              key: key,
-                              update_expression: update_expression,
-                              expression_attribute_values: expression_attribute_values,
-                            })
+      registration = Registrations.find("#{competition_id}-#{user_id}")
+      updated_lanes = registration.lanes.map { |lane|
+        if lane.name == "Competing"
+          lane.lane_state = status
+        end
+        lane
+      }
+      puts updated_lanes.to_json
+      registration.update_attributes(lanes: updated_lanes)
+      # Render a success response
       render json: { status: 'ok' }
-    rescue Aws::DynamoDB::Errors::ServiceError => e
+    rescue StandardError => e
+      # Render an error response
       puts e
       Metrics.registration_dynamodb_errors_counter.increment
-      render json: { status: 'Failed to update registration data' }, status: :internal_server_error
+      render json: { status: "Error Updating Registration: #{e.message}" },
+             status: :internal_server_error
     end
   end
 
   def delete
-    competitor_id = params[:competitor_id]
+    user_id = params[:competitor_id]
     competition_id = params[:competition_id]
 
-    unless validate_request(competitor_id, competition_id)
+    unless validate_request(user_id, competition_id)
       Metrics.registration_validation_errors_counter.increment
       return render json: { status: 'User cannot register, wrong format' }, status: :forbidden
     end
-
-    # Define the key of the item to delete
-    key = {
-      'competition_id' => competition_id,
-      'competitor_id' => competitor_id,
-    }
-
     begin
-      # Call the delete_item method to delete the item from the table
-      $dynamodb.delete_item(
-        table_name: 'Registrations',
-        key: key,
-      )
-
+      registration = Registrations.find("#{competition_id}-#{user_id}")
+      updated_lanes = registration.lanes.map { |lane|
+        if lane.name == "Competing"
+          lane.lane_state = "deleted"
+        end
+        lane
+      }
+      puts updated_lanes.to_json
+      registration.update_attributes(lanes: updated_lanes)
       # Render a success response
       render json: { status: 'ok' }
-    rescue Aws::DynamoDB::Errors::ServiceError => e
+    rescue StandardError => e
       # Render an error response
       puts e
       Metrics.registration_dynamodb_errors_counter.increment
@@ -110,9 +129,10 @@ def delete
   def list
     competition_id = params[:competition_id]
     registrations = get_registrations(competition_id)
+
     # Render a success response
     render json: registrations
-  rescue Aws::DynamoDB::Errors::ServiceError => e
+  rescue StandardError => e
     # Render an error response
     puts e
     Metrics.registration_dynamodb_errors_counter.increment
@@ -152,15 +172,8 @@ def validate_request(competitor_id, competition_id, status = 'waiting')
     end
 
     def get_registrations(competition_id)
-      # Query DynamoDB for registrations with the given competition_id
-      resp = $dynamodb.query({
-                               table_name: 'Registrations',
-                               key_condition_expression: '#ci = :cid',
-                               expression_attribute_names: { '#ci' => 'competition_id' },
-                               expression_attribute_values: { ':cid' => competition_id },
-                             })
-
-      # Return the items from the response
-      resp.items
+      # Query DynamoDB for registrations with the given competition_id using the Global Secondary Index
+      # TODO make this more beautiful and not break if there are more then one lane
+      Registrations.where(competition_id: competition_id).all.map { |x| { competitor_id: x["user_id"], event_ids: x["lanes"][0].step_details["event_ids"], registration_status: x["lanes"][0].lane_state } }
     end
 end

app/helpers/lane_factory.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class LaneFactory
+  def self.competing_lane(event_ids = {})
+    competing_lane = Lane.new({})
+    competing_lane.name = "Competing"
+    if event_ids != {}
+      competing_lane.completed_steps = ["Event Registration"]
+      competing_lane.step_details = {
+        event_ids: event_ids,
+      }
+    end
+    competing_lane
+  end
+end

app/models/lane.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class Lane
+  attr_accessor :name, :lane_state, :completed_steps, :step_details
+
+  def initialize(args)
+    @name = args["name"]
+    @lane_state = args["lane_state"] || "waiting"
+    @completed_steps = args["completed_steps"] || []
+    @step_details = args["step_details"] || {}
+  end
+
+  def dynamoid_dump
+    self.to_json
+  end
+
+  def self.dynamoid_load(serialized_str)
+    parsed = JSON.parse serialized_str
+    Lane.new(parsed)
+  end
+end

app/models/registrations.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative 'lane'
+class Registrations
+  include Dynamoid::Document
+
+  # We autoscale dynamodb in production
+  if ENV.fetch("CODE_ENVIRONMENT", "development") == "staging"
+    table name: 'registrations-staging', read_capacity: 5, write_capacity: 5, key: :attendee_id
+  else
+    table name: "registrations", capacity_mode: nil, key: :attendee_id
+  end
+
+  # Fields
+  field :user_id, :string
+  field :competition_id, :string
+  field :is_attending, :boolean
+  field :hide_name_publicly, :boolean
+  field :lane_states, :map
+  field :lanes, :array, of: Lane
+
+  global_secondary_index hash_key: :user_id, projected_attributes: :all
+  global_secondary_index hash_key: :competition_id, projected_attributes: :all
+end

app/worker/queue_poller.rb

@@ -32,6 +32,7 @@ def self.perform
                Aws::SQS::Client.new
              end
     queue_url = ENV["QUEUE_URL"] || @sqs.get_queue_url(queue_name: 'registrations.fifo').queue_url
+    processor = RegistrationProcessor.new
     poller = Aws::SQS::QueuePoller.new(queue_url)
     poller.poll(wait_time_seconds: WAIT_TIME, max_number_of_messages: MAX_MESSAGES) do |messages|
       messages.each do |msg|
@@ -40,7 +41,7 @@ def self.perform
         puts "Message body: #{msg.body}"
         body = JSON.parse msg.body
         begin
-          RegistrationProcessor.process_message(body)
+          processor.process_message(body)
           registrations_counter.increment
         rescue StandardError => e
           # unexpected error occurred while processing messages,

app/worker/registration_processor.rb

@@ -1,31 +1,48 @@
 # frozen_string_literal: true
 
 require 'aws-sdk-dynamodb'
+require 'dynamoid'
+require_relative '../helpers/lane_factory'
 
 class RegistrationProcessor
-  def self.process_message(message)
-    @dynamodb ||= if ENV['LOCALSTACK_ENDPOINT']
-                    Aws::DynamoDB::Client.new(endpoint: ENV['LOCALSTACK_ENDPOINT'])
-                  else
-                    Aws::DynamoDB::Client.new
-                  end
-    # implement your message processing logic here
-    puts "Working on Message: #{message}"
-    return unless message['step'] == 'Event Registration'
-
-    registration = {
-      competitor_id: message['competitor_id'],
-      competition_id: message['competition_id'],
-      event_ids: message['event_ids'],
-      registration_status: 'waiting',
-    }
-    save_registration(registration)
+  def initialize
+    Dynamoid.configure do |config|
+      config.region = ENV.fetch("AWS_REGION", 'us-west-2')
+      config.namespace = nil
+      if ENV.fetch("CODE_ENVIRONMENT", "development") == "development"
+        config.endpoint = ENV.fetch('LOCALSTACK_ENDPOINT', nil)
+      else
+        config.credentials = Aws::ECSCredentials.new(retries: 3)
+      end
+    end
+    # We have to require the model after we initialized dynamoid
+    require_relative '../models/registrations'
   end
 
-  def self.save_registration(registration)
-    @dynamodb.put_item({
-                         table_name: 'Registrations',
-                         item: registration,
-                       })
+  def process_message(message)
+    puts "Working on Message: #{message}"
+    if message['step'] == 'Lane Init'
+      lane_init(message['competition_id'], message['user_id'])
+    elsif message['step'] == 'Event Registration'
+      event_registration(message['competition_id'], message['user_id'], message['step_details']['event_ids'])
+    end
   end
+
+  private
+
+    def lane_init(competition_id, user_id)
+      empty_registration = Registrations.new(attendee_id: "#{competition_id}-#{user_id}", competition_id: competition_id, user_id: user_id)
+      empty_registration.save!
+    end
+
+    def event_registration(competition_id, user_id, event_ids)
+      registration = Registrations.find("#{competition_id}-#{user_id}")
+      competing_lane = LaneFactory.competing_lane(event_ids)
+      if registration.lanes.nil?
+        registration.update_attributes(lanes: [competing_lane])
+      else
+        registration.update_attributes(lanes: registration.lanes.append(competing_lane))
+      end
+      registration.save!
+    end
 end

config/initializers/dynamoid.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'dynamoid'
+
+Dynamoid.configure do |config|
+  config.region = ENV.fetch("AWS_REGION", 'us-west-2')
+  config.namespace = nil
+  if Rails.env.production?
+    config.credentials = Aws::ECSCredentials.new(retries: 3)
+  else
+    config.endpoint = ENV.fetch('LOCALSTACK_ENDPOINT', nil)
+  end
+end