[SYSTEMDS-3701] Additional scuro data representations

Closes #2111.

src/main/python/systemds/scuro/__init__.py

@@ -18,3 +18,54 @@
 # under the License.
 #
 # -------------------------------------------------------------
+from systemds.scuro.representations.representation import Representation
+from systemds.scuro.representations.average import Average
+from systemds.scuro.representations.concatenation import Concatenation
+from systemds.scuro.representations.fusion import Fusion
+from systemds.scuro.representations.sum import Sum
+from systemds.scuro.representations.max import RowMax
+from systemds.scuro.representations.multiplication import Multiplication
+from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
+from systemds.scuro.representations.resnet import ResNet
+from systemds.scuro.representations.bert import Bert
+from systemds.scuro.representations.unimodal import UnimodalRepresentation
+from systemds.scuro.representations.lstm import LSTM
+from systemds.scuro.representations.utils import NPY, Pickle, HDF5, JSON
+from systemds.scuro.models.model import Model
+from systemds.scuro.models.discrete_model import DiscreteModel
+from systemds.scuro.modality.aligned_modality import AlignedModality
+from systemds.scuro.modality.audio_modality import AudioModality
+from systemds.scuro.modality.video_modality import VideoModality
+from systemds.scuro.modality.text_modality import TextModality
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.aligner.dr_search import DRSearch
+from systemds.scuro.aligner.task import Task
+
+
+__all__ = ["Representation",
+           "Average",
+           "Concatenation",
+           "Fusion",
+           "Sum",
+           "RowMax",
+           "Multiplication",
+           "MelSpectrogram",
+           "ResNet",
+           "Bert",
+           "UnimodalRepresentation",
+           "LSTM",
+           "NPY",
+           "Pickle",
+           "HDF5",
+           "JSON",
+           "Model",
+           "DiscreteModel",
+           "AlignedModality",
+           "AudioModality",
+           "VideoModality",
+           "TextModality",
+           "Modality",
+           "DRSearch",
+           "Task"
+           ]
+

src/main/python/systemds/scuro/aligner/dr_search.py

@@ -19,19 +19,24 @@
 #
 # -------------------------------------------------------------
 import itertools
+import random
 from typing import List
 
-from aligner.task import Task
-from modality.aligned_modality import AlignedModality
-from modality.modality import Modality
-from representations.representation import Representation
+from systemds.scuro.aligner.task import Task
+from systemds.scuro.modality.aligned_modality import AlignedModality
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.representations.representation import Representation
+
+import warnings
+
+warnings.filterwarnings('ignore')
 
 
 def get_modalities_by_name(modalities, name):
     for modality in modalities:
         if modality.name == name:
             return modality
-    
+
     raise 'Modality ' + name + 'not in modalities'
 
 
@@ -51,9 +56,9 @@ def __init__(self, modalities: List[Modality], task: Task, representations: List
         self.best_modalities = None
         self.best_representation = None
         self.best_score = -1
-        
+
     def set_best_params(self, modality_name: str, representation: Representation,
-                        score: float, modality_names: List[str]):
+                        scores: List[float], modality_names: List[str]):
         """
         Updates the best parameters for given modalities, representation, and score
         :param modality_name: The name of the aligned modality
@@ -62,43 +67,66 @@ def set_best_params(self, modality_name: str, representation: Representation,
         :param modality_names: List of modality names used in this setting
         :return:
         """
-        
+
         # check if modality name is already in dictionary
         if modality_name not in self.scores.keys():
             # if not add it to dictionary
             self.scores[modality_name] = {}
-        
+
         # set score for representation
-        self.scores[modality_name][representation] = score
-        
+        self.scores[modality_name][representation] = scores
+
         # compare current score with best score
-        if score > self.best_score:
-            self.best_score = score
+        if scores[1] > self.best_score:
+            self.best_score = scores[1]
             self.best_representation = representation
             self.best_modalities = modality_names
-
-    def fit(self):
+
+    def reset_best_params(self):
+        self.best_score = -1
+        self.best_modalities = None
+        self.best_representation = None
+        self.scores = {}
+
+    def fit_random(self, seed=-1):
+        """
+        This method randomly selects a modality or combination of modalities and representation
+        """
+        if seed != -1:
+            random.seed(seed)
+
+        modalities = []
+        for M in range(1, len(self.modalities) + 1):
+            for combination in itertools.combinations(self.modalities, M):
+                modalities.append(combination)
+
+        modality_combination = random.choice(modalities)
+        representation = random.choice(self.representations)
+
+        modality = AlignedModality(representation, list(modality_combination))  # noqa
+        modality.combine()
+
+        scores = self.task.run(modality.data)
+        self.set_best_params(modality.name, representation, scores, modality.get_modality_names())
+
+        return self.best_representation, self.best_score, self.best_modalities
+
+    def fit_enumerate_all(self):
         """
         This method finds the best representation out of a given List of uni-modal modalities and
         representations
         :return: The best parameters found in the search procedure
         """
-        
+
         for M in range(1, len(self.modalities) + 1):
             for combination in itertools.combinations(self.modalities, M):
-                if len(combination) == 1:
-                    modality = combination[0]
-                    score = self.task.run(modality.representation.scale_data(modality.data, self.task.train_indices))
-                    self.set_best_params(modality.name, modality.representation.name, score, [modality.name])
-                    self.scores[modality] = score
-                else:
-                    for representation in self.representations:
-                        modality = AlignedModality(representation, list(combination)) # noqa
-                        modality.combine(self.task.train_indices)
-
-                        score = self.task.run(modality.data)
-                        self.set_best_params(modality.name, representation, score, modality.get_modality_names())
-
+                for representation in self.representations:
+                    modality = AlignedModality(representation, list(combination))  # noqa
+                    modality.combine()
+
+                    scores = self.task.run(modality.data)
+                    self.set_best_params(modality.name, representation, scores, modality.get_modality_names())
+
         return self.best_representation, self.best_score, self.best_modalities
 
     def transform(self, modalities: List[Modality]):
@@ -108,17 +136,16 @@ def transform(self, modalities: List[Modality]):
         :param modalities: List of uni-modal modalities
         :return: aligned data
         """
-        
+
         if self.best_score == -1:
             raise 'Please fit representations first!'
-        
+
         used_modalities = []
-        
+
         for modality_name in self.best_modalities:
             used_modalities.append(get_modalities_by_name(modalities, modality_name))
-        
+
         modality = AlignedModality(self.best_representation, used_modalities)  # noqa
         modality.combine(self.task.train_indices)
-        
+
         return modality.data
-

src/main/python/systemds/scuro/aligner/task.py

@@ -20,7 +20,7 @@
 # -------------------------------------------------------------
 from typing import List
 
-from models.model import Model
+from systemds.scuro.models.model import Model
 
 
 class Task:

src/main/python/systemds/scuro/main.py

@@ -22,16 +22,16 @@
 import json
 from datetime import datetime
 
-from representations.average import Averaging
-from representations.concatenation import Concatenation
-from modality.aligned_modality import AlignedModality
-from modality.text_modality import TextModality
-from modality.video_modality import VideoModality
-from modality.audio_modality import AudioModality
-from representations.unimodal import Pickle, JSON, HDF5, NPY
-from models.discrete_model import DiscreteModel
-from aligner.task import Task
-from aligner.dr_search import DRSearch
+from systemds.scuro.representations.average import Average
+from systemds.scuro.representations.concatenation import Concatenation
+from systemds.scuro.modality.aligned_modality import AlignedModality
+from systemds.scuro.modality.text_modality import TextModality
+from systemds.scuro.modality.video_modality import VideoModality
+from systemds.scuro.modality.audio_modality import AudioModality
+from systemds.scuro.representations.unimodal import Pickle, JSON, HDF5, NPY
+from systemds.scuro.models.discrete_model import DiscreteModel
+from systemds.scuro.aligner.task import Task
+from systemds.scuro.aligner.dr_search import DRSearch
 
 
 class CustomTask(Task):
@@ -66,8 +66,8 @@ def run(self, data):
 
 model = DiscreteModel()
 custom_task = CustomTask(model, labels, train_indices, val_indices)
-representations = [Concatenation(), Averaging()]
+representations = [Concatenation(), Average()]
 
 dr_search = DRSearch(modalities, custom_task, representations)
-best_representation, best_score, best_modalities = dr_search.fit()
+best_representation, best_score, best_modalities = dr_search.fit_random()
 aligned_representation = dr_search.transform(modalities)

src/main/python/systemds/scuro/modality/__init__.py

@@ -18,11 +18,3 @@
 # under the License.
 #
 # -------------------------------------------------------------
-from systemds.scuro.modality.aligned_modality import AlignedModality
-from systemds.scuro.modality.audio_modality import AudioModality
-from systemds.scuro.modality.video_modality import VideoModality
-from systemds.scuro.modality.test_modality import TextModality
-from systemds.scuro.modality.modality import Modality
-
-
-__all__ = ["AlignedModality", "AudioModality", "VideoModality", "TextModality", "Modality"]

src/main/python/systemds/scuro/modality/aligned_modality.py

@@ -20,8 +20,8 @@
 # -------------------------------------------------------------
 from typing import List
 
-from modality.modality import Modality
-from representations.fusion import Fusion
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.representations.fusion import Fusion
 
 
 class AlignedModality(Modality):
@@ -36,9 +36,16 @@ def __init__(self, representation: Fusion, modalities: List[Modality]):
             name += modality.name
         super().__init__(representation, modality_name=name)
         self.modalities = modalities
-    
+
     def combine(self):
         """
         Initiates the call to fuse the given modalities depending on the Fusion type
         """
         self.data = self.representation.fuse(self.modalities) # noqa
+
+    def get_modality_names(self):
+        names = []
+        for modality in self.modalities:
+            names.append(modality.name)
+
+        return names

src/main/python/systemds/scuro/modality/audio_modality.py

@@ -20,8 +20,8 @@
 # -------------------------------------------------------------
 import os
 
-from modality.modality import Modality
-from representations.unimodal import UnimodalRepresentation
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.representations.unimodal import UnimodalRepresentation
 
 
 class AudioModality(Modality):

src/main/python/systemds/scuro/modality/modality.py

@@ -19,7 +19,7 @@
 #
 # -------------------------------------------------------------
 
-from representations.representation import Representation
+from systemds.scuro.representations.representation import Representation
 
 
 class Modality:

src/main/python/systemds/scuro/modality/text_modality.py

@@ -20,8 +20,8 @@
 # -------------------------------------------------------------
 import os
 
-from modality.modality import Modality
-from representations.unimodal import UnimodalRepresentation
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.representations.unimodal import UnimodalRepresentation
 
 
 class TextModality(Modality):

src/main/python/systemds/scuro/modality/video_modality.py

@@ -20,8 +20,8 @@
 # -------------------------------------------------------------
 import os
 
-from modality.modality import Modality
-from representations.unimodal import UnimodalRepresentation
+from systemds.scuro.modality.modality import Modality
+from systemds.scuro.representations.unimodal import UnimodalRepresentation
 
 
 class VideoModality(Modality):

src/main/python/systemds/scuro/models/__init__.py

@@ -18,8 +18,3 @@
 # under the License.
 #
 # -------------------------------------------------------------
-from systemds.scuro.models import Model
-from systemds.scuro.discrete_model import DiscreteModel
-
-
-__all__ = ["Model", "DiscreteModel"]

src/main/python/systemds/scuro/models/discrete_model.py

@@ -18,7 +18,7 @@
 # under the License.
 #
 # -------------------------------------------------------------
-from models.model import Model
+from systemds.scuro.models.model import Model
 
 
 class DiscreteModel(Model):

src/main/python/systemds/scuro/representations/__init__.py

@@ -18,21 +18,3 @@
 # under the License.
 #
 # -------------------------------------------------------------
-from systemds.scuro.representations.representation import Representation
-from systemds.scuro.representations.average import Average
-from systemds.scuro.representations.concatenation import Concatenation
-from systemds.scuro.representations.fusion import Fusion
-from systemds.scuro.representations.unimodal import UnimodalRepresentation, HDF5, NPY, Pickle, JSON
-from systemds.scuro.representations.lstm import LSTM
-
-
-__all__ = ["Representation",
-           "Average",
-           "Concatenation",
-           "Fusion",
-           "UnimodalRepresentation",
-           "HDF5",
-           "NPY",
-           "Pickle",
-           "JSON",
-           "LSTM"]