diff --git a/adam/continuous.py b/adam/continuous.py index b330a1dfa..0b5f64ae1 100644 --- a/adam/continuous.py +++ b/adam/continuous.py @@ -4,10 +4,11 @@ from abc import abstractmethod import logging from math import sqrt -from typing import Tuple +from typing import Tuple, Iterable from typing_extensions import Protocol +from adam.perception.perception_utils import dist from attr import attrs, attrib from attr.validators import instance_of @@ -106,8 +107,131 @@ def match_score(self, value: float) -> float: consistently across different matcher types. """ standard_deviation = sqrt(self.sample_variance) - return 2.0 * norm.cdf( - self._mean - abs(value - self._mean), loc=self._mean, scale=standard_deviation + return ( + 1.0 + if standard_deviation == 0 and value == self._mean + else 2.0 + * norm.cdf( + self._mean - abs(value - self._mean), + loc=self._mean, + scale=standard_deviation, + ) + ) + + @staticmethod + def _calculate_new_values( + mean: float, + sum_of_squared_differences: float, + n_observations: int, + observation: float, + ) -> Tuple[float, float]: + new_mean = mean + (observation - mean) / (n_observations + 1) + new_sum_squared = sum_of_squared_differences + (observation - mean) * ( + observation - new_mean + ) + return new_mean, new_sum_squared + + def update_on_observation(self, value: float) -> None: + """ + Update the matcher's distribution to account for the given value. + """ + # With some help from Wikipedia. :) + # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + new_mean, new_sum_squared = self._calculate_new_values( + self._mean, self._sum_of_squared_differences, self._n_observations, value + ) + + self._mean = new_mean + self._sum_of_squared_differences = new_sum_squared + self._n_observations += 1 + + def merge(self, other: "ContinuousValueMatcher") -> None: + # pylint: disable=protected-access + # Pylint doesn't realize the "client class" whose private members we're accessing is this + # same class + if isinstance(other, GaussianContinuousValueMatcher): + if self._n_observations == 1: + # Treat our own mean as a single observation (because it is) and calculate the new + # mean and sum of squares from the other matcher's perspective. + new_mean, new_sum_squared = self._calculate_new_values( + other._mean, + other._sum_of_squared_differences, + other._n_observations, + self._mean, + ) + self._mean = new_mean + self._sum_of_squared_differences = new_sum_squared + self._n_observations += other._n_observations + elif other._n_observations == 1: + self.update_on_observation(other.mean) + else: + raise ValueError( + f"Cannot merge two matchers that both have multiple observations (self with " + f"{self._n_observations} and other with {other._n_observations})." + ) + else: + raise ValueError( + f"Cannot merge {type(self)} with matcher of foreign type {type(other)}" + ) + + +@attrs +class GaussianContinuousValueMatcher(ContinuousValueMatcher): + """ + Implements soft value matching where we pretend values come from a Gaussian distribution. + """ + + _mean: float = attrib(validator=instance_of(float)) + _sum_of_squared_differences: float = attrib( + validator=instance_of(float), init=False, default=0.0 + ) + """ + Also called M_{2,n} e.g. on the Wikipedia page. + https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + """ + _n_observations: int = attrib(validator=instance_of(int), init=False, default=1) + + @property + def mean(self) -> float: + return self._mean + + @property + def sample_variance(self) -> float: + if self._n_observations < 2: + return float("nan") + else: + return self._sum_of_squared_differences / (self._n_observations - 1) + + @property + def n_observations(self) -> int: + return self._n_observations + + @staticmethod + def from_observation(value) -> "GaussianContinuousValueMatcher": + """ + Return a new Gaussian continuous matcher created from the given single observation. + + This exists for clarity more than anything. + """ + return GaussianContinuousValueMatcher(value) + + def match_score(self, value: float) -> float: + """ + Return a score representing how closely the given value matches this distribution. + + This score should fall into the interval [0, 1] so that learners can threshold scores + consistently across different matcher types. + """ + standard_deviation = sqrt(self.sample_variance) + return ( + 1.0 + if standard_deviation == 0 and value == self._mean + else 2.0 + * norm.cdf( + self._mean - abs(value - self._mean), + loc=self._mean, + scale=standard_deviation, + ) ) @staticmethod @@ -165,3 +289,88 @@ def merge(self, other: "ContinuousValueMatcher") -> None: raise ValueError( f"Cannot merge {type(self)} with matcher of foreign type {type(other)}" ) + + +@attrs +class MultidimensionalGaussianContinuousValueMatcher(GaussianContinuousValueMatcher): + """ + Extend Gaussian continuous value matcher to >1 dimensional data + """ + _root_coordinates: Iterable[float] = attrib(validator = instance_of(tuple), init=True) + _mean: float = attrib(validator=instance_of(float), init=False, default=0.0) + + @staticmethod + def from_observation(point: Iterable[float]) -> "MultidimensionalGaussianContinuousValueMatcher": + """ + Return a new Multidimensional Gaussian continuous matcher created from the given single point. + + This exists for clarity more than anything. + """ + return MultidimensionalGaussianContinuousValueMatcher(point) + + + def match_score(self, point: Iterable[float]) -> float: + """ + Return a score representing how closely the given value matches this distribution. + + This score should fall into the interval [0, 1] so that learners can threshold scores + consistently across different matcher types. + """ + standard_deviation = sqrt(self.sample_variance) + value = dist(self._root_coordinates, point) + return ( + 1.0 + if standard_deviation == 0 and value == self._mean + else 0.0 if standard_deviation == 0 else + 2.0 * norm.cdf( + self._mean - abs(value - self._mean), + loc=self._mean, + scale=standard_deviation, + ) + ) + + def update_on_observation(self, point: Iterable[float]) -> None: + """ + Update the matcher's distribution to account for the given value. + """ + value = dist(self._root_coordinates, point) + # With some help from Wikipedia. :) + # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm + new_mean, new_sum_squared = self._calculate_new_values( + self._mean, self._sum_of_squared_differences, self._n_observations, value + ) + + self._mean = new_mean + self._sum_of_squared_differences = new_sum_squared + self._n_observations += 1 + + def merge(self, other: "ContinuousValueMatcher") -> None: + # pylint: disable=protected-access + # Pylint doesn't realize the "client class" whose private members we're accessing is this + # same class + if isinstance(other, MultidimensionalGaussianContinuousValueMatcher): + if self._n_observations == 1: + # Treat our own mean as a single observation (because it is) and calculate the new + # mean and sum of squares from the other matcher's perspective. + new_mean, new_sum_squared = self._calculate_new_values( + other._mean, + other._sum_of_squared_differences, + other._n_observations, + dist(self._root_coordinates, other._root_coordinates), + ) + self._mean = new_mean + self._sum_of_squared_differences = new_sum_squared + self._n_observations += other._n_observations + self._root_coordinates = other._root_coordinates + elif other._n_observations == 1: + self.update_on_observation(other._root_coordinates) + else: + return + raise ValueError( + f"Cannot merge two matchers that both have multiple observations (self with " + f"{self._n_observations} and other with {other._n_observations})." + ) + else: + raise ValueError( + f"Cannot merge {type(self)} with matcher of foreign type {type(other)}" + ) diff --git a/adam/curriculum/curriculum_from_files.py b/adam/curriculum/curriculum_from_files.py index 0073803ce..aee04c85d 100644 --- a/adam/curriculum/curriculum_from_files.py +++ b/adam/curriculum/curriculum_from_files.py @@ -25,9 +25,16 @@ PHASE_3_TRAINING_CURRICULUM_OPTIONS = [ "m4_core", "m4_stretch", + "m5_objects_v0_with_mugs", + "m5_objects_v0_apples_oranges_bananas" ] -PHASE_3_TESTING_CURRICULUM_OPTIONS = ["m4_core_eval", "m4_stretch_eval"] +PHASE_3_TESTING_CURRICULUM_OPTIONS = [ + "m4_core_eval", + "m4_stretch_eval", + "m5_objects_v0_with_mugs_eval", + "m5_objects_v0_apples_oranges_bananas_eval", +] TRAINING_CUR = "training" TESTING_CUR = "testing" @@ -92,7 +99,7 @@ def phase3_load_from_disk( # pylint: disable=unused-argument ) as situation_description_file: situation_description = yaml.safe_load(situation_description_file) language_tuple = tuple(situation_description["language"].split(" ")) - feature_yamls = sorted(situation_dir.glob("feature_*")) + feature_yamls = sorted(situation_dir.glob("feature*")) situation = SimulationSituation( language=language_tuple, scene_images_png=sorted(situation_dir.glob("rgb_*")), diff --git a/adam/perception/perception_graph.py b/adam/perception/perception_graph.py index a6f3fed32..0f6d8643d 100644 --- a/adam/perception/perception_graph.py +++ b/adam/perception/perception_graph.py @@ -88,6 +88,7 @@ CategoricalNode, ContinuousNode, RgbColorNode, + CielabColorNode, GraphNode, ObjectStroke, StrokeGNNRecognitionNode, @@ -110,6 +111,7 @@ CategoricalPredicate, ContinuousPredicate, RgbColorPredicate, + CielabColorPredicate, ObjectStrokePredicate, StrokeGNNRecognitionPredicate, DistributionalContinuousPredicate, @@ -585,6 +587,8 @@ def _to_dot_node( label = f"axis:{unwrapped_perception_node.debug_name}" elif isinstance(unwrapped_perception_node, RgbColorPerception): label = unwrapped_perception_node.hex + elif isinstance(unwrapped_perception_node, RgbColorPerception): + label = unwrapped_perception_node.hex elif isinstance(unwrapped_perception_node, OntologyNode): label = unwrapped_perception_node.handle elif isinstance(unwrapped_perception_node, Geon): @@ -605,7 +609,13 @@ def _to_dot_node( label = f"Stroke: [{', '.join(str(point) for point in unwrapped_perception_node.normalized_coordinates)}]" elif isinstance( unwrapped_perception_node, - (ContinuousNode, CategoricalNode, RgbColorNode, StrokeGNNRecognitionNode), + ( + ContinuousNode, + CategoricalNode, + RgbColorNode, + CielabColorNode, + StrokeGNNRecognitionNode, + ), ): label = str(unwrapped_perception_node) else: @@ -1058,6 +1068,10 @@ def map_node(node: Any) -> "NodePredicate": perception_node_to_pattern_node[key] = RgbColorPredicate.from_node( node ) + elif isinstance(node, CielabColorNode): + perception_node_to_pattern_node[key] = CielabColorPredicate.from_node( + node, min_match_score=min_continuous_feature_match_score + ) elif isinstance(node, ObjectStroke): perception_node_to_pattern_node[ key @@ -2279,10 +2293,12 @@ def _translate_region( DistributionalContinuousPredicate, ContinuousPredicate, RgbColorPredicate, + CielabColorPredicate, ObjectStroke, CategoricalNode, ContinuousNode, RgbColorNode, + CielabColorNode, # Paths are rare, match them next IsPathPredicate, PathOperatorPredicate, @@ -2315,6 +2331,7 @@ def _pattern_matching_node_order(node_node_data_tuple) -> int: CategoricalNode, ContinuousNode, RgbColorNode, + CielabColorNode, SpatialPath, PathOperator, OntologyNode, diff --git a/adam/perception/perception_graph_nodes.py b/adam/perception/perception_graph_nodes.py index ed8899410..77ac4f7f9 100644 --- a/adam/perception/perception_graph_nodes.py +++ b/adam/perception/perception_graph_nodes.py @@ -3,6 +3,8 @@ from attr import attrs, attrib from attr.validators import instance_of, in_, optional, deep_iterable +from colormath.color_conversions import convert_color +from colormath.color_objects import sRGBColor, LabColor from immutablecollections import ImmutableSet from immutablecollections.converter_utils import _to_immutableset @@ -126,6 +128,54 @@ def __str__(self) -> str: return f"#{hex(self.red)[2:]}{hex(self.green)[2:]}{hex(self.blue)[2:]}" +@attrs(frozen=True, slots=True, eq=False) +class CielabColorNode(GraphNode): + """A node representing a CIELAB perception value.""" + + lab_l: float = attrib(validator=instance_of(float)) + lab_a: float = attrib(validator=instance_of(float)) + lab_b: float = attrib(validator=instance_of(float)) + + def dot_label(self): + return f"CielabColorNode({self})" + + def __str__(self) -> str: + return f"Lab=({self.lab_l:.2f}, {self.lab_a:.2f}, {self.lab_b:.2f})" + + def to_tuple(self) -> tuple: + return self.lab_l, self.lab_a, self.lab_b + + @staticmethod + def from_rgb(node: RgbColorNode) -> "CielabColorNode": + rgb_color: sRGBColor = sRGBColor( + node.red, node.green, node.blue, is_upscaled=True + ) + lab_color: LabColor = convert_color( + color=rgb_color, target_cs=LabColor, target_illuminant="d65" + ) + return CielabColorNode( + lab_l=lab_color.lab_l, + lab_a=lab_color.lab_a, + lab_b=lab_color.lab_b, + weight=node.weight, + ) + + @staticmethod + def from_colors( + red: float, green: float, blue: float, *, weight: float = 1.0 + ) -> "CielabColorNode": + rgb_color: sRGBColor = sRGBColor(red, green, blue, is_upscaled=True) + lab_color: LabColor = convert_color( + color=rgb_color, target_cs=LabColor, target_illuminant="d65" + ) + return CielabColorNode( + lab_l=lab_color.lab_l, + lab_a=lab_color.lab_a, + lab_b=lab_color.lab_b, + weight=weight, + ) + + @attrs(frozen=True, slots=True, eq=False) class StrokeGNNRecognitionNode(GraphNode): """A property node indicating Stroke GNN object recognition.""" diff --git a/adam/perception/perception_graph_predicates.py b/adam/perception/perception_graph_predicates.py index 4803cbd84..4232e40b1 100644 --- a/adam/perception/perception_graph_predicates.py +++ b/adam/perception/perception_graph_predicates.py @@ -7,7 +7,7 @@ from immutablecollections.converter_utils import _to_tuple, _to_immutableset from adam.axis import GeonAxis -from adam.continuous import ContinuousValueMatcher, GaussianContinuousValueMatcher +from adam.continuous import ContinuousValueMatcher, GaussianContinuousValueMatcher, MultidimensionalGaussianContinuousValueMatcher from adam.geon import Geon, CrossSection from adam.math_3d import Point from adam.ontology import OntologyNode @@ -27,11 +27,13 @@ CategoricalNode, ContinuousNode, RgbColorNode, + CielabColorNode, ObjectStroke, StrokeGNNRecognitionNode, JointPointNode, TrajectoryRecognitionNode, ) +from adam.perception.perception_utils import dist # Perception graph predicate nodes are defined below. # These match the graph nodes defined above when using computer vision inputs @@ -411,6 +413,97 @@ def from_node(node: RgbColorNode) -> "RgbColorPredicate": return RgbColorPredicate(red=node.red, green=node.green, blue=node.blue) +@attrs(slots=True, eq=False) +class CielabColorPredicate(NodePredicate): + """ + Matches a node where the CIELAB value matches closely using a Gaussian distribution. + """ + + lab_l: float = attrib(validator=instance_of(float)) + lab_a: float = attrib(validator=instance_of(float)) + lab_b: float = attrib(validator=instance_of(float)) + matcher: ContinuousValueMatcher = attrib( + validator=instance_of(ContinuousValueMatcher) + ) + min_match_score: float = attrib(validator=instance_of(float)) + _fallback_tolerance: float = attrib(validator=instance_of(float)) + _weight: float = attrib( + kw_only=True, default=1.0, validator=instance_of(float), eq=False + ) + + @property + def weight(self) -> float: + return self._weight + + @staticmethod + def from_node( + node: CielabColorNode, *, min_match_score: float, fallback_tolerance: float = 0.25 + ) -> "CielabColorPredicate": + return CielabColorPredicate( + lab_l=node.lab_l, + lab_a=node.lab_a, + lab_b=node.lab_b, + matcher=MultidimensionalGaussianContinuousValueMatcher.from_observation(node.to_tuple()), + min_match_score=min_match_score, + fallback_tolerance=fallback_tolerance, + weight=1.0, + ) + + def __call__(self, graph_node: PerceptionGraphNode) -> bool: + if isinstance(graph_node, CielabColorNode): + # If we have enough samples, do score-based value matching. + if self.matcher.n_observations > 2: + return ( + self.matcher.match_score(graph_node.to_tuple()) + >= self.min_match_score + ) + # Otherwise, fall back on value + tolerance matching. + else: + return ( + -self._fallback_tolerance + <= dist(self.to_tuple(), graph_node.to_tuple()) + <= self._fallback_tolerance + ) + return False + + def dot_label(self) -> str: + return f"CielabColorFeature(L={self.lab_l}, a={self.lab_a}, b={self.lab_b}) <{self._weight}>" + + def is_equivalent(self, other) -> bool: + return isinstance(other, CielabColorPredicate) + + def matches_predicate(self, predicate_node: NodePredicate) -> bool: + return isinstance(predicate_node, CielabColorPredicate) + + def confirm_match(self, node: Union[PerceptionGraphNode, "NodePredicate"]) -> None: + if isinstance(node, CielabColorPredicate): + self._merge_with_compatible_node(node) + else: + raise ValueError( + f"Can't confirm match with apparently non-matching node {node}." + ) + + def _merge_with_compatible_node(self, node: "CielabColorPredicate") -> None: + # pylint: disable=protected-access + # Pylint doesn't realize the "client class" whose private members we're accessing is this + # same class + # + # We don't statically know the concrete type of either matcher, so we check the types match + # exactly instead of using isinstance. + if type(self.matcher) == type( # noqa: E721 pylint: disable=unidiomatic-typecheck + node.matcher + ): + self.matcher.merge(node.matcher) + + else: + raise ValueError( + f"Can't merge distributions of different types {self.matcher} and {node.matcher}." + ) + + def to_tuple(self) -> tuple: + return self.lab_l, self.lab_a, self.lab_b + + @attrs(frozen=True, slots=True, eq=False) class ObjectStrokePredicate(NodePredicate): """Matches an Object Stroke""" diff --git a/adam/perception/perception_utils.py b/adam/perception/perception_utils.py new file mode 100644 index 000000000..dfd75320d --- /dev/null +++ b/adam/perception/perception_utils.py @@ -0,0 +1,11 @@ +from math import sqrt +from typing import Iterable + + +def dist(list_1: Iterable, list_2: Iterable) -> float: + """ + Computes Euclidean distance between two iterables. + + Taken from Python 3.8: https://docs.python.org/3/library/math.html#math.dist""" + + return sqrt(sum((px - qx) ** 2.0 for px, qx in zip(list_1, list_2))) diff --git a/adam/perception/visual_perception.py b/adam/perception/visual_perception.py index aea753429..3042f57c3 100644 --- a/adam/perception/visual_perception.py +++ b/adam/perception/visual_perception.py @@ -22,6 +22,7 @@ from adam.perception.perception_graph_nodes import ( GraphNode, RgbColorNode, + CielabColorNode, CategoricalNode, ContinuousNode, ObjectStroke, @@ -131,6 +132,14 @@ def from_mapping( if color_is_rgb else color_as_category(color_property) ] + if color_is_rgb: + properties.append( + CielabColorNode.from_colors( + red=color_property[0], + green=color_property[1], + blue=color_property[2], + ) + ) properties.extend( CategoricalNode(label=entry, value=cluster_map[entry], weight=1.0) for entry in CATEGORY_PROPERTY_KEYS diff --git a/parameters/experiments/p3/m5_objects_v0_apples_oranges_bananas.params b/parameters/experiments/p3/m5_objects_v0_apples_oranges_bananas.params new file mode 100644 index 000000000..de44e65ae --- /dev/null +++ b/parameters/experiments/p3/m5_objects_v0_apples_oranges_bananas.params @@ -0,0 +1,61 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + beam_size: 10 + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m5_objects_v0_apples_oranges_bananas" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m5_objects_v0_apples_oranges_bananas_eval" + color_is_rgb: True + +# Experiment Configuration +experiment: "m5_objects_v0_apples_oranges_bananas" +experiment_group_dir: '%adam_experiment_root%/color_experiment/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true \ No newline at end of file diff --git a/parameters/experiments/p3/m5_objects_v0_with_mugs_subset.params b/parameters/experiments/p3/m5_objects_v0_with_mugs_subset.params new file mode 100644 index 000000000..75780ad3c --- /dev/null +++ b/parameters/experiments/p3/m5_objects_v0_with_mugs_subset.params @@ -0,0 +1,61 @@ +_includes: + - "../../root.params" + +# Learner Configuration +learner: simulated-integrated-learner-params +object_learner: + learner_type: "subset" + beam_size: 10 + ontology: "phase3" + min_continuous_feature_match_score: 0.05 +attribute_learner: + learner_type: "none" +relation_learner: + learner_type: "none" +action_learner: + learner_type: "none" +plural_learner: + learner_type: "none" +affordance_learner: + learner_type: "none" +include_functional_learner: false +include_generics_learner: false +suppress_error: false + +# Curriculum Configuration +curriculum: "phase3" +train_curriculum: + curriculum_type: "training" + curriculum: "m5_objects_v0_with_mugs" + color_is_rgb: True +test_curriculum: + curriculum_type: "testing" + curriculum: "m5_objects_v0_with_mugs_eval" + color_is_rgb: True + +# Experiment Configuration +experiment: "m5_objects_v0_with_mugs" +experiment_group_dir: '%adam_experiment_root%/learners/%learner%/experiments/%train_curriculum.curriculum%/' +log_learner_state: true +experiment_type: simulated + +# Hypothesis Logging +hypothesis_log_dir: "%experiment_group_dir%/hypotheses" +log_hypothesis_every_n_steps: 250 + +# Debug Configuration +debug_log_directory: "%experiment_group_dir%/graphs" +debug_perception_log_dir: "%experiment_group_dir%/perception_graphs" + +# Observer Params +post_observer: + experiment_output_path: "%experiment_group_dir%" + copy_curriculum: true + file_name: "post_decode" + +test_observer: + experiment_output_path: "%post_observer.experiment_output_path%/test_curriculums/%test_curriculum.curriculum%/" + copy_curriculum: true + file_name: "post_decode" + calculate_accuracy_by_language: true + calculate_overall_accuracy: true \ No newline at end of file diff --git a/requirements_lock.txt b/requirements_lock.txt index ccee61ae9..bd877b3e5 100644 --- a/requirements_lock.txt +++ b/requirements_lock.txt @@ -7,6 +7,7 @@ certifi==2021.10.8 charset-normalizer==2.0.12 click==8.1.2 click-default-group==1.2.2 +colormath==3.0.0 contexttimer==0.3.3 coverage==6.3.2 deprecation==2.1.0 @@ -70,7 +71,7 @@ temppathlib==1.2.0 toml==0.10.2 tomli==2.0.1 towncrier==21.9.0 -typed-ast==1.5.2 +typed-ast==1.5.3 typing_extensions==4.1.1 urllib3==1.26.9 vistautils==0.24.0