Restrict the scope of facial expression recognition by thresholding likelihood (#38)

* Apply thresholding to restrict the scope of facial expression recognition

* fix test dict faces

* remove approx

* do not ignore data in subdirs

* where does test_display come from

* remove face analysis duplication

* imageai sneaked into ci

Co-authored-by: Inga Ulusoy <inga.ulusoy@uni-heidelberg.de>

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = E203, F401, E402, E501
+ignore = E203, F401, E402, E501, W503
 exclude = .git,__pycache__,.ipynb_checkpoints
 max-line-length = 90
 max-complexity = 18

.gitignore

@@ -129,4 +129,4 @@ dmypy.json
 .pyre/
 
 # data folder
-data/
+/data/

misinformation/faces.py

@@ -84,14 +84,22 @@ retinaface_model = DownloadResource(
 
 
 class EmotionDetector(utils.AnalysisMethod):
-    def __init__(self, subdict: dict) -> None:
+    def __init__(
+        self, subdict: dict, emotion_threshold=50.0, race_threshold=50.0
+    ) -> None:
         super().__init__(subdict)
         self.subdict.update(self.set_keys())
-        self.emotion_threshold = 25.0
+        self.emotion_threshold = emotion_threshold
-        self.race_threshold = 80.0
+        self.race_threshold = race_threshold
-        self.negative_emotion = ["angry", "disgust", "fear", "sad"]
+        self.emotion_categories = {
-        self.positive_emotion = ["happy"]
+            "angry": "Negative",
-        self.neutral_emotion = ["surprise", "neutral"]
+            "disgust": "Negative",
+            "fear": "Negative",
+            "sad": "Negative",
+            "happy": "Positive",
+            "surprise": "Neutral",
+            "neutral": "Neutral",
+        }
 
     def set_keys(self) -> dict:
         params = {
@@ -191,37 +199,28 @@ class EmotionDetector(utils.AnalysisMethod):
                 self.subdict["emotion"].append(None)
                 self.subdict["emotion (category)"].append(None)
             elif not result[person]["wears_mask"]:
-                # also assign categories based on threshold
+                # Check whether the race threshold was exceeded
-                cumulative = [
+                if (
-                    sum(
+                    result[person]["race"][result[person]["dominant_race"]]
-                        result[person]["emotion"][key]
+                    > self.race_threshold
-                        for key in result[person]["emotion"].keys()
+                ):
-                        if key in self.negative_emotion
-                    )
-                ]
-                cumulative.append(
-                    sum(
-                        result[person]["emotion"][key]
-                        for key in result[person]["emotion"].keys()
-                        if key in self.positive_emotion
-                    )
-                )
-                cumulative.append(
-                    sum(
-                        result[person]["emotion"][key]
-                        for key in result[person]["emotion"].keys()
-                        if key in self.neutral_emotion
-                    )
-                )
-                expression = ["Negative", "Positive", "Neutral"]
-                # now zip the two lists and sort according to highest contribution
-                category = sorted(zip(cumulative, expression), reverse=True)[0][1]
                     self.subdict["race"].append(result[person]["dominant_race"])
-                dominant = result[person]["dominant_emotion"]
+                else:
-                self.subdict["emotion"].append(
+                    self.subdict["race"].append(None)
-                    (dominant, result[person]["emotion"][dominant])
+
+                # Check whether the emotion threshold was exceeded
+                if (
+                    result[person]["emotion"][result[person]["dominant_emotion"]]
+                    > self.emotion_threshold
+                ):
+                    self.subdict["emotion"].append(result[person]["dominant_emotion"])
+                    self.subdict["emotion (category)"].append(
+                        self.emotion_categories[result[person]["dominant_emotion"]]
                     )
-                self.subdict["emotion (category)"].append(category)
+                else:
+                    self.subdict["emotion"].append(None)
+                    self.subdict["emotion (category)"].append(None)
+
         return self.subdict
 
     def wears_mask(self, face: np.ndarray) -> bool:

misinformation/test/data/example_analysis_faces.json

@@ -1 +0,0 @@
-{"IMG_2746": {"filename": "./test/data/IMG_2746.png", "face": "Yes", "multiple_faces": "Yes", "no_faces": 11, "wears_mask": ["No", "No", "Yes"], "age": [36, 35, 33], "gender": ["Man", "Man", "Man"], "race": ["white", "white", null], "emotion": [["sad", 73.24262697950762], ["fear", 84.20094847679138], null], "emotion (category)": ["Negative", "Negative", null]}}

misinformation/test/data/example_faces.json

@@ -7,6 +7,6 @@
         "age": [36, 35, 33], 
         "gender": ["Man", "Man", "Man"], 
         "race": ["white", "white", null], 
-        "emotion": [["sad", 73.24264486090212], ["fear", 84.20093247879356], null], 
+        "emotion": ["sad", "fear", null], 
         "emotion (category)": ["Negative", "Negative", null]
 }    

misinformation/test/test_display.py

@@ -6,22 +6,13 @@ from pytest import approx
 def test_explore_analysis_faces():
     mydict = {"IMG_2746": {"filename": "./test/data/IMG_2746.png"}}
     explore_analysis(mydict, identify="faces")
-    with open("./test/data/example_analysis_faces.json", "r") as file:
+    with open("./test/data/example_faces.json", "r") as file:
         outs = json.load(file)
 
     for im_key in mydict.keys():
         sub_dict = mydict[im_key]
         for key in sub_dict.keys():
-            if key != "emotion":
+            assert sub_dict[key] == outs[key]
-                assert sub_dict[key] == outs[im_key][key]
-        # json can't handle tuples natively
-        for i in range(0, len(sub_dict["emotion"])):
-            temp = (
-                list(sub_dict["emotion"][i])
-                if type(sub_dict["emotion"][i]) == tuple
-                else sub_dict["emotion"][i]
-            )
-            assert approx(temp) == outs[im_key]["emotion"][i]
 
 
 def test_explore_analysis_objects():

misinformation/test/test_faces.py

@@ -8,19 +8,9 @@ def test_analyse_faces():
         "filename": "./test/data/IMG_2746.png",
     }
     mydict = fc.EmotionDetector(mydict).analyse_image()
-    print(mydict)
 
     with open("./test/data/example_faces.json", "r") as file:
         out_dict = json.load(file)
 
     for key in mydict.keys():
-        if key != "emotion":
         assert mydict[key] == out_dict[key]
-    # json can't handle tuples natively
-    for i in range(0, len(mydict["emotion"])):
-        temp = (
-            list(mydict["emotion"][i])
-            if type(mydict["emotion"][i]) == tuple
-            else mydict["emotion"][i]
-        )
-        assert approx(temp) == out_dict["emotion"][i]