From f99d15cf40feb658cc461236806d6367b9267d99 Mon Sep 17 00:00:00 2001 From: Dominic Kempf Date: Tue, 13 Dec 2022 13:15:56 +0100 Subject: [PATCH] Restrict the scope of facial expression recognition by thresholding likelihood (#38) * Apply thresholding to restrict the scope of facial expression recognition * fix test dict faces * remove approx * do not ignore data in subdirs * where does test_display come from * remove face analysis duplication * imageai sneaked into ci Co-authored-by: Inga Ulusoy --- .flake8 | 2 +- .gitignore | 2 +- misinformation/faces.py | 71 +++++++++---------- .../test/data/example_analysis_faces.json | 1 - misinformation/test/data/example_faces.json | 2 +- misinformation/test/test_display.py | 13 +--- misinformation/test/test_faces.py | 12 +--- 7 files changed, 41 insertions(+), 62 deletions(-) delete mode 100644 misinformation/test/data/example_analysis_faces.json diff --git a/.flake8 b/.flake8 index 1dc56f9..07619f9 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -ignore = E203, F401, E402, E501 +ignore = E203, F401, E402, E501, W503 exclude = .git,__pycache__,.ipynb_checkpoints max-line-length = 90 max-complexity = 18 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8090558..97b0bf0 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,4 @@ dmypy.json .pyre/ # data folder -data/ \ No newline at end of file +/data/ \ No newline at end of file diff --git a/misinformation/faces.py b/misinformation/faces.py index 23f3c25..58c66d6 100644 --- a/misinformation/faces.py +++ b/misinformation/faces.py @@ -84,14 +84,22 @@ retinaface_model = DownloadResource( class EmotionDetector(utils.AnalysisMethod): - def __init__(self, subdict: dict) -> None: + def __init__( + self, subdict: dict, emotion_threshold=50.0, race_threshold=50.0 + ) -> None: super().__init__(subdict) self.subdict.update(self.set_keys()) - self.emotion_threshold = 25.0 - self.race_threshold = 80.0 - self.negative_emotion = ["angry", "disgust", "fear", "sad"] - self.positive_emotion = ["happy"] - self.neutral_emotion = ["surprise", "neutral"] + self.emotion_threshold = emotion_threshold + self.race_threshold = race_threshold + self.emotion_categories = { + "angry": "Negative", + "disgust": "Negative", + "fear": "Negative", + "sad": "Negative", + "happy": "Positive", + "surprise": "Neutral", + "neutral": "Neutral", + } def set_keys(self) -> dict: params = { @@ -191,37 +199,28 @@ class EmotionDetector(utils.AnalysisMethod): self.subdict["emotion"].append(None) self.subdict["emotion (category)"].append(None) elif not result[person]["wears_mask"]: - # also assign categories based on threshold - cumulative = [ - sum( - result[person]["emotion"][key] - for key in result[person]["emotion"].keys() - if key in self.negative_emotion + # Check whether the race threshold was exceeded + if ( + result[person]["race"][result[person]["dominant_race"]] + > self.race_threshold + ): + self.subdict["race"].append(result[person]["dominant_race"]) + else: + self.subdict["race"].append(None) + + # Check whether the emotion threshold was exceeded + if ( + result[person]["emotion"][result[person]["dominant_emotion"]] + > self.emotion_threshold + ): + self.subdict["emotion"].append(result[person]["dominant_emotion"]) + self.subdict["emotion (category)"].append( + self.emotion_categories[result[person]["dominant_emotion"]] ) - ] - cumulative.append( - sum( - result[person]["emotion"][key] - for key in result[person]["emotion"].keys() - if key in self.positive_emotion - ) - ) - cumulative.append( - sum( - result[person]["emotion"][key] - for key in result[person]["emotion"].keys() - if key in self.neutral_emotion - ) - ) - expression = ["Negative", "Positive", "Neutral"] - # now zip the two lists and sort according to highest contribution - category = sorted(zip(cumulative, expression), reverse=True)[0][1] - self.subdict["race"].append(result[person]["dominant_race"]) - dominant = result[person]["dominant_emotion"] - self.subdict["emotion"].append( - (dominant, result[person]["emotion"][dominant]) - ) - self.subdict["emotion (category)"].append(category) + else: + self.subdict["emotion"].append(None) + self.subdict["emotion (category)"].append(None) + return self.subdict def wears_mask(self, face: np.ndarray) -> bool: diff --git a/misinformation/test/data/example_analysis_faces.json b/misinformation/test/data/example_analysis_faces.json deleted file mode 100644 index 55d66f9..0000000 --- a/misinformation/test/data/example_analysis_faces.json +++ /dev/null @@ -1 +0,0 @@ -{"IMG_2746": {"filename": "./test/data/IMG_2746.png", "face": "Yes", "multiple_faces": "Yes", "no_faces": 11, "wears_mask": ["No", "No", "Yes"], "age": [36, 35, 33], "gender": ["Man", "Man", "Man"], "race": ["white", "white", null], "emotion": [["sad", 73.24262697950762], ["fear", 84.20094847679138], null], "emotion (category)": ["Negative", "Negative", null]}} \ No newline at end of file diff --git a/misinformation/test/data/example_faces.json b/misinformation/test/data/example_faces.json index cc44e77..8465ca1 100644 --- a/misinformation/test/data/example_faces.json +++ b/misinformation/test/data/example_faces.json @@ -7,6 +7,6 @@ "age": [36, 35, 33], "gender": ["Man", "Man", "Man"], "race": ["white", "white", null], - "emotion": [["sad", 73.24264486090212], ["fear", 84.20093247879356], null], + "emotion": ["sad", "fear", null], "emotion (category)": ["Negative", "Negative", null] } \ No newline at end of file diff --git a/misinformation/test/test_display.py b/misinformation/test/test_display.py index 10dbd21..ea634f9 100644 --- a/misinformation/test/test_display.py +++ b/misinformation/test/test_display.py @@ -6,22 +6,13 @@ from pytest import approx def test_explore_analysis_faces(): mydict = {"IMG_2746": {"filename": "./test/data/IMG_2746.png"}} explore_analysis(mydict, identify="faces") - with open("./test/data/example_analysis_faces.json", "r") as file: + with open("./test/data/example_faces.json", "r") as file: outs = json.load(file) for im_key in mydict.keys(): sub_dict = mydict[im_key] for key in sub_dict.keys(): - if key != "emotion": - assert sub_dict[key] == outs[im_key][key] - # json can't handle tuples natively - for i in range(0, len(sub_dict["emotion"])): - temp = ( - list(sub_dict["emotion"][i]) - if type(sub_dict["emotion"][i]) == tuple - else sub_dict["emotion"][i] - ) - assert approx(temp) == outs[im_key]["emotion"][i] + assert sub_dict[key] == outs[key] def test_explore_analysis_objects(): diff --git a/misinformation/test/test_faces.py b/misinformation/test/test_faces.py index 27c40a1..9682c22 100644 --- a/misinformation/test/test_faces.py +++ b/misinformation/test/test_faces.py @@ -8,19 +8,9 @@ def test_analyse_faces(): "filename": "./test/data/IMG_2746.png", } mydict = fc.EmotionDetector(mydict).analyse_image() - print(mydict) with open("./test/data/example_faces.json", "r") as file: out_dict = json.load(file) for key in mydict.keys(): - if key != "emotion": - assert mydict[key] == out_dict[key] - # json can't handle tuples natively - for i in range(0, len(mydict["emotion"])): - temp = ( - list(mydict["emotion"][i]) - if type(mydict["emotion"][i]) == tuple - else mydict["emotion"][i] - ) - assert approx(temp) == out_dict["emotion"][i] + assert mydict[key] == out_dict[key]