зеркало из
https://github.com/ssciwr/AMMICO.git
synced 2025-10-29 05:04:14 +02:00
f99d15cf40
* Apply thresholding to restrict the scope of facial expression recognition * fix test dict faces * remove approx * do not ignore data in subdirs * where does test_display come from * remove face analysis duplication * imageai sneaked into ci Co-authored-by: Inga Ulusoy <inga.ulusoy@uni-heidelberg.de>
270 строки
9.6 KiB
Python
270 строки
9.6 KiB
Python
import cv2
|
|
import numpy as np
|
|
import os
|
|
import shutil
|
|
import pathlib
|
|
import ipywidgets
|
|
|
|
from tensorflow.keras.models import load_model
|
|
from tensorflow.keras.applications.mobilenet_v2 import preprocess_input
|
|
from tensorflow.keras.preprocessing.image import img_to_array
|
|
from deepface import DeepFace
|
|
from retinaface import RetinaFace
|
|
|
|
from misinformation.utils import DownloadResource
|
|
import misinformation.utils as utils
|
|
|
|
DEEPFACE_PATH = ".deepface"
|
|
|
|
|
|
def deepface_symlink_processor(name):
|
|
def _processor(fname, action, pooch):
|
|
if not os.path.exists(name):
|
|
# symlink does not work on windows
|
|
# use copy if running on windows
|
|
if os.name != "nt":
|
|
os.symlink(fname, name)
|
|
else:
|
|
shutil.copy(fname, name)
|
|
return fname
|
|
|
|
return _processor
|
|
|
|
|
|
face_mask_model = DownloadResource(
|
|
url="https://github.com/chandrikadeb7/Face-Mask-Detection/raw/v1.0.0/mask_detector.model",
|
|
known_hash="sha256:d0b30e2c7f8f187c143d655dee8697fcfbe8678889565670cd7314fb064eadc8",
|
|
)
|
|
|
|
deepface_age_model = DownloadResource(
|
|
url="https://github.com/serengil/deepface_models/releases/download/v1.0/age_model_weights.h5",
|
|
known_hash="sha256:0aeff75734bfe794113756d2bfd0ac823d51e9422c8961125b570871d3c2b114",
|
|
processor=deepface_symlink_processor(
|
|
pathlib.Path.home().joinpath(DEEPFACE_PATH, "weights", "age_model_weights.h5")
|
|
),
|
|
)
|
|
|
|
deepface_face_expression_model = DownloadResource(
|
|
url="https://github.com/serengil/deepface_models/releases/download/v1.0/facial_expression_model_weights.h5",
|
|
known_hash="sha256:e8e8851d3fa05c001b1c27fd8841dfe08d7f82bb786a53ad8776725b7a1e824c",
|
|
processor=deepface_symlink_processor(
|
|
pathlib.Path.home().joinpath(
|
|
".deepface", "weights", "facial_expression_model_weights.h5"
|
|
)
|
|
),
|
|
)
|
|
|
|
deepface_gender_model = DownloadResource(
|
|
url="https://github.com/serengil/deepface_models/releases/download/v1.0/gender_model_weights.h5",
|
|
known_hash="sha256:45513ce5678549112d25ab85b1926fb65986507d49c674a3d04b2ba70dba2eb5",
|
|
processor=deepface_symlink_processor(
|
|
pathlib.Path.home().joinpath(
|
|
DEEPFACE_PATH, "weights", "gender_model_weights.h5"
|
|
)
|
|
),
|
|
)
|
|
|
|
deepface_race_model = DownloadResource(
|
|
url="https://github.com/serengil/deepface_models/releases/download/v1.0/race_model_single_batch.h5",
|
|
known_hash="sha256:eb22b28b1f6dfce65b64040af4e86003a5edccb169a1a338470dde270b6f5e54",
|
|
processor=deepface_symlink_processor(
|
|
pathlib.Path.home().joinpath(
|
|
DEEPFACE_PATH, "weights", "race_model_single_batch.h5"
|
|
)
|
|
),
|
|
)
|
|
|
|
retinaface_model = DownloadResource(
|
|
url="https://github.com/serengil/deepface_models/releases/download/v1.0/retinaface.h5",
|
|
known_hash="sha256:ecb2393a89da3dd3d6796ad86660e298f62a0c8ae7578d92eb6af14e0bb93adf",
|
|
processor=deepface_symlink_processor(
|
|
pathlib.Path.home().joinpath(DEEPFACE_PATH, "weights", "retinaface.h5")
|
|
),
|
|
)
|
|
|
|
|
|
class EmotionDetector(utils.AnalysisMethod):
|
|
def __init__(
|
|
self, subdict: dict, emotion_threshold=50.0, race_threshold=50.0
|
|
) -> None:
|
|
super().__init__(subdict)
|
|
self.subdict.update(self.set_keys())
|
|
self.emotion_threshold = emotion_threshold
|
|
self.race_threshold = race_threshold
|
|
self.emotion_categories = {
|
|
"angry": "Negative",
|
|
"disgust": "Negative",
|
|
"fear": "Negative",
|
|
"sad": "Negative",
|
|
"happy": "Positive",
|
|
"surprise": "Neutral",
|
|
"neutral": "Neutral",
|
|
}
|
|
|
|
def set_keys(self) -> dict:
|
|
params = {
|
|
"face": "No",
|
|
"multiple_faces": "No",
|
|
"no_faces": 0,
|
|
"wears_mask": ["No"],
|
|
"age": [None],
|
|
"gender": [None],
|
|
"race": [None],
|
|
"emotion": [None],
|
|
"emotion (category)": [None],
|
|
}
|
|
return params
|
|
|
|
def analyse_image(self):
|
|
return self.facial_expression_analysis()
|
|
|
|
def analyze_single_face(self, face: np.ndarray) -> dict:
|
|
fresult = {}
|
|
|
|
# Determine whether the face wears a mask
|
|
fresult["wears_mask"] = self.wears_mask(face)
|
|
|
|
# Adapt the features we are looking for depending on whether a mask is
|
|
# worn. White masks screw race detection, emotion detection is useless.
|
|
actions = ["age", "gender"]
|
|
if not fresult["wears_mask"]:
|
|
actions = actions + ["race", "emotion"]
|
|
|
|
# Ensure that all data has been fetched by pooch
|
|
deepface_age_model.get()
|
|
deepface_face_expression_model.get()
|
|
deepface_gender_model.get()
|
|
deepface_race_model.get()
|
|
|
|
# Run the full DeepFace analysis
|
|
fresult.update(
|
|
DeepFace.analyze(
|
|
img_path=face,
|
|
actions=actions,
|
|
prog_bar=False,
|
|
detector_backend="skip",
|
|
)
|
|
)
|
|
|
|
# We remove the region, as the data is not correct - after all we are
|
|
# running the analysis on a subimage.
|
|
del fresult["region"]
|
|
|
|
return fresult
|
|
|
|
def facial_expression_analysis(self) -> dict:
|
|
|
|
# Find (multiple) faces in the image and cut them
|
|
retinaface_model.get()
|
|
faces = RetinaFace.extract_faces(self.subdict["filename"])
|
|
|
|
# If no faces are found, we return empty keys
|
|
if len(faces) == 0:
|
|
return self.subdict
|
|
|
|
# Sort the faces by sight to prioritize prominent faces
|
|
faces = list(reversed(sorted(faces, key=lambda f: f.shape[0] * f.shape[1])))
|
|
|
|
self.subdict["face"] = "Yes"
|
|
self.subdict["multiple_faces"] = "Yes" if len(faces) > 1 else "No"
|
|
self.subdict["no_faces"] = len(faces) if len(faces) <= 15 else 99
|
|
# note number of faces being identified
|
|
result = {"number_faces": len(faces) if len(faces) <= 3 else 3}
|
|
# We limit ourselves to three faces
|
|
for i, face in enumerate(faces[:3]):
|
|
result[f"person{ i+1 }"] = self.analyze_single_face(face)
|
|
|
|
self.clean_subdict(result)
|
|
|
|
return self.subdict
|
|
|
|
def clean_subdict(self, result: dict) -> dict:
|
|
# each person subdict converted into list for keys
|
|
self.subdict["wears_mask"] = []
|
|
self.subdict["age"] = []
|
|
self.subdict["gender"] = []
|
|
self.subdict["race"] = []
|
|
self.subdict["emotion"] = []
|
|
self.subdict["emotion (category)"] = []
|
|
for i in range(result["number_faces"]):
|
|
person = "person{}".format(i + 1)
|
|
self.subdict["wears_mask"].append(
|
|
"Yes" if result[person]["wears_mask"] else "No"
|
|
)
|
|
self.subdict["age"].append(result[person]["age"])
|
|
self.subdict["gender"].append(result[person]["gender"])
|
|
# race, emotion only detected if person does not wear mask
|
|
if result[person]["wears_mask"]:
|
|
self.subdict["race"].append(None)
|
|
self.subdict["emotion"].append(None)
|
|
self.subdict["emotion (category)"].append(None)
|
|
elif not result[person]["wears_mask"]:
|
|
# Check whether the race threshold was exceeded
|
|
if (
|
|
result[person]["race"][result[person]["dominant_race"]]
|
|
> self.race_threshold
|
|
):
|
|
self.subdict["race"].append(result[person]["dominant_race"])
|
|
else:
|
|
self.subdict["race"].append(None)
|
|
|
|
# Check whether the emotion threshold was exceeded
|
|
if (
|
|
result[person]["emotion"][result[person]["dominant_emotion"]]
|
|
> self.emotion_threshold
|
|
):
|
|
self.subdict["emotion"].append(result[person]["dominant_emotion"])
|
|
self.subdict["emotion (category)"].append(
|
|
self.emotion_categories[result[person]["dominant_emotion"]]
|
|
)
|
|
else:
|
|
self.subdict["emotion"].append(None)
|
|
self.subdict["emotion (category)"].append(None)
|
|
|
|
return self.subdict
|
|
|
|
def wears_mask(self, face: np.ndarray) -> bool:
|
|
global mask_detection_model
|
|
|
|
# Preprocess the face to match the assumptions of the face mask
|
|
# detection model
|
|
face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
|
|
face = cv2.resize(face, (224, 224))
|
|
face = img_to_array(face)
|
|
face = preprocess_input(face)
|
|
face = np.expand_dims(face, axis=0)
|
|
|
|
# Lazily load the model
|
|
mask_detection_model = load_model(face_mask_model.get())
|
|
|
|
# Run the model (ignoring output)
|
|
with NocatchOutput():
|
|
mask, without_mask = mask_detection_model.predict(face)[0]
|
|
|
|
# Convert from np.bool_ to bool to later be able to serialize the result
|
|
return bool(mask > without_mask)
|
|
|
|
|
|
class NocatchOutput(ipywidgets.Output):
|
|
"""An output container that suppresses output, but not exceptions
|
|
|
|
Taken from https://github.com/jupyter-widgets/ipywidgets/issues/3208#issuecomment-1070836153
|
|
"""
|
|
|
|
def __exit__(self, *args, **kwargs):
|
|
super().__exit__(*args, **kwargs)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
files = utils.find_files(
|
|
path="/home/inga/projects/misinformation-project/misinformation/data/test_no_text/"
|
|
)
|
|
# files = [
|
|
# "/home/inga/projects/misinformation-project/misinformation/data/test_no_text/102141_1_eng.png"
|
|
# ]
|
|
mydict = utils.initialize_dict(files)
|
|
image_ids = [key for key in mydict.keys()]
|
|
for i in image_ids:
|
|
mydict[i] = EmotionDetector(mydict[i]).analyse_image()
|
|
print(mydict)
|