Accuracy test (#32)

* Create ci.yml

* include pytest

* Update pyproject.toml

* include pytest-cov

* use approx in pytest

* Update test_faces.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add coverage yaml

* reduce passing grade

* use copy instead of symlink on windows

* crude attempt at calculating deviations

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

misinformation/accuracy.py

@@ -77,13 +77,24 @@ class LabelManager:
                 # substitute the values that are not boolean
                 if self.map[key]["variable_coding"] != "Bool":
                     mapped_subdict[key] = mydict_current
+                # take only first value in lists - this applies to faces,
+                # reported are up to three in a list, we compare only the
+                # largest one here
+                if isinstance(mydict_current, list):
+                    mapped_subdict[key] = 1 if mydict_current[0] == mydict_value else 0
+                    # also cut out the likelihood for detected emotion
+                    if isinstance(mydict_current[0], tuple):
+                        mapped_subdict[key] = (
+                            1 if mydict_current[0][0] == mydict_value else 0
+                        )
             mapped_dict[id] = mapped_subdict
         return mapped_dict
 
 
 if __name__ == "__main__":
     files = utils.find_files(
-        path="/home/inga/projects/misinformation-project/misinformation/misinformation/test/data/Europe APRMAY20 visual data/cropped images"
+        path="/home/inga/projects/misinformation-project/misinformation/misinformation/test/data/Europe APRMAY20 visual data/cropped images",
+        limit=500,
     )
     mydict = utils.initialize_dict(files)
     # analyze faces
@@ -102,8 +113,35 @@ if __name__ == "__main__":
     orders = lm.get_orders()
     # map mydict to the specified variable names and values
     mydict_map = lm.map_dict(mydict)
-    print(mydict_map)
     lm.filter_from_order([1, 2, 3] + orders)
 
     labels = lm.gen_dict()
-    print(labels)
+    comp = {}
+    for key in labels.keys():
+        if str(key) not in mydict_map:
+            print("Key {} not found.".format(key))
+            continue
+        print("ref: {}".format(labels[key]))
+        print("com: {}".format(mydict_map[str(key)]))
+        for subkey in labels[key]:
+            if type(labels[key][subkey]) != int:
+                continue
+            if type(mydict_map[str(key)][subkey]) != int:
+                continue
+            comp[subkey] = comp.get(subkey, 0) + abs(
+                labels[key][subkey] - mydict_map[str(key)][subkey]
+            )
+    print("summary: ")
+    # why v9_5a not there - bec reads in as float from the csv
+    print(comp)
+    # summary:
+    # {'v9_4': 42, 'v9_5b': 1579, 'v9_6': 229, 'v9_7': 45, 'v9_8': 39, 'v9_8a': 31, 'v9_9': 58, 'v9_10': 33, 'v9_11': 22, 'v9_12': 2, 'v9_13': 24, 'v11_3': 39}
+    # Important here is:
+    # Overall positive - 'v9_8': 39 deviations
+    # Overall negative - 'v9_9': 58
+    # happy - 'v9_8a': 31
+    # fear - 'v9_10': 33
+    # angry - 'v9_11': 22
+    # disgust - 'v9_12': 2
+    # sad - 'v9_13': 24
+    # respect of rules = wears mask - 'v11_3': 39

misinformation/data/map_test_set.json

@@ -1,5 +1,5 @@
 {
-    "V9_4": {
+    "v9_4": {
         "order": 169,
         "variable_label": "4=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Person visible",
@@ -8,7 +8,7 @@
         "variable_mydict": "face",
         "value_mydict": "Yes"
     },
-    "V9_5a": {
+    "v9_5a": {
         "order": 170,
         "variable_label": "5a=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "More than one person shown",
@@ -17,7 +17,7 @@
         "variable_mydict": "multiple_faces",
         "value_mydict": "Yes"
     },
-    "V9_5b": {
+    "v9_5b": {
         "order": 171,
         "variable_label": "5b=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "How many people shown?",
@@ -26,7 +26,7 @@
         "variable_mydict": "no_faces",
         "value_mydict": "0"
     },
-    "V9_6": {
+    "v9_6": {
         "order": 172,
         "variable_label": "6=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Face fully visible",
@@ -35,7 +35,7 @@
         "variable_mydict": "wears_mask",
         "value_mydict": "No"
     },
-    "V9_7": {
+    "v9_7": {
         "order": 173,
         "variable_label": "7=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Face ONLY partially visible",
@@ -44,7 +44,7 @@
         "variable_mydict": "wears_mask",
         "value_mydict": "Yes"
     },
-    "V9_8": {
+    "v9_8": {
         "order": 174,
         "variable_label": "8=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Facial positive expression",
@@ -53,7 +53,7 @@
         "variable_mydict": "emotion (category)",
         "value_mydict": "Positive"
     },
-    "V9_8a": {
+    "v9_8a": {
         "order": 175,
         "variable_label": "8a=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Positive expression: happiness",
@@ -62,7 +62,7 @@
         "variable_mydict": "emotion",
         "value_mydict": "happy"
     },
-    "V9_9": {
+    "v9_9": {
         "order": 176,
         "variable_label": "9=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Facial negative expression",
@@ -71,7 +71,7 @@
         "variable_mydict": "emotion (category)",
         "value_mydict": "Negative"
     },
-    "V9_10": {
+    "v9_10": {
         "order": 177,
         "variable_label": "10=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Negative expression: anxiety",
@@ -80,7 +80,7 @@
         "variable_mydict": "emotion",
         "value_mydict": "fear"
     },
-    "V9_11": {
+    "v9_11": {
         "order": 178,
         "variable_label": "11=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Negative expression: anger",
@@ -89,7 +89,7 @@
         "variable_mydict": "emotion",
         "value_mydict": "angry"
     },
-    "V9_12": {
+    "v9_12": {
         "order": 179,
         "variable_label": "12=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Negative expression: disgust",
@@ -98,7 +98,7 @@
         "variable_mydict": "emotion",
         "value_mydict": "disgust"
     },
-    "V9_13": {
+    "v9_13": {
         "order": 180,
         "variable_label": "13=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Negative expression: other, specify",
@@ -107,7 +107,7 @@
         "variable_mydict": "emotion",
         "value_mydict": "sad"
     },
-    "V9_13_text": {
+    "v9_13_text": {
         "order": 181,
         "variable_label": "13=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Negative expression: other, specify",
@@ -115,7 +115,7 @@
         "variable_mydict": "emotion",
         "value_mydict": ""
     },
-    "V11_3": {
+    "v11_3": {
         "order": 189,
         "variable_label": "111_3=PICTURE_SPECIFIC_VisualONLY",
         "variable_explanation": "Respect of the rules",

misinformation/faces.py

@@ -1,6 +1,7 @@
 import cv2
 import numpy as np
 import os
+import shutil
 import pathlib
 import ipywidgets
 
@@ -17,7 +18,12 @@ import misinformation.utils as utils
 def deepface_symlink_processor(name):
     def _processor(fname, action, pooch):
         if not os.path.exists(name):
+            # symlink does not work on windows
+            # use copy if running on windows
+            if not os.name == "nt":
                 os.symlink(fname, name)
+            else:
+                shutil.copy(fname, name)
         return fname
 
     return _processor