changed summmary.py logic (#121)

* changed summmary.py logic

* fixing test_summary

* added macos for testing

* fixed_display_test

* fixed docs and exceptions

* added dropout menu for summary

* added new SummaryDetector to AnalysisExplorer

* bug fixing

* code improving

* fixed test_display

* fixed code smells

* reduce tests for macos

* added some tests and exceptions for summary init

* changed CI, runs pytest independently

* exclude test_analysisExplorer from macos in CI

* moved some tests from test_init_summary to test_advanced_init_summary and mark them as long

---------

Co-authored-by: Inga Ulusoy <inga.ulusoy@uni-heidelberg.de>

.github/workflows/ci.yml

@@ -27,16 +27,42 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install -e .
-    - name: Run pytest linux (linux-only)
+    - name: Run pytest test_colors
-      if: matrix.os == 'ubuntu-22.04'
       run: |
         cd ammico
-        python -m pytest -m "not gcv and not long" -svv --cov=. --cov-report=xml
+        python -m pytest test/test_colors.py -svv --cov=. --cov-report=xml
-    - name: Run pytest windows(windows-only)
+    - name: Run pytest test_cropposts
-      if: matrix.os == 'windows-latest'
       run: |
         cd ammico
-        python -m pytest -m "not gcv and not long and not win_skip" -svv --cov=. --cov-report=xml
+        python -m pytest test/test_cropposts.py -svv --cov=. --cov-report=xml
+    - name: Run pytest test_display
+      run: |
+        cd ammico
+        python -m pytest test/test_display.py -svv --cov=. --cov-report=xml
+    - name: Run pytest test_faces
+      run: |
+        cd ammico
+        python -m pytest test/test_faces.py -svv --cov=. --cov-report=xml
+    - name: Run pytest test_multimodal_search
+      run: |
+        cd ammico
+        python -m pytest test/test_multimodal_search.py -m "not long" -svv --cov=. --cov-report=xml
+    - name: Run pytest test_objects
+      run: |
+        cd ammico
+        python -m pytest test/test_objects.py -svv --cov=. --cov-report=xml
+    - name: Run pytest test_summary
+      run: |
+        cd ammico
+        python -m pytest test/test_summary.py -m "not long" -svv --cov=. --cov-report=xml
+    - name: Run pytest test_text
+      run: |
+        cd ammico
+        python -m pytest test/test_text.py -m "not gcv" -svv --cov=. --cov-report=xml
+    - name: Run pytest test_utils
+      run: |
+        cd ammico
+        python -m pytest test/test_utils.py -svv --cov=. --cov-report=xml
     - name: Upload coverage
       if: matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.9'
       uses: codecov/codecov-action@v3

ammico/display.py

@@ -24,6 +24,8 @@ COLOR_SCHEMES = [
     "CAM16-UCS",
     "DIN99",
 ]
+SUMMARY_ANALYSIS_TYPE = ["summary_and_questions", "summary", "questions"]
+SUMMARY_MODEL = ["base", "large"]
 
 
 class AnalysisExplorer:
@@ -111,6 +113,9 @@ class AnalysisExplorer:
             State("setting_Emotion_emotion_threshold", "value"),
             State("setting_Emotion_race_threshold", "value"),
             State("setting_Color_delta_e_method", "value"),
+            State("setting_Summary_analysis_type", "value"),
+            State("setting_Summary_model", "value"),
+            State("setting_Summary_list_of_questions", "value"),
             prevent_initial_call=True,
         )(self._right_output_analysis)
 
@@ -118,6 +123,7 @@ class AnalysisExplorer:
             Output("settings_TextDetector", "style"),
             Output("settings_EmotionDetector", "style"),
             Output("settings_ColorDetector", "style"),
+            Output("settings_Summary_Detector", "style"),
             Input("Dropdown_select_Detector", "value"),
         )(self._update_detector_setting)
 
@@ -240,6 +246,60 @@ class AnalysisExplorer:
                         )
                     ],
                 ),
+                html.Div(
+                    id="settings_Summary_Detector",
+                    style={"display": "none"},
+                    children=[
+                        html.Div(
+                            [
+                                dcc.Dropdown(
+                                    options=SUMMARY_ANALYSIS_TYPE,
+                                    value="summary_and_questions",
+                                    id="setting_Summary_analysis_type",
+                                )
+                            ],
+                            style={
+                                "width": "33%",
+                                "display": "inline-block",
+                            },
+                        ),
+                        html.Div(
+                            [
+                                dcc.Dropdown(
+                                    options=SUMMARY_MODEL,
+                                    value="base",
+                                    id="setting_Summary_model",
+                                )
+                            ],
+                            style={
+                                "width": "33%",
+                                "display": "inline-block",
+                                "margin-top": "10px",
+                            },
+                        ),
+                        html.Div(
+                            [
+                                html.Div(
+                                    "Please enter a question",
+                                    style={
+                                        "height": "50px",
+                                        "margin-top": "5px",
+                                    },
+                                ),
+                                dcc.Input(
+                                    type="text",
+                                    id="setting_Summary_list_of_questions",
+                                    style={"height": "auto", "margin-bottom": "auto"},
+                                ),
+                            ],
+                            style={
+                                "width": "33%",
+                                "display": "inline-block",
+                                "margin-top": "10px",
+                            },
+                        ),
+                    ],
+                ),
             ],
         )
         return settings_layout
@@ -334,16 +394,19 @@ class AnalysisExplorer:
         }
 
         if setting_input == "TextDetector":
-            return display_flex, display_none, display_none
+            return display_flex, display_none, display_none, display_none
 
         if setting_input == "EmotionDetector":
-            return display_none, display_flex, display_none
+            return display_none, display_flex, display_none, display_none
 
         if setting_input == "ColorDetector":
-            return display_none, display_none, display_flex
+            return display_none, display_none, display_flex, display_none
+
+        if setting_input == "SummaryDetector":
+            return display_none, display_none, display_none, display_flex
 
         else:
-            return display_none, display_none, display_none
+            return display_none, display_none, display_none, display_none
 
     def _right_output_analysis(
         self,
@@ -355,6 +418,9 @@ class AnalysisExplorer:
         setting_emotion_emotion_threshold: int,
         setting_emotion_race_threshold: int,
         setting_color_delta_e_method: str,
+        setting_summary_analysis_type: str,
+        setting_summary_model: str,
+        setting_summary_list_of_questions: str,
     ) -> dict:
         """Callback function to perform analysis on the selected image and return the output.
 
@@ -396,6 +462,15 @@ class AnalysisExplorer:
                 image_copy,
                 delta_e_method=setting_color_delta_e_method,
             )
+        elif detector_value == "SummaryDetector":
+            detector_class = identify_function(
+                image_copy,
+                analysis_type=setting_summary_analysis_type,
+                summary_model_type=setting_summary_model,
+                list_of_questions=[setting_summary_list_of_questions]
+                if (setting_summary_list_of_questions is not None)
+                else None,
+            )
         else:
             detector_class = identify_function(image_copy)
         return detector_class.analyse_image()

ammico/summary.py

@@ -5,9 +5,91 @@ from lavis.models import load_model_and_preprocess
 
 
 class SummaryDetector(AnalysisMethod):
-    def __init__(self, subdict: dict) -> None:
+    def __init__(
+        self,
+        subdict: dict = {},
+        summary_model_type: str = "base",
+        analysis_type: str = "summary_and_questions",
+        list_of_questions: str = None,
+        summary_model=None,
+        summary_vis_processors=None,
+        summary_vqa_model=None,
+        summary_vqa_vis_processors=None,
+        summary_vqa_txt_processors=None,
+    ) -> None:
+        """
+        SummaryDetector class for analysing images using the blip_caption model.
+
+        Args:
+            subdict (dict, optional): Dictionary containing the image to be analysed. Defaults to {}.
+            summary_model_type (str, optional): Type of blip_caption model to use. Can be "base" or "large". Defaults to "base".
+            analysis_type (str, optional): Type of analysis to perform. Can be "summary", "questions" or "summary_and_questions". Defaults to "summary_and_questions".
+            list_of_questions (list, optional): List of questions to answer. Defaults to ["Are there people in the image?", "What is this picture about?"].
+            summary_model ([type], optional): blip_caption model. Defaults to None.
+            summary_vis_processors ([type], optional): Preprocessors for visual inputs. Defaults to None.
+            summary_vqa_model ([type], optional): blip_vqa model. Defaults to None.
+            summary_vqa_vis_processors ([type], optional): Preprocessors for vqa visual inputs. Defaults to None.
+            summary_vqa_txt_processors ([type], optional): Preprocessors for vqa text inputs. Defaults to None.
+
+        Raises:
+            ValueError: If analysis_type is not one of "summary", "questions" or "summary_and_questions".
+
+        Returns:
+            None.
+        """
+
         super().__init__(subdict)
+        if analysis_type not in ["summary", "questions", "summary_and_questions"]:
+            raise ValueError(
+                "analysis_type must be one of 'summary', 'questions' or 'summary_and_questions'"
+            )
         self.summary_device = "cuda" if cuda.is_available() else "cpu"
+        allowed_model_types = ["base", "large"]
+        if summary_model_type not in allowed_model_types:
+            raise ValueError(
+                "Model type is not allowed - please select one of {}".format(
+                    allowed_model_types
+                )
+            )
+        self.summary_model_type = summary_model_type
+        self.analysis_type = analysis_type
+        if list_of_questions is None:
+            self.list_of_questions = [
+                "Are there people in the image?",
+                "What is this picture about?",
+            ]
+        elif (not isinstance(list_of_questions, list)) or (
+            not all(isinstance(i, str) for i in list_of_questions)
+        ):
+            raise ValueError("list_of_questions must be a list of string (questions)")
+        else:
+            self.list_of_questions = list_of_questions
+        if (
+            (summary_model is None)
+            and (summary_vis_processors is None)
+            and (analysis_type != "questions")
+        ):
+            self.summary_model, self.summary_vis_processors = self.load_model(
+                model_type=summary_model_type
+            )
+        else:
+            self.summary_model = summary_model
+            self.summary_vis_processors = summary_vis_processors
+        if (
+            (summary_vqa_model is None)
+            and (summary_vqa_vis_processors is None)
+            and (summary_vqa_txt_processors is None)
+            and (analysis_type != "summary")
+        ):
+            (
+                self.summary_vqa_model,
+                self.summary_vqa_vis_processors,
+                self.summary_vqa_txt_processors,
+            ) = self.load_vqa_model()
+        else:
+            self.summary_vqa_model = summary_vqa_model
+            self.summary_vqa_vis_processors = summary_vqa_vis_processors
+            self.summary_vqa_txt_processors = summary_vqa_txt_processors
 
     def load_model_base(self):
         """
@@ -63,32 +145,71 @@ class SummaryDetector(AnalysisMethod):
         summary_model, summary_vis_processors = select_model[model_type](self)
         return summary_model, summary_vis_processors
 
-    def analyse_image(self, summary_model=None, summary_vis_processors=None):
+    def load_vqa_model(self):
+        """
+        Load blip_vqa model and preprocessors for visual and text inputs from lavis.models.
+
+        Args:
+
+        Returns:
+            model (torch.nn.Module): model.
+            vis_processors (dict): preprocessors for visual inputs.
+            txt_processors (dict): preprocessors for text inputs.
+
+        """
+        (
+            summary_vqa_model,
+            summary_vqa_vis_processors,
+            summary_vqa_txt_processors,
+        ) = load_model_and_preprocess(
+            name="blip_vqa",
+            model_type="vqav2",
+            is_eval=True,
+            device=self.summary_device,
+        )
+        return summary_vqa_model, summary_vqa_vis_processors, summary_vqa_txt_processors
+
+    def analyse_image(self):
+        """
+        Analyse image with blip_caption model.
+
+        Args:
+
+        Returns:
+            self.subdict (dict): dictionary with analysis results.
+        """
+        if self.analysis_type == "summary_and_questions":
+            self.analyse_summary()
+            self.analyse_questions(self.list_of_questions)
+        elif self.analysis_type == "summary":
+            self.analyse_summary()
+        elif self.analysis_type == "questions":
+            self.analyse_questions(self.list_of_questions)
+
+        return self.subdict
+
+    def analyse_summary(self):
         """
         Create 1 constant and 3 non deterministic captions for image.
 
         Args:
-            summary_model (str): model.
-            summary_vis_processors (str): preprocessors for visual inputs.
 
         Returns:
-            self.subdict (dict): dictionary with constant image summary and 3 non deterministic summary.
+            self.subdict (dict): dictionary with analysis results.
         """
-        if summary_model is None and summary_vis_processors is None:
-            summary_model, summary_vis_processors = self.load_model_base()
 
         path = self.subdict["filename"]
         raw_image = Image.open(path).convert("RGB")
         image = (
-            summary_vis_processors["eval"](raw_image)
+            self.summary_vis_processors["eval"](raw_image)
             .unsqueeze(0)
             .to(self.summary_device)
         )
         with no_grad():
-            self.subdict["const_image_summary"] = summary_model.generate(
+            self.subdict["const_image_summary"] = self.summary_model.generate(
                 {"image": image}
             )[0]
-            self.subdict["3_non-deterministic summary"] = summary_model.generate(
+            self.subdict["3_non-deterministic summary"] = self.summary_model.generate(
                 {"image": image}, use_nucleus_sampling=True, num_captions=3
             )
         return self.subdict
@@ -103,32 +224,37 @@ class SummaryDetector(AnalysisMethod):
         Returns:
             self.subdict (dict): dictionary with answers to questions.
         """
-        (
+        if (
-            summary_vqa_model,
+            (self.summary_vqa_model is None)
-            summary_vqa_vis_processors,
+            and (self.summary_vqa_vis_processors is None)
-            summary_vqa_txt_processors,
+            and (self.summary_vqa_txt_processors is None)
-        ) = load_model_and_preprocess(
+        ):
-            name="blip_vqa",
+            (
-            model_type="vqav2",
+                self.summary_vqa_model,
-            is_eval=True,
+                self.summary_vqa_vis_processors,
-            device=self.summary_device,
+                self.summary_vqa_txt_processors,
-        )
+            ) = load_model_and_preprocess(
+                name="blip_vqa",
+                model_type="vqav2",
+                is_eval=True,
+                device=self.summary_device,
+            )
         if len(list_of_questions) > 0:
             path = self.subdict["filename"]
             raw_image = Image.open(path).convert("RGB")
             image = (
-                summary_vqa_vis_processors["eval"](raw_image)
+                self.summary_vqa_vis_processors["eval"](raw_image)
                 .unsqueeze(0)
                 .to(self.summary_device)
             )
             question_batch = []
             for quest in list_of_questions:
-                question_batch.append(summary_vqa_txt_processors["eval"](quest))
+                question_batch.append(self.summary_vqa_txt_processors["eval"](quest))
             batch_size = len(list_of_questions)
             image_batch = image.repeat(batch_size, 1, 1, 1)
 
             with no_grad():
-                answers_batch = summary_vqa_model.predict_answers(
+                answers_batch = self.summary_vqa_model.predict_answers(
                     samples={"image": image_batch, "text_input": question_batch},
                     inference_method="generate",
                 )

ammico/test/test_display.py

@@ -1,6 +1,7 @@
 import json
 import ammico.display as ammico_display
 import pytest
+import sys
 
 
 def test_explore_analysis_faces(get_path):
@@ -25,6 +26,7 @@ def test_explore_analysis_objects(get_path):
             assert sub_dict[key] == outs[key]
 
 
+@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac")
 def test_AnalysisExplorer(get_path):
     path_img_1 = get_path + "IMG_2809.png"
     path_img_2 = get_path + "IMG_2746.png"
@@ -46,18 +48,59 @@ def test_AnalysisExplorer(get_path):
     assert analysis_explorer.update_picture(None) is None
 
     analysis_explorer._right_output_analysis(
-        2, all_options_dict, path_img_1, "ObjectDetector", True, 50, 50, "CIE 1976"
+        2,
+        all_options_dict,
+        path_img_1,
+        "ObjectDetector",
+        True,
+        50,
+        50,
+        "CIE 1976",
+        "summary_and_questions",
+        "base",
+        "How many people are in the picture?",
     )
 
     analysis_explorer._right_output_analysis(
-        2, all_options_dict, path_img_1, "EmotionDetector", True, 50, 50, "CIE 1976"
+        2,
-    )
+        all_options_dict,
-    analysis_explorer._right_output_analysis(
+        path_img_1,
-        2, all_options_dict, path_img_1, "SummaryDetector", True, 50, 50, "CIE 1976"
+        "EmotionDetector",
+        True,
+        50,
+        50,
+        "CIE 1976",
+        "summary_and_questions",
+        "base",
+        "How many people are in the picture?",
     )
 
     analysis_explorer._right_output_analysis(
-        2, all_options_dict, path_img_1, "ColorDetector", True, 50, 50, "CIE 1976"
+        2,
+        all_options_dict,
+        path_img_1,
+        "SummaryDetector",
+        True,
+        50,
+        50,
+        "CIE 1976",
+        "summary_and_questions",
+        "base",
+        "How many people are in the picture?",
+    )
+
+    analysis_explorer._right_output_analysis(
+        2,
+        all_options_dict,
+        path_img_1,
+        "ColorDetector",
+        True,
+        50,
+        50,
+        "CIE 1976",
+        "summary_and_questions",
+        "base",
+        "How many people are in the picture?",
     )
 
     with pytest.raises(EnvironmentError):

ammico/test/test_objects.py

@@ -2,6 +2,7 @@ import json
 import pytest
 import ammico.objects as ob
 import ammico.objects_cvlib as ob_cvlib
+import sys
 
 OBJECT_1 = "cell phone"
 OBJECT_2 = "motorcycle"
@@ -25,6 +26,7 @@ def test_objects_from_cvlib(default_objects):
     assert str(objects) == str(out_objects)
 
 
+@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac")
 def test_analyse_image_cvlib(get_path):
     mydict = {"filename": get_path + TEST_IMAGE_1}
     ob_cvlib.ObjectCVLib().analyse_image(mydict)
@@ -55,6 +57,7 @@ def test_init_default_objects():
         assert init_objects[obj] == "no"
 
 
+@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac")
 def test_analyse_image_from_file_cvlib(get_path):
     file_path = get_path + TEST_IMAGE_1
     objs = ob_cvlib.ObjectCVLib().analyse_image_from_file(file_path)
@@ -66,6 +69,7 @@ def test_analyse_image_from_file_cvlib(get_path):
         assert objs[key] == out_dict[key]
 
 
+@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac")
 def test_detect_objects_cvlib(get_path):
     file_path = get_path + TEST_IMAGE_1
     objs = ob_cvlib.ObjectCVLib().detect_objects_cvlib(file_path)
@@ -82,6 +86,7 @@ def test_set_keys(default_objects, get_path):
     assert str(default_objects) == str(key_objs)
 
 
+@pytest.mark.skipif(sys.platform == "darwin", reason="segmentation fault on mac")
 def test_analyse_image(get_path):
     mydict = {"filename": get_path + TEST_IMAGE_1}
     ob.ObjectDetector.set_client_to_cvlib()

ammico/test/test_summary.py

@@ -40,7 +40,7 @@ def get_dict(get_path):
     return mydict
 
 
-@pytest.mark.long
+# @pytest.mark.long
 def test_analyse_image(get_dict):
     reference_results = {
         "run1": {
@@ -63,9 +63,12 @@ def test_analyse_image(get_dict):
         )
         # run two different images
         for key in get_dict.keys():
-            get_dict[key] = sm.SummaryDetector(get_dict[key]).analyse_image(
+            get_dict[key] = sm.SummaryDetector(
-                summary_model, summary_vis_processors
+                get_dict[key],
-            )
+                analysis_type="summary",
+                summary_model=summary_model,
+                summary_vis_processors=summary_vis_processors,
+            ).analyse_image()
         assert len(get_dict) == 2
         for key in get_dict.keys():
             assert len(get_dict[key]["3_non-deterministic summary"]) == 3
@@ -84,9 +87,11 @@ def test_analyse_questions(get_dict):
         "What happends on the picture?",
     ]
     for key in get_dict:
-        get_dict[key] = sm.SummaryDetector(get_dict[key]).analyse_questions(
+        get_dict[key] = sm.SummaryDetector(
-            list_of_questions
+            get_dict[key],
-        )
+            analysis_type="questions",
+            list_of_questions=list_of_questions,
+        ).analyse_image()
     assert len(get_dict) == 2
     list_of_questions_ans = ["2", "100"]
     list_of_questions_ans2 = ["flood", "festival"]
@@ -97,3 +102,50 @@ def test_analyse_questions(get_dict):
         test_answers2.append(get_dict[key][list_of_questions[1]])
     assert sorted(test_answers) == sorted(list_of_questions_ans)
     assert sorted(test_answers2) == sorted(list_of_questions_ans2)
+
+
+def test_init_summary():
+    sd = sm.SummaryDetector({}, analysis_type="summary")
+    assert sd.analysis_type == "summary"
+    with pytest.raises(ValueError):
+        sm.SummaryDetector({}, analysis_type="something")
+    list_of_questions = ["Question 1", "Question 2"]
+    sd = sm.SummaryDetector({}, list_of_questions=list_of_questions)
+    assert sd.list_of_questions == list_of_questions
+    with pytest.raises(ValueError):
+        sm.SummaryDetector({}, list_of_questions={})
+    with pytest.raises(ValueError):
+        sm.SummaryDetector({}, list_of_questions=[None])
+    with pytest.raises(ValueError):
+        sm.SummaryDetector({}, list_of_questions=[0.1])
+
+
+@pytest.mark.long
+def test_advanced_init_summary():
+    sd = sm.SummaryDetector({})
+    assert sd.summary_model
+    assert sd.summary_vis_processors
+    sd = sm.SummaryDetector({}, summary_model_type="large")
+    assert sd.summary_model
+    assert sd.summary_vis_processors
+    with pytest.raises(ValueError):
+        sm.SummaryDetector({}, summary_model_type="bla")
+    (
+        summary_vqa_model,
+        summary_vqa_vis_processors,
+        summary_vqa_txt_processors,
+    ) = load_model_and_preprocess(
+        name="blip_vqa",
+        model_type="vqav2",
+        is_eval=True,
+        device="cpu",
+    )
+    sd = sm.SummaryDetector(
+        {},
+        summary_vqa_model=summary_vqa_model,
+        summary_vqa_vis_processors=summary_vqa_vis_processors,
+        summary_vqa_txt_processors=summary_vqa_txt_processors,
+    )
+    assert sd.summary_vqa_model
+    assert sd.summary_vqa_vis_processors
+    assert sd.summary_vqa_txt_processors

ammico/testing_all.py

@@ -1,13 +0,0 @@
-import ammico
-
-if __name__ == "__main__":
-    images = ammico.find_files(path=".")
-    mydict = ammico.initialize_dict(images)
-    for key in mydict:
-        mydict[key] = ammico.TextDetector(
-            mydict[key], analyse_text=True
-        ).analyse_image()
-    print(mydict)
-    outdict = ammico.append_data_to_dict(mydict)
-    df = ammico.dump_df(outdict)
-    df.to_csv("data_out.csv")