from misinformation.utils import AnalysisMethod import torch from PIL import Image from lavis.models import load_model_and_preprocess class SummaryDetector(AnalysisMethod): def __init__( self, subdict: dict ) -> None: super().__init__(subdict) self.subdict.update(self.set_keys()) self.image_summary = { "const_image_summary": None, "3_non-deterministic summary": None, } summary_device = torch.device("cuda" if torch.cuda.is_available() else "cpu") summary_model, summary_vis_processors, _ = load_model_and_preprocess( name="blip_caption", model_type="base_coco", is_eval=True, device=summary_device ) def set_keys(self) -> dict: params = { "const_image_summary": None, "3_non-deterministic summary": None, } return params def analyse_image(self): path = self.subdict["filename"] raw_image = Image.open(path).convert("RGB") image = self.summary_vis_processors["eval"](raw_image).unsqueeze(0).to(self.summary_device) self.image_summary["const_image_summary"] = self.summary_model.generate({"image": image})[0] self.image_summary["3_non-deterministic summary"] = self.summary_model.generate({"image": image}, use_nucleus_sampling=True, num_captions=3) for key in self.image_summary: self.subdict[key] = self.image_summary[key] return self.subdict summary_VQA_model, summary_VQA_vis_processors, summary_VQA_txt_processors = load_model_and_preprocess(name="blip_vqa", model_type="vqav2", is_eval=True, device=summary_device) def analyse_questions(self, list_of_questions): if (len(list_of_questions)>0): path = self.subdict["filename"] raw_image = Image.open(path).convert("RGB") image = self.summary_VQA_vis_processors["eval"](raw_image).unsqueeze(0).to(self.summary_device) question_batch =[] for quest in list_of_questions: question_batch.append(self.summary_VQA_txt_processors["eval"](quest)) batch_size = len(list_of_questions) image_batch = image.repeat(batch_size, 1, 1, 1) answers_batch = self.summary_VQA_model.predict_answers(samples={"image": image_batch, "text_input": question_batch}, inference_method="generate") for q,a in zip(question_batch,answers_batch): self.image_summary[q] = a for key in self.image_summary: self.subdict[key] = self.image_summary[key] else: print("Please, enter list of questions") return self.subdict