locsec/AMMICO
1
Форкнуть 0
зеркало из https://github.com/ssciwr/AMMICO.git synced 2025-10-30 21:46:04 +02:00
Код Задачи Пакеты Проекты Релизы Вики Активность

add model change function to summary

Просмотр исходного кода
Этот коммит содержится в:
Petr Andriushchenko 2023-02-23 14:09:19 +01:00
родитель c208039b7c
Коммит c136b91fba
1 изменённых файлов: 41 добавлений и 17 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

58
misinformation/summary.py
Просмотреть файл

@ -15,9 +15,23 @@ class SummaryDetector(AnalysisMethod):
summary_device = torch.device("cuda" if torch.cuda.is_available() else "cpu") summary_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
summary_model, summary_vis_processors, _ = load_model_and_preprocess( def load_model_base(self):
name="blip_caption", model_type="base_coco", is_eval=True, device=summary_device summary_model, summary_vis_processors, _ = load_model_and_preprocess(
) name="blip_caption",
model_type="base_coco",
is_eval=True,
device=SummaryDetector.summary_device,
)
return summary_model, summary_vis_processors
def load_model_large(self):
summary_model, summary_vis_processors, _ = load_model_and_preprocess(
name="blip_caption",
model_type="large_coco",
is_eval=True,
device=SummaryDetector.summary_device,
)
return summary_model, summary_vis_processors
def set_keys(self) -> dict: def set_keys(self) -> dict:
params = { params = {
@ -26,21 +40,29 @@ class SummaryDetector(AnalysisMethod):
} }
return params return params
def analyse_image(self): def analyse_image(self, model_type):
select_model = {
"base": self.load_model_base,
"large": self.load_model_large,
}
summary_model, summary_vis_processors = select_model[model_type]()
path = self.subdict["filename"] path = self.subdict["filename"]
raw_image = Image.open(path).convert("RGB") raw_image = Image.open(path).convert("RGB")
image = ( image = (
self.summary_vis_processors["eval"](raw_image) summary_vis_processors["eval"](raw_image)
.unsqueeze(0) .unsqueeze(0)
.to(self.summary_device) .to(self.summary_device)
) )
self.image_summary["const_image_summary"] = self.summary_model.generate( with torch.no_grad():
{"image": image} self.image_summary["const_image_summary"] = summary_model.generate(
)[0] {"image": image}
self.image_summary["3_non-deterministic summary"] = self.summary_model.generate( )[0]
{"image": image}, use_nucleus_sampling=True, num_captions=3 self.image_summary[
) "3_non-deterministic summary"
] = summary_model.generate(
{"image": image}, use_nucleus_sampling=True, num_captions=3
)
for key in self.image_summary: for key in self.image_summary:
self.subdict[key] = self.image_summary[key] self.subdict[key] = self.image_summary[key]
return self.subdict return self.subdict
@ -53,12 +75,13 @@ class SummaryDetector(AnalysisMethod):
name="blip_vqa", model_type="vqav2", is_eval=True, device=summary_device name="blip_vqa", model_type="vqav2", is_eval=True, device=summary_device
) )
def analyse_questions(self, list_of_questions): def analyse_questions(self, model_type, list_of_questions):
if len(list_of_questions) > 0: if len(list_of_questions) > 0:
path = self.subdict["filename"] path = self.subdict["filename"]
raw_image = Image.open(path).convert("RGB") raw_image = Image.open(path).convert("RGB")
image = ( image = (
self.summary_VQA_vis_processors["eval"](raw_image) summary_VQA_vis_processors["eval"](raw_image)
.unsqueeze(0) .unsqueeze(0)
.to(self.summary_device) .to(self.summary_device)
) )
@ -68,10 +91,11 @@ class SummaryDetector(AnalysisMethod):
batch_size = len(list_of_questions) batch_size = len(list_of_questions)
image_batch = image.repeat(batch_size, 1, 1, 1) image_batch = image.repeat(batch_size, 1, 1, 1)
answers_batch = self.summary_VQA_model.predict_answers( with torch.no_grad():
samples={"image": image_batch, "text_input": question_batch}, answers_batch = self.summary_VQA_model.predict_answers(
inference_method="generate", samples={"image": image_batch, "text_input": question_batch},
) inference_method="generate",
)
for q, a in zip(question_batch, answers_batch): for q, a in zip(question_batch, answers_batch):
self.image_summary[q] = a self.image_summary[q] = a