зеркало из
https://github.com/ssciwr/AMMICO.git
synced 2025-10-30 13:36:04 +02:00
96 строки
3.5 KiB
Python
96 строки
3.5 KiB
Python
from ammico.utils import AnalysisMethod
|
|
from torch import device, cuda, no_grad
|
|
from PIL import Image
|
|
from lavis.models import load_model_and_preprocess
|
|
|
|
|
|
class SummaryDetector(AnalysisMethod):
|
|
def __init__(self, subdict: dict) -> None:
|
|
super().__init__(subdict)
|
|
self.summary_device = device("cuda" if cuda.is_available() else "cpu")
|
|
|
|
def load_model_base(self):
|
|
summary_model, summary_vis_processors, _ = load_model_and_preprocess(
|
|
name="blip_caption",
|
|
model_type="base_coco",
|
|
is_eval=True,
|
|
device=self.summary_device,
|
|
)
|
|
return summary_model, summary_vis_processors
|
|
|
|
def load_model_large(self):
|
|
summary_model, summary_vis_processors, _ = load_model_and_preprocess(
|
|
name="blip_caption",
|
|
model_type="large_coco",
|
|
is_eval=True,
|
|
device=self.summary_device,
|
|
)
|
|
return summary_model, summary_vis_processors
|
|
|
|
def load_model(self, model_type):
|
|
# self.summary_device = device("cuda" if cuda.is_available() else "cpu")
|
|
select_model = {
|
|
"base": SummaryDetector.load_model_base,
|
|
"large": SummaryDetector.load_model_large,
|
|
}
|
|
summary_model, summary_vis_processors = select_model[model_type](self)
|
|
return summary_model, summary_vis_processors
|
|
|
|
def analyse_image(self, summary_model=None, summary_vis_processors=None):
|
|
if summary_model is None and summary_vis_processors is None:
|
|
summary_model, summary_vis_processors = self.load_model_base()
|
|
|
|
path = self.subdict["filename"]
|
|
raw_image = Image.open(path).convert("RGB")
|
|
image = (
|
|
summary_vis_processors["eval"](raw_image)
|
|
.unsqueeze(0)
|
|
.to(self.summary_device)
|
|
)
|
|
with no_grad():
|
|
self.subdict["const_image_summary"] = summary_model.generate(
|
|
{"image": image}
|
|
)[0]
|
|
self.subdict["3_non-deterministic summary"] = summary_model.generate(
|
|
{"image": image}, use_nucleus_sampling=True, num_captions=3
|
|
)
|
|
return self.subdict
|
|
|
|
def analyse_questions(self, list_of_questions):
|
|
(
|
|
summary_vqa_model,
|
|
summary_vqa_vis_processors,
|
|
summary_vqa_txt_processors,
|
|
) = load_model_and_preprocess(
|
|
name="blip_vqa",
|
|
model_type="vqav2",
|
|
is_eval=True,
|
|
device=self.summary_device,
|
|
)
|
|
if len(list_of_questions) > 0:
|
|
path = self.subdict["filename"]
|
|
raw_image = Image.open(path).convert("RGB")
|
|
image = (
|
|
summary_vqa_vis_processors["eval"](raw_image)
|
|
.unsqueeze(0)
|
|
.to(self.summary_device)
|
|
)
|
|
question_batch = []
|
|
for quest in list_of_questions:
|
|
question_batch.append(summary_vqa_txt_processors["eval"](quest))
|
|
batch_size = len(list_of_questions)
|
|
image_batch = image.repeat(batch_size, 1, 1, 1)
|
|
|
|
with no_grad():
|
|
answers_batch = summary_vqa_model.predict_answers(
|
|
samples={"image": image_batch, "text_input": question_batch},
|
|
inference_method="generate",
|
|
)
|
|
|
|
for q, a in zip(list_of_questions, answers_batch):
|
|
self.subdict[q] = a
|
|
|
|
else:
|
|
print("Please, enter list of questions")
|
|
return self.subdict
|