AMMICO/notebooks/multimodal_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f10ad6c9-b1a0-4043-8c5d-ed660d77be37",
   "metadata": {},
   "outputs": [],
   "source": [
    "import misinformation\n",
    "import misinformation.multimodal_search as ms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d3fe589-ff3c-4575-b8f5-650db85596bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "images = misinformation.find_files(\n",
    "    path=\"../data/images/\",\n",
    "    limit=1000,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adf3db21-1f8b-4d44-bbef-ef0acf4623a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "mydict = misinformation.utils.initialize_dict(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d98b6227-886d-41b8-a377-896dd8ab3c2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mydict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e516efbe-ea51-418f-8d3f-377490ffbdfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(mydict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c27b04d4-0359-485d-aa00-4631e9952f75",
   "metadata": {},
   "outputs": [],
   "source": [
    "# image_keys = mydict.keys()\n",
    "# image_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca095404-57d0-4f5d-aeb0-38c232252b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "image_keys, image_names, features_image_stacked = ms.MultimodalSearch.parsing_images(\n",
    "    mydict, \"blip\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ffbc08b-0176-4eca-aeb1-2d358cbd6670",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_image_stacked.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c40e93f0-6bea-4886-b904-8b46ed6ec819",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_image_stacked2 = ms.MultimodalSearch.load_tensors(\"saved_features_image.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ab45184-17ee-4fa7-b031-1c7832847eb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "search_query = [\n",
    "    \"сrime scene\",\n",
    "    \"world map \",\n",
    "    \"virus\",\n",
    "    \"a politician makes a speech\",\n",
    "    \"a dog\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f7dc52f-7ee9-4590-96b7-e0d9d3b82378",
   "metadata": {},
   "outputs": [],
   "source": [
    "ms.MultimodalSearch.multimodal_search(\n",
    "    mydict, \"blip\", image_keys, features_image_stacked, search_query\n",
    ");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81fd6534-6478-4819-aa65-e5df6368706b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ms.MultimodalSearch.show_results(mydict, \"virus\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "318221fa-f2d6-482d-a74b-fb88bb59f21d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}