{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Test area for AMITT code" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dict_keys(['df_phases', 'df_frameworks', 'df_techniques', 'df_tasks', 'df_incidents', 'df_counters', 'df_detections', 'df_actortypes', 'df_resources', 'df_responsetypes', 'df_metatechniques', 'it', 'df_tactics', 'df_techniques_per_tactic', 'df_counters_per_tactic', 'phases', 'tactics', 'techniques', 'counters', 'metatechniques', 'actortypes', 'resources', 'num_tactics', 'cross_counterid_techniqueid', 'cross_counterid_resourceid', 'cross_counterid_actortypeid'])\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
amitt_idtechnique_id
0C00006T0007
0C00006T0015
0C00006T0018
0C00006T0043
0C00006T0053
.........
135C00219T0025
136C00220
137C00221
138C00222
139C00223
\n", "

898 rows × 2 columns

\n", "
" ], "text/plain": [ " amitt_id technique_id\n", "0 C00006 T0007\n", "0 C00006 T0015\n", "0 C00006 T0018\n", "0 C00006 T0043\n", "0 C00006 T0053\n", ".. ... ...\n", "135 C00219 T0025\n", "136 C00220 \n", "137 C00221 \n", "138 C00222 \n", "139 C00223 \n", "\n", "[898 rows x 2 columns]" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "import sqlite3 as sql\n", "from generate_amitt_ttps import Amitt\n", "\n", "\n", "# Generate AMITT datasets\n", "amitt = Amitt()\n", "\n", "# Check which amitt variables we can see from here\n", "print('{}'.format(vars(amitt).keys()))\n", "vars(amitt)['cross_counterid_techniqueid']" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
amitt_idtechnique_id
0C00006T0007
0C00006T0015
0C00006T0018
0C00006T0043
0C00006T0053
.........
135C00219T0025
136C00220
137C00221
138C00222
139C00223
\n", "

898 rows × 2 columns

\n", "
" ], "text/plain": [ " amitt_id technique_id\n", "0 C00006 T0007\n", "0 C00006 T0015\n", "0 C00006 T0018\n", "0 C00006 T0043\n", "0 C00006 T0053\n", ".. ... ...\n", "135 C00219 T0025\n", "136 C00220 \n", "137 C00221 \n", "138 C00222 \n", "139 C00223 \n", "\n", "[898 rows x 2 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "amitt.cross_counterid_techniqueid" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
amitt_idtactic_idnamesummaryid
0T0001TA015Ds (dismiss, distort, distract, dismay, divide)Nimmo's \"4Ds of propaganda\": dismiss, distort,...1
1T0002TA01Facilitate State PropagandaOrganize citizens around pro-state messaging. ...2
2T0003TA01Leverage Existing NarrativesUse or adapt existing narrative themes, where ...3
3T0004TA01Competing NarrativesAdvance competing narratives connected to same...4
4T0005TA02Center of Gravity AnalysisRecon/research to identify \"the source of powe...5
..................
59T0060TA11Continue to amplifycontinue narrative or message amplification af...60
60T0061TA10Sell merchandisingSell hats, t-shirts, flags and other branded c...61
61T0062TA12Behaviour changesMonitor and evaluate behaviour changes from mi...62
62T0063TA12Message reachMonitor and evaluate message reach in misinfor...63
63T0064TA12Social media engagementMonitor and evaluate social media engagement i...64
\n", "

64 rows × 5 columns

\n", "
" ], "text/plain": [ " amitt_id tactic_id name \\\n", "0 T0001 TA01 5Ds (dismiss, distort, distract, dismay, divide) \n", "1 T0002 TA01 Facilitate State Propaganda \n", "2 T0003 TA01 Leverage Existing Narratives \n", "3 T0004 TA01 Competing Narratives \n", "4 T0005 TA02 Center of Gravity Analysis \n", ".. ... ... ... \n", "59 T0060 TA11 Continue to amplify \n", "60 T0061 TA10 Sell merchandising \n", "61 T0062 TA12 Behaviour changes \n", "62 T0063 TA12 Message reach \n", "63 T0064 TA12 Social media engagement \n", "\n", " summary id \n", "0 Nimmo's \"4Ds of propaganda\": dismiss, distort,... 1 \n", "1 Organize citizens around pro-state messaging. ... 2 \n", "2 Use or adapt existing narrative themes, where ... 3 \n", "3 Advance competing narratives connected to same... 4 \n", "4 Recon/research to identify \"the source of powe... 5 \n", ".. ... .. \n", "59 continue narrative or message amplification af... 60 \n", "60 Sell hats, t-shirts, flags and other branded c... 61 \n", "61 Monitor and evaluate behaviour changes from mi... 62 \n", "62 Monitor and evaluate message reach in misinfor... 63 \n", "63 Monitor and evaluate social media engagement i... 64 \n", "\n", "[64 rows x 5 columns]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Generate minimal sqlite database from the Amitt variables\n", "conn = sql.connect('amittsite.sqlite')\n", "\n", "def add_table(dataframe, tablename, columns): \n", " # Create sql table\n", " colnames = ', '.join(['{} TEXT NOT NULL'.format(col) for col in columns])\n", " conn.execute(\"DROP TABLE IF EXISTS {}\".format(tablename))\n", " conn.execute('''CREATE TABLE {} (id INTEGER PRIMARY KEY AUTOINCREMENT, {});'''.format(tablename, colnames))\n", " #populate table from dataframe\n", " newtable = dataframe[columns].copy().applymap(str)\n", " newtable['id'] = range(1,len(newtable)+1)\n", " newtable.to_sql(tablename, conn, index=False, if_exists='append')\n", " conn.commit()\n", " return newtable\n", "\n", "#newtable = add_table(amitt.df_actortypes, 'actor_type', ['amitt_id', 'sector_id', 'framework_id', 'name', 'summary'])\n", "# counter\n", "newtable = add_table(amitt.df_counters, 'counter', ['amitt_id', 'tactic_id', 'metatechnique_id', 'name', 'summary'])\n", "# dataset\n", "newtable = add_table(amitt.df_frameworks, 'framework', ['amitt_id', 'name', 'summary'])\n", "# incident\n", "# metatechnique\n", "newtable = add_table(amitt.df_metatechniques, 'metatechnique', ['amitt_id', 'name', 'summary'])\n", "newtable = add_table(amitt.df_phases, 'phase', ['amitt_id', 'name', 'rank', 'summary'])\n", "# playbook\n", "# reference\n", "# response_type\n", "# sector\n", "newtable = add_table(amitt.df_tactics, 'tactic', ['amitt_id', 'phase_id', 'name', 'rank', 'summary'])\n", "newtable = add_table(amitt.df_tasks, 'task', ['amitt_id', 'tactic_id', 'framework_id', 'name', 'summary'])\n", "newtable = add_table(amitt.df_techniques, 'technique', ['amitt_id', 'tactic_id', 'name', 'summary'])\n", "# techniques_counters\n", "\n", "conn.execute(\"DROP TABLE IF EXISTS {}\".format('user'))\n", "conn.execute('''CREATE TABLE user (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL UNIQUE, password TEXT NOT NULL);''')\n", "\n", "conn.close()\n", "newtable" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "df_phases loaded\n", "df_techniques loaded\n", "df_tasks loaded\n", "df_incidents loaded\n", "df_counters loaded\n", "df_detections loaded\n", "df_actors loaded\n", "df_resources loaded\n", "df_responsetypes loaded\n", "df_metatechniques loaded\n", "it loaded\n", "df_tactics loaded\n", "df_techniques_per_tactic loaded\n", "df_counters_per_tactic loaded\n", "phases not loaded\n", "tactics not loaded\n", "techniques not loaded\n", "counters not loaded\n", "metatechniques not loaded\n", "actors not loaded\n", "resources not loaded\n", "num_tactics not loaded\n", "cross_counterid_techniqueid loaded\n", "cross_counterid_resourceid loaded\n", "cross_counterid_actorid loaded\n" ] } ], "source": [ "# Generate full sqlite database from the Amitt variables\n", "conn = sql.connect('amitt_sqlite.db')\n", "for tablename, table in vars(amitt).items():\n", " if type(table) == pd.core.frame.DataFrame:\n", " table.applymap(str).to_sql(tablename, conn)\n", " print('{} loaded'.format(tablename))\n", " else:\n", " print('{} not loaded'.format(tablename))" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idactor_id
0C00006A033
1C00008A007
2C00009A016
2C00009A006
3C00010A020
.........
135C00219
136C00220
137C00221
138C00222
139C00223
\n", "

166 rows × 2 columns

\n", "
" ], "text/plain": [ " id actor_id\n", "0 C00006 A033\n", "1 C00008 A007\n", "2 C00009 A016\n", "2 C00009 A006\n", "3 C00010 A020\n", ".. ... ...\n", "135 C00219 \n", "136 C00220 \n", "137 C00221 \n", "138 C00222 \n", "139 C00223 \n", "\n", "[166 rows x 2 columns]" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "amitt.cross_counterid_actorid" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idtechnique_idWeight
1C00008TA011
1C00008TA061
1C00008TA081
1C00008T00061
1C00008T00091
............
134C00216T00181
134C00216T00571
135C00219T00241
135C00219T00261
135C00219T00251
\n", "

717 rows × 3 columns

\n", "
" ], "text/plain": [ " id technique_id Weight\n", "1 C00008 TA01 1\n", "1 C00008 TA06 1\n", "1 C00008 TA08 1\n", "1 C00008 T0006 1\n", "1 C00008 T0009 1\n", ".. ... ... ...\n", "134 C00216 T0018 1\n", "134 C00216 T0057 1\n", "135 C00219 T0024 1\n", "135 C00219 T0026 1\n", "135 C00219 T0025 1\n", "\n", "[717 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ct = amitt.cross_counterid_techniqueid\n", "ct['Weight'] = 1\n", "ct = ct[ct['technique_id'].str.len() > 0]\n", "ct.to_csv('../visualisations/cross_counterid_techniqueid.csv', index=False, header=['Source','Target', 'Weight'])\n", "ct" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# fix the problem with excelfile changes" ] }, { "cell_type": "code", "execution_count": 141, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ ":9: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n", " pd.set_option('display.max_colwidth', -1)\n" ] } ], "source": [ "import pandas as pd\n", "import sqlite3 as sql\n", "from generate_amitt_ttps import Amitt\n", "import pandas as pd\n", "import numpy as np\n", "import os\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "pd.set_option('display.max_rows', 1000)\n", "pd.set_option('display.max_colwidth', -1)\n", "\n", "newfile = '../AMITT_MASTER_DATA/AMITT_TTPs_MASTER.xlsx'\n", "oldfile = 'AMITT_TTPs_MASTER_github_version.xlsx'\n", "\n", "# Load dfs from file\n", "newdfs = {}\n", "newxlsx = pd.ExcelFile(newfile)\n", "for sheetname in newxlsx.sheet_names:\n", " newdfs[sheetname] = newxlsx.parse(sheetname)\n", " newdfs[sheetname].fillna('', inplace=True)\n", "\n", "olddfs = {}\n", "oldxlsx = pd.ExcelFile(oldfile)\n", "for sheetname in oldxlsx.sheet_names:\n", " olddfs[sheetname] = oldxlsx.parse(sheetname)\n", " olddfs[sheetname].fillna('', inplace=True)\n", "\n", "addedtables = newdfs.keys() - olddfs.keys()\n", "losttables = olddfs.keys() - newdfs.keys()\n", "if len(addedtables) + len(losttables) > 0:\n", " print('Table changes: new tables are {}, lost tables are {}'.format(addedtables, losttables))\n", "\n", "def investigate_table(table):\n", " print('\\n\\nTable {} is changed'.format(table))\n", " # Column headings\n", " coldiffs = set(newdfs[table].columns).symmetric_difference(set(olddfs[table].columns))\n", " if len(coldiffs) > 0:\n", " print('column differences: {}'.format(coldiffs))\n", " # length\n", " if len(newdfs[table]) != len(olddfs[table]):\n", " print('length differences: new {} old {}'.format(len(newdfs[table]), len(olddfs[table])))\n", "\n", " # column by column\n", " for column in newdfs[table].columns:\n", " coldiffs = newdfs[table][column] != olddfs[table][column]\n", " if len(newdfs[table][coldiffs]) > 0:\n", " print('Differences in column {}'.format(column))\n", " return\n", "\n", "for table in newdfs.keys():\n", " if newdfs[table].equals(olddfs[table]) == False:\n", " investigate_table(table) " ] }, { "cell_type": "code", "execution_count": 142, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
amitt_idnewold
\n", "
" ], "text/plain": [ "Empty DataFrame\n", "Columns: [amitt_id, new, old]\n", "Index: []" ] }, "execution_count": 142, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Look at individual table differences\n", "table = 'countermeasures'\n", "column = 'summary'\n", "coldiffs = newdfs[table][column] != olddfs[table][column]\n", "diffcols = pd.DataFrame()\n", "diffcols['amitt_id'] = newdfs[table][coldiffs]['amitt_id']\n", "diffcols['new'] = newdfs[table][coldiffs][column]\n", "diffcols['old'] = olddfs[table][coldiffs][column]\n", "diffcols[diffcols['old'] != '']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.3" } }, "nbformat": 4, "nbformat_minor": 4 }