From f3336513cc69db818b3f262aa95defe68eb82924 Mon Sep 17 00:00:00 2001 From: Sara-Jayne Terp Date: Fri, 25 Jun 2021 08:52:18 +0100 Subject: [PATCH] more sqlite code --- .../AMITT_code_testbed-checkpoint.ipynb | 604 +++++------------- ...nerate_AMITT_github_files-checkpoint.ipynb | 10 +- HTML_GENERATING_CODE/AMITT_code_testbed.ipynb | 510 +++++---------- .../Generate_AMITT_github_files.ipynb | 10 +- .../generate_amitt_ttps.cpython-38.pyc | Bin 23598 -> 23637 bytes HTML_GENERATING_CODE/generate_amitt_ttps.py | 1 + 6 files changed, 316 insertions(+), 819 deletions(-) diff --git a/HTML_GENERATING_CODE/.ipynb_checkpoints/AMITT_code_testbed-checkpoint.ipynb b/HTML_GENERATING_CODE/.ipynb_checkpoints/AMITT_code_testbed-checkpoint.ipynb index f6b2b53..e79abe2 100644 --- a/HTML_GENERATING_CODE/.ipynb_checkpoints/AMITT_code_testbed-checkpoint.ipynb +++ b/HTML_GENERATING_CODE/.ipynb_checkpoints/AMITT_code_testbed-checkpoint.ipynb @@ -9,34 +9,120 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "dict_keys(['df_phases', 'df_techniques', 'df_tasks', 'df_incidents', 'df_counters', 'df_detections', 'df_actortypes', 'df_resources', 'df_responsetypes', 'df_metatechniques', 'it', 'df_tactics', 'df_techniques_per_tactic', 'df_counters_per_tactic', 'phases', 'tactics', 'techniques', 'counters', 'metatechniques', 'actortypes', 'resources', 'num_tactics', 'cross_counterid_techniqueid', 'cross_counterid_resourceid', 'cross_counterid_actortypeid'])\n" + "dict_keys(['df_phases', 'df_frameworks', 'df_techniques', 'df_tasks', 'df_incidents', 'df_counters', 'df_detections', 'df_actortypes', 'df_resources', 'df_responsetypes', 'df_metatechniques', 'it', 'df_tactics', 'df_techniques_per_tactic', 'df_counters_per_tactic', 'phases', 'tactics', 'techniques', 'counters', 'metatechniques', 'actortypes', 'resources', 'num_tactics', 'cross_counterid_techniqueid', 'cross_counterid_resourceid', 'cross_counterid_actortypeid'])\n" ] }, { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
amitt_idtechnique_id
0C00006T0007
0C00006T0015
0C00006T0018
0C00006T0043
0C00006T0053
.........
135C00219T0025
136C00220
137C00221
138C00222
139C00223
\n", + "

898 rows × 2 columns

\n", + "
" + ], "text/plain": [ - "{'TA01': 'Strategic Planning',\n", - " 'TA02': 'Objective Planning',\n", - " 'TA03': 'Develop People',\n", - " 'TA04': 'Develop Networks',\n", - " 'TA05': 'Microtargeting',\n", - " 'TA06': 'Develop Content',\n", - " 'TA07': 'Channel Selection',\n", - " 'TA08': 'Pump Priming',\n", - " 'TA09': 'Exposure',\n", - " 'TA10': 'Go Physical',\n", - " 'TA11': 'Persistence',\n", - " 'TA12': 'Measure Effectiveness'}" + " amitt_id technique_id\n", + "0 C00006 T0007\n", + "0 C00006 T0015\n", + "0 C00006 T0018\n", + "0 C00006 T0043\n", + "0 C00006 T0053\n", + ".. ... ...\n", + "135 C00219 T0025\n", + "136 C00220 \n", + "137 C00221 \n", + "138 C00222 \n", + "139 C00223 \n", + "\n", + "[898 rows x 2 columns]" ] }, - "execution_count": 1, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -52,12 +138,12 @@ "\n", "# Check which amitt variables we can see from here\n", "print('{}'.format(vars(amitt).keys()))\n", - "vars(amitt)['tactics']" + "vars(amitt)['cross_counterid_techniqueid']" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 3, "metadata": { "scrolled": true }, @@ -84,407 +170,99 @@ " \n", " \n", " amitt_id\n", - " name\n", - " metatechnique\n", - " summary\n", - " playbooks\n", - " actortypes\n", - " resources_needed\n", - " how_found\n", - " references\n", - " incident_ids\n", - " tactic\n", - " responsetype\n", - " notes\n", - " techniques\n", - " longname\n", - " tactic_id\n", - " tactic_name\n", - " metatechnique_id\n", - " metatechnique_name\n", + " technique_id\n", " \n", " \n", " \n", " \n", " 0\n", " C00006\n", - " Charge for social media\n", - " M004 - friction\n", - " Include a paid-for privacy option, e.g. pay Fa...\n", - " \n", - " A033 - social media platform owner\n", - " \n", - " 2019-11-workshop\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D2 Deny\n", - " \n", - " \n", - " C00006 - Charge for social media\n", - " TA01\n", - " Strategic Planning\n", - " M004\n", - " - friction\n", + " T0007\n", " \n", " \n", - " 8\n", + " 0\n", " C00006\n", - " Censorship\n", - " M005 - removal\n", - " Alter and/or block the publication/disseminati...\n", - " \n", - " A031 - social media platform administrator\n", - " \n", - " grugq\n", - " Taylor81\n", - " \n", - " TA01 Strategic Planning\n", - " D2 Deny\n", - " \n", - " T0009 - Create fake experts\\nT0008 - Create fa...\n", - " C00006 - Censorship\n", - " TA01\n", - " Strategic Planning\n", - " M005\n", - " - removal\n", + " T0015\n", " \n", " \n", - " 1\n", - " C00008\n", - " Create shared fact-checking database\n", - " M006 - scoring\n", - " Snopes is best-known example\n", - " \n", - " A007 - factchecker\n", - " \n", - " 2019-11-workshop\\n2019-11-search\n", - " \n", - " I00049,I00050\n", - " TA01 Strategic Planning\n", - " D4 Degrade\n", - " \n", - " TA01 - Strategic Planning\\nTA06 - Develop Cont...\n", - " C00008 - Create shared fact-checking database\n", - " TA01\n", - " Strategic Planning\n", - " M006\n", - " - scoring\n", + " 0\n", + " C00006\n", + " T0018\n", " \n", " \n", - " 2\n", - " C00009\n", - " Educate high profile influencers on best pract...\n", - " M001 - resilience\n", - " \n", - " \n", - " A016 - influencer,A006 - educator\n", - " \n", - " 2019-11-workshop\n", - " \n", - " \n", - " TA02 Objective Planning\n", - " D2 Deny\n", - " \n", - " TA08 - Pump Priming\\nT0010 - Cultivate ignoran...\n", - " C00009 - Educate high profile influencers on b...\n", - " TA02\n", - " Objective Planning\n", - " M001\n", - " - resilience\n", + " 0\n", + " C00006\n", + " T0043\n", " \n", " \n", - " 3\n", - " C00010\n", - " Enhanced privacy regulation for social media\n", - " M004 - friction\n", - " Privacy standards\n", - " \n", - " A020 - policy maker\n", - " \n", - " 2019-11-workshop\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D2 Deny\n", - " \n", - " TA05 - Microtargeting\\nT00018 - Paid targeted ads\n", - " C00010 - Enhanced privacy regulation for socia...\n", - " TA01\n", - " Strategic Planning\n", - " M004\n", - " - friction\n", + " 0\n", + " C00006\n", + " T0053\n", " \n", " \n", " ...\n", " ...\n", " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", " \n", " \n", " 135\n", " C00219\n", - " Add metadata to content that’s out of the cont...\n", - " M003 - daylight\n", - " Steganography. Adding date, signatures etc to ...\n", - " Add date and source to images\n", - " \n", - " \n", - " grugq\n", - " \n", - " \n", - " TA06 Develop Content\n", - " D4 Degrade\n", - " \n", - " T0024 - Create fake videos and images\\nT0026 -...\n", - " C00219 - Add metadata to content that’s out of...\n", - " TA06\n", - " Develop Content\n", - " M003\n", - " - daylight\n", + " T0025\n", " \n", " \n", " 136\n", " C00220\n", - " Develop a monitoring and intelligence plan\n", - " M007 - metatechnique\n", " \n", - " \n", - " \n", - " \n", - " Counters cleanup\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00220 - Develop a monitoring and intelligence...\n", - " TA01\n", - " Strategic Planning\n", - " M007\n", - " - metatechnique\n", " \n", " \n", " 137\n", " C00221\n", - " Run a disinformation red team, and design miti...\n", - " M007 - metatechnique\n", - " Include PACE plans - Primary, Alternate, Conti...\n", " \n", - " \n", - " \n", - " Counters cleanup\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00221 - Run a disinformation red team, and de...\n", - " TA01\n", - " Strategic Planning\n", - " M007\n", - " - metatechnique\n", " \n", " \n", " 138\n", " C00222\n", - " Tabletop simulations\n", - " M007 - metatechnique\n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " TA02 Objective Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00222 - Tabletop simulations\n", - " TA02\n", - " Objective Planning\n", - " M007\n", - " - metatechnique\n", " \n", " \n", " 139\n", " C00223\n", - " Strengthen Trust in social media platforms\n", - " M001 - resilience\n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00223 - Strengthen Trust in social media plat...\n", - " TA01\n", - " Strategic Planning\n", - " M001\n", - " - resilience\n", " \n", " \n", "\n", - "

140 rows × 19 columns

\n", + "

898 rows × 2 columns

\n", "" ], "text/plain": [ - " amitt_id name \\\n", - "0 C00006 Charge for social media \n", - "8 C00006 Censorship \n", - "1 C00008 Create shared fact-checking database \n", - "2 C00009 Educate high profile influencers on best pract... \n", - "3 C00010 Enhanced privacy regulation for social media \n", - ".. ... ... \n", - "135 C00219 Add metadata to content that’s out of the cont... \n", - "136 C00220 Develop a monitoring and intelligence plan \n", - "137 C00221 Run a disinformation red team, and design miti... \n", - "138 C00222 Tabletop simulations \n", - "139 C00223 Strengthen Trust in social media platforms \n", + " amitt_id technique_id\n", + "0 C00006 T0007\n", + "0 C00006 T0015\n", + "0 C00006 T0018\n", + "0 C00006 T0043\n", + "0 C00006 T0053\n", + ".. ... ...\n", + "135 C00219 T0025\n", + "136 C00220 \n", + "137 C00221 \n", + "138 C00222 \n", + "139 C00223 \n", "\n", - " metatechnique summary \\\n", - "0 M004 - friction Include a paid-for privacy option, e.g. pay Fa... \n", - "8 M005 - removal Alter and/or block the publication/disseminati... \n", - "1 M006 - scoring Snopes is best-known example \n", - "2 M001 - resilience \n", - "3 M004 - friction Privacy standards \n", - ".. ... ... \n", - "135 M003 - daylight Steganography. Adding date, signatures etc to ... \n", - "136 M007 - metatechnique \n", - "137 M007 - metatechnique Include PACE plans - Primary, Alternate, Conti... \n", - "138 M007 - metatechnique \n", - "139 M001 - resilience \n", - "\n", - " playbooks \\\n", - "0 \n", - "8 \n", - "1 \n", - "2 \n", - "3 \n", - ".. ... \n", - "135 Add date and source to images \n", - "136 \n", - "137 \n", - "138 \n", - "139 \n", - "\n", - " actortypes resources_needed \\\n", - "0 A033 - social media platform owner \n", - "8 A031 - social media platform administrator \n", - "1 A007 - factchecker \n", - "2 A016 - influencer,A006 - educator \n", - "3 A020 - policy maker \n", - ".. ... ... \n", - "135 \n", - "136 \n", - "137 \n", - "138 \n", - "139 \n", - "\n", - " how_found references incident_ids \\\n", - "0 2019-11-workshop \n", - "8 grugq Taylor81 \n", - "1 2019-11-workshop\\n2019-11-search I00049,I00050 \n", - "2 2019-11-workshop \n", - "3 2019-11-workshop \n", - ".. ... ... ... \n", - "135 grugq \n", - "136 Counters cleanup \n", - "137 Counters cleanup \n", - "138 \n", - "139 \n", - "\n", - " tactic responsetype notes \\\n", - "0 TA01 Strategic Planning D2 Deny \n", - "8 TA01 Strategic Planning D2 Deny \n", - "1 TA01 Strategic Planning D4 Degrade \n", - "2 TA02 Objective Planning D2 Deny \n", - "3 TA01 Strategic Planning D2 Deny \n", - ".. ... ... ... \n", - "135 TA06 Develop Content D4 Degrade \n", - "136 TA01 Strategic Planning D3 Disrupt \n", - "137 TA01 Strategic Planning D3 Disrupt \n", - "138 TA02 Objective Planning D3 Disrupt \n", - "139 TA01 Strategic Planning D3 Disrupt \n", - "\n", - " techniques \\\n", - "0 \n", - "8 T0009 - Create fake experts\\nT0008 - Create fa... \n", - "1 TA01 - Strategic Planning\\nTA06 - Develop Cont... \n", - "2 TA08 - Pump Priming\\nT0010 - Cultivate ignoran... \n", - "3 TA05 - Microtargeting\\nT00018 - Paid targeted ads \n", - ".. ... \n", - "135 T0024 - Create fake videos and images\\nT0026 -... \n", - "136 \n", - "137 \n", - "138 \n", - "139 \n", - "\n", - " longname tactic_id \\\n", - "0 C00006 - Charge for social media TA01 \n", - "8 C00006 - Censorship TA01 \n", - "1 C00008 - Create shared fact-checking database TA01 \n", - "2 C00009 - Educate high profile influencers on b... TA02 \n", - "3 C00010 - Enhanced privacy regulation for socia... TA01 \n", - ".. ... ... \n", - "135 C00219 - Add metadata to content that’s out of... TA06 \n", - "136 C00220 - Develop a monitoring and intelligence... TA01 \n", - "137 C00221 - Run a disinformation red team, and de... TA01 \n", - "138 C00222 - Tabletop simulations TA02 \n", - "139 C00223 - Strengthen Trust in social media plat... TA01 \n", - "\n", - " tactic_name metatechnique_id metatechnique_name \n", - "0 Strategic Planning M004 - friction \n", - "8 Strategic Planning M005 - removal \n", - "1 Strategic Planning M006 - scoring \n", - "2 Objective Planning M001 - resilience \n", - "3 Strategic Planning M004 - friction \n", - ".. ... ... ... \n", - "135 Develop Content M003 - daylight \n", - "136 Strategic Planning M007 - metatechnique \n", - "137 Strategic Planning M007 - metatechnique \n", - "138 Objective Planning M007 - metatechnique \n", - "139 Strategic Planning M001 - resilience \n", - "\n", - "[140 rows x 19 columns]" + "[898 rows x 2 columns]" ] }, - "execution_count": 45, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "amitt.df_counters" + "amitt.cross_counterid_techniqueid" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 10, "metadata": { "scrolled": true }, @@ -587,7 +365,7 @@ " T0062\n", " TA12\n", " Behaviour changes\n", - " \n", + " Monitor and evaluate behaviour changes from mi...\n", " 62\n", " \n", " \n", @@ -595,7 +373,7 @@ " T0063\n", " TA12\n", " Message reach\n", - " \n", + " Monitor and evaluate message reach in misinfor...\n", " 63\n", " \n", " \n", @@ -603,7 +381,7 @@ " T0064\n", " TA12\n", " Social media engagement\n", - " \n", + " Monitor and evaluate social media engagement i...\n", " 64\n", " \n", " \n", @@ -634,21 +412,21 @@ ".. ... .. \n", "59 continue narrative or message amplification af... 60 \n", "60 Sell hats, t-shirts, flags and other branded c... 61 \n", - "61 62 \n", - "62 63 \n", - "63 64 \n", + "61 Monitor and evaluate behaviour changes from mi... 62 \n", + "62 Monitor and evaluate message reach in misinfor... 63 \n", + "63 Monitor and evaluate social media engagement i... 64 \n", "\n", "[64 rows x 5 columns]" ] }, - "execution_count": 44, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Generate minimal sqlite database from the Amitt variables\n", - "conn = sql.connect('test_amitt_sqlite.db')\n", + "conn = sql.connect('amittsite.sqlite')\n", "\n", "def add_table(dataframe, tablename, columns): \n", " # Create sql table\n", @@ -664,10 +442,12 @@ "\n", "#newtable = add_table(amitt.df_actortypes, 'actor_type', ['amitt_id', 'sector_id', 'framework_id', 'name', 'summary'])\n", "# counter\n", + "newtable = add_table(amitt.df_counters, 'counter', ['amitt_id', 'tactic_id', 'metatechnique_id', 'name', 'summary'])\n", "# dataset\n", - "# framework\n", + "newtable = add_table(amitt.df_frameworks, 'framework', ['amitt_id', 'name', 'summary'])\n", "# incident\n", "# metatechnique\n", + "newtable = add_table(amitt.df_metatechniques, 'metatechnique', ['amitt_id', 'name', 'summary'])\n", "newtable = add_table(amitt.df_phases, 'phase', ['amitt_id', 'name', 'rank', 'summary'])\n", "# playbook\n", "# reference\n", @@ -996,21 +776,15 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 141, "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "\n", - "\n", - "Table countermeasures is changed\n", - "Differences in column name\n", - "Differences in column summary\n", - "Differences in column responsetype\n", - "Differences in column techniques\n", - "Differences in column longname\n" + ":9: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n", + " pd.set_option('display.max_colwidth', -1)\n" ] } ], @@ -1022,6 +796,8 @@ "import numpy as np\n", "import os\n", "from sklearn.feature_extraction.text import CountVectorizer\n", + "pd.set_option('display.max_rows', 1000)\n", + "pd.set_option('display.max_colwidth', -1)\n", "\n", "newfile = '../AMITT_MASTER_DATA/AMITT_TTPs_MASTER.xlsx'\n", "oldfile = 'AMITT_TTPs_MASTER_github_version.xlsx'\n", @@ -1068,7 +844,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 142, "metadata": {}, "outputs": [ { @@ -1098,81 +874,17 @@ " \n", " \n", " \n", - " \n", - " 27\n", - " C00047\n", - " Coordinated inauthentics\n", - " Honeypot with coordinated inauthentics\n", - " \n", - " \n", - " 32\n", - " C00056\n", - " Get off social media\n", - " Encourage people to leave social media\n", - " \n", - " \n", - " 34\n", - " C00059\n", - " Verification of project before posting (counte...\n", - " Verification of project before posting fund re...\n", - " \n", - " \n", - " 42\n", - " C00072\n", - " Remove non-relevant content from special inter...\n", - " Remove non-relevant content from special inter...\n", - " \n", - " \n", - " 68\n", - " C00105\n", - " Buy more advertising than misinformation creators\n", - " Buy more advertising than the adversary to shi...\n", - " \n", - " \n", - " 74\n", - " C00113\n", - " Debunk and defuse a fake expert / credentials.\n", - " Debunk and defuse a fake expert / credentials....\n", - " \n", - " \n", - " 80\n", - " C00119\n", - " Engage payload and debunk.\n", - " Engage payload and debunk. Provide link to fac...\n", - " \n", - " \n", - " 101\n", - " C00147\n", - " Make amplification of social media posts expir...\n", - " Make amplification of social media ports expir...\n", - " \n", " \n", "\n", "" ], "text/plain": [ - " amitt_id new \\\n", - "27 C00047 Coordinated inauthentics \n", - "32 C00056 Get off social media \n", - "34 C00059 Verification of project before posting (counte... \n", - "42 C00072 Remove non-relevant content from special inter... \n", - "68 C00105 Buy more advertising than misinformation creators \n", - "74 C00113 Debunk and defuse a fake expert / credentials. \n", - "80 C00119 Engage payload and debunk. \n", - "101 C00147 Make amplification of social media posts expir... \n", - "\n", - " old \n", - "27 Honeypot with coordinated inauthentics \n", - "32 Encourage people to leave social media \n", - "34 Verification of project before posting fund re... \n", - "42 Remove non-relevant content from special inter... \n", - "68 Buy more advertising than the adversary to shi... \n", - "74 Debunk and defuse a fake expert / credentials.... \n", - "80 Engage payload and debunk. Provide link to fac... \n", - "101 Make amplification of social media ports expir... " + "Empty DataFrame\n", + "Columns: [amitt_id, new, old]\n", + "Index: []" ] }, - "execution_count": 125, + "execution_count": 142, "metadata": {}, "output_type": "execute_result" } @@ -1180,13 +892,13 @@ "source": [ "# Look at individual table differences\n", "table = 'countermeasures'\n", - "column = 'name'\n", + "column = 'summary'\n", "coldiffs = newdfs[table][column] != olddfs[table][column]\n", "diffcols = pd.DataFrame()\n", "diffcols['amitt_id'] = newdfs[table][coldiffs]['amitt_id']\n", "diffcols['new'] = newdfs[table][coldiffs][column]\n", "diffcols['old'] = olddfs[table][coldiffs][column]\n", - "diffcols" + "diffcols[diffcols['old'] != '']" ] }, { diff --git a/HTML_GENERATING_CODE/.ipynb_checkpoints/Generate_AMITT_github_files-checkpoint.ipynb b/HTML_GENERATING_CODE/.ipynb_checkpoints/Generate_AMITT_github_files-checkpoint.ipynb index 8de3ecc..d22e476 100644 --- a/HTML_GENERATING_CODE/.ipynb_checkpoints/Generate_AMITT_github_files-checkpoint.ipynb +++ b/HTML_GENERATING_CODE/.ipynb_checkpoints/Generate_AMITT_github_files-checkpoint.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": { "scrolled": true }, @@ -48,8 +48,10 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, + "execution_count": 10, + "metadata": { + "scrolled": true + }, "outputs": [ { "data": { @@ -437,7 +439,7 @@ "[140 rows x 18 columns]" ] }, - "execution_count": 8, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } diff --git a/HTML_GENERATING_CODE/AMITT_code_testbed.ipynb b/HTML_GENERATING_CODE/AMITT_code_testbed.ipynb index ac73699..e79abe2 100644 --- a/HTML_GENERATING_CODE/AMITT_code_testbed.ipynb +++ b/HTML_GENERATING_CODE/AMITT_code_testbed.ipynb @@ -9,34 +9,120 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "dict_keys(['df_phases', 'df_techniques', 'df_tasks', 'df_incidents', 'df_counters', 'df_detections', 'df_actortypes', 'df_resources', 'df_responsetypes', 'df_metatechniques', 'it', 'df_tactics', 'df_techniques_per_tactic', 'df_counters_per_tactic', 'phases', 'tactics', 'techniques', 'counters', 'metatechniques', 'actortypes', 'resources', 'num_tactics', 'cross_counterid_techniqueid', 'cross_counterid_resourceid', 'cross_counterid_actortypeid'])\n" + "dict_keys(['df_phases', 'df_frameworks', 'df_techniques', 'df_tasks', 'df_incidents', 'df_counters', 'df_detections', 'df_actortypes', 'df_resources', 'df_responsetypes', 'df_metatechniques', 'it', 'df_tactics', 'df_techniques_per_tactic', 'df_counters_per_tactic', 'phases', 'tactics', 'techniques', 'counters', 'metatechniques', 'actortypes', 'resources', 'num_tactics', 'cross_counterid_techniqueid', 'cross_counterid_resourceid', 'cross_counterid_actortypeid'])\n" ] }, { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
amitt_idtechnique_id
0C00006T0007
0C00006T0015
0C00006T0018
0C00006T0043
0C00006T0053
.........
135C00219T0025
136C00220
137C00221
138C00222
139C00223
\n", + "

898 rows × 2 columns

\n", + "
" + ], "text/plain": [ - "{'TA01': 'Strategic Planning',\n", - " 'TA02': 'Objective Planning',\n", - " 'TA03': 'Develop People',\n", - " 'TA04': 'Develop Networks',\n", - " 'TA05': 'Microtargeting',\n", - " 'TA06': 'Develop Content',\n", - " 'TA07': 'Channel Selection',\n", - " 'TA08': 'Pump Priming',\n", - " 'TA09': 'Exposure',\n", - " 'TA10': 'Go Physical',\n", - " 'TA11': 'Persistence',\n", - " 'TA12': 'Measure Effectiveness'}" + " amitt_id technique_id\n", + "0 C00006 T0007\n", + "0 C00006 T0015\n", + "0 C00006 T0018\n", + "0 C00006 T0043\n", + "0 C00006 T0053\n", + ".. ... ...\n", + "135 C00219 T0025\n", + "136 C00220 \n", + "137 C00221 \n", + "138 C00222 \n", + "139 C00223 \n", + "\n", + "[898 rows x 2 columns]" ] }, - "execution_count": 1, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -52,12 +138,12 @@ "\n", "# Check which amitt variables we can see from here\n", "print('{}'.format(vars(amitt).keys()))\n", - "vars(amitt)['tactics']" + "vars(amitt)['cross_counterid_techniqueid']" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 3, "metadata": { "scrolled": true }, @@ -84,407 +170,99 @@ " \n", " \n", " amitt_id\n", - " name\n", - " metatechnique\n", - " summary\n", - " playbooks\n", - " actortypes\n", - " resources_needed\n", - " how_found\n", - " references\n", - " incident_ids\n", - " tactic\n", - " responsetype\n", - " notes\n", - " techniques\n", - " longname\n", - " tactic_id\n", - " tactic_name\n", - " metatechnique_id\n", - " metatechnique_name\n", + " technique_id\n", " \n", " \n", " \n", " \n", " 0\n", " C00006\n", - " Charge for social media\n", - " M004 - friction\n", - " Include a paid-for privacy option, e.g. pay Fa...\n", - " \n", - " A033 - social media platform owner\n", - " \n", - " 2019-11-workshop\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D2 Deny\n", - " \n", - " \n", - " C00006 - Charge for social media\n", - " TA01\n", - " Strategic Planning\n", - " M004\n", - " - friction\n", + " T0007\n", " \n", " \n", - " 8\n", + " 0\n", " C00006\n", - " Censorship\n", - " M005 - removal\n", - " Alter and/or block the publication/disseminati...\n", - " \n", - " A031 - social media platform administrator\n", - " \n", - " grugq\n", - " Taylor81\n", - " \n", - " TA01 Strategic Planning\n", - " D2 Deny\n", - " \n", - " T0009 - Create fake experts\\nT0008 - Create fa...\n", - " C00006 - Censorship\n", - " TA01\n", - " Strategic Planning\n", - " M005\n", - " - removal\n", + " T0015\n", " \n", " \n", - " 1\n", - " C00008\n", - " Create shared fact-checking database\n", - " M006 - scoring\n", - " Snopes is best-known example\n", - " \n", - " A007 - factchecker\n", - " \n", - " 2019-11-workshop\\n2019-11-search\n", - " \n", - " I00049,I00050\n", - " TA01 Strategic Planning\n", - " D4 Degrade\n", - " \n", - " TA01 - Strategic Planning\\nTA06 - Develop Cont...\n", - " C00008 - Create shared fact-checking database\n", - " TA01\n", - " Strategic Planning\n", - " M006\n", - " - scoring\n", + " 0\n", + " C00006\n", + " T0018\n", " \n", " \n", - " 2\n", - " C00009\n", - " Educate high profile influencers on best pract...\n", - " M001 - resilience\n", - " \n", - " \n", - " A016 - influencer,A006 - educator\n", - " \n", - " 2019-11-workshop\n", - " \n", - " \n", - " TA02 Objective Planning\n", - " D2 Deny\n", - " \n", - " TA08 - Pump Priming\\nT0010 - Cultivate ignoran...\n", - " C00009 - Educate high profile influencers on b...\n", - " TA02\n", - " Objective Planning\n", - " M001\n", - " - resilience\n", + " 0\n", + " C00006\n", + " T0043\n", " \n", " \n", - " 3\n", - " C00010\n", - " Enhanced privacy regulation for social media\n", - " M004 - friction\n", - " Privacy standards\n", - " \n", - " A020 - policy maker\n", - " \n", - " 2019-11-workshop\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D2 Deny\n", - " \n", - " TA05 - Microtargeting\\nT00018 - Paid targeted ads\n", - " C00010 - Enhanced privacy regulation for socia...\n", - " TA01\n", - " Strategic Planning\n", - " M004\n", - " - friction\n", + " 0\n", + " C00006\n", + " T0053\n", " \n", " \n", " ...\n", " ...\n", " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", - " ...\n", " \n", " \n", " 135\n", " C00219\n", - " Add metadata to content that’s out of the cont...\n", - " M003 - daylight\n", - " Steganography. Adding date, signatures etc to ...\n", - " Add date and source to images\n", - " \n", - " \n", - " grugq\n", - " \n", - " \n", - " TA06 Develop Content\n", - " D4 Degrade\n", - " \n", - " T0024 - Create fake videos and images\\nT0026 -...\n", - " C00219 - Add metadata to content that’s out of...\n", - " TA06\n", - " Develop Content\n", - " M003\n", - " - daylight\n", + " T0025\n", " \n", " \n", " 136\n", " C00220\n", - " Develop a monitoring and intelligence plan\n", - " M007 - metatechnique\n", " \n", - " \n", - " \n", - " \n", - " Counters cleanup\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00220 - Develop a monitoring and intelligence...\n", - " TA01\n", - " Strategic Planning\n", - " M007\n", - " - metatechnique\n", " \n", " \n", " 137\n", " C00221\n", - " Run a disinformation red team, and design miti...\n", - " M007 - metatechnique\n", - " Include PACE plans - Primary, Alternate, Conti...\n", " \n", - " \n", - " \n", - " Counters cleanup\n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00221 - Run a disinformation red team, and de...\n", - " TA01\n", - " Strategic Planning\n", - " M007\n", - " - metatechnique\n", " \n", " \n", " 138\n", " C00222\n", - " Tabletop simulations\n", - " M007 - metatechnique\n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " TA02 Objective Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00222 - Tabletop simulations\n", - " TA02\n", - " Objective Planning\n", - " M007\n", - " - metatechnique\n", " \n", " \n", " 139\n", " C00223\n", - " Strengthen Trust in social media platforms\n", - " M001 - resilience\n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " TA01 Strategic Planning\n", - " D3 Disrupt\n", - " \n", - " \n", - " C00223 - Strengthen Trust in social media plat...\n", - " TA01\n", - " Strategic Planning\n", - " M001\n", - " - resilience\n", " \n", " \n", "\n", - "

140 rows × 19 columns

\n", + "

898 rows × 2 columns

\n", "" ], "text/plain": [ - " amitt_id name \\\n", - "0 C00006 Charge for social media \n", - "8 C00006 Censorship \n", - "1 C00008 Create shared fact-checking database \n", - "2 C00009 Educate high profile influencers on best pract... \n", - "3 C00010 Enhanced privacy regulation for social media \n", - ".. ... ... \n", - "135 C00219 Add metadata to content that’s out of the cont... \n", - "136 C00220 Develop a monitoring and intelligence plan \n", - "137 C00221 Run a disinformation red team, and design miti... \n", - "138 C00222 Tabletop simulations \n", - "139 C00223 Strengthen Trust in social media platforms \n", + " amitt_id technique_id\n", + "0 C00006 T0007\n", + "0 C00006 T0015\n", + "0 C00006 T0018\n", + "0 C00006 T0043\n", + "0 C00006 T0053\n", + ".. ... ...\n", + "135 C00219 T0025\n", + "136 C00220 \n", + "137 C00221 \n", + "138 C00222 \n", + "139 C00223 \n", "\n", - " metatechnique summary \\\n", - "0 M004 - friction Include a paid-for privacy option, e.g. pay Fa... \n", - "8 M005 - removal Alter and/or block the publication/disseminati... \n", - "1 M006 - scoring Snopes is best-known example \n", - "2 M001 - resilience \n", - "3 M004 - friction Privacy standards \n", - ".. ... ... \n", - "135 M003 - daylight Steganography. Adding date, signatures etc to ... \n", - "136 M007 - metatechnique \n", - "137 M007 - metatechnique Include PACE plans - Primary, Alternate, Conti... \n", - "138 M007 - metatechnique \n", - "139 M001 - resilience \n", - "\n", - " playbooks \\\n", - "0 \n", - "8 \n", - "1 \n", - "2 \n", - "3 \n", - ".. ... \n", - "135 Add date and source to images \n", - "136 \n", - "137 \n", - "138 \n", - "139 \n", - "\n", - " actortypes resources_needed \\\n", - "0 A033 - social media platform owner \n", - "8 A031 - social media platform administrator \n", - "1 A007 - factchecker \n", - "2 A016 - influencer,A006 - educator \n", - "3 A020 - policy maker \n", - ".. ... ... \n", - "135 \n", - "136 \n", - "137 \n", - "138 \n", - "139 \n", - "\n", - " how_found references incident_ids \\\n", - "0 2019-11-workshop \n", - "8 grugq Taylor81 \n", - "1 2019-11-workshop\\n2019-11-search I00049,I00050 \n", - "2 2019-11-workshop \n", - "3 2019-11-workshop \n", - ".. ... ... ... \n", - "135 grugq \n", - "136 Counters cleanup \n", - "137 Counters cleanup \n", - "138 \n", - "139 \n", - "\n", - " tactic responsetype notes \\\n", - "0 TA01 Strategic Planning D2 Deny \n", - "8 TA01 Strategic Planning D2 Deny \n", - "1 TA01 Strategic Planning D4 Degrade \n", - "2 TA02 Objective Planning D2 Deny \n", - "3 TA01 Strategic Planning D2 Deny \n", - ".. ... ... ... \n", - "135 TA06 Develop Content D4 Degrade \n", - "136 TA01 Strategic Planning D3 Disrupt \n", - "137 TA01 Strategic Planning D3 Disrupt \n", - "138 TA02 Objective Planning D3 Disrupt \n", - "139 TA01 Strategic Planning D3 Disrupt \n", - "\n", - " techniques \\\n", - "0 \n", - "8 T0009 - Create fake experts\\nT0008 - Create fa... \n", - "1 TA01 - Strategic Planning\\nTA06 - Develop Cont... \n", - "2 TA08 - Pump Priming\\nT0010 - Cultivate ignoran... \n", - "3 TA05 - Microtargeting\\nT00018 - Paid targeted ads \n", - ".. ... \n", - "135 T0024 - Create fake videos and images\\nT0026 -... \n", - "136 \n", - "137 \n", - "138 \n", - "139 \n", - "\n", - " longname tactic_id \\\n", - "0 C00006 - Charge for social media TA01 \n", - "8 C00006 - Censorship TA01 \n", - "1 C00008 - Create shared fact-checking database TA01 \n", - "2 C00009 - Educate high profile influencers on b... TA02 \n", - "3 C00010 - Enhanced privacy regulation for socia... TA01 \n", - ".. ... ... \n", - "135 C00219 - Add metadata to content that’s out of... TA06 \n", - "136 C00220 - Develop a monitoring and intelligence... TA01 \n", - "137 C00221 - Run a disinformation red team, and de... TA01 \n", - "138 C00222 - Tabletop simulations TA02 \n", - "139 C00223 - Strengthen Trust in social media plat... TA01 \n", - "\n", - " tactic_name metatechnique_id metatechnique_name \n", - "0 Strategic Planning M004 - friction \n", - "8 Strategic Planning M005 - removal \n", - "1 Strategic Planning M006 - scoring \n", - "2 Objective Planning M001 - resilience \n", - "3 Strategic Planning M004 - friction \n", - ".. ... ... ... \n", - "135 Develop Content M003 - daylight \n", - "136 Strategic Planning M007 - metatechnique \n", - "137 Strategic Planning M007 - metatechnique \n", - "138 Objective Planning M007 - metatechnique \n", - "139 Strategic Planning M001 - resilience \n", - "\n", - "[140 rows x 19 columns]" + "[898 rows x 2 columns]" ] }, - "execution_count": 45, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "amitt.df_counters" + "amitt.cross_counterid_techniqueid" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 10, "metadata": { "scrolled": true }, @@ -587,7 +365,7 @@ " T0062\n", " TA12\n", " Behaviour changes\n", - " \n", + " Monitor and evaluate behaviour changes from mi...\n", " 62\n", " \n", " \n", @@ -595,7 +373,7 @@ " T0063\n", " TA12\n", " Message reach\n", - " \n", + " Monitor and evaluate message reach in misinfor...\n", " 63\n", " \n", " \n", @@ -603,7 +381,7 @@ " T0064\n", " TA12\n", " Social media engagement\n", - " \n", + " Monitor and evaluate social media engagement i...\n", " 64\n", " \n", " \n", @@ -634,21 +412,21 @@ ".. ... .. \n", "59 continue narrative or message amplification af... 60 \n", "60 Sell hats, t-shirts, flags and other branded c... 61 \n", - "61 62 \n", - "62 63 \n", - "63 64 \n", + "61 Monitor and evaluate behaviour changes from mi... 62 \n", + "62 Monitor and evaluate message reach in misinfor... 63 \n", + "63 Monitor and evaluate social media engagement i... 64 \n", "\n", "[64 rows x 5 columns]" ] }, - "execution_count": 44, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Generate minimal sqlite database from the Amitt variables\n", - "conn = sql.connect('test_amitt_sqlite.db')\n", + "conn = sql.connect('amittsite.sqlite')\n", "\n", "def add_table(dataframe, tablename, columns): \n", " # Create sql table\n", @@ -664,10 +442,12 @@ "\n", "#newtable = add_table(amitt.df_actortypes, 'actor_type', ['amitt_id', 'sector_id', 'framework_id', 'name', 'summary'])\n", "# counter\n", + "newtable = add_table(amitt.df_counters, 'counter', ['amitt_id', 'tactic_id', 'metatechnique_id', 'name', 'summary'])\n", "# dataset\n", - "# framework\n", + "newtable = add_table(amitt.df_frameworks, 'framework', ['amitt_id', 'name', 'summary'])\n", "# incident\n", "# metatechnique\n", + "newtable = add_table(amitt.df_metatechniques, 'metatechnique', ['amitt_id', 'name', 'summary'])\n", "newtable = add_table(amitt.df_phases, 'phase', ['amitt_id', 'name', 'rank', 'summary'])\n", "# playbook\n", "# reference\n", diff --git a/HTML_GENERATING_CODE/Generate_AMITT_github_files.ipynb b/HTML_GENERATING_CODE/Generate_AMITT_github_files.ipynb index 8de3ecc..d22e476 100644 --- a/HTML_GENERATING_CODE/Generate_AMITT_github_files.ipynb +++ b/HTML_GENERATING_CODE/Generate_AMITT_github_files.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": { "scrolled": true }, @@ -48,8 +48,10 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, + "execution_count": 10, + "metadata": { + "scrolled": true + }, "outputs": [ { "data": { @@ -437,7 +439,7 @@ "[140 rows x 18 columns]" ] }, - "execution_count": 8, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } diff --git a/HTML_GENERATING_CODE/__pycache__/generate_amitt_ttps.cpython-38.pyc b/HTML_GENERATING_CODE/__pycache__/generate_amitt_ttps.cpython-38.pyc index 69ec091a761036ac05a5ee9208659c3a9eaf0745..b9db96cc0dc1f48321a959f32fbd87b20cbd0786 100644 GIT binary patch delta 940 zcmaiyUr19?9LMkXZu8H*yPKPvIh~uE)9FkL3KNRVpr`&Bw*Jg;j*gnwwBa8NT1+T{ zs6likDx$V}F{03p4}uRp^iYPB{RiJP2qcK82cefb_przZqkG`;``qvMe9!m5eKd{t z(@?^SA_?qy`{Z56%Sk09kDw z9S56BxZ-3{x#=Qe&45U(@VcSm4AQQQpq3Dcl=gsCb*DsjXsKTCpU$#XPB(Kp)v|<; zR7+Wim3W9ZWlS3hGEhB?|3vDCbEE41pD*oYUK1~0&Mu>qlant?xt#o2k~t+SvS{ZR z$P&9NyH9173OH2_1@*{tx>C)Q8b&fJmD4S{1d>C_i7#VhT4Kcgkb&i}G=%tIwS)fjf zS{p18g>*&#Dr44XqP;`^G@7IB-l79SJw zkWQ3*2WIF!_ZeW8dOhrf51QBGq>sH1fKRmD7YDx5SN@43^51VygxwOtJ)yi z^|?rU9h`%g-mabmLRw2`9u3r28y3-`jnsX{VXeLXId;E>Go*l2jyl8J3=+}#9R&`$69f)sm~)AM^T17p;-|2}Ys z&a_O47^m^pW#9#kg!_SyboS63u%ul(vdmIPj~)QFXm28S@k<+Sn=)<6Gv(noyF3j% delta 905 zcmZ{hPe>F|9LL{xbj+P~c6D9<%&NPtyRNSGk0RNFWClf0X<;k3R&7u-j2h+?`8@0Df zwZzuFv*)D#iX(nOlh@s?mNh4II9$IRoi!8V*^O~8LmQkOXRjtaLX6B^+<;)$a zrlg4k%Nu6NksxL_baZuhww`XA&FDVUdcKYSQ7;1>tbHnBkv-cP!y>P#y9WH^Q@h-N!oSoH zry-li_9SF+EwaB3=x6s2e8+%TOZlUN0|4Wp#)rTNPaGPTFv@Q=EdXzLU$_hS#D6uv f1LoNCqYI*S^w?pbhAka`fPZZG#JIdIQ_jSHnsf?% diff --git a/HTML_GENERATING_CODE/generate_amitt_ttps.py b/HTML_GENERATING_CODE/generate_amitt_ttps.py index f366df3..51a9750 100644 --- a/HTML_GENERATING_CODE/generate_amitt_ttps.py +++ b/HTML_GENERATING_CODE/generate_amitt_ttps.py @@ -92,6 +92,7 @@ class Amitt: # Create individual tables and dictionaries self.df_phases = metadata['phases'] + self.df_frameworks = metadata['frameworks'] self.df_techniques = metadata['techniques'] self.df_tasks = metadata['tasks'] self.df_incidents = metadata['incidents']