{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Test area for AMITT code" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dict_keys(['df_phases', 'df_techniques', 'df_tasks', 'df_incidents', 'df_counters', 'df_detections', 'df_actortypes', 'df_resources', 'df_responsetypes', 'df_metatechniques', 'it', 'df_tactics', 'df_techniques_per_tactic', 'df_counters_per_tactic', 'phases', 'tactics', 'techniques', 'counters', 'metatechniques', 'actortypes', 'resources', 'num_tactics', 'cross_counterid_techniqueid', 'cross_counterid_resourceid', 'cross_counterid_actortypeid'])\n" ] }, { "data": { "text/plain": [ "{'TA01': 'Strategic Planning',\n", " 'TA02': 'Objective Planning',\n", " 'TA03': 'Develop People',\n", " 'TA04': 'Develop Networks',\n", " 'TA05': 'Microtargeting',\n", " 'TA06': 'Develop Content',\n", " 'TA07': 'Channel Selection',\n", " 'TA08': 'Pump Priming',\n", " 'TA09': 'Exposure',\n", " 'TA10': 'Go Physical',\n", " 'TA11': 'Persistence',\n", " 'TA12': 'Measure Effectiveness'}" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "import sqlite3 as sql\n", "from generate_amitt_ttps import Amitt\n", "\n", "\n", "# Generate AMITT datasets\n", "amitt = Amitt()\n", "\n", "# Check which amitt variables we can see from here\n", "print('{}'.format(vars(amitt).keys()))\n", "vars(amitt)['tactics']" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
amitt_idnamemetatechniquesummaryplaybooksactortypesresources_neededhow_foundreferencesincident_idstacticresponsetypenotestechniqueslongnametactic_idtactic_namemetatechnique_idmetatechnique_name
0C00006Charge for social mediaM004 - frictionInclude a paid-for privacy option, e.g. pay Fa...A033 - social media platform owner2019-11-workshopTA01 Strategic PlanningD2 DenyC00006 - Charge for social mediaTA01Strategic PlanningM004- friction
8C00006CensorshipM005 - removalAlter and/or block the publication/disseminati...A031 - social media platform administratorgrugqTaylor81TA01 Strategic PlanningD2 DenyT0009 - Create fake experts\\nT0008 - Create fa...C00006 - CensorshipTA01Strategic PlanningM005- removal
1C00008Create shared fact-checking databaseM006 - scoringSnopes is best-known exampleA007 - factchecker2019-11-workshop\\n2019-11-searchI00049,I00050TA01 Strategic PlanningD4 DegradeTA01 - Strategic Planning\\nTA06 - Develop Cont...C00008 - Create shared fact-checking databaseTA01Strategic PlanningM006- scoring
2C00009Educate high profile influencers on best pract...M001 - resilienceA016 - influencer,A006 - educator2019-11-workshopTA02 Objective PlanningD2 DenyTA08 - Pump Priming\\nT0010 - Cultivate ignoran...C00009 - Educate high profile influencers on b...TA02Objective PlanningM001- resilience
3C00010Enhanced privacy regulation for social mediaM004 - frictionPrivacy standardsA020 - policy maker2019-11-workshopTA01 Strategic PlanningD2 DenyTA05 - Microtargeting\\nT00018 - Paid targeted adsC00010 - Enhanced privacy regulation for socia...TA01Strategic PlanningM004- friction
............................................................
135C00219Add metadata to content that’s out of the cont...M003 - daylightSteganography. Adding date, signatures etc to ...Add date and source to imagesgrugqTA06 Develop ContentD4 DegradeT0024 - Create fake videos and images\\nT0026 -...C00219 - Add metadata to content that’s out of...TA06Develop ContentM003- daylight
136C00220Develop a monitoring and intelligence planM007 - metatechniqueCounters cleanupTA01 Strategic PlanningD3 DisruptC00220 - Develop a monitoring and intelligence...TA01Strategic PlanningM007- metatechnique
137C00221Run a disinformation red team, and design miti...M007 - metatechniqueInclude PACE plans - Primary, Alternate, Conti...Counters cleanupTA01 Strategic PlanningD3 DisruptC00221 - Run a disinformation red team, and de...TA01Strategic PlanningM007- metatechnique
138C00222Tabletop simulationsM007 - metatechniqueTA02 Objective PlanningD3 DisruptC00222 - Tabletop simulationsTA02Objective PlanningM007- metatechnique
139C00223Strengthen Trust in social media platformsM001 - resilienceTA01 Strategic PlanningD3 DisruptC00223 - Strengthen Trust in social media plat...TA01Strategic PlanningM001- resilience
\n", "

140 rows × 19 columns

\n", "
" ], "text/plain": [ " amitt_id name \\\n", "0 C00006 Charge for social media \n", "8 C00006 Censorship \n", "1 C00008 Create shared fact-checking database \n", "2 C00009 Educate high profile influencers on best pract... \n", "3 C00010 Enhanced privacy regulation for social media \n", ".. ... ... \n", "135 C00219 Add metadata to content that’s out of the cont... \n", "136 C00220 Develop a monitoring and intelligence plan \n", "137 C00221 Run a disinformation red team, and design miti... \n", "138 C00222 Tabletop simulations \n", "139 C00223 Strengthen Trust in social media platforms \n", "\n", " metatechnique summary \\\n", "0 M004 - friction Include a paid-for privacy option, e.g. pay Fa... \n", "8 M005 - removal Alter and/or block the publication/disseminati... \n", "1 M006 - scoring Snopes is best-known example \n", "2 M001 - resilience \n", "3 M004 - friction Privacy standards \n", ".. ... ... \n", "135 M003 - daylight Steganography. Adding date, signatures etc to ... \n", "136 M007 - metatechnique \n", "137 M007 - metatechnique Include PACE plans - Primary, Alternate, Conti... \n", "138 M007 - metatechnique \n", "139 M001 - resilience \n", "\n", " playbooks \\\n", "0 \n", "8 \n", "1 \n", "2 \n", "3 \n", ".. ... \n", "135 Add date and source to images \n", "136 \n", "137 \n", "138 \n", "139 \n", "\n", " actortypes resources_needed \\\n", "0 A033 - social media platform owner \n", "8 A031 - social media platform administrator \n", "1 A007 - factchecker \n", "2 A016 - influencer,A006 - educator \n", "3 A020 - policy maker \n", ".. ... ... \n", "135 \n", "136 \n", "137 \n", "138 \n", "139 \n", "\n", " how_found references incident_ids \\\n", "0 2019-11-workshop \n", "8 grugq Taylor81 \n", "1 2019-11-workshop\\n2019-11-search I00049,I00050 \n", "2 2019-11-workshop \n", "3 2019-11-workshop \n", ".. ... ... ... \n", "135 grugq \n", "136 Counters cleanup \n", "137 Counters cleanup \n", "138 \n", "139 \n", "\n", " tactic responsetype notes \\\n", "0 TA01 Strategic Planning D2 Deny \n", "8 TA01 Strategic Planning D2 Deny \n", "1 TA01 Strategic Planning D4 Degrade \n", "2 TA02 Objective Planning D2 Deny \n", "3 TA01 Strategic Planning D2 Deny \n", ".. ... ... ... \n", "135 TA06 Develop Content D4 Degrade \n", "136 TA01 Strategic Planning D3 Disrupt \n", "137 TA01 Strategic Planning D3 Disrupt \n", "138 TA02 Objective Planning D3 Disrupt \n", "139 TA01 Strategic Planning D3 Disrupt \n", "\n", " techniques \\\n", "0 \n", "8 T0009 - Create fake experts\\nT0008 - Create fa... \n", "1 TA01 - Strategic Planning\\nTA06 - Develop Cont... \n", "2 TA08 - Pump Priming\\nT0010 - Cultivate ignoran... \n", "3 TA05 - Microtargeting\\nT00018 - Paid targeted ads \n", ".. ... \n", "135 T0024 - Create fake videos and images\\nT0026 -... \n", "136 \n", "137 \n", "138 \n", "139 \n", "\n", " longname tactic_id \\\n", "0 C00006 - Charge for social media TA01 \n", "8 C00006 - Censorship TA01 \n", "1 C00008 - Create shared fact-checking database TA01 \n", "2 C00009 - Educate high profile influencers on b... TA02 \n", "3 C00010 - Enhanced privacy regulation for socia... TA01 \n", ".. ... ... \n", "135 C00219 - Add metadata to content that’s out of... TA06 \n", "136 C00220 - Develop a monitoring and intelligence... TA01 \n", "137 C00221 - Run a disinformation red team, and de... TA01 \n", "138 C00222 - Tabletop simulations TA02 \n", "139 C00223 - Strengthen Trust in social media plat... TA01 \n", "\n", " tactic_name metatechnique_id metatechnique_name \n", "0 Strategic Planning M004 - friction \n", "8 Strategic Planning M005 - removal \n", "1 Strategic Planning M006 - scoring \n", "2 Objective Planning M001 - resilience \n", "3 Strategic Planning M004 - friction \n", ".. ... ... ... \n", "135 Develop Content M003 - daylight \n", "136 Strategic Planning M007 - metatechnique \n", "137 Strategic Planning M007 - metatechnique \n", "138 Objective Planning M007 - metatechnique \n", "139 Strategic Planning M001 - resilience \n", "\n", "[140 rows x 19 columns]" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ "amitt.df_counters" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
amitt_idtactic_idnamesummaryid
0T0001TA015Ds (dismiss, distort, distract, dismay, divide)Nimmo's \"4Ds of propaganda\": dismiss, distort,...1
1T0002TA01Facilitate State PropagandaOrganize citizens around pro-state messaging. ...2
2T0003TA01Leverage Existing NarrativesUse or adapt existing narrative themes, where ...3
3T0004TA01Competing NarrativesAdvance competing narratives connected to same...4
4T0005TA02Center of Gravity AnalysisRecon/research to identify \"the source of powe...5
..................
59T0060TA11Continue to amplifycontinue narrative or message amplification af...60
60T0061TA10Sell merchandisingSell hats, t-shirts, flags and other branded c...61
61T0062TA12Behaviour changes62
62T0063TA12Message reach63
63T0064TA12Social media engagement64
\n", "

64 rows × 5 columns

\n", "
" ], "text/plain": [ " amitt_id tactic_id name \\\n", "0 T0001 TA01 5Ds (dismiss, distort, distract, dismay, divide) \n", "1 T0002 TA01 Facilitate State Propaganda \n", "2 T0003 TA01 Leverage Existing Narratives \n", "3 T0004 TA01 Competing Narratives \n", "4 T0005 TA02 Center of Gravity Analysis \n", ".. ... ... ... \n", "59 T0060 TA11 Continue to amplify \n", "60 T0061 TA10 Sell merchandising \n", "61 T0062 TA12 Behaviour changes \n", "62 T0063 TA12 Message reach \n", "63 T0064 TA12 Social media engagement \n", "\n", " summary id \n", "0 Nimmo's \"4Ds of propaganda\": dismiss, distort,... 1 \n", "1 Organize citizens around pro-state messaging. ... 2 \n", "2 Use or adapt existing narrative themes, where ... 3 \n", "3 Advance competing narratives connected to same... 4 \n", "4 Recon/research to identify \"the source of powe... 5 \n", ".. ... .. \n", "59 continue narrative or message amplification af... 60 \n", "60 Sell hats, t-shirts, flags and other branded c... 61 \n", "61 62 \n", "62 63 \n", "63 64 \n", "\n", "[64 rows x 5 columns]" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Generate minimal sqlite database from the Amitt variables\n", "conn = sql.connect('test_amitt_sqlite.db')\n", "\n", "def add_table(dataframe, tablename, columns): \n", " # Create sql table\n", " colnames = ', '.join(['{} TEXT NOT NULL'.format(col) for col in columns])\n", " conn.execute(\"DROP TABLE IF EXISTS {}\".format(tablename))\n", " conn.execute('''CREATE TABLE {} (id INTEGER PRIMARY KEY AUTOINCREMENT, {});'''.format(tablename, colnames))\n", " #populate table from dataframe\n", " newtable = dataframe[columns].copy().applymap(str)\n", " newtable['id'] = range(1,len(newtable)+1)\n", " newtable.to_sql(tablename, conn, index=False, if_exists='append')\n", " conn.commit()\n", " return newtable\n", "\n", "#newtable = add_table(amitt.df_actortypes, 'actor_type', ['amitt_id', 'sector_id', 'framework_id', 'name', 'summary'])\n", "# counter\n", "# dataset\n", "# framework\n", "# incident\n", "# metatechnique\n", "newtable = add_table(amitt.df_phases, 'phase', ['amitt_id', 'name', 'rank', 'summary'])\n", "# playbook\n", "# reference\n", "# response_type\n", "# sector\n", "newtable = add_table(amitt.df_tactics, 'tactic', ['amitt_id', 'phase_id', 'name', 'rank', 'summary'])\n", "newtable = add_table(amitt.df_tasks, 'task', ['amitt_id', 'tactic_id', 'framework_id', 'name', 'summary'])\n", "newtable = add_table(amitt.df_techniques, 'technique', ['amitt_id', 'tactic_id', 'name', 'summary'])\n", "# techniques_counters\n", "\n", "conn.execute(\"DROP TABLE IF EXISTS {}\".format('user'))\n", "conn.execute('''CREATE TABLE user (id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL UNIQUE, password TEXT NOT NULL);''')\n", "\n", "conn.close()\n", "newtable" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "df_phases loaded\n", "df_techniques loaded\n", "df_tasks loaded\n", "df_incidents loaded\n", "df_counters loaded\n", "df_detections loaded\n", "df_actors loaded\n", "df_resources loaded\n", "df_responsetypes loaded\n", "df_metatechniques loaded\n", "it loaded\n", "df_tactics loaded\n", "df_techniques_per_tactic loaded\n", "df_counters_per_tactic loaded\n", "phases not loaded\n", "tactics not loaded\n", "techniques not loaded\n", "counters not loaded\n", "metatechniques not loaded\n", "actors not loaded\n", "resources not loaded\n", "num_tactics not loaded\n", "cross_counterid_techniqueid loaded\n", "cross_counterid_resourceid loaded\n", "cross_counterid_actorid loaded\n" ] } ], "source": [ "# Generate full sqlite database from the Amitt variables\n", "conn = sql.connect('amitt_sqlite.db')\n", "for tablename, table in vars(amitt).items():\n", " if type(table) == pd.core.frame.DataFrame:\n", " table.applymap(str).to_sql(tablename, conn)\n", " print('{} loaded'.format(tablename))\n", " else:\n", " print('{} not loaded'.format(tablename))" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idactor_id
0C00006A033
1C00008A007
2C00009A016
2C00009A006
3C00010A020
.........
135C00219
136C00220
137C00221
138C00222
139C00223
\n", "

166 rows × 2 columns

\n", "
" ], "text/plain": [ " id actor_id\n", "0 C00006 A033\n", "1 C00008 A007\n", "2 C00009 A016\n", "2 C00009 A006\n", "3 C00010 A020\n", ".. ... ...\n", "135 C00219 \n", "136 C00220 \n", "137 C00221 \n", "138 C00222 \n", "139 C00223 \n", "\n", "[166 rows x 2 columns]" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "amitt.cross_counterid_actorid" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idtechnique_idWeight
1C00008TA011
1C00008TA061
1C00008TA081
1C00008T00061
1C00008T00091
............
134C00216T00181
134C00216T00571
135C00219T00241
135C00219T00261
135C00219T00251
\n", "

717 rows × 3 columns

\n", "
" ], "text/plain": [ " id technique_id Weight\n", "1 C00008 TA01 1\n", "1 C00008 TA06 1\n", "1 C00008 TA08 1\n", "1 C00008 T0006 1\n", "1 C00008 T0009 1\n", ".. ... ... ...\n", "134 C00216 T0018 1\n", "134 C00216 T0057 1\n", "135 C00219 T0024 1\n", "135 C00219 T0026 1\n", "135 C00219 T0025 1\n", "\n", "[717 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ct = amitt.cross_counterid_techniqueid\n", "ct['Weight'] = 1\n", "ct = ct[ct['technique_id'].str.len() > 0]\n", "ct.to_csv('../visualisations/cross_counterid_techniqueid.csv', index=False, header=['Source','Target', 'Weight'])\n", "ct" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# fix the problem with excelfile changes" ] }, { "cell_type": "code", "execution_count": 141, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ ":9: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n", " pd.set_option('display.max_colwidth', -1)\n" ] } ], "source": [ "import pandas as pd\n", "import sqlite3 as sql\n", "from generate_amitt_ttps import Amitt\n", "import pandas as pd\n", "import numpy as np\n", "import os\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "pd.set_option('display.max_rows', 1000)\n", "pd.set_option('display.max_colwidth', -1)\n", "\n", "newfile = '../AMITT_MASTER_DATA/AMITT_TTPs_MASTER.xlsx'\n", "oldfile = 'AMITT_TTPs_MASTER_github_version.xlsx'\n", "\n", "# Load dfs from file\n", "newdfs = {}\n", "newxlsx = pd.ExcelFile(newfile)\n", "for sheetname in newxlsx.sheet_names:\n", " newdfs[sheetname] = newxlsx.parse(sheetname)\n", " newdfs[sheetname].fillna('', inplace=True)\n", "\n", "olddfs = {}\n", "oldxlsx = pd.ExcelFile(oldfile)\n", "for sheetname in oldxlsx.sheet_names:\n", " olddfs[sheetname] = oldxlsx.parse(sheetname)\n", " olddfs[sheetname].fillna('', inplace=True)\n", "\n", "addedtables = newdfs.keys() - olddfs.keys()\n", "losttables = olddfs.keys() - newdfs.keys()\n", "if len(addedtables) + len(losttables) > 0:\n", " print('Table changes: new tables are {}, lost tables are {}'.format(addedtables, losttables))\n", "\n", "def investigate_table(table):\n", " print('\\n\\nTable {} is changed'.format(table))\n", " # Column headings\n", " coldiffs = set(newdfs[table].columns).symmetric_difference(set(olddfs[table].columns))\n", " if len(coldiffs) > 0:\n", " print('column differences: {}'.format(coldiffs))\n", " # length\n", " if len(newdfs[table]) != len(olddfs[table]):\n", " print('length differences: new {} old {}'.format(len(newdfs[table]), len(olddfs[table])))\n", "\n", " # column by column\n", " for column in newdfs[table].columns:\n", " coldiffs = newdfs[table][column] != olddfs[table][column]\n", " if len(newdfs[table][coldiffs]) > 0:\n", " print('Differences in column {}'.format(column))\n", " return\n", "\n", "for table in newdfs.keys():\n", " if newdfs[table].equals(olddfs[table]) == False:\n", " investigate_table(table) " ] }, { "cell_type": "code", "execution_count": 142, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
amitt_idnewold
\n", "
" ], "text/plain": [ "Empty DataFrame\n", "Columns: [amitt_id, new, old]\n", "Index: []" ] }, "execution_count": 142, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Look at individual table differences\n", "table = 'countermeasures'\n", "column = 'summary'\n", "coldiffs = newdfs[table][column] != olddfs[table][column]\n", "diffcols = pd.DataFrame()\n", "diffcols['amitt_id'] = newdfs[table][coldiffs]['amitt_id']\n", "diffcols['new'] = newdfs[table][coldiffs][column]\n", "diffcols['old'] = olddfs[table][coldiffs][column]\n", "diffcols[diffcols['old'] != '']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.3" } }, "nbformat": 4, "nbformat_minor": 4 }