explore_text_attack.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example 1: provide data and use existing model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n",
      "\u001b[34;1mtextattack\u001b[0m: Loading pre-trained model from HuggingFace model repository: \u001b[94mtextattack/bert-base-uncased-rotten-tomatoes\u001b[0m\n",
      "Downloading: 100%|██████████████████████████████| 487/487 [00:00<00:00, 202kB/s]\n",
      "Using /tmp/tfhub_modules to cache modules.\n",
      "Downloading TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder/4'.\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 40.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 70.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 110.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 150.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 190.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 240.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 270.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 310.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 360.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 390.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 430.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 460.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 500.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 530.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 570.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 590.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 630.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 640.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 670.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 720.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 770.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 800.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 860.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 890.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 930.00MB\n",
      "Downloading https://tfhub.dev/google/universal-sentence-encoder/4: 970.00MB\n",
      "Downloaded https://tfhub.dev/google/universal-sentence-encoder/4, Total size: 987.47MB\n",
      "Downloaded TF-Hub Module 'https://tfhub.dev/google/universal-sentence-encoder/4'.\n",
      "\u001b[34;1mtextattack\u001b[0m: Unknown if model of class <class 'textattack.models.wrappers.huggingface_model_wrapper.HuggingFaceModelWrapper'> compatible with goal function <class 'textattack.goal_functions.classification.untargeted_classification.UntargetedClassification'>.\n",
      "Attack(\n",
      "  (search_method): GreedyWordSwapWIR(\n",
      "    (wir_method):  delete\n",
      "  )\n",
      "  (goal_function):  UntargetedClassification\n",
      "  (transformation):  WordSwapEmbedding(\n",
      "    (max_candidates):  50\n",
      "    (embedding_type):  paragramcf\n",
      "  )\n",
      "  (constraints): \n",
      "    (0): WordEmbeddingDistance(\n",
      "        (embedding_type):  paragramcf\n",
      "        (min_cos_sim):  0.5\n",
      "        (cased):  False\n",
      "        (include_unknown_words):  True\n",
      "        (compare_against_original):  True\n",
      "      )\n",
      "    (1): PartOfSpeech(\n",
      "        (tagger_type):  nltk\n",
      "        (tagset):  universal\n",
      "        (allow_verb_noun_swap):  True\n",
      "        (compare_against_original):  True\n",
      "      )\n",
      "    (2): UniversalSentenceEncoder(\n",
      "        (metric):  angular\n",
      "        (threshold):  0.840845057\n",
      "        (window_size):  15\n",
      "        (skip_text_shorter_than_window):  True\n",
      "        (compare_against_original):  False\n",
      "      )\n",
      "    (3): RepeatModification\n",
      "    (4): StopwordModification\n",
      "    (5): InputColumnModification(\n",
      "        (matching_column_labels):  ['premise', 'hypothesis']\n",
      "        (columns_to_ignore):  {'premise'}\n",
      "      )\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n",
      "\u001b[34;1mtextattack\u001b[0m: Load time: 511.0323598384857s\n",
      "\u001b[34;1mtextattack\u001b[0m: Loading model and tokenizer from file: None\n",
      "\u001b[34;1mtextattack\u001b[0m: Loading module from `\u001b[94mtest_data.py\u001b[0m`.\n",
      "  0%|                                                     | 0/5 [00:00<?, ?it/s]--------------------------------------------- Result 1 ---------------------------------------------\n",
      "\u001b[91m0 (82%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n",
      "\n",
      "Wall St. Bears Claw Back Into the Black ( Reuters ) . Reuters - Short - sellers , Wall Street 's dwindling band of ultra - cynics , are seeing green again .\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 0 / 0 / 1:  20%|█    | 1/5 [00:00<00:01,  3.28it/s]--------------------------------------------- Result 2 ---------------------------------------------\n",
      "\u001b[91m0 (87%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n",
      "\n",
      "Carlyle Looks Toward Commercial Aerospace ( Reuters ) . Reuters - Private investment firm Carlyle Group , which has a reputation for making well - timed and occasionally controversial plays in the defense industry , has quietly placed its bets on another part of the market .\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 0 / 0 / 2:  40%|██   | 2/5 [00:00<00:00,  6.14it/s]/home/qc/anaconda3/lib/python3.7/site-packages/textattack/constraints/semantics/sentence_encoders/sentence_encoder.py:149: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  embeddings[len(transformed_texts) :]\n",
      "--------------------------------------------- Result 3 ---------------------------------------------\n",
      "\u001b[91m0 (99%)\u001b[0m --> \u001b[92m1 (99%)\u001b[0m\n",
      "\n",
      "Oil and Economy \u001b[91mCloud\u001b[0m Stocks ' Outlook ( Reuters ) . Reuters - Soaring crude prices plus \u001b[91mworries\u001b[0m about the economy and the outlook for earnings are expected to \u001b[91mhang\u001b[0m over the stock market next week during the depth of the \u001b[91msummer\u001b[0m \u001b[91mdoldrums\u001b[0m .\n",
      "\n",
      "Oil and Economy \u001b[92mClouds\u001b[0m Stocks ' Outlook ( Reuters ) . Reuters - Soaring crude prices plus \u001b[92mirritates\u001b[0m about the economy and the outlook for earnings are expected to \u001b[92mheng\u001b[0m over the stock market next week during the depth of the \u001b[92mhsia\u001b[0m \u001b[92madversity\u001b[0m .\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 1 / 0 / 3:  60%|███  | 3/5 [00:02<00:01,  1.18it/s]--------------------------------------------- Result 4 ---------------------------------------------\n",
      "\u001b[91m0 (85%)\u001b[0m --> \u001b[92m1 (51%)\u001b[0m\n",
      "\n",
      "Iraq \u001b[91mHalts\u001b[0m Oil \u001b[91mExports\u001b[0m from Main \u001b[91mSouthern\u001b[0m Pipeline ( Reuters ) . Reuters - Authorities have halted oil export flows from the main pipeline in southern Iraq after intelligence \u001b[91mshowed\u001b[0m a rebel militia could strike infrastructure , an oil official said on Saturday .\n",
      "\n",
      "Iraq \u001b[92mArrested\u001b[0m Oil \u001b[92mExits\u001b[0m from Main \u001b[92mEasterly\u001b[0m Pipeline ( Reuters ) . Reuters - Authorities have halted oil export flows from the main pipeline in southern Iraq after intelligence \u001b[92mdiscovered\u001b[0m a rebel militia could strike infrastructure , an oil official said on Saturday .\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 2 / 0 / 4:  80%|████ | 4/5 [00:04<00:01,  1.09s/it]--------------------------------------------- Result 5 ---------------------------------------------\n",
      "\u001b[91m0 (74%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n",
      "\n",
      "Oil prices soar to all - time record , posing new menace to US economy ( AFP ) . AFP - Tearaway world oil prices , toppling records and straining wallets , present a new economic menace barely three months before the US presidential elections .\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 2 / 0 / 5: 100%|█████| 5/5 [00:04<00:00,  1.14it/s]\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 2      |\n",
      "| Number of failed attacks:     | 0      |\n",
      "| Number of skipped attacks:    | 3      |\n",
      "| Original accuracy:            | 40.0%  |\n",
      "| Accuracy under attack:        | 0.0%   |\n",
      "| Attack success rate:          | 100.0% |\n",
      "| Average perturbed word %:     | 11.84% |\n",
      "| Average num. words per input: | 35.2   |\n",
      "| Avg num queries:              | 154.5  |\n",
      "+-------------------------------+--------+\n",
      "\u001b[34;1mtextattack\u001b[0m: Attack time: 4.387859106063843s\n"
     ]
    }
   ],
   "source": [
    "!textattack attack --recipe textfooler --dataset-from-file \"test_data.py\" --model bert-base-uncased-mr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example2:provide model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34;1mtextattack\u001b[0m: Loading model and tokenizer from file: \u001b[94mmodel1.py\u001b[0m\n",
      "\u001b[34;1mtextattack\u001b[0m: Loading module from `\u001b[94mmodel1.py\u001b[0m`.\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/qc/anaconda3/bin/textattack\", line 8, in <module>\n",
      "    sys.exit(main())\n",
      "  File \"/home/qc/anaconda3/lib/python3.7/site-packages/textattack/commands/textattack_cli.py\", line 42, in main\n",
      "    args.func.run(args)\n",
      "  File \"/home/qc/anaconda3/lib/python3.7/site-packages/textattack/commands/attack/attack_command.py\", line 42, in run\n",
      "    run_single_threaded(args)\n",
      "  File \"/home/qc/anaconda3/lib/python3.7/site-packages/textattack/commands/attack/run_attack_single_threaded.py\", line 47, in run\n",
      "    attack = parse_attack_from_args(args)\n",
      "  File \"/home/qc/anaconda3/lib/python3.7/site-packages/textattack/commands/attack/attack_args_helpers.py\", line 190, in parse_attack_from_args\n",
      "    model = parse_model_from_args(args)\n",
      "  File \"/home/qc/anaconda3/lib/python3.7/site-packages/textattack/commands/attack/attack_args_helpers.py\", line 272, in parse_model_from_args\n",
      "    model = model.to(textattack.shared.utils.device)\n",
      "AttributeError: 'LogisticRegression' object has no attribute 'to'\n"
     ]
    }
   ],
   "source": [
    "!textattack attack --model-from-file model1.py --recipe textfooler --num-examples 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## this is torch syntax, wrapper needed!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example 3: sklearn model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n",
      "\u001b[34;1mtextattack\u001b[0m: Loading model and tokenizer from file: \u001b[94mmodel2.py\u001b[0m\n",
      "\u001b[34;1mtextattack\u001b[0m: Loading module from `\u001b[94mmodel2.py\u001b[0m`.\n",
      "\u001b[34;1mtextattack\u001b[0m: Unknown if model of class <class 'temp_1596169849.9316492.SklearnModelWrapper'> compatible with goal function <class 'textattack.goal_functions.classification.untargeted_classification.UntargetedClassification'>.\n",
      "Attack(\n",
      "  (search_method): GreedyWordSwapWIR(\n",
      "    (wir_method):  unk\n",
      "  )\n",
      "  (goal_function):  UntargetedClassification\n",
      "  (transformation):  CompositeTransformation(\n",
      "    (0): WordSwapNeighboringCharacterSwap(\n",
      "        (random_one):  True\n",
      "      )\n",
      "    (1): WordSwapRandomCharacterSubstitution(\n",
      "        (random_one):  True\n",
      "      )\n",
      "    (2): WordSwapRandomCharacterDeletion(\n",
      "        (random_one):  True\n",
      "      )\n",
      "    (3): WordSwapRandomCharacterInsertion(\n",
      "        (random_one):  True\n",
      "      )\n",
      "    )\n",
      "  (constraints): \n",
      "    (0): LevenshteinEditDistance(\n",
      "        (max_edit_distance):  30\n",
      "        (compare_against_original):  True\n",
      "      )\n",
      "    (1): RepeatModification\n",
      "    (2): StopwordModification\n",
      "  (is_black_box):  True\n",
      ") \n",
      "\n",
      "\u001b[34;1mtextattack\u001b[0m: Load time: 0.09342098236083984s\n",
      "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mnlp\u001b[0m dataset \u001b[94mimdb\u001b[0m, split \u001b[94mtrain\u001b[0m.\n",
      "  0%|                                                    | 0/10 [00:00<?, ?it/s]--------------------------------------------- Result 1 ---------------------------------------------\n",
      "\u001b[91mNegative (81%)\u001b[0m --> \u001b[92mPositive (51%)\u001b[0m\n",
      "\n",
      "I was very \u001b[91mdisappointed\u001b[0m in this movie. Plotwise it was \u001b[91mweak\u001b[0m bordering on \u001b[91msilly\u001b[0m: Souls who can affect reality in the way they do? A mission \u001b[91mapparently\u001b[0m critical to the Soul Hunters entrusted to one of their younger members? And the whole B-story with the \"holobrothel\" and the lawsuit against the station was so \u001b[91mawful\u001b[0m that at one point I blurted out to the television, \"\u001b[91mWhy\u001b[0m are you \u001b[91mwasting\u001b[0m my time with this?\"<br /><br />Thematically, \"River of Souls\" didn't really go into the question of the soul in any more depth than the \u001b[91moriginal\u001b[0m episode \"Soul Hunter\" did. We see that Soul Hunters can make mistakes, but we still don't get a feeling for their culture. (Are there any female Soul Hunters?)<br /><br />The \u001b[91macting\u001b[0m was \u001b[91mokay\u001b[0m, given the \u001b[91mmaterial\u001b[0m they had to work with, and the special effects - especially the planetscapes in the first act - were very impressive. But overall, I'd say give this one a miss.\n",
      "\n",
      "I was very \u001b[92mdiasppointed\u001b[0m in this movie. Plotwise it was \u001b[92meak\u001b[0m bordering on \u001b[92milly\u001b[0m: Souls who can affect reality in the way they do? A mission \u001b[92mapparentMly\u001b[0m critical to the Soul Hunters entrusted to one of their younger members? And the whole B-story with the \"holobrothel\" and the lawsuit against the station was so \u001b[92mafwul\u001b[0m that at one point I blurted out to the television, \"\u001b[92mWhP\u001b[0m are you \u001b[92mGwasting\u001b[0m my time with this?\"<br /><br />Thematically, \"River of Souls\" didn't really go into the question of the soul in any more depth than the \u001b[92morgiinal\u001b[0m episode \"Soul Hunter\" did. We see that Soul Hunters can make mistakes, but we still don't get a feeling for their culture. (Are there any female Soul Hunters?)<br /><br />The \u001b[92maccting\u001b[0m was \u001b[92meokay\u001b[0m, given the \u001b[92mVaterial\u001b[0m they had to work with, and the special effects - especially the planetscapes in the first act - were very impressive. But overall, I'd say give this one a miss.\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 1 / 0 / 1:  10%|▎  | 1/10 [03:59<35:58, 239.82s/it]--------------------------------------------- Result 2 ---------------------------------------------\n",
      "\u001b[92mPositive (77%)\u001b[0m --> \u001b[91mNegative (51%)\u001b[0m\n",
      "\n",
      "Very few movies have had the impact on American culture the way Urban Cowboy has. Thank god it was temporary. But UC is almost in a class by itself as one of those flicks that when you're flipping channels at 3:00AM you just can't take your eyes off....my top three are Animal House and Walking Tall BTW but that's beside the point. I \u001b[92mremember\u001b[0m Urban Cowboy hit the theaters and overnite there were honkytonks being opened on every corner, men were sporting cowboy hats with their penny loafers, and if you didnt know how to two-step you were considered a social moron! Personally I think it's a \u001b[92mgreat\u001b[0m movie. Travolta really \u001b[92msurprised\u001b[0m me on the heels of SatNiteFever. Who'd a thunk. He's actually \u001b[92mbelievable\u001b[0m too. The \u001b[92msoundtrack\u001b[0m is \u001b[92mawesome\u001b[0m. Too bad Charlene Tilton, of TV's Dallas fame ruined Johnny Lee's career cause that guy was just \u001b[92mterrific\u001b[0m. The show stealer here is Scott Glenn as the greaser ex-con redneck cowboy. And I ain't got nothing against greaser ex-con cowboys semi-being one myself, but I've \u001b[92malways\u001b[0m envied what I feel to be the \u001b[92mgreatest\u001b[0m power line of all time....\"Pack-at S*#t!\" Sort of like Clint's \"make my day.\" And watching him slap Sissy around is the closest thing I'll see to my Julia Roberts fantasy so..... Like I said, \u001b[92mbeautiful\u001b[0m. 9/10\n",
      "\n",
      "Very few movies have had the impact on American culture the way Urban Cowboy has. Thank god it was temporary. But UC is almost in a class by itself as one of those flicks that when you're flipping channels at 3:00AM you just can't take your eyes off....my top three are Animal House and Walking Tall BTW but that's beside the point. I \u001b[91memember\u001b[0m Urban Cowboy hit the theaters and overnite there were honkytonks being opened on every corner, men were sporting cowboy hats with their penny loafers, and if you didnt know how to two-step you were considered a social moron! Personally I think it's a \u001b[91mZreat\u001b[0m movie. Travolta really \u001b[91msurprisWed\u001b[0m me on the heels of SatNiteFever. Who'd a thunk. He's actually \u001b[91mbeZievable\u001b[0m too. The \u001b[91msouCdtrack\u001b[0m is \u001b[91maewsome\u001b[0m. Too bad Charlene Tilton, of TV's Dallas fame ruined Johnny Lee's career cause that guy was just \u001b[91mtMrrific\u001b[0m. The show stealer here is Scott Glenn as the greaser ex-con redneck cowboy. And I ain't got nothing against greaser ex-con cowboys semi-being one myself, but I've \u001b[91malwaWys\u001b[0m envied what I feel to be the \u001b[91mgAreatest\u001b[0m power line of all time....\"Pack-at S*#t!\" Sort of like Clint's \"make my day.\" And watching him slap Sissy around is the closest thing I'll see to my Julia Roberts fantasy so..... Like I said, \u001b[91mbAeautiful\u001b[0m. 9/10\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 2 / 0 / 2:  20%|▌  | 2/10 [08:48<35:15, 264.49s/it]--------------------------------------------- Result 3 ---------------------------------------------\n",
      "\u001b[91mNegative (55%)\u001b[0m --> \u001b[37m[SKIPPED]\u001b[0m\n",
      "\n",
      "Okay. Yes, this was a very-tight-budget movie with continuity errors (like single scenes obviously filmed in sunshine and then in shadow and then mixed together), and as much as I love Nick Mancuso he was often a little too good at the burnt-out part, and some of the minor supporting cast was really bad (plus at least one actor was used for two different but conspicuous roles). But come on. Richard Grieco was hysterical (his hair alone is worth the trip). Steven Ford was very likable. Mancuso had some great lines, while Nancy Allen, ironically, was completely bland and uninteresting. Classic? No. Bad parts? Yes. Entertaining? Big yes. I would have loved to have been on-set the day they decided what kind of hairstyle Grieco would have. \"Are you fast?\" ... \"Y'ain't THAT fast.\"\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 2 / 0 / 3:  30%|▉  | 3/10 [08:54<20:46, 178.03s/it]--------------------------------------------- Result 4 ---------------------------------------------\n",
      "\u001b[92mPositive (79%)\u001b[0m --> \u001b[91mNegative (50%)\u001b[0m\n",
      "\n",
      "Although the word megalmania is used a \u001b[92mlot\u001b[0m to describe Gene Kelly, and \u001b[92msometimes\u001b[0m his dancing is way too stiff, you have to admit the guy knows how to put on a show. In American In Paris, he choreographs some \u001b[92moutstanding\u001b[0m numbers, some which stall the plot, but are nonetheless \u001b[92mamazing\u001b[0m to look at. (Check out Gene Kelly's \"Getting Out Of Bed Routine\" for starters)<br /><br />Gene Kelly stars as a GI who is based out of Paris, he stayed there to paint, soon he is a rich woman's gigolo, but he really LOVES SOMEONE ELSE! Hoary story sure, but the musical numbers save the show here! I really \u001b[92mloved\u001b[0m Georges Gu¨¦tary's voice work in this one. His 'Stairway to Paradise' and his duet with Le Gene on 'S \u001b[92mWonderful'\u001b[0m is 's \u001b[92mmarvelous'\u001b[0m. \u001b[92mOscar\u001b[0m Levant and Leslie Caron I can take or leave. All in all, a pretty good, but not dynamite movie.\n",
      "\n",
      "Although the word megalmania is used a \u001b[91mlo\u001b[0m to describe Gene Kelly, and \u001b[91mometimes\u001b[0m his dancing is way too stiff, you have to admit the guy knows how to put on a show. In American In Paris, he choreographs some \u001b[91moutstTanding\u001b[0m numbers, some which stall the plot, but are nonetheless \u001b[91mPamazing\u001b[0m to look at. (Check out Gene Kelly's \"Getting Out Of Bed Routine\" for starters)<br /><br />Gene Kelly stars as a GI who is based out of Paris, he stayed there to paint, soon he is a rich woman's gigolo, but he really LOVES SOMEONE ELSE! Hoary story sure, but the musical numbers save the show here! I really \u001b[91mloed\u001b[0m Georges Gu¨¦tary's voice work in this one. His 'Stairway to Paradise' and his duet with Le Gene on 'S \u001b[91mWondOerful'\u001b[0m is 's \u001b[91mmaYvelous'\u001b[0m. \u001b[91mOsacr\u001b[0m Levant and Leslie Caron I can take or leave. All in all, a pretty good, but not dynamite movie.\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 3 / 0 / 4:  40%|█▏ | 4/10 [12:25<18:38, 186.43s/it]--------------------------------------------- Result 5 ---------------------------------------------\n",
      "\u001b[92mPositive (79%)\u001b[0m --> \u001b[91mNegative (50%)\u001b[0m\n",
      "\n",
      "James J. Corbett, heavyweight champion of the \u001b[92mworld\u001b[0m from 1892 to 1897, turned out to be Errol \u001b[92mFlynn's\u001b[0m \u001b[92mfavorite\u001b[0m role. Possibly because he didn't have to wield a sword or be in a western. He grew tired of the swashbucklers and he said in his autobiography that he felt he was miscast in westerns and couldn't understand why people \u001b[92mliked\u001b[0m him in them. It is an \u001b[92menjoyable\u001b[0m film, but hardly does tell the real story of James J. Corbett.<br /><br />As portrayed Corbett was the first scientific boxer to win in the heavyweight division, a man who used brains and speed more than brute strength to win. He defeated John L. Sullivan and lost to Bob Fitzsimmons the heavyweight crown. He was \u001b[92malso\u001b[0m a compulsive womanizer and in that was a \u001b[92mlot\u001b[0m like the man \u001b[92mportraying\u001b[0m him. Of course that was not shown on the screen. The character that Alexis Smith plays, the banker's daughter who falls for him, has no basis in reality. Corbett was in fact married twice and was flagrantly unfaithful with both of his wives.<br /><br />\u001b[92mAlso\u001b[0m \u001b[92mthough\u001b[0m after he lost his title and after the events of this film are concluded he suffered a \u001b[92mgreat\u001b[0m personal tragedy. His father in a moment of depression, probably over finances because he lost heavily betting on his son to beat Bob Fitzsimmons, shot his mother and then turned the gun on himself. The murder/suicide hardly squares with the happy clan that Alan Hale presided over.<br /><br />One thing though that I did like about Gentleman Jim. Ward Bond got the career role of his \u001b[92mlife\u001b[0m in playing John L. Sullivan. Director Raoul Walsh got a \u001b[92mgreat\u001b[0m \u001b[92mperformance\u001b[0m out of Bond as the blustering, but lovable Sullivan. Even given some of Sullivan's bad points that don't make it to the screen like rabid racism, Bond's \u001b[92mportrayal\u001b[0m is the quintessential John L. and the \u001b[92mbest\u001b[0m thing about Gentleman Jim.<br /><br />Speaking of racism, one thing that should have been told was the fact that Sullivan while champion refused on grounds of race to meet Peter Jackson who was black and from Australia and probably the \u001b[92mbest\u001b[0m heavyweight of his time. I say probably because as a challenger Corbett met him and fought him for 61 rounds to a draw. That fight more than any other created a demand for a Sullivan-Corbett title match. Of course when Corbett was champion he refused to give Jackson a title shot. Maybe he didn't want another 61 round marathon with someone who \u001b[92mmay\u001b[0m have been better on that particular \u001b[92mday\u001b[0m.<br /><br />Gentleman Jim is not Jim Corbett's story, it is a movie of Errol \u001b[92mFlynn\u001b[0m playing at being James J. Corbett. But Corbett had he been alive in 1942 no doubt would have \u001b[92mloved\u001b[0m the movie and \u001b[92mloved\u001b[0m \u001b[92mFlynn's\u001b[0m \u001b[92mportrayal\u001b[0m of his \u001b[92mlife\u001b[0m. It's as he would have \u001b[92mliked\u001b[0m to have been remembered.\n",
      "\n",
      "James J. Corbett, heavyweight champion of the \u001b[91mUorld\u001b[0m from 1892 to 1897, turned out to be Errol \u001b[91mFlyn's\u001b[0m \u001b[91mfVvorite\u001b[0m role. Possibly because he didn't have to wield a sword or be in a western. He grew tired of the swashbucklers and he said in his autobiography that he felt he was miscast in westerns and couldn't understand why people \u001b[91mlied\u001b[0m him in them. It is an \u001b[91menjoyaRle\u001b[0m film, but hardly does tell the real story of James J. Corbett.<br /><br />As portrayed Corbett was the first scientific boxer to win in the heavyweight division, a man who used brains and speed more than brute strength to win. He defeated John L. Sullivan and lost to Bob Fitzsimmons the heavyweight crown. He was \u001b[91malSso\u001b[0m a compulsive womanizer and in that was a \u001b[91mlo\u001b[0m like the man \u001b[91mportrAaying\u001b[0m him. Of course that was not shown on the screen. The character that Alexis Smith plays, the banker's daughter who falls for him, has no basis in reality. Corbett was in fact married twice and was flagrantly unfaithful with both of his wives.<br /><br />\u001b[91mAlo\u001b[0m \u001b[91mChough\u001b[0m after he lost his title and after the events of this film are concluded he suffered a \u001b[91mOreat\u001b[0m personal tragedy. His father in a moment of depression, probably over finances because he lost heavily betting on his son to beat Bob Fitzsimmons, shot his mother and then turned the gun on himself. The murder/suicide hardly squares with the happy clan that Alan Hale presided over.<br /><br />One thing though that I did like about Gentleman Jim. Ward Bond got the career role of his \u001b[91mgife\u001b[0m in playing John L. Sullivan. Director Raoul Walsh got a \u001b[91mgerat\u001b[0m \u001b[91mpefrormance\u001b[0m out of Bond as the blustering, but lovable Sullivan. Even given some of Sullivan's bad points that don't make it to the screen like rabid racism, Bond's \u001b[91mportrLayal\u001b[0m is the quintessential John L. and the \u001b[91mbAest\u001b[0m thing about Gentleman Jim.<br /><br />Speaking of racism, one thing that should have been told was the fact that Sullivan while champion refused on grounds of race to meet Peter Jackson who was black and from Australia and probably the \u001b[91mbes\u001b[0m heavyweight of his time. I say probably because as a challenger Corbett met him and fought him for 61 rounds to a draw. That fight more than any other created a demand for a Sullivan-Corbett title match. Of course when Corbett was champion he refused to give Jackson a title shot. Maybe he didn't want another 61 round marathon with someone who \u001b[91mcmay\u001b[0m have been better on that particular \u001b[91mdad\u001b[0m.<br /><br />Gentleman Jim is not Jim Corbett's story, it is a movie of Errol \u001b[91mFlnn\u001b[0m playing at being James J. Corbett. But Corbett had he been alive in 1942 no doubt would have \u001b[91mloDed\u001b[0m the movie and \u001b[91mlXoved\u001b[0m \u001b[91mFXlynn's\u001b[0m \u001b[91mportraEal\u001b[0m of his \u001b[91mlie\u001b[0m. It's as he would have \u001b[91mIliked\u001b[0m to have been remembered.\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 4 / 0 / 5:  50%|█▌ | 5/10 [23:27<23:27, 281.41s/it]--------------------------------------------- Result 6 ---------------------------------------------\n",
      "\u001b[91mNegative (96%)\u001b[0m --> \u001b[92mPositive (54%)\u001b[0m\n",
      "\n",
      "Well then, what is it?! I found Nicholson's character shallow and most \u001b[91munfortunately\u001b[0m \u001b[91muninteresting\u001b[0m. Angelica Huston's character drained my power. And Kathleen Turner is a filthy no good slut. It's not that I \"don't get it\". It's not that I don't think that some of the ideas could've lead to something more. This is a film with \u001b[91mnothing\u001b[0m but the notion that we're \u001b[91msupposed\u001b[0m to accept these ideas, and that's what the movie has going for it. That Nicholson \u001b[91mfalls\u001b[0m for Turner is \u001b[91mabsurd\u001b[0m, but then again, it is intended to be so. This however does not strike me as a.)funny, or b.)...\u001b[91meven\u001b[0m \u001b[91mremotely\u001b[0m \u001b[91minteresting\u001b[0m!!! This was a \u001b[91mwaste\u001b[0m of my time, so don't let the \u001b[91mhype\u001b[0m get the best of you...it is a \u001b[91mwaste\u001b[0m of your time! With all that being said, the opening church sequence is quite beautiful...\n",
      "\n",
      "Well then, what is it?! I found Nicholson's character shallow and most \u001b[92munfobtunately\u001b[0m \u001b[92munineresting\u001b[0m. Angelica Huston's character drained my power. And Kathleen Turner is a filthy no good slut. It's not that I \"don't get it\". It's not that I don't think that some of the ideas could've lead to something more. This is a film with \u001b[92mnothong\u001b[0m but the notion that we're \u001b[92msupjosed\u001b[0m to accept these ideas, and that's what the movie has going for it. That Nicholson \u001b[92mVfalls\u001b[0m for Turner is \u001b[92maAsurd\u001b[0m, but then again, it is intended to be so. This however does not strike me as a.)funny, or b.)...\u001b[92mevAen\u001b[0m \u001b[92mremoetly\u001b[0m \u001b[92mIinteresting\u001b[0m!!! This was a \u001b[92mwante\u001b[0m of my time, so don't let the \u001b[92mhVpe\u001b[0m get the best of you...it is a \u001b[92mvwaste\u001b[0m of your time! With all that being said, the opening church sequence is quite beautiful...\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 5 / 0 / 6:  60%|█▊ | 6/10 [27:20<18:13, 273.34s/it]--------------------------------------------- Result 7 ---------------------------------------------\n",
      "\u001b[91mNegative (62%)\u001b[0m --> \u001b[92mPositive (50%)\u001b[0m\n",
      "\n",
      "Although there's Flying Guillotines as part of the title of this movie, it has no connections to the original Flying Guillotines (1975) and its sequel Flying Guillotines II (1978). The two originals are masterpieces of kung-fu movie and still stands out as a classic. This is a much inferior copy of the original, and \u001b[91meven\u001b[0m as a regular kung-fu movie, it's below average.<br /><br />First of all, this movie doesn't have much acting. It's one senseless fight scene after another, and flying guillotine doesn't \u001b[91meven\u001b[0m play a major part in them. Story is about some Shaolin monks who are tracking down some villains who've took off with a sacred book, and an evil prince who owns part of this book is part of the \u001b[91mplot\u001b[0m. The same evil prince has plans to lure the monks in and use the flying guillotines on them.<br /><br />There are four movies with Flying Guillotine as part of its title. This in my opinion is of \u001b[91mleast\u001b[0m quality. The design of the flying guillotine in this movie is different from the other three indicating that this movie was produced by a different entity from the other three.<br /><br />The movie has no chemistry asides from being unintentionally funny due to \u001b[91mpoor\u001b[0m production. <br /><br />Best \u001b[91mskip\u001b[0m this and watch the two originals.\n",
      "\n",
      "Although there's Flying Guillotines as part of the title of this movie, it has no connections to the original Flying Guillotines (1975) and its sequel Flying Guillotines II (1978). The two originals are masterpieces of kung-fu movie and still stands out as a classic. This is a much inferior copy of the original, and \u001b[92meRen\u001b[0m as a regular kung-fu movie, it's below average.<br /><br />First of all, this movie doesn't have much acting. It's one senseless fight scene after another, and flying guillotine doesn't \u001b[92mevean\u001b[0m play a major part in them. Story is about some Shaolin monks who are tracking down some villains who've took off with a sacred book, and an evil prince who owns part of this book is part of the \u001b[92mkplot\u001b[0m. The same evil prince has plans to lure the monks in and use the flying guillotines on them.<br /><br />There are four movies with Flying Guillotine as part of its title. This in my opinion is of \u001b[92mlast\u001b[0m quality. The design of the flying guillotine in this movie is different from the other three indicating that this movie was produced by a different entity from the other three.<br /><br />The movie has no chemistry asides from being unintentionally funny due to \u001b[92mpooj\u001b[0m production. <br /><br />Best \u001b[92mkip\u001b[0m this and watch the two originals.\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 6 / 0 / 7:  70%|██ | 7/10 [31:35<13:32, 270.74s/it]--------------------------------------------- Result 8 ---------------------------------------------\n",
      "\u001b[91mNegative (94%)\u001b[0m --> \u001b[92mPositive (51%)\u001b[0m\n",
      "\n",
      "Sure, we all like \u001b[91mbad\u001b[0m movies at one time or another, and we in fact enjoy them, This however, wasn't \u001b[91meven\u001b[0m a guilty pleasure, it was just \u001b[91mcrap\u001b[0m. Some guy, vince offer, who is conceited \u001b[91menough\u001b[0m to \u001b[91mmake\u001b[0m himself the main character while probably got drunk/high--probably both--and thought it was a great \u001b[91midea\u001b[0m to \u001b[91mmake\u001b[0m a movie. He then proceeded to show his \u001b[91mscript\u001b[0m to equally high/drunk individuals. Overall, this movie was so \u001b[91mbad\u001b[0m, \u001b[91mpredictable\u001b[0m, and unoriginal I couldn't get through 20 \u001b[91mminutes\u001b[0m of it before I turned it off. It makes You Got Served look like Citizen Kane. Bat Man? WTF...Some guy that walks around with a bat, real \u001b[91moriginal\u001b[0m. Almost as good as calling him Fat Man, and having a fat guy \u001b[91mwalk\u001b[0m around in a superhero outfit.\n",
      "\n",
      "Sure, we all like \u001b[92mbd\u001b[0m movies at one time or another, and we in fact enjoy them, This however, wasn't \u001b[92mveen\u001b[0m a guilty pleasure, it was just \u001b[92mErap\u001b[0m. Some guy, vince offer, who is conceited \u001b[92menAough\u001b[0m to \u001b[92mmak\u001b[0m himself the main character while probably got drunk/high--probably both--and thought it was a great \u001b[92midVea\u001b[0m to \u001b[92mOmake\u001b[0m a movie. He then proceeded to show his \u001b[92mscipt\u001b[0m to equally high/drunk individuals. Overall, this movie was so \u001b[92mabd\u001b[0m, \u001b[92mOredictable\u001b[0m, and unoriginal I couldn't get through 20 \u001b[92mmTinutes\u001b[0m of it before I turned it off. It makes You Got Served look like Citizen Kane. Bat Man? WTF...Some guy that walks around with a bat, real \u001b[92morgginal\u001b[0m. Almost as good as calling him Fat Man, and having a fat guy \u001b[92malk\u001b[0m around in a superhero outfit.\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 7 / 0 / 8:  80%|██▍| 8/10 [35:21<08:50, 265.25s/it]--------------------------------------------- Result 9 ---------------------------------------------\n",
      "\u001b[92mPositive (75%)\u001b[0m --> \u001b[91mNegative (50%)\u001b[0m\n",
      "\n",
      "There are some \u001b[92mwonderful\u001b[0m things about this movie. Marion Davies could act, given the right property; she is \u001b[92mwonderful\u001b[0m in comedic roles. William Haines could act, and you can see why he was one of the screen's most popular leading men. (Until a potential scandal forced him from the business).<br /><br />The story is a \u001b[92mbit\u001b[0m trite, but handled so \u001b[92mbeautifully\u001b[0m that you don't notice. King Vidor's direction is one of the principle reasons for this. The producer? The boy \u001b[92mgenius\u001b[0m, Irving Thalberg.<br /><br />It's about movie making, and you get to \u001b[92msee\u001b[0m the process as it was done in 1928, the cameras, sets, directors directing and actors emoting. You get to \u001b[92msee\u001b[0m (briefly) some of the major stars of the \u001b[92mday\u001b[0m; even Charlie Chaplin does a turn as himself, seeking an autograph. You \u001b[92malso\u001b[0m catch glimpses of Eleanor Boardman, Elinor Glyn, Claire Windsor, King Vidor, and many others who are otherwise just names and old photographs.<br /><br />Please, even if you're not a fan of the silents, take the time to catch this film when you can. It's really a \u001b[92mterrific\u001b[0m trip back in time.\n",
      "\n",
      "There are some \u001b[91mwoderful\u001b[0m things about this movie. Marion Davies could act, given the right property; she is \u001b[91mwUonderful\u001b[0m in comedic roles. William Haines could act, and you can see why he was one of the screen's most popular leading men. (Until a potential scandal forced him from the business).<br /><br />The story is a \u001b[91mbi\u001b[0m trite, but handled so \u001b[91mbeLutifully\u001b[0m that you don't notice. King Vidor's direction is one of the principle reasons for this. The producer? The boy \u001b[91meenius\u001b[0m, Irving Thalberg.<br /><br />It's about movie making, and you get to \u001b[91mese\u001b[0m the process as it was done in 1928, the cameras, sets, directors directing and actors emoting. You get to \u001b[91mese\u001b[0m (briefly) some of the major stars of the \u001b[91mdad\u001b[0m; even Charlie Chaplin does a turn as himself, seeking an autograph. You \u001b[91mNlso\u001b[0m catch glimpses of Eleanor Boardman, Elinor Glyn, Claire Windsor, King Vidor, and many others who are otherwise just names and old photographs.<br /><br />Please, even if you're not a fan of the silents, take the time to catch this film when you can. It's really a \u001b[91mtbrrific\u001b[0m trip back in time.\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 8 / 0 / 9:  90%|██▋| 9/10 [39:36<04:24, 264.08s/it]--------------------------------------------- Result 10 ---------------------------------------------\n",
      "\u001b[92mPositive (70%)\u001b[0m --> \u001b[91mNegative (52%)\u001b[0m\n",
      "\n",
      "Apparently Ruggero Deodato figured out, early on, that his story wouldn't work if he approached it too seriously, so he decided to camp it up. The result is a film that can be viewed as either a ludicrous sword-and-sorcery epic or as a very \u001b[92mentertaining\u001b[0m comedy! And I think I'll go the second way. The brief gore moments are \u001b[92mwell-done\u001b[0m, the \u001b[92mPaul\u001b[0m Brothers openly mock the material (they even bark at each other in one scene!), and there is \u001b[92malso\u001b[0m a charming, spirited, \u001b[92mgood-natured\u001b[0m \u001b[92mperformance\u001b[0m by Eva La Rue, as the girl who tags along with the \"boys\". Plus, where else can you see the insides of a dragon lighted like a discotheque?? (***)\n",
      "\n",
      "Apparently Ruggero Deodato figured out, early on, that his story wouldn't work if he approached it too seriously, so he decided to camp it up. The result is a film that can be viewed as either a ludicrous sword-and-sorcery epic or as a very \u001b[91mYntertaining\u001b[0m comedy! And I think I'll go the second way. The brief gore moments are \u001b[91mewll-done\u001b[0m, the \u001b[91mPXul\u001b[0m Brothers openly mock the material (they even bark at each other in one scene!), and there is \u001b[91manso\u001b[0m a charming, spirited, \u001b[91mgTood-natured\u001b[0m \u001b[91mperfomance\u001b[0m by Eva La Rue, as the girl who tags along with the \"boys\". Plus, where else can you see the insides of a dragon lighted like a discotheque?? (***)\n",
      "\n",
      "\n",
      "[Succeeded / Failed / Total] 9 / 0 / 10: 100%|█| 10/10 [42:12<00:00, 253.27s/it]\n",
      "\n",
      "+-------------------------------+--------+\n",
      "| Attack Results                |        |\n",
      "+-------------------------------+--------+\n",
      "| Number of successful attacks: | 9      |\n",
      "| Number of failed attacks:     | 0      |\n",
      "| Number of skipped attacks:    | 1      |\n",
      "| Original accuracy:            | 90.0%  |\n",
      "| Accuracy under attack:        | 0.0%   |\n",
      "| Attack success rate:          | 100.0% |\n",
      "| Average perturbed word %:     | 5.93%  |\n",
      "| Average num. words per input: | 195.6  |\n",
      "| Avg num queries:              | 248.22 |\n",
      "+-------------------------------+--------+\n",
      "\u001b[34;1mtextattack\u001b[0m: Attack time: 2533.5819494724274s\n"
     ]
    }
   ],
   "source": [
    "!textattack attack --model-from-file model2.py --recipe deepwordbug --num-examples 10 --dataset-from-nlp imdb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example 4: bypass command line api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import textattack\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "num_remaining_attacks = 1\n",
    "worklist = deque(range(0, num_remaining_attacks))\n",
    "worklist_tail = worklist[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Bunch(object):\n",
    "    def __init__(self, adict):\n",
    "        self.__dict__.update(adict)\n",
    "\n",
    "\n",
    "class SklearnTokenizer:\n",
    "    def __init__(self, tokenizer_path):\n",
    "        # load tokenizer from file\n",
    "        self.tokenizer = pickle.load(open(tokenizer_path, \"rb\"))\n",
    "\n",
    "    def batch_encode(self, text_list):\n",
    "        if isinstance(text_list[0], tuple):  # unroll tuples passed in by TextAttack\n",
    "            text_list = [x[0] for x in text_list]\n",
    "        encoded_text_matrix = self.tokenizer.transform(text_list)\n",
    "        return torch.tensor(encoded_text_matrix.toarray())\n",
    "\n",
    "    def encode(self, text):\n",
    "        return self.batch_encode([text])\n",
    "\n",
    "\n",
    "class SklearnModelWrapper:\n",
    "    def __init__(self, model_path, feature_names):\n",
    "        self.model = pickle.load(open(model_path, 'rb'))\n",
    "        self.feature_names = feature_names\n",
    "\n",
    "        # dummy attribute that will also not be necessary in the future\n",
    "        def dummy_params():\n",
    "            yield Bunch({'device': torch.device('cuda')})\n",
    "\n",
    "        setattr(self.model, 'parameters', dummy_params)\n",
    "\n",
    "    def __call__(self, tokenized_text_input):\n",
    "        tokenized_text_df = pd.DataFrame(tokenized_text_input, columns=self.feature_names)\n",
    "        return torch.tensor(self.model.predict_proba(tokenized_text_df))\n",
    "\n",
    "    def to(self, *args):\n",
    "        # dummy method that will not be necessary in the future\n",
    "        return self\n",
    "\n",
    "\n",
    "tokenizer = SklearnTokenizer('/home/qc/work/imdb_adversarial_attack/tf_idf_vectorizor2.pkl')\n",
    "feature_names = tokenizer.tokenizer.get_feature_names()\n",
    "model = SklearnModelWrapper('/home/qc/work/imdb_adversarial_attack/tf_idf_logistic_reg.pkl', feature_names)\n",
    "\n",
    "\n",
    "model = model.to(textattack.shared.utils.device)\n",
    "setattr(model, \"tokenizer\", tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textattack.attack_recipes import DeepWordBugGao2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34;1mtextattack\u001b[0m: Unknown if model of class <class '__main__.SklearnModelWrapper'> compatible with goal function <class 'textattack.goal_functions.classification.untargeted_classification.UntargetedClassification'>.\n"
     ]
    }
   ],
   "source": [
    "recipe = DeepWordBugGao2018(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Attack(\n",
       "  (search_method): GreedyWordSwapWIR(\n",
       "    (wir_method):  unk\n",
       "  )\n",
       "  (goal_function):  UntargetedClassification\n",
       "  (transformation):  CompositeTransformation(\n",
       "    (0): WordSwapNeighboringCharacterSwap(\n",
       "        (random_one):  True\n",
       "      )\n",
       "    (1): WordSwapRandomCharacterSubstitution(\n",
       "        (random_one):  True\n",
       "      )\n",
       "    (2): WordSwapRandomCharacterDeletion(\n",
       "        (random_one):  True\n",
       "      )\n",
       "    (3): WordSwapRandomCharacterInsertion(\n",
       "        (random_one):  True\n",
       "      )\n",
       "    )\n",
       "  (constraints): \n",
       "    (0): LevenshteinEditDistance(\n",
       "        (max_edit_distance):  30\n",
       "        (compare_against_original):  True\n",
       "      )\n",
       "    (1): RepeatModification\n",
       "    (2): StopwordModification\n",
       "  (is_black_box):  True\n",
       ")"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "recipe.goal_function.model_batch_size = 32\n",
    "recipe.goal_function.model_cache_size = 262144\n",
    "recipe.constraint_cache_size = 262144"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get data set\n",
    "_df = pd.read_csv('test_data.csv', header=None)\n",
    "_df.columns = ['label', 'text']\n",
    "dataset = _df[['text', 'label']].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run things"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "for result in recipe.attack_dataset(dataset, indices=worklist):\n",
    "    results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[92m1 (52%)\u001b[0m --> \u001b[91m0 (51%)\u001b[0m\n",
      "\n",
      "Wall St. \u001b[92mBears\u001b[0m Claw Back Into the Black ( Reuters ) . Reuters - Short - sellers , Wall Street 's dwindling band of ultra - cynics , are seeing green again .\n",
      "\n",
      "Wall St. \u001b[91mBeajrs\u001b[0m Claw Back Into the Black ( Reuters ) . Reuters - Short - sellers , Wall Street 's dwindling band of ultra - cynics , are seeing green again .\n"
     ]
    }
   ],
   "source": [
    "print(results[0].__str__(\"ansi\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].original_result.ground_truth_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.48340441672921686"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].original_result.score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AttackedText \"Wall St. Bears Claw Back Into the Black ( Reuters ) . Reuters - Short - sellers , Wall Street 's dwindling band of ultra - cynics , are seeing green again .\">"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].original_result.attacked_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].perturbed_result.num_queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].perturbed_result.output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5102398402251678"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].perturbed_result.score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{2}"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results[0].perturbed_result.attacked_text.all_words_diff(results[0].original_result.attacked_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}