{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0f9f666f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification, Trainer, TrainingArguments\n",
    "from datasets import load_dataset\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2f35116b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3bdffef37cd4d5aaa090640d5384825",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/25000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load the IMDb dataset\n",
    "dataset = load_dataset(\"imdb\")\n",
    "\n",
    "# Tokenizer function\n",
    "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True, max_length=512)\n",
    "\n",
    "# Tokenize the dataset\n",
    "tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
    "\n",
    "# Format for PyTorch\n",
    "train_dataset = tokenized_datasets[\"train\"].shuffle(seed=42).select(range(10000)) # Subset for training\n",
    "test_dataset = tokenized_datasets[\"test\"].shuffle(seed=42).select(range(1000)) # Subset for testing\n",
    "\n",
    "train_dataset.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "test_dataset.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "93d6a61b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_projector.bias', 'vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_transform.weight']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'pre_classifier.bias', 'classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased', num_labels=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "58400de8",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',\n",
    "    num_train_epochs=3,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=64,\n",
    "    warmup_steps=500,\n",
    "    weight_decay=0.01,\n",
    "    logging_dir='./logs',\n",
    "    evaluation_strategy='steps',  \n",
    "    save_strategy='steps',        \n",
    "    load_best_model_at_end=True,\n",
    "    logging_steps=50,             \n",
    "    save_steps=50                 \n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3389ad91",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\n",
    "        'accuracy': acc,\n",
    "        'f1': f1,\n",
    "        'precision': precision,\n",
    "        'recall': recall\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0d68d5ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "C:\\Users\\saime\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\transformers\\optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 10000\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 1875\n",
      "  Number of trainable parameters = 66955010\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1875' max='1875' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1875/1875 14:06:36, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.688800</td>\n",
       "      <td>0.680938</td>\n",
       "      <td>0.661000</td>\n",
       "      <td>0.543742</td>\n",
       "      <td>0.792157</td>\n",
       "      <td>0.413934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>0.629000</td>\n",
       "      <td>0.465259</td>\n",
       "      <td>0.841000</td>\n",
       "      <td>0.819113</td>\n",
       "      <td>0.920716</td>\n",
       "      <td>0.737705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>0.371200</td>\n",
       "      <td>0.323407</td>\n",
       "      <td>0.868000</td>\n",
       "      <td>0.867470</td>\n",
       "      <td>0.850394</td>\n",
       "      <td>0.885246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.336300</td>\n",
       "      <td>0.374150</td>\n",
       "      <td>0.857000</td>\n",
       "      <td>0.836197</td>\n",
       "      <td>0.948052</td>\n",
       "      <td>0.747951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>0.336700</td>\n",
       "      <td>0.312763</td>\n",
       "      <td>0.865000</td>\n",
       "      <td>0.871795</td>\n",
       "      <td>0.812389</td>\n",
       "      <td>0.940574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>0.311800</td>\n",
       "      <td>0.296506</td>\n",
       "      <td>0.889000</td>\n",
       "      <td>0.882540</td>\n",
       "      <td>0.912473</td>\n",
       "      <td>0.854508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>0.309800</td>\n",
       "      <td>0.286319</td>\n",
       "      <td>0.886000</td>\n",
       "      <td>0.886228</td>\n",
       "      <td>0.863813</td>\n",
       "      <td>0.909836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.272300</td>\n",
       "      <td>0.292773</td>\n",
       "      <td>0.890000</td>\n",
       "      <td>0.884696</td>\n",
       "      <td>0.905579</td>\n",
       "      <td>0.864754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>0.315100</td>\n",
       "      <td>0.419856</td>\n",
       "      <td>0.854000</td>\n",
       "      <td>0.831019</td>\n",
       "      <td>0.954787</td>\n",
       "      <td>0.735656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.350900</td>\n",
       "      <td>0.298303</td>\n",
       "      <td>0.862000</td>\n",
       "      <td>0.869565</td>\n",
       "      <td>0.807018</td>\n",
       "      <td>0.942623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>550</td>\n",
       "      <td>0.355200</td>\n",
       "      <td>0.333094</td>\n",
       "      <td>0.870000</td>\n",
       "      <td>0.852608</td>\n",
       "      <td>0.954315</td>\n",
       "      <td>0.770492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.279900</td>\n",
       "      <td>0.282081</td>\n",
       "      <td>0.887000</td>\n",
       "      <td>0.879915</td>\n",
       "      <td>0.913907</td>\n",
       "      <td>0.848361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>650</td>\n",
       "      <td>0.279200</td>\n",
       "      <td>0.288312</td>\n",
       "      <td>0.892000</td>\n",
       "      <td>0.883621</td>\n",
       "      <td>0.931818</td>\n",
       "      <td>0.840164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>0.198600</td>\n",
       "      <td>0.338301</td>\n",
       "      <td>0.876000</td>\n",
       "      <td>0.863736</td>\n",
       "      <td>0.931280</td>\n",
       "      <td>0.805328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>750</td>\n",
       "      <td>0.195600</td>\n",
       "      <td>0.292916</td>\n",
       "      <td>0.897000</td>\n",
       "      <td>0.897512</td>\n",
       "      <td>0.872340</td>\n",
       "      <td>0.924180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.243400</td>\n",
       "      <td>0.289307</td>\n",
       "      <td>0.899000</td>\n",
       "      <td>0.900883</td>\n",
       "      <td>0.864407</td>\n",
       "      <td>0.940574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>850</td>\n",
       "      <td>0.193000</td>\n",
       "      <td>0.304464</td>\n",
       "      <td>0.897000</td>\n",
       "      <td>0.894359</td>\n",
       "      <td>0.895277</td>\n",
       "      <td>0.893443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>0.214500</td>\n",
       "      <td>0.257609</td>\n",
       "      <td>0.899000</td>\n",
       "      <td>0.895337</td>\n",
       "      <td>0.905660</td>\n",
       "      <td>0.885246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>950</td>\n",
       "      <td>0.228000</td>\n",
       "      <td>0.279465</td>\n",
       "      <td>0.887000</td>\n",
       "      <td>0.891659</td>\n",
       "      <td>0.837838</td>\n",
       "      <td>0.952869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.208100</td>\n",
       "      <td>0.230380</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>0.908537</td>\n",
       "      <td>0.901210</td>\n",
       "      <td>0.915984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1050</td>\n",
       "      <td>0.200600</td>\n",
       "      <td>0.307765</td>\n",
       "      <td>0.901000</td>\n",
       "      <td>0.902077</td>\n",
       "      <td>0.871893</td>\n",
       "      <td>0.934426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>0.210600</td>\n",
       "      <td>0.278725</td>\n",
       "      <td>0.901000</td>\n",
       "      <td>0.901493</td>\n",
       "      <td>0.876209</td>\n",
       "      <td>0.928279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1150</td>\n",
       "      <td>0.208200</td>\n",
       "      <td>0.283095</td>\n",
       "      <td>0.912000</td>\n",
       "      <td>0.909836</td>\n",
       "      <td>0.909836</td>\n",
       "      <td>0.909836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.201000</td>\n",
       "      <td>0.256353</td>\n",
       "      <td>0.901000</td>\n",
       "      <td>0.895238</td>\n",
       "      <td>0.925602</td>\n",
       "      <td>0.866803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1250</td>\n",
       "      <td>0.186200</td>\n",
       "      <td>0.249205</td>\n",
       "      <td>0.909000</td>\n",
       "      <td>0.906282</td>\n",
       "      <td>0.910973</td>\n",
       "      <td>0.901639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>0.080400</td>\n",
       "      <td>0.367344</td>\n",
       "      <td>0.902000</td>\n",
       "      <td>0.900609</td>\n",
       "      <td>0.891566</td>\n",
       "      <td>0.909836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1350</td>\n",
       "      <td>0.152700</td>\n",
       "      <td>0.323376</td>\n",
       "      <td>0.905000</td>\n",
       "      <td>0.900315</td>\n",
       "      <td>0.922581</td>\n",
       "      <td>0.879098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.100400</td>\n",
       "      <td>0.416915</td>\n",
       "      <td>0.888000</td>\n",
       "      <td>0.891892</td>\n",
       "      <td>0.843066</td>\n",
       "      <td>0.946721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1450</td>\n",
       "      <td>0.108800</td>\n",
       "      <td>0.324885</td>\n",
       "      <td>0.908000</td>\n",
       "      <td>0.907258</td>\n",
       "      <td>0.892857</td>\n",
       "      <td>0.922131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>0.066700</td>\n",
       "      <td>0.378826</td>\n",
       "      <td>0.902000</td>\n",
       "      <td>0.901210</td>\n",
       "      <td>0.886905</td>\n",
       "      <td>0.915984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1550</td>\n",
       "      <td>0.078500</td>\n",
       "      <td>0.368980</td>\n",
       "      <td>0.906000</td>\n",
       "      <td>0.901674</td>\n",
       "      <td>0.920940</td>\n",
       "      <td>0.883197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>0.081500</td>\n",
       "      <td>0.364918</td>\n",
       "      <td>0.909000</td>\n",
       "      <td>0.907048</td>\n",
       "      <td>0.904277</td>\n",
       "      <td>0.909836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1650</td>\n",
       "      <td>0.062600</td>\n",
       "      <td>0.386855</td>\n",
       "      <td>0.905000</td>\n",
       "      <td>0.903943</td>\n",
       "      <td>0.892216</td>\n",
       "      <td>0.915984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>0.067000</td>\n",
       "      <td>0.392243</td>\n",
       "      <td>0.906000</td>\n",
       "      <td>0.905051</td>\n",
       "      <td>0.892430</td>\n",
       "      <td>0.918033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1750</td>\n",
       "      <td>0.047400</td>\n",
       "      <td>0.409893</td>\n",
       "      <td>0.910000</td>\n",
       "      <td>0.908350</td>\n",
       "      <td>0.902834</td>\n",
       "      <td>0.913934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.108200</td>\n",
       "      <td>0.401962</td>\n",
       "      <td>0.909000</td>\n",
       "      <td>0.907801</td>\n",
       "      <td>0.897796</td>\n",
       "      <td>0.918033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1850</td>\n",
       "      <td>0.105400</td>\n",
       "      <td>0.390589</td>\n",
       "      <td>0.912000</td>\n",
       "      <td>0.910020</td>\n",
       "      <td>0.908163</td>\n",
       "      <td>0.911885</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-50\n",
      "Configuration saved in ./results\\checkpoint-50\\config.json\n",
      "Model weights saved in ./results\\checkpoint-50\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-100\n",
      "Configuration saved in ./results\\checkpoint-100\\config.json\n",
      "Model weights saved in ./results\\checkpoint-100\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-150\n",
      "Configuration saved in ./results\\checkpoint-150\\config.json\n",
      "Model weights saved in ./results\\checkpoint-150\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-200\n",
      "Configuration saved in ./results\\checkpoint-200\\config.json\n",
      "Model weights saved in ./results\\checkpoint-200\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-250\n",
      "Configuration saved in ./results\\checkpoint-250\\config.json\n",
      "Model weights saved in ./results\\checkpoint-250\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-300\n",
      "Configuration saved in ./results\\checkpoint-300\\config.json\n",
      "Model weights saved in ./results\\checkpoint-300\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-350\n",
      "Configuration saved in ./results\\checkpoint-350\\config.json\n",
      "Model weights saved in ./results\\checkpoint-350\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-400\n",
      "Configuration saved in ./results\\checkpoint-400\\config.json\n",
      "Model weights saved in ./results\\checkpoint-400\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-450\n",
      "Configuration saved in ./results\\checkpoint-450\\config.json\n",
      "Model weights saved in ./results\\checkpoint-450\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-500\n",
      "Configuration saved in ./results\\checkpoint-500\\config.json\n",
      "Model weights saved in ./results\\checkpoint-500\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-550\n",
      "Configuration saved in ./results\\checkpoint-550\\config.json\n",
      "Model weights saved in ./results\\checkpoint-550\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-600\n",
      "Configuration saved in ./results\\checkpoint-600\\config.json\n",
      "Model weights saved in ./results\\checkpoint-600\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-650\n",
      "Configuration saved in ./results\\checkpoint-650\\config.json\n",
      "Model weights saved in ./results\\checkpoint-650\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-700\n",
      "Configuration saved in ./results\\checkpoint-700\\config.json\n",
      "Model weights saved in ./results\\checkpoint-700\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-750\n",
      "Configuration saved in ./results\\checkpoint-750\\config.json\n",
      "Model weights saved in ./results\\checkpoint-750\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-800\n",
      "Configuration saved in ./results\\checkpoint-800\\config.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Model weights saved in ./results\\checkpoint-800\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-850\n",
      "Configuration saved in ./results\\checkpoint-850\\config.json\n",
      "Model weights saved in ./results\\checkpoint-850\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-900\n",
      "Configuration saved in ./results\\checkpoint-900\\config.json\n",
      "Model weights saved in ./results\\checkpoint-900\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-950\n",
      "Configuration saved in ./results\\checkpoint-950\\config.json\n",
      "Model weights saved in ./results\\checkpoint-950\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1000\n",
      "Configuration saved in ./results\\checkpoint-1000\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1000\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1050\n",
      "Configuration saved in ./results\\checkpoint-1050\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1050\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1100\n",
      "Configuration saved in ./results\\checkpoint-1100\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1100\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1150\n",
      "Configuration saved in ./results\\checkpoint-1150\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1150\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1200\n",
      "Configuration saved in ./results\\checkpoint-1200\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1200\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1250\n",
      "Configuration saved in ./results\\checkpoint-1250\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1250\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1300\n",
      "Configuration saved in ./results\\checkpoint-1300\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1300\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1350\n",
      "Configuration saved in ./results\\checkpoint-1350\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1350\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1400\n",
      "Configuration saved in ./results\\checkpoint-1400\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1400\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1450\n",
      "Configuration saved in ./results\\checkpoint-1450\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1450\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1500\n",
      "Configuration saved in ./results\\checkpoint-1500\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1500\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1550\n",
      "Configuration saved in ./results\\checkpoint-1550\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1550\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1600\n",
      "Configuration saved in ./results\\checkpoint-1600\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1600\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1650\n",
      "Configuration saved in ./results\\checkpoint-1650\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1650\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1700\n",
      "Configuration saved in ./results\\checkpoint-1700\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1700\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1750\n",
      "Configuration saved in ./results\\checkpoint-1750\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1750\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1800\n",
      "Configuration saved in ./results\\checkpoint-1800\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1800\\pytorch_model.bin\n",
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./results\\checkpoint-1850\n",
      "Configuration saved in ./results\\checkpoint-1850\\config.json\n",
      "Model weights saved in ./results\\checkpoint-1850\\pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./results\\checkpoint-1000 (score: 0.23037973046302795).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1875, training_loss=0.22492422332763673, metrics={'train_runtime': 50814.837, 'train_samples_per_second': 0.59, 'train_steps_per_second': 0.037, 'total_flos': 3974021959680000.0, 'train_loss': 0.22492422332763673, 'epoch': 3.0})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=test_dataset,\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "\n",
    "trainer.train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e2b3a88e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set don't have a corresponding argument in `DistilBertForSequenceClassification.forward` and have been ignored: text. If text are not expected by `DistilBertForSequenceClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 1000\n",
      "  Batch size = 64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='16' max='16' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [16/16 07:01]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.23037973046302795,\n",
       " 'eval_accuracy': 0.91,\n",
       " 'eval_f1': 0.9085365853658537,\n",
       " 'eval_precision': 0.9012096774193549,\n",
       " 'eval_recall': 0.9159836065573771,\n",
       " 'eval_runtime': 450.0402,\n",
       " 'eval_samples_per_second': 2.222,\n",
       " 'eval_steps_per_second': 0.036,\n",
       " 'epoch': 3.0}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a15f4208",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Configuration saved in ./saved_model\\config.json\n",
      "Model weights saved in ./saved_model\\pytorch_model.bin\n",
      "tokenizer config file saved in ./saved_model\\tokenizer_config.json\n",
      "Special tokens file saved in ./saved_model\\special_tokens_map.json\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('./saved_model\\\\tokenizer_config.json',\n",
       " './saved_model\\\\special_tokens_map.json',\n",
       " './saved_model\\\\vocab.txt',\n",
       " './saved_model\\\\added_tokens.json',\n",
       " './saved_model\\\\tokenizer.json')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.save_pretrained('./saved_model')\n",
    "tokenizer.save_pretrained('./saved_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "eb978982",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "positive\n"
     ]
    }
   ],
   "source": [
    "def predict_sentiment(text):\n",
    "    inputs = tokenizer(text, return_tensors=\"pt\", padding=True, truncation=True, max_length=512)\n",
    "    inputs = {k: v.to(model.device) for k, v in inputs.items()}\n",
    "    with torch.no_grad():\n",
    "        logits = model(**inputs).logits\n",
    "    prediction = logits.argmax(-1).item()\n",
    "    return 'positive' if prediction == 1 else 'negative'\n",
    "\n",
    "# Test with a new sentence\n",
    "print(predict_sentiment(\"This movie was great! I loved it.\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "30dac866",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[1;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241m.\u001b[39msave_pretrained(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./Sentimental_Analysis\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      2\u001b[0m tokenizer\u001b[38;5;241m.\u001b[39msave_pretrained(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./Sentimental_Analysis\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "model.save_pretrained('./Sentimental_Analysis')\n",
    "tokenizer.save_pretrained('./Sentimental_Analysis')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3b53c73",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}