Adding notebooks for hugging face transformers

.gitignore

@@ -0,0 +1,143 @@
+
+# https://github.com/jupyter/notebook/blob/main/.gitignore
+
+**/*.ipynb_checkpoints/
+**/*.jupyter/
+
+*.bundle.*
+lib/
+node_modules/
+*.egg-info/
+.ipynb_checkpoints
+*.tsbuildinfo
+
+# Created by https://www.gitignore.io/api/python
+# Edit at https://www.gitignore.io/?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# OS X stuff
+*.DS_Store
+
+# End of https://www.gitignore.io/api/python
+
+_temp_extension
+junit.xml
+[uU]ntitled*
+notebook/static/*
+!notebook/static/favicons
+notebook/labextension
+notebook/schemas
+docs/source/changelog.md
+docs/source/contributing.md
+
+# playwright
+ui-tests/test-results
+ui-tests/playwright-report
+
+# VSCode
+.vscode
+
+# RTC
+.jupyter_ystore.db
+
+# yarn >=2.x local files
+.yarn/*
+.pnp.*
+ui-tests/.yarn/*
+ui-tests/.pnp.*

.ipynb_checkpoints/Example-Notebook-checkpoint.ipynb

@@ -1,64 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "79519eae-15e4-4f94-8ec5-c2a7724a3275",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# My First Notebook for Hugging face"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "39677f86-b552-458f-9d40-366b11ea2f0e",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "24"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "12+12"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7ed8225a-3777-48ba-a76a-aa6cc903d083",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.18"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

.jupyter/desktop-settings.json

@@ -1,3 +0,0 @@
-{
-  "pythonPath": "/Users/milindchawre/Library/jupyterlab-desktop/jlab_server/bin/python"
-}

.jupyter/desktop-workspaces/default-37a8.jupyterlab-workspace

@@ -1 +0,0 @@
-{"data":{"layout-restorer:data":{"main":{"dock":{"type":"tab-area","currentIndex":1,"widgets":["notebook:Example-Notebook.ipynb"]},"current":"notebook:Example-Notebook.ipynb"},"down":{"size":0,"widgets":[]},"left":{"collapsed":false,"visible":true,"current":"filebrowser","widgets":["filebrowser","running-sessions","@jupyterlab/toc:plugin","extensionmanager.main-view"]},"right":{"collapsed":true,"visible":true,"widgets":["jp-property-inspector","debugger-sidebar"]},"relativeSizes":[0.26227795193312436,0.7377220480668757,0],"top":{"simpleVisibility":true}},"notebook:Example-Notebook.ipynb":{"data":{"path":"Example-Notebook.ipynb","factory":"Notebook"}}},"metadata":{"id":"default"}}

01-Transformers/00-HF-basics.ipynb

@@ -0,0 +1,653 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "27b0322e-d6a8-4202-9f78-8d2754ebdd97",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#!pip list | grep hugging"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "da82a90f-7098-4d0c-9fe8-3e0cfc39671d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#!pip install transformers datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "829575c2-c292-4455-8cc6-48764e64c4b0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#!pip install torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "ba0ced0b-35cd-40fd-934f-1013d4a1364d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/milindchawre/.pyenv/versions/3.12.2/envs/hugging-face/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import transformers\n",
+    "import datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d196c435-fa5a-4c3b-bec4-0181aa00e8bb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'4.44.0'"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "transformers.__version__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "55ddfdaa-3a22-4eab-ad36-24355cbb7fee",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'2.21.0'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "datasets.__version__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "13b8669b-5cc5-40b8-bd22-a7ff44fa43f3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a722a796-84b7-4c45-a104-3d863d52cbb5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "reviews = load_dataset('rotten_tomatoes')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "607c352e-70c8-4697-b9a1-a4c68e55d502",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "datasets.dataset_dict.DatasetDict"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(reviews)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e0637ff3-90b9-41cf-bdd0-ba3dbe185225",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DatasetDict({\n",
+      "    train: Dataset({\n",
+      "        features: ['text', 'label'],\n",
+      "        num_rows: 8530\n",
+      "    })\n",
+      "    validation: Dataset({\n",
+      "        features: ['text', 'label'],\n",
+      "        num_rows: 1066\n",
+      "    })\n",
+      "    test: Dataset({\n",
+      "        features: ['text', 'label'],\n",
+      "        num_rows: 1066\n",
+      "    })\n",
+      "})\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(reviews)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "79f60106-7628-4605-920f-6bb8375e6cb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text</th>\n",
+       "      <th>label</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>the rock is destined to be the 21st century's ...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>the gorgeously elaborate continuation of \" the...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>effective but too-tepid biopic</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>if you sometimes like to go to the movies to h...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>emerges as something rare , an issue movie tha...</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8525</th>\n",
+       "      <td>any enjoyment will be hinge from a personal th...</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8526</th>\n",
+       "      <td>if legendary shlockmeister ed wood had ever ma...</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8527</th>\n",
+       "      <td>hardly a nuanced portrait of a young woman's b...</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8528</th>\n",
+       "      <td>interminably bleak , to say nothing of boring .</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8529</th>\n",
+       "      <td>things really get weird , though not particula...</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8530 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                   text  label\n",
+       "0     the rock is destined to be the 21st century's ...      1\n",
+       "1     the gorgeously elaborate continuation of \" the...      1\n",
+       "2                        effective but too-tepid biopic      1\n",
+       "3     if you sometimes like to go to the movies to h...      1\n",
+       "4     emerges as something rare , an issue movie tha...      1\n",
+       "...                                                 ...    ...\n",
+       "8525  any enjoyment will be hinge from a personal th...      0\n",
+       "8526  if legendary shlockmeister ed wood had ever ma...      0\n",
+       "8527  hardly a nuanced portrait of a young woman's b...      0\n",
+       "8528    interminably bleak , to say nothing of boring .      0\n",
+       "8529  things really get weird , though not particula...      0\n",
+       "\n",
+       "[8530 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reviews['train'].to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "8971be5a-f5ba-4cb9-be88-7fa509b201ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'the rock is destined to be the 21st century\\'s new \" conan \" and that he\\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reviews['train'].to_pandas()['text'][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "74db724b-063c-4c91-9681-37231a5a09fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "import torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "4a22ba6c-2db8-4dff-9abd-037f1a08fc7a",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No model was supplied, defaulted to distilbert/distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english).\n",
+      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
+      "/Users/milindchawre/.pyenv/versions/3.12.2/envs/hugging-face/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "classifier = pipeline(\"sentiment-analysis\", device=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "2d64c34d-c2d8-4cda-9c04-91e5497304a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'POSITIVE', 'score': 0.9998668432235718}]"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "classifier(\"This was great movie!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "fc11b406-3770-44ea-9654-4ec2e2e1081b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'POSITIVE', 'score': 0.9998465776443481}]"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "classifier(\"This was great ok!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "682a57fc-d2f6-4ad5-b05c-ad4a311216fa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'NEGATIVE', 'score': 0.9997976422309875}]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "classifier(\"This was not that good movie!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "3e790b3e-cb65-46a8-9bcb-74503742fcb6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'NEGATIVE', 'score': 0.9997455477714539}]"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "classifier(\"This was worst movie!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "56f1f502-f683-4c3a-924d-9a16e5d6c55c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'NEGATIVE', 'score': 0.9991676807403564}]"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "classifier(\"In the movie, the acting was fine, but the story was bad, while the costume was good but the daigloues are boring!\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "a8755bd0-5c4b-4da7-84d5-fb1986188d4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def score(review_text):\n",
+    "    return classifier(review_text)[0]['label']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "1de830b3-bb2d-42ce-90b6-b2868017d499",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_df = reviews['test'].to_pandas()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "0b0004ea-562c-493b-8a0c-125dd20a185f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_df['model_prediction'] = test_df['text'].apply(score)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "21715d4f-51f2-4118-b5d8-23b5d432d3b6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text</th>\n",
+       "      <th>label</th>\n",
+       "      <th>model_prediction</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>lovingly photographed in the manner of a golde...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>POSITIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>consistently clever and suspenseful .</td>\n",
+       "      <td>1</td>\n",
+       "      <td>POSITIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>it's like a \" big chill \" reunion of the baade...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>NEGATIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>the story gives ample opportunity for large-sc...</td>\n",
+       "      <td>1</td>\n",
+       "      <td>POSITIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>red dragon \" never cuts corners .</td>\n",
+       "      <td>1</td>\n",
+       "      <td>POSITIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1061</th>\n",
+       "      <td>a terrible movie that some people will neverth...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NEGATIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1062</th>\n",
+       "      <td>there are many definitions of 'time waster' bu...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NEGATIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1063</th>\n",
+       "      <td>as it stands , crocodile hunter has the hurrie...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NEGATIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1064</th>\n",
+       "      <td>the thing looks like a made-for-home-video qui...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NEGATIVE</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1065</th>\n",
+       "      <td>enigma is well-made , but it's just too dry an...</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NEGATIVE</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1066 rows × 3 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                   text  label  \\\n",
+       "0     lovingly photographed in the manner of a golde...      1   \n",
+       "1                 consistently clever and suspenseful .      1   \n",
+       "2     it's like a \" big chill \" reunion of the baade...      1   \n",
+       "3     the story gives ample opportunity for large-sc...      1   \n",
+       "4                     red dragon \" never cuts corners .      1   \n",
+       "...                                                 ...    ...   \n",
+       "1061  a terrible movie that some people will neverth...      0   \n",
+       "1062  there are many definitions of 'time waster' bu...      0   \n",
+       "1063  as it stands , crocodile hunter has the hurrie...      0   \n",
+       "1064  the thing looks like a made-for-home-video qui...      0   \n",
+       "1065  enigma is well-made , but it's just too dry an...      0   \n",
+       "\n",
+       "     model_prediction  \n",
+       "0            POSITIVE  \n",
+       "1            POSITIVE  \n",
+       "2            NEGATIVE  \n",
+       "3            POSITIVE  \n",
+       "4            POSITIVE  \n",
+       "...               ...  \n",
+       "1061         NEGATIVE  \n",
+       "1062         NEGATIVE  \n",
+       "1063         NEGATIVE  \n",
+       "1064         NEGATIVE  \n",
+       "1065         NEGATIVE  \n",
+       "\n",
+       "[1066 rows x 3 columns]"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3c916443-92ad-4dc6-a5bf-10aad52d8b4c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

01-Transformers/01-Pipelines-for-NLP-Tasks.ipynb

@@ -0,0 +1,711 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b89da7e6-431a-4659-a5b3-45323d11fd03",
+   "metadata": {},
+   "source": [
+    "# Pipelines for NLP Tasks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "76d9d1e3-05e1-456d-b564-5d096896a778",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/milindchawre/.pyenv/versions/3.12.2/envs/hugging-face/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import transformers\n",
+    "from transformers import pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "cf038c7c-13ce-4231-8acb-2a6b8de67de6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "4.44.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(transformers.__version__)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1dbc51c2-1efd-4811-929a-54ef8424c30c",
+   "metadata": {},
+   "source": [
+    "## Loading Tasks\n",
+    "\n",
+    "The task defining which pipeline will be returned. Currently accepted tasks are:\n",
+    "    \n",
+    "    - `\"audio-classification\"`: will return a [`AudioClassificationPipeline`].\n",
+    "    - `\"automatic-speech-recognition\"`: will return a [`AutomaticSpeechRecognitionPipeline`].\n",
+    "    - `\"conversational\"`: will return a [`ConversationalPipeline`].\n",
+    "    - `\"depth-estimation\"`: will return a [`DepthEstimationPipeline`].\n",
+    "    - `\"document-question-answering\"`: will return a [`DocumentQuestionAnsweringPipeline`].\n",
+    "    - `\"feature-extraction\"`: will return a [`FeatureExtractionPipeline`].\n",
+    "    - `\"fill-mask\"`: will return a [`FillMaskPipeline`]:.\n",
+    "    - `\"image-classification\"`: will return a [`ImageClassificationPipeline`].\n",
+    "    - `\"image-segmentation\"`: will return a [`ImageSegmentationPipeline`].\n",
+    "    - `\"image-to-text\"`: will return a [`ImageToTextPipeline`].\n",
+    "    - `\"object-detection\"`: will return a [`ObjectDetectionPipeline`].\n",
+    "    - `\"question-answering\"`: will return a [`QuestionAnsweringPipeline`].\n",
+    "    - `\"summarization\"`: will return a [`SummarizationPipeline`].\n",
+    "    - `\"table-question-answering\"`: will return a [`TableQuestionAnsweringPipeline`].\n",
+    "    - `\"text2text-generation\"`: will return a [`Text2TextGenerationPipeline`].\n",
+    "    - `\"text-classification\"` (alias `\"sentiment-analysis\"` available): will return a\n",
+    "      [`TextClassificationPipeline`].\n",
+    "    - `\"text-generation\"`: will return a [`TextGenerationPipeline`]:.\n",
+    "    - `\"token-classification\"` (alias `\"ner\"` available): will return a [`TokenClassificationPipeline`].\n",
+    "    - `\"translation\"`: will return a [`TranslationPipeline`].\n",
+    "    - `\"translation_xx_to_yy\"`: will return a [`TranslationPipeline`].\n",
+    "    - `\"video-classification\"`: will return a [`VideoClassificationPipeline`].\n",
+    "    - `\"visual-question-answering\"`: will return a [`VisualQuestionAnsweringPipeline`].\n",
+    "    - `\"zero-shot-classification\"`: will return a [`ZeroShotClassificationPipeline`].\n",
+    "    - `\"zero-shot-image-classification\"`: will return a [`ZeroShotImageClassificationPipeline`].\n",
+    "    - `\"zero-shot-object-detection\"`: will return a [`ZeroShotObjectDetectionPipeline`]."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aaa893bc-026c-456a-99f0-8f56a47da96e",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "## Classification \n",
+    "\n",
+    "### Default Models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "c3498239-01f0-49b1-bf3e-d9cba22d41ac",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No model was supplied, defaulted to distilbert/distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english).\n",
+      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
+      "/Users/milindchawre/.pyenv/versions/3.12.2/envs/hugging-face/lib/python3.12/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'POSITIVE', 'score': 0.9998236298561096}]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pipe = pipeline(task=\"text-classification\",device=0)\n",
+    "pipe(\"This restaurant is ok\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e88492a4-003a-4387-9590-0f3f621278ba",
+   "metadata": {},
+   "source": [
+    "### Specific Models\n",
+    "\n",
+    "Perhaps you want to use a different model for different categories or text types, for example, financial news: https://huggingface.co/ProsusAI/finbert\n",
+    "\n",
+    "You can explore more details in the paper: https://arxiv.org/pdf/1908.10063"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "bcf0966e-892f-46ca-9321-c0b31c9862ab",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
+      "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n"
+     ]
+    }
+   ],
+   "source": [
+    "pipe = pipeline(model=\"ProsusAI/finbert\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "fdf85b86-1132-4ef7-80c8-36e729be2910",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'positive', 'score': 0.9350943565368652}]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pipe(\"Shares of food delivery companies surged despite the catastrophic impact of coronavirus on global markets.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "6eca0d8e-1d32-4801-945a-34e9e7bbf83d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tweets = ['Gonna buy AAPL, its about to surge up!',\n",
+    "          'Gotta sell AAPL, its gonna plummet!']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "bfc8d3e1-d7d4-457f-a61c-25c023f4851a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': 'positive', 'score': 0.523411750793457},\n",
+       " {'label': 'neutral', 'score': 0.5528597831726074}]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pipe(tweets)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d2e8be7b-2cb7-425d-ae25-fa0b57f67f6a",
+   "metadata": {},
+   "source": [
+    "# Named Entity Recognition\n",
+    "\n",
+    "Let's explore another NLP task, such as NER - Named Entity Recognition\n",
+    "\n",
+    "**Note, this is a much larger model! If you run this it will download about 1.5 GB on to your computer inside of a cache folder!**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1486daa6-6108-4179-8fd3-682c17ad8f56",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No model was supplied, defaulted to distilbert/distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english).\n",
+      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
+      "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n"
+     ]
+    }
+   ],
+   "source": [
+    "pipe = pipeline(task=\"text-classification\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "293aa275-9fde-4017-9345-ce7a4debf315",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No model was supplied, defaulted to dbmdz/bert-large-cased-finetuned-conll03-english and revision f2482bf (https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english).\n",
+      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
+      "Some weights of the model checkpoint at dbmdz/bert-large-cased-finetuned-conll03-english were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
+      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n"
+     ]
+    }
+   ],
+   "source": [
+    "ner_tag_pipe = pipeline('ner')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "cbaaa69a-9dab-42a3-a1f5-1b109251c8c6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = ner_tag_pipe(\"After working at Tesla I started to study Nikola Tesla a lot more, especially at university in the USA.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "88e7d85a-6811-47a5-b400-2b36eb32e87e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#sentence =\"\"\"After working at Tomtom I started to study AI a lot more, especially at home in the Mumbai, Topics like RAG, hugging face and data science interest me more, Eating food like snacks and packed food with my laptop is my working setup.\"\"\"\n",
+    "#result = ner_tag_pipe(sentence)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "1be4c2c4-c1ae-446a-9ce3-680934e7da9c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'entity': 'I-ORG',\n",
+       "  'score': 0.9137765,\n",
+       "  'index': 4,\n",
+       "  'word': 'Te',\n",
+       "  'start': 17,\n",
+       "  'end': 19},\n",
+       " {'entity': 'I-ORG',\n",
+       "  'score': 0.3789888,\n",
+       "  'index': 5,\n",
+       "  'word': '##sla',\n",
+       "  'start': 19,\n",
+       "  'end': 22},\n",
+       " {'entity': 'I-PER',\n",
+       "  'score': 0.99693346,\n",
+       "  'index': 10,\n",
+       "  'word': 'Nikola',\n",
+       "  'start': 42,\n",
+       "  'end': 48},\n",
+       " {'entity': 'I-PER',\n",
+       "  'score': 0.9901416,\n",
+       "  'index': 11,\n",
+       "  'word': 'Te',\n",
+       "  'start': 49,\n",
+       "  'end': 51},\n",
+       " {'entity': 'I-PER',\n",
+       "  'score': 0.8931826,\n",
+       "  'index': 12,\n",
+       "  'word': '##sla',\n",
+       "  'start': 51,\n",
+       "  'end': 54},\n",
+       " {'entity': 'I-LOC',\n",
+       "  'score': 0.9997478,\n",
+       "  'index': 22,\n",
+       "  'word': 'USA',\n",
+       "  'start': 99,\n",
+       "  'end': 102}]"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88b446e2-8b25-4736-932b-4a7570c2570b",
+   "metadata": {},
+   "source": [
+    "# Question Answering"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "2ab7745a-1fe3-4e5d-89cf-8399185acd9d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No model was supplied, defaulted to distilbert/distilbert-base-cased-distilled-squad and revision 626af31 (https://huggingface.co/distilbert/distilbert-base-cased-distilled-squad).\n",
+      "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
+      "Hardware accelerator e.g. GPU is available in the environment, but no `device` argument is passed to the `Pipeline` object. Model will be on CPU.\n"
+     ]
+    }
+   ],
+   "source": [
+    "qa_bot = pipeline('question-answering')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "4380b8e5-4807-48da-bd3e-65e78c42e967",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text = \"\"\"\n",
+    "D-Day, marked on June 6, 1944, stands as one of the most significant military operations in history, \n",
+    "initiating the Allied invasion of Nazi-occupied Europe during World War II. Known as Operation Overlord, \n",
+    "this massive amphibious assault involved nearly 160,000 Allied troops landing on the beaches of Normandy, \n",
+    "France, across five sectors: Utah, Omaha, Gold, Juno, and Sword. Supported by over 5,000 ships and 13,000 \n",
+    "aircraft, the operation was preceded by extensive aerial and naval bombardment and an airborne assault. \n",
+    "The invasion set the stage for the liberation of Western Europe from Nazi control, despite the heavy \n",
+    "casualties and formidable German defenses. This day not only demonstrated the logistical prowess \n",
+    "and courage of the Allied forces but also marked a turning point in the war, leading to the eventual \n",
+    "defeat of Nazi Germany.\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "c185e089-0966-44b6-b4d8-882faf504e3c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "question = \"What were the five beach sectors on D-Day?\"\n",
+    "\n",
+    "result = qa_bot(question=question,context=text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "id": "720fcd66-2e85-4034-a81a-f49901d2fbb7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#\n",
+    "#question = \"Who is sherlock holmes?\"\n",
+    "#result = qa_bot(question=question,context=text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "8d1918a0-70f3-4d6c-8a17-bac0d07a9761",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'score': 0.9430821537971497,\n",
+       " 'start': 345,\n",
+       " 'end': 379,\n",
+       " 'answer': 'Utah, Omaha, Gold, Juno, and Sword'}"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f536e540-4080-4135-84e8-86fe13dc2fe6",
+   "metadata": {},
+   "source": [
+    "## Translations\n",
+    "\n",
+    "Translates from one language to another.\n",
+    "\n",
+    "This translation pipeline can currently be loaded from pipeline() using the following task identifier: \"translation_xx_to_yy\".\n",
+    "\n",
+    "The models that this pipeline can use are models that have been fine-tuned on a translation task. See the up-to-date list of available models on www.huggingface.co/models.  \n",
+    "\n",
+    "Note: You would typically call a specific model for translations: https://huggingface.co/models?pipeline_tag=translation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "24f282e5-1f0e-44ca-9b92-22b211208274",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "No model was supplied, defaulted to t5-base and revision 686f1db (https://huggingface.co/t5-base).\n",
+      "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "453587fb51494963bf1c3ba521c405f4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "config.json:   0%|          | 0.00/1.21k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\Marcial\\AppData\\Roaming\\Python\\Python39\\site-packages\\huggingface_hub\\file_download.py:148: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\Marcial\\.cache\\huggingface\\hub\\models--t5-base. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
+      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
+      "  warnings.warn(message)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e099a45ebcfa45c48dfecbcfb3eec137",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "pytorch_model.bin:   0%|          | 0.00/892M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6607a675437748f586b034cbcbebd046",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "eeaea7f972ce4f968a30bcf0c8cf7a30",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "21b2cfc97ada4a41bb3cefb2e2b95170",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "tokenizer.json:   0%|          | 0.00/1.39M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\Marcial\\AppData\\Roaming\\Python\\Python39\\site-packages\\transformers\\models\\t5\\tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
+      "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
+      "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
+      "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
+      "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import pipeline\n",
+    "translate = pipeline('translation_en_to_fr')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "88c1074c-d1c2-4957-9a32-bb0275b9a3c2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\Marcial\\AppData\\Roaming\\Python\\Python39\\site-packages\\transformers\\generation\\utils.py:1186: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation)\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = translate(\"Hello, my name is Jose. What is your name?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "f57e175b-c6e3-4645-941d-2d415295ff14",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'translation_text': 'Bonjour, mon nom est Jose, quel est votre nom ?'}]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "619c77f5-9eda-47a0-b721-2bf4efa9cec5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = translate(\"Hello, my name is Jose.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "10eea677-5ac7-477f-a67e-8f36c31e2acb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'translation_text': 'Bonjour, mon nom est Jose.'}]"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "fb287c23-f3e2-47e9-b92c-a0714937e0cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = translate(\"Hello, I am called Jose.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "ca2a23c2-eed8-45ef-9d96-208dc2370b34",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'translation_text': \"Bonjour, je m'appelle Jose.\"}]"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1b6b3d58-694b-4a0e-808d-84be7d5745ff",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}