{ "metadata": { "kernelspec": { "language": "python", "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python", "version": "3.7.12", "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py" }, "kaggle": { "accelerator": "gpu", "dataSources": [ { "sourceId": 9679350, "sourceType": "datasetVersion", "datasetId": 5916065 } ], "dockerImageVersionId": 30302, "isInternetEnabled": true, "language": "python", "sourceType": "notebook", "isGpuEnabled": true }, "colab": { "name": "Hack49-Training", "provenance": [] } }, "nbformat_minor": 0, "nbformat": 4, "cells": [ { "source": [ "# IMPORTANT: SOME KAGGLE DATA SOURCES ARE PRIVATE\n", "# RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES.\n", "import kagglehub\n", "kagglehub.login()\n" ], "metadata": { "id": "qWlTgQVSmgbJ" }, "cell_type": "code", "outputs": [], "execution_count": null }, { "source": [ "# IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES,\n", "# THEN FEEL FREE TO DELETE THIS CELL.\n", "# NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON\n", "# ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR\n", "# NOTEBOOK.\n", "\n", "datajediai_hack49_alzheimer_dataset_path = kagglehub.dataset_download('datajediai/hack49-alzheimer-dataset')\n", "\n", "print('Data source import complete.')\n" ], "metadata": { "id": "1uar05ngmgbJ" }, "cell_type": "code", "outputs": [], "execution_count": null }, { "cell_type": "code", "source": [ "pip install boto3" ], "metadata": { "execution": { "iopub.status.busy": "2024-10-21T05:31:24.709096Z", "iopub.execute_input": "2024-10-21T05:31:24.709572Z", "iopub.status.idle": "2024-10-21T05:31:37.120182Z", "shell.execute_reply.started": "2024-10-21T05:31:24.709539Z", "shell.execute_reply": "2024-10-21T05:31:37.118905Z" }, "trusted": true, "id": "Mm8ZSAxkmgbK", "outputId": "e4930bdb-3291-4ae3-ffb1-2cee2f31c0d8" }, "execution_count": null, "outputs": [ { "name": "stdout", "text": "Requirement already satisfied: boto3 in /opt/conda/lib/python3.7/site-packages (1.24.93)\nRequirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from boto3) (0.6.0)\nRequirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.7/site-packages (from boto3) (1.0.1)\nRequirement already satisfied: botocore<1.28.0,>=1.27.93 in /opt/conda/lib/python3.7/site-packages (from boto3) (1.27.93)\nRequirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/conda/lib/python3.7/site-packages (from botocore<1.28.0,>=1.27.93->boto3) (1.26.12)\nRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/lib/python3.7/site-packages (from botocore<1.28.0,>=1.27.93->boto3) (2.8.2)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.93->boto3) (1.15.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mNote: you may need to restart the kernel to use updated packages.\n", "output_type": "stream" } ] }, { "cell_type": "code", "source": [ "import os\n", "import pandas as pd\n", "import torch\n", "import torchaudio\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "# Custom collate function for DataLoader\n", "import torch.nn.functional as F\n", "from torch.utils.data import Dataset, DataLoader, random_split\n", "dataset_dir = '/kaggle/input/hack49-alzheimer-dataset/Hack49-Alzheimer-Dataset'\n", "# Custom Dataset class\n", "class HealthAudioDataset(Dataset):\n", " def __init__(self, root_dir, device='cpu'):\n", " self.root_dir = root_dir\n", " self.file_list = []\n", " self.labels = []\n", " self.dataframe = []\n", " self.device = device\n", " for label, subdir in enumerate(['Healthy', 'NotHealthy']):\n", " subdir_path = os.path.join(root_dir, subdir)\n", " for wav_file in os.listdir(subdir_path):\n", " if wav_file.endswith('.wav'):\n", " self.file_list.append(os.path.join(subdir_path, wav_file))\n", " self.labels.append(label)\n", " self.dataframe = pd.DataFrame({'file_path': self.file_list, 'label': self.labels})\n", "\n", " def __len__(self):\n", " return len(self.file_list)\n", "\n", " def __getitem__(self, idx):\n", " wav_path = self.file_list[idx]\n", " label = self.labels[idx]\n", " waveform, sample_rate = torchaudio.load(wav_path)\n", " waveform = waveform.to(self.device)\n", " if sample_rate != bundle.sample_rate:\n", " waveform = torchaudio.functional.resample(waveform, sample_rate, bundle.sample_rate)\n", " return waveform, label\n", "\n", " def get_dataframe(self):\n", " return self.dataframe\n", "\n", "\n", "class SelfAttention(nn.Module):\n", " def __init__(self, input_dim, num_heads):\n", " super(SelfAttention, self).__init__()\n", " self.multihead_attn = nn.MultiheadAttention(embed_dim=input_dim, num_heads=num_heads)\n", " self.layer_norm = nn.LayerNorm(input_dim)\n", "\n", " def forward(self, x):\n", " attn_output, _ = self.multihead_attn(x, x, x)\n", " x = x + attn_output # Add & Normalize\n", " x = self.layer_norm(x)\n", " return x\n", "\n", "class TimeSeriesClassifier(nn.Module):\n", " def __init__(self, input_dim, num_heads, hidden_dim, output_dim):\n", " super(TimeSeriesClassifier, self).__init__()\n", " self.self_attention = SelfAttention(input_dim, num_heads)\n", " self.fc1 = nn.Linear(input_dim, hidden_dim)\n", " self.fc2 = nn.Linear(hidden_dim, output_dim)\n", "\n", " def forward(self, x):\n", " # x: [batch_size, seq_len, input_dim]\n", " x = x.permute(1, 0, 2) # Change to [seq_len, batch_size, input_dim]\n", " x = self.self_attention(x)\n", " x = x.permute(1, 0, 2) # Change back to [batch_size, seq_len, input_dim]\n", " x = torch.mean(x, dim=1) # Global Average Pooling over the time dimension\n", " x = torch.relu(self.fc1(x))\n", " x = torch.sigmoid(self.fc2(x)) # Sigmoid for binary classification\n", " return x\n", "\n", "\n", "# Define the combined Encoder-Decoder model\n", "class EncoderDecoder(nn.Module):\n", " def __init__(self, encoder, decoder):\n", " super(EncoderDecoder, self).__init__()\n", " self.encoder = encoder\n", " self.decoder = decoder\n", " def forward(self, x):\n", " emission, _ = self.encoder(x)\n", " x = self.decoder(emission)\n", " return x\n", "\n", "\n", "\n", "\n", "def collate_fn(batch):\n", " waveforms = [item[0] for item in batch]\n", " labels = [item[1] for item in batch]\n", "\n", " # Find the length of the longest waveform in the batch\n", " max_length = max(waveform.size(1) for waveform in waveforms)\n", "\n", " # Pad all waveforms to the length of the longest waveform\n", " padded_waveforms = [F.pad(waveform, (0, max_length - waveform.size(1))) for waveform in waveforms]\n", " padded_waveforms = torch.stack(padded_waveforms)\n", "\n", " labels = torch.tensor(labels)\n", " return padded_waveforms, labels\n", "\n", "\n", "\n", "\n" ], "metadata": { "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "execution": { "iopub.status.busy": "2024-10-21T05:31:37.122433Z", "iopub.execute_input": "2024-10-21T05:31:37.122772Z", "iopub.status.idle": "2024-10-21T05:31:37.149038Z", "shell.execute_reply.started": "2024-10-21T05:31:37.122737Z", "shell.execute_reply": "2024-10-21T05:31:37.148219Z" }, "trusted": true, "id": "slo_6EJUmgbK" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# Initialize everything\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "bundle = torchaudio.pipelines.WAV2VEC2_ASR_BASE_960H\n", "encoder = bundle.get_model().to(device)\n", "\n", "# Example usage\n", "batch_size = 32\n", "seq_len = 10 # Number of time steps\n", "input_dim = 29\n", "hidden_dim = 15\n", "# Creating a dummy input tensor with shape [batch_size, seq_len, input_dim]\n", "dummy_input = torch.randn(batch_size, seq_len, input_dim)\n", "\n", "decoder = TimeSeriesClassifier(input_dim = input_dim,\n", " num_heads = input_dim,\n", " hidden_dim = hidden_dim,\n", " output_dim = 1)\n", "output = decoder(dummy_input)\n", "print(output)\n", "decoder = decoder.to(device)\n", "model = EncoderDecoder(encoder, decoder).to(device)\n", "\n", "# Freeze the encoder parameters\n", "for param in model.encoder.parameters():\n", " param.requires_grad = False\n", "\n", "# Dataset and DataLoaders\n", "\n", "dataset = HealthAudioDataset(dataset_dir, device)\n", "train_size = int(0.8 * len(dataset))\n", "test_size = len(dataset) - train_size\n", "train_dataset, test_dataset = random_split(dataset, [train_size, test_size])\n", "\n", "train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True, collate_fn=collate_fn)\n", "test_loader = DataLoader(test_dataset, batch_size=2, shuffle=True, collate_fn=collate_fn)\n", "\n", "# Print the contents of the train_loader" ], "metadata": { "execution": { "iopub.status.busy": "2024-10-21T05:31:37.150151Z", "iopub.execute_input": "2024-10-21T05:31:37.15093Z", "iopub.status.idle": "2024-10-21T05:31:38.413704Z", "shell.execute_reply.started": "2024-10-21T05:31:37.150901Z", "shell.execute_reply": "2024-10-21T05:31:38.412655Z" }, "trusted": true, "id": "__Tb1u1wmgbK", "outputId": "d6b97111-e800-40db-8873-fff376726885" }, "execution_count": null, "outputs": [ { "name": "stdout", "text": "tensor([[0.5726],\n [0.5791],\n [0.5756],\n [0.5718],\n [0.6160],\n [0.5886],\n [0.5634],\n [0.5896],\n [0.5754],\n [0.6070],\n [0.5597],\n [0.5673],\n [0.5942],\n [0.5967],\n [0.5815],\n [0.5582],\n [0.5873],\n [0.6225],\n [0.5830],\n [0.5879],\n [0.5958],\n [0.5540],\n [0.5812],\n [0.6039],\n [0.5825],\n [0.5635],\n [0.6145],\n [0.5483],\n [0.5770],\n [0.5952],\n [0.6086],\n [0.5806]], grad_fn=)\n", "output_type": "stream" } ] }, { "cell_type": "code", "source": [ "# Loss and optimizer\n", "criterion = nn.BCELoss()\n", "optimizer = optim.Adam(model.parameters(), lr=1e-4)\n", "\n", "def verification(model):\n", " # Testing script\n", " test_loss = 0.0\n", " with torch.no_grad():\n", " for i, (inputs_batch, labels_batch) in enumerate(test_loader):\n", " for batch_idx in range(inputs_batch.size(0)):\n", " inputs, labels = inputs_batch[batch_idx], labels_batch[batch_idx]\n", " inputs = inputs.to(device)\n", " labels = labels.to(device).float().unsqueeze(0) # Convert to float and add batch dimension\n", " outputs = model(inputs)\n", " labels = labels.view(outputs.shape) # Ensure labels match the shape of outputs\n", " loss = criterion(outputs, labels)\n", " test_loss += loss.item()\n", " ret = test_loss / len(test_loader)\n", " print(f'Test Loss: {ret:.4f}')\n", " return ret\n", "verification(model)\n", "# save the model\n" ], "metadata": { "execution": { "iopub.status.busy": "2024-10-21T05:31:38.416523Z", "iopub.execute_input": "2024-10-21T05:31:38.417174Z", "iopub.status.idle": "2024-10-21T05:31:39.180667Z", "shell.execute_reply.started": "2024-10-21T05:31:38.417142Z", "shell.execute_reply": "2024-10-21T05:31:39.179716Z" }, "trusted": true, "id": "fXi2w6_dmgbL", "outputId": "de44529f-7406-42b4-ec62-c01f85b4c675" }, "execution_count": null, "outputs": [ { "name": "stdout", "text": "Test Loss: 1.6243\n", "output_type": "stream" }, { "execution_count": 99, "output_type": "execute_result", "data": { "text/plain": "1.6242794593175252" }, "metadata": {} } ] }, { "cell_type": "code", "source": [ "print(f\"Training started... Using {device}\")\n", "num_epochs = 0\n", "while verification(model) > 0.10:\n", " model.train()\n", " running_loss = 0.0\n", " for i, (inputs_batch, labels_batch) in enumerate(train_loader):\n", "# print(f'Batch {i + 1}:')\n", " for batch_idx in range(inputs_batch.size(0)):\n", " inputs, labels = inputs_batch[batch_idx], labels_batch[batch_idx]\n", " inputs = inputs.to(device)\n", " labels = labels.to(device).float().unsqueeze(0) # Convert to float and add batch dimension\n", "\n", " # print(f' Input {batch_idx + 1}: {inputs.size()}, Label: {labels.size()}')\n", " # print(f' Input data type: {inputs.dtype}, Label data type: {labels.dtype}')\n", "\n", " optimizer.zero_grad()\n", " outputs = model(inputs)\n", " # print(f' Output: {outputs.size()}')\n", " labels = labels.view(outputs.shape) # Ensure labels match the shape of outputs\n", " loss = criterion(outputs, labels)\n", " loss.backward()\n", " optimizer.step()\n", " running_loss += loss.item()\n", "\n", "\n", " print(f'Epoch [{epoch + 1}/{num_epochs}], Batch [{i + 1}], Loss: {running_loss / 100:.4f}')\n", " epoch = epoch + 1\n", "\n", "print(\"Training completed.\")\n", "# now generate the testing script\n", "\n" ], "metadata": { "execution": { "iopub.status.busy": "2024-10-21T05:39:17.441427Z", "iopub.execute_input": "2024-10-21T05:39:17.441812Z", "iopub.status.idle": "2024-10-21T05:39:59.551312Z", "shell.execute_reply.started": "2024-10-21T05:39:17.44178Z", "shell.execute_reply": "2024-10-21T05:39:59.550324Z" }, "trusted": true, "id": "BRtV6wDfmgbL", "outputId": "93af8a83-48c1-41b2-e1f3-b27afa75c285" }, "execution_count": null, "outputs": [ { "name": "stdout", "text": "Training started... Using cuda\nTest Loss: 0.1040\nEpoch [283/0], Batch [12], Loss: 0.0445\nTest Loss: 0.1708\nEpoch [284/0], Batch [12], Loss: 0.0302\nTest Loss: 0.1152\nEpoch [285/0], Batch [12], Loss: 0.0298\nTest Loss: 0.1269\nEpoch [286/0], Batch [12], Loss: 0.0319\nTest Loss: 0.1675\nEpoch [287/0], Batch [12], Loss: 0.0276\nTest Loss: 0.1064\nEpoch [288/0], Batch [12], Loss: 0.0250\nTest Loss: 0.1328\nEpoch [289/0], Batch [12], Loss: 0.0220\nTest Loss: 0.1303\nEpoch [290/0], Batch [12], Loss: 0.0219\nTest Loss: 0.1164\nEpoch [291/0], Batch [12], Loss: 0.0200\nTest Loss: 0.1139\nEpoch [292/0], Batch [12], Loss: 0.0342\nTest Loss: 0.1391\nEpoch [293/0], Batch [12], Loss: 0.0213\nTest Loss: 0.1166\nEpoch [294/0], Batch [12], Loss: 0.0271\nTest Loss: 0.0904\nTraining completed.\n", "output_type": "stream" } ] }, { "cell_type": "code", "source": [], "metadata": { "trusted": true, "id": "FU8OK0LumgbL" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "model" ], "metadata": { "execution": { "iopub.status.busy": "2024-10-21T05:39:12.138603Z", "iopub.status.idle": "2024-10-21T05:39:12.138955Z", "shell.execute_reply.started": "2024-10-21T05:39:12.138776Z", "shell.execute_reply": "2024-10-21T05:39:12.138793Z" }, "trusted": true, "id": "jIbFxLW-mgbM" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "torch.save(model.state_dict(), 'hack49_encoder_decoder_model.pth')\n", "import boto3\n", "\n", "def upload_to_s3(file_path, bucket_name, object_name, access_key, secret_key):\n", " # Initialize a session using your AWS credentials\n", " s3_client = boto3.client('s3',\n", " region_name='us-east-2',\n", " aws_access_key_id=access_key,\n", " aws_secret_access_key=secret_key)\n", "\n", " try:\n", " # Uploads the given file using a managed uploader\n", " s3_client.upload_file(file_path, bucket_name, object_name)\n", " print(f'Successfully uploaded {file_path} to {bucket_name}/{object_name}')\n", " except Exception as e:\n", " print(f'Error uploading file: {e}')\n", "\n", "# Example usage\n", "file_path = 'hack49_encoder_decoder_model.pth'\n", "bucket_name = 'my-ai-models-darcy'\n", "# get time\n", "import datetime\n", "now = datetime.datetime.now()\n", "object_name = f'hack49_encoder_decoder_model_{now.strftime(\"%Y-%m-%d_%H-%M-%S\")}.pth'\n", "access_key = 'XXXX'\n", "secret_key = 'XXXX'\n", "\n", "upload_to_s3(file_path, bucket_name, object_name, access_key, secret_key)\n" ], "metadata": { "execution": { "iopub.status.busy": "2024-10-21T05:40:04.454771Z", "iopub.execute_input": "2024-10-21T05:40:04.455706Z", "iopub.status.idle": "2024-10-21T05:40:09.278691Z", "shell.execute_reply.started": "2024-10-21T05:40:04.455668Z", "shell.execute_reply": "2024-10-21T05:40:09.277704Z" }, "trusted": true, "id": "zDZBAM2omgbM", "outputId": "ec41c422-578a-4a27-bfbb-c5a3d569744a" }, "execution_count": null, "outputs": [ { "name": "stdout", "text": "Successfully uploaded hack49_encoder_decoder_model.pth to my-ai-models-darcy/hack49_encoder_decoder_model_2024-10-21_05-40-05.pth\n", "output_type": "stream" } ] } ] }