1. ML Overview.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"width: 100%; overflow: hidden;\">\n",
    "    <div style=\"width: 150px; float: left;\"> <img src=\"data/D4Sci_logo_ball.png\" alt=\"Data For Science, Inc\" align=\"left\" border=\"0\"> </div>\n",
    "    <div style=\"float: left; margin-left: 10px;\"> <h1>Machine Learning with PyTorch for Developers</h1>\n",
    "<h1>Machine Learning Overview</h1>\n",
    "        <p>Bruno Gonçalves<br/>\n",
    "        <a href=\"http://www.data4sci.com/\">www.data4sci.com</a><br/>\n",
    "            @bgoncalves, @data4sci</p></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from pprint import pprint\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import watermark\n",
    "\n",
    "%load_ext watermark\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We start by print out the versions of the libraries we're using for future reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.13.2\n",
      "IPython version      : 9.0.0\n",
      "\n",
      "Compiler    : Clang 16.0.0 (clang-1600.0.26.6)\n",
      "OS          : Darwin\n",
      "Release     : 24.3.0\n",
      "Machine     : arm64\n",
      "Processor   : arm\n",
      "CPU cores   : 16\n",
      "Architecture: 64bit\n",
      "\n",
      "Git hash: ad15a2b1c0a4ef5f594dd15d8bc57ae64aca5725\n",
      "\n",
      "pandas    : 2.2.3\n",
      "torch     : 2.6.0\n",
      "numpy     : 2.2.3\n",
      "matplotlib: 3.10.1\n",
      "watermark : 2.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%watermark -n -v -m -g -iv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load default figure style"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('d4sci.mplstyle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creation\n",
    "Tensors can be created in a number of ways:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From Python list\n",
    "list_tensor = torch.tensor([[1, 2], [3, 4]])\n",
    "\n",
    "# Factory methods\n",
    "range_tensor = torch.arange(0, 10, step=2)\n",
    "linspace_tensor = torch.linspace(0, 10, steps=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also create tensors directly from numpy arrays:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "numpy_tensor = torch.from_numpy(np.array([[1, 2], [3, 4]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, `torch.from_numpy` shares memory with the underlying NumPy array, so in-place operations on one side will affect the other (if they are on the CPU)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Arithmetic and Indexing\n",
    "\n",
    "Arithmetic operations are as straightforward as in numpy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Element-wise addition:  tensor([4., 4., 4.])\n",
      "Element-wise subtraction:  tensor([-2.,  0.,  2.])\n",
      "Element-wise multiplication:  tensor([3., 4., 3.])\n",
      "Element-wise division:  tensor([0.3333, 1.0000, 3.0000])\n",
      "Matrix multiplication:  tensor(10.)\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([1.0, 2.0, 3.0])\n",
    "y = torch.tensor([3.0, 2.0, 1.0])\n",
    "\n",
    "sum_xy = x + y \n",
    "print(\"Element-wise addition: \", sum_xy)\n",
    "\n",
    "diff_xy = x - y\n",
    "print(\"Element-wise subtraction: \", diff_xy)\n",
    "\n",
    "mul_xy = x * y\n",
    "print(\"Element-wise multiplication: \", mul_xy)\n",
    "\n",
    "div_xy = x / y\n",
    "print(\"Element-wise division: \", div_xy)\n",
    "\n",
    "mm_xy = torch.matmul(x, y)\n",
    "print(\"Matrix multiplication: \", mm_xy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As as indexing and slicing. Let us start by defining a 2D tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = torch.tensor([[1, 2, 3], \n",
    "                  [4, 5, 6]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can easily access the first element"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1)\n"
     ]
    }
   ],
   "source": [
    "first_element = M[0, 0]\n",
    "print(first_element)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As well as any subset of elements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1, 2, 3])\n"
     ]
    }
   ],
   "source": [
    "first_row = M[0, :]\n",
    "print(first_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([3, 6])\n"
     ]
    }
   ],
   "source": [
    "last_column = M[:, -1]\n",
    "print(last_column)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transpose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original:\n",
      "tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n",
      "\n",
      "Transpose:\n",
      "tensor([[1, 4],\n",
      "        [2, 5],\n",
      "        [3, 6]])\n"
     ]
    }
   ],
   "source": [
    "print(f\"Original:\\n{M}\\n\\nTranspose:\\n{M.t()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reshaping\n",
    "\n",
    "Reshaping can be done as a `view` (for contiguous storage) or a `reshape` (generic) operations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M:tensor([[1, 2, 3],\n",
      "        [4, 5, 6]])\n",
      "\n",
      "M_flat:tensor([1, 2, 3, 4, 5, 6])\n",
      "\n",
      "M_reshaped:tensor([[1, 2],\n",
      "        [3, 4],\n",
      "        [5, 6]])\n",
      "\n",
      "M_reshaped_alt:tensor([[1, 2],\n",
      "        [3, 4],\n",
      "        [5, 6]])\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(f\"M:{M}\\n\")\n",
    "\n",
    "M_flat = M.view(-1)\n",
    "print(f\"M_flat:{M_flat}\\n\")\n",
    "\n",
    "M_reshaped = M.view(3, 2)\n",
    "print(f\"M_reshaped:{M_reshaped}\\n\")\n",
    "\n",
    "M_reshaped_alt = M.reshape(3, 2)\n",
    "print(f\"M_reshaped_alt:{M_reshaped_alt}\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Algebra"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us defined some toy matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32)\n",
    "B = torch.tensor([[5, 6], [7, 8]], dtype=torch.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Matrix multiplication comes in two flavors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=tensor([[19., 22.],\n",
      "        [43., 50.]])\n",
      "\n",
      "C_alt=tensor([[19., 22.],\n",
      "        [43., 50.]])\n"
     ]
    }
   ],
   "source": [
    "C = A @ B\n",
    "print(f\"C={C}\")\n",
    "\n",
    "C_alt = torch.matmul(A, B)\n",
    "print(f\"\\nC_alt={C_alt}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Determinant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(-2.)\n"
     ]
    }
   ],
   "source": [
    "det_A = torch.linalg.det(A)\n",
    "print(det_A)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "inv_A = torch.linalg.inv(A)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Eigenvalues and singular values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eigenvalues: tensor([-0.3723+0.j,  5.3723+0.j])\n",
      "Eigenvectors: tensor([[-0.8246+0.j, -0.4160+0.j],\n",
      "        [ 0.5658+0.j, -0.9094+0.j]])\n",
      "\n",
      "U: tensor([[-0.4046, -0.9145],\n",
      "        [-0.9145,  0.4046]])\n",
      "S: tensor([5.4650, 0.3660])\n",
      "Wt: tensor([[-0.5760, -0.8174],\n",
      "        [ 0.8174, -0.5760]])\n"
     ]
    }
   ],
   "source": [
    "eigenvals, eigenvecs = torch.linalg.eig(A)\n",
    "print(f\"Eigenvalues: {eigenvals}\")\n",
    "print(f\"Eigenvectors: {eigenvecs}\")\n",
    "\n",
    "U, S, Wt = torch.linalg.svd(A)\n",
    "print(f\"\\nU: {U}\\nS: {S}\\nWt: {Wt}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch Operations\n",
    "\n",
    "PyTorch’s linear algebra routines have the native ability to batch operations. When you provide a 3D (or higher-dimensional) Tensor, PyTorch will treat the first dimention as being the batch and perform the operation across each sub-matrix.\n",
    "\n",
    "If A is of shape (batch_size, m, n) and B is of shape (batch_size, n, p), then A @ B yields a Tensor of shape (batch_size, m, p)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C_batched: tensor([[[-2.9937, -0.1721,  6.6884],\n",
      "         [ 2.6640,  0.7627, -0.5547],\n",
      "         [ 0.8741,  3.4144,  0.7436],\n",
      "         [-1.3956, -0.9773,  5.6701],\n",
      "         [ 0.0518, -3.9272,  5.9910]],\n",
      "\n",
      "        [[ 0.1881,  7.0937,  1.1628],\n",
      "         [ 2.8885,  3.7015,  1.4655],\n",
      "         [-2.5885, -0.6995,  0.3287],\n",
      "         [ 0.2270,  0.1224,  3.6411],\n",
      "         [-0.6234,  4.0266, -3.3933]],\n",
      "\n",
      "        [[ 3.1847,  2.3944, -0.4251],\n",
      "         [ 1.0032, -0.2744, -0.2651],\n",
      "         [-1.5299, -4.4197,  0.3445],\n",
      "         [-2.0277, -0.5207, -0.1652],\n",
      "         [-1.2571, -1.6274, -2.8720]]])\n"
     ]
    }
   ],
   "source": [
    "A_batched = torch.randn((3, 5, 7)) # 3 batches of 5x7 matrices\n",
    "B_batched = torch.randn((3, 7, 3)) # 3 batches of 7x3 matrices\n",
    "C_batched = A_batched @ B_batched\n",
    "print(f\"C_batched: {C_batched}\") # 3 batches of 5x3 matrices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hardware acceleration\n",
    "\n",
    "PyTorch supports GPUs and MPS devices straight out of the box:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"mps\" if torch.backends.mps.is_available() else \"cpu\") # Uncomment this line if you have a MPS device\n",
    "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # Uncomment this line if you have a GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='mps')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tensors can be created directly within a device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_mps = torch.randn((1000, 1000), device=device)\n",
    "y_mps = torch.randn((1000, 1000), device=device)\n",
    "z_mps = x_mps @ y_mps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confirm that we used the right device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='mps', index=0)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z_mps.device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Or transfered between devices:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "z_cpu = z_mps.to(\"cpu\") # Move tensor to CPU\n",
    "print(z_cpu.device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Automatic Differentiation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We start by defining a matrix and a vector with grad tracking enabled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = torch.tensor([[3.0, 1.0],\n",
    "                  [2.0, 4.0]], requires_grad=True)\n",
    "x = torch.tensor([5.0, 6.0], requires_grad=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perform some operations tand obtain a scalar output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(55., grad_fn=<SumBackward0>)\n"
     ]
    }
   ],
   "source": [
    "y = A @ x             \n",
    "z = y.sum()\n",
    "print(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "z.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The gradient with respect to each of the variables is stored within those variables:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gradient of z with respect to x: tensor([5., 5.])\n",
      "\n",
      "Gradient of z with respect to A: tensor([[5., 6.],\n",
      "        [5., 6.]])\n"
     ]
    }
   ],
   "source": [
    "print(f\"\\nGradient of z with respect to x: {x.grad}\")\n",
    "print(f\"\\nGradient of z with respect to A: {A.grad}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center>\n",
    "     <img src=\"data/D4Sci_logo_full.png\" alt=\"Data For Science, Inc\" border=\"0\" width=300px> \n",
    "</center>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}