CIFAR10_equalization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "95a3e37d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "#from prototype import prototype\n",
    "#import scipy.spatial.distance as ssd\n",
    "#from dissimilarity import dissimilarity\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pickle\n",
    "def makefile(what,filename):\n",
    "    with open(filename,\"wb\") as f3:\n",
    "        pickle.dump(what,f3)\n",
    "\n",
    "def readfile(filename):\n",
    "    with open(filename,\"rb\") as f4:\n",
    "        ans=pickle.load(f4)\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4962e8d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.datasets import fashion_mnist\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7d2f54fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "train=cifar10.load_data()[0][0]\n",
    "train_label=cifar10.load_data()[0][1]\n",
    "x_test=cifar10.load_data()[1][0]\n",
    "y_test=cifar10.load_data()[1][1]\n",
    "y_test=y_test.reshape(len(y_test))\n",
    "train_label=train_label.reshape(len(train_label))\n",
    "C=3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa3e16a9",
   "metadata": {},
   "source": [
    "Defined histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cedc1af4",
   "metadata": {},
   "outputs": [],
   "source": [
    "for value in range(8):\n",
    "    entropy=value+1\n",
    "    area=2**entropy\n",
    "    Q=int(train.shape[1]*train.shape[2]/area)\n",
    "    mod=train.shape[1]*train.shape[2]%area\n",
    "    pixel_set=[]\n",
    "    for n in range(area):\n",
    "        if n < mod:\n",
    "            for i in range(Q+1):\n",
    "                pixel_set.append(n)\n",
    "        else:\n",
    "            for i in range(Q):\n",
    "                pixel_set.append(n)\n",
    "    equalized=np.zeros([train.shape[0],train.shape[1]*train.shape[2],C])\n",
    "    a=train.reshape(-1,train.shape[1]*train.shape[2],C)\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set)):\n",
    "        equalized[ind2==i]=int(pixel_set[i])\n",
    "    equalized=equalized.reshape(train.shape[0],train.shape[1],train.shape[2],C)\n",
    "    equalized_test=np.zeros([x_test.shape[0],x_test.shape[1]*x_test.shape[2],C])\n",
    "    a=x_test.reshape(-1,x_test.shape[1]*x_test.shape[2],C)\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set)):\n",
    "        equalized_test[ind2==i]=int(pixel_set[i])\n",
    "    equalized_test=equalized_test.reshape(x_test.shape[0],x_test.shape[1],x_test.shape[2],C)\n",
    "    makefile(equalized,\"Data/CIFAR10_edit/x_train\"+str(entropy)+\".pkl\")\n",
    "    makefile(equalized_test,\"Data/CIFAR10_edit/x_test\"+str(entropy)+\".pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dba42e6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "72f03ab8",
   "metadata": {},
   "source": [
    "Average CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "733a610c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train=cifar10.load_data()[0][0]\n",
    "train_label=cifar10.load_data()[0][1]\n",
    "x_test=cifar10.load_data()[1][0]\n",
    "y_test=cifar10.load_data()[1][1]\n",
    "y_test=y_test.reshape(len(y_test))\n",
    "train_label=train_label.reshape(len(train_label))\n",
    "C=3\n",
    "for label in range(10):\n",
    "    print(label)\n",
    "    x_train=train[train_label==label]\n",
    "    pixel_set_R=[]\n",
    "    pixel_set_G=[]\n",
    "    pixel_set_B=[]\n",
    "    for c in range(C):\n",
    "        pixel_list=dict()\n",
    "        for n in range(256):\n",
    "            pixel_list[n]=0\n",
    "        for data in x_train:\n",
    "            for raw in data:\n",
    "                for pixel in raw:\n",
    "                    pixel_list[pixel[c]]+=1\n",
    "        hist_list=dict()\n",
    "        for num in pixel_list:\n",
    "            hist_list[num]=pixel_list[num]/len(x_train)\n",
    "        hist_num=dict()\n",
    "        residual=0\n",
    "        for num in hist_list:\n",
    "            hist_num[num]=int(np.round(hist_list[num]+residual,0))\n",
    "            residual=hist_list[num]+residual-int(np.round(hist_list[num]+residual,0))\n",
    "\n",
    "        if c==0:\n",
    "            for num in hist_num:\n",
    "                repeat=hist_num[num]\n",
    "                for n in range(repeat):\n",
    "                    pixel_set_R.append(num)\n",
    "        elif c==1:\n",
    "            for num in hist_num:\n",
    "                repeat=hist_num[num]\n",
    "                for n in range(repeat):\n",
    "                    pixel_set_G.append(num)\n",
    "        else:\n",
    "            for num in hist_num:\n",
    "                repeat=hist_num[num]\n",
    "                for n in range(repeat):\n",
    "                    pixel_set_B.append(num)\n",
    "    red=np.zeros([train.shape[0],train.shape[1]*train.shape[2]])\n",
    "    green=np.zeros([train.shape[0],train.shape[1]*train.shape[2]])\n",
    "    blue=np.zeros([train.shape[0],train.shape[1]*train.shape[2]])\n",
    "\n",
    "    a=train[:,:,:,0].reshape(-1,train.shape[1]*train.shape[2])\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set_R)):\n",
    "        red[ind2==i]=int(pixel_set_R[i])\n",
    "\n",
    "    a=train[:,:,:,1].reshape(-1,train.shape[1]*train.shape[2])\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set_G)):\n",
    "        green[ind2==i]=int(pixel_set_G[i])\n",
    "    a=train[:,:,:,2].reshape(-1,train.shape[1]*train.shape[2])\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set_B)):\n",
    "        blue[ind2==i]=int(pixel_set_B[i])\n",
    "    equalized=np.concatenate([red.reshape(train.shape[0],train.shape[1],train.shape[2],1),green.reshape(train.shape[0],train.shape[1],train.shape[2],1),blue.reshape(train.shape[0],train.shape[1],train.shape[2],1)], axis=3)\n",
    "\n",
    "    makefile(equalized,\"Data/CIFAR10_edit/x_train_average_\"+str(label)+\".pkl\")\n",
    "    red=np.zeros([x_test.shape[0],x_test.shape[1]*x_test.shape[2]])\n",
    "    green=np.zeros([x_test.shape[0],x_test.shape[1]*x_test.shape[2]])\n",
    "    blue=np.zeros([x_test.shape[0],x_test.shape[1]*x_test.shape[2]])\n",
    "\n",
    "    a=x_test[:,:,:,0].reshape(-1,x_test.shape[1]*x_test.shape[2])\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set_R)):\n",
    "        red[ind2==i]=int(pixel_set_R[i])\n",
    "\n",
    "    a=x_test[:,:,:,1].reshape(-1,x_test.shape[1]*x_test.shape[2])\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set_G)):\n",
    "        green[ind2==i]=int(pixel_set_G[i])\n",
    "    a=x_test[:,:,:,2].reshape(-1,x_test.shape[1]*x_test.shape[2])\n",
    "    ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "    ind2=np.argsort(ind,axis=1)\n",
    "    for i in range(len(pixel_set_B)):\n",
    "        blue[ind2==i]=int(pixel_set_B[i])\n",
    "    equalized_test=np.concatenate([red.reshape(x_test.shape[0],x_test.shape[1],x_test.shape[2],1),green.reshape(x_test.shape[0],x_test.shape[1],x_test.shape[2],1),blue.reshape(x_test.shape[0],x_test.shape[1],x_test.shape[2],1)], axis=3)\n",
    "    makefile(equalized_test,\"Data/CIFAR10_edit/x_test_average_\"+str(label)+\".pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09b223bf",
   "metadata": {},
   "source": [
    "Average all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a08e0ccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "train=cifar10.load_data()[0][0]\n",
    "train_label=cifar10.load_data()[0][1]\n",
    "x_test=cifar10.load_data()[1][0]\n",
    "y_test=cifar10.load_data()[1][1]\n",
    "y_test=y_test.reshape(len(y_test))\n",
    "train_label=train_label.reshape(len(train_label))\n",
    "C=3\n",
    "\n",
    "x_train=train\n",
    "pixel_set_R=[]\n",
    "pixel_set_G=[]\n",
    "pixel_set_B=[]\n",
    "for c in range(C):\n",
    "    pixel_list=dict()\n",
    "    for n in range(256):\n",
    "        pixel_list[n]=0\n",
    "    for data in x_train:\n",
    "        for raw in data:\n",
    "            for pixel in raw:\n",
    "                pixel_list[pixel[c]]+=1\n",
    "    hist_list=dict()\n",
    "    for num in pixel_list:\n",
    "        hist_list[num]=pixel_list[num]/len(x_train)\n",
    "    hist_num=dict()\n",
    "    residual=0\n",
    "    for num in hist_list:\n",
    "        hist_num[num]=int(np.round(hist_list[num]+residual,0))\n",
    "        residual=hist_list[num]+residual-int(np.round(hist_list[num]+residual,0))\n",
    "\n",
    "    if c==0:\n",
    "        for num in hist_num:\n",
    "            repeat=hist_num[num]\n",
    "            for n in range(repeat):\n",
    "                pixel_set_R.append(num)\n",
    "    elif c==1:\n",
    "        for num in hist_num:\n",
    "            repeat=hist_num[num]\n",
    "            for n in range(repeat):\n",
    "                pixel_set_G.append(num)\n",
    "    else:\n",
    "        for num in hist_num:\n",
    "            repeat=hist_num[num]\n",
    "            for n in range(repeat):\n",
    "                pixel_set_B.append(num)\n",
    "red=np.zeros([train.shape[0],train.shape[1]*train.shape[2]])\n",
    "green=np.zeros([train.shape[0],train.shape[1]*train.shape[2]])\n",
    "blue=np.zeros([train.shape[0],train.shape[1]*train.shape[2]])\n",
    "\n",
    "a=train[:,:,:,0].reshape(-1,train.shape[1]*train.shape[2])\n",
    "ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "ind2=np.argsort(ind,axis=1)\n",
    "for i in range(len(pixel_set_R)):\n",
    "    red[ind2==i]=int(pixel_set_R[i])\n",
    "\n",
    "a=train[:,:,:,1].reshape(-1,train.shape[1]*train.shape[2])\n",
    "ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "ind2=np.argsort(ind,axis=1)\n",
    "for i in range(len(pixel_set_G)):\n",
    "    green[ind2==i]=int(pixel_set_G[i])\n",
    "a=train[:,:,:,2].reshape(-1,train.shape[1]*train.shape[2])\n",
    "ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "ind2=np.argsort(ind,axis=1)\n",
    "for i in range(len(pixel_set_B)):\n",
    "    blue[ind2==i]=int(pixel_set_B[i])\n",
    "equalized=np.concatenate([red.reshape(train.shape[0],train.shape[1],train.shape[2],1),green.reshape(train.shape[0],train.shape[1],train.shape[2],1),blue.reshape(train.shape[0],train.shape[1],train.shape[2],1)], axis=3)\n",
    "\n",
    "makefile(equalized,\"Data/CIFAR10_edit/x_train_average_all.pkl\")\n",
    "red=np.zeros([x_test.shape[0],x_test.shape[1]*x_test.shape[2]])\n",
    "green=np.zeros([x_test.shape[0],x_test.shape[1]*x_test.shape[2]])\n",
    "blue=np.zeros([x_test.shape[0],x_test.shape[1]*x_test.shape[2]])\n",
    "\n",
    "a=x_test[:,:,:,0].reshape(-1,x_test.shape[1]*x_test.shape[2])\n",
    "ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "ind2=np.argsort(ind,axis=1)\n",
    "for i in range(len(pixel_set_R)):\n",
    "    red[ind2==i]=int(pixel_set_R[i])\n",
    "\n",
    "a=x_test[:,:,:,1].reshape(-1,x_test.shape[1]*x_test.shape[2])\n",
    "ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "ind2=np.argsort(ind,axis=1)\n",
    "for i in range(len(pixel_set_G)):\n",
    "    green[ind2==i]=int(pixel_set_G[i])\n",
    "a=x_test[:,:,:,2].reshape(-1,x_test.shape[1]*x_test.shape[2])\n",
    "ind=np.argsort(a,axis=1,kind=\"mergesort\")\n",
    "ind2=np.argsort(ind,axis=1)\n",
    "for i in range(len(pixel_set_B)):\n",
    "    blue[ind2==i]=int(pixel_set_B[i])\n",
    "equalized_test=np.concatenate([red.reshape(x_test.shape[0],x_test.shape[1],x_test.shape[2],1),green.reshape(x_test.shape[0],x_test.shape[1],x_test.shape[2],1),blue.reshape(x_test.shape[0],x_test.shape[1],x_test.shape[2],1)], axis=3)\n",
    "makefile(equalized_test,\"Data/CIFAR10_edit/x_test_average_all.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b194dee",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}