Creating_HF_README.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e3720c49-df16-47c1-be1d-09fb92cc6eac",
   "metadata": {},
   "source": [
    "### Notebook to check the Licenses of all Zenodo Records in our Cache and upload a ReadMe to Huggingface with the Corresponding Licencse Terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3fd869f3-38b0-48f9-97f2-84a7bb1b0c5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "def get_zenodo_license(record_id):\n",
    "    url = f\"https://zenodo.org/api/records/{record_id}\"\n",
    "    response = requests.get(url)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        data = response.json()\n",
    "        license_info = data.get(\"metadata\", {}).get(\"license\", {})\n",
    "        \n",
    "        if license_info:\n",
    "            license_id = license_info.get(\"id\", \"No ID available\")\n",
    "            return license_id\n",
    "        else:\n",
    "            return \"No license information found.\"\n",
    "    else:\n",
    "        return f\"Error: Unable to fetch data (Status Code: {response.status_code})\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3469424-3654-4b70-b89f-c09c897858b1",
   "metadata": {},
   "source": [
    "## Load the Data\n",
    "Load the DataSet from Huggingface and convert it to a pandas data frame (automatically done by the function)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0bf219e0-2c02-4125-8bd9-a75a4865c4d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b2e0ad51398457085fa7ba93b4fef2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/5.78k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Error during conversion: ChunkedEncodingError(ProtocolError(\"Connection broken: InvalidChunkLength(got length b'', 0 bytes read)\", InvalidChunkLength(got length b'', 0 bytes read)))\n",
      "Repo card metadata block was not found. Setting CardData to empty.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "913364a7412346c99f37bdbaf8dca645",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train-00000-of-00001.parquet:   0%|          | 0.00/34.1M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c332cd5ae29e4956a0eb312058368881",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split:   0%|          | 0/2617 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from caching import load_full_hf_cache\n",
    "import pandas as pd\n",
    "\n",
    "repo_name = \"ScaDS-AI/SlightInsight_Cache\"\n",
    "\n",
    "df = load_full_hf_cache(repo_name=repo_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cb5b2087-9451-43d7-891a-369299362016",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>zenodo_record_id</th>\n",
       "      <th>zenodo_filename</th>\n",
       "      <th>page_number</th>\n",
       "      <th>text</th>\n",
       "      <th>visual</th>\n",
       "      <th>mixed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>record10008464_pdf1_slide1</td>\n",
       "      <td>10008464</td>\n",
       "      <td>2023-Moore-N4BI-AHM-Welcome.pdf</td>\n",
       "      <td>1</td>\n",
       "      <td>[0.8054575, -0.4204579, 0.111390926, 0.2636367...</td>\n",
       "      <td>[0.22024887800216675, 0.6564452648162842, 0.02...</td>\n",
       "      <td>[0.22330284, -0.5643485, 0.32713842, -0.074667...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>record10008464_pdf1_slide2</td>\n",
       "      <td>10008464</td>\n",
       "      <td>2023-Moore-N4BI-AHM-Welcome.pdf</td>\n",
       "      <td>2</td>\n",
       "      <td>[1.1522328, 0.02467385, 0.23145455, 0.17258961...</td>\n",
       "      <td>[-0.3846272826194763, -0.01668522134423256, -0...</td>\n",
       "      <td>[0.34757608, -0.6562839, 0.5335755, -0.1906935...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>record10008464_pdf1_slide3</td>\n",
       "      <td>10008464</td>\n",
       "      <td>2023-Moore-N4BI-AHM-Welcome.pdf</td>\n",
       "      <td>3</td>\n",
       "      <td>[1.0057557, 0.18344171, 0.03796136, 0.23549518...</td>\n",
       "      <td>[-0.32376205921173096, 0.06897055357694626, 0....</td>\n",
       "      <td>[0.06456853, -0.79703176, 0.871621, -0.8768049...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>record10008464_pdf1_slide4</td>\n",
       "      <td>10008464</td>\n",
       "      <td>2023-Moore-N4BI-AHM-Welcome.pdf</td>\n",
       "      <td>4</td>\n",
       "      <td>[1.2563236, 0.25261688, 0.0040982994, 0.202414...</td>\n",
       "      <td>[-0.03493745997548103, -0.11242333054542542, 0...</td>\n",
       "      <td>[0.3643795, -0.40593308, 0.97395766, -0.495041...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>record10008464_pdf1_slide5</td>\n",
       "      <td>10008464</td>\n",
       "      <td>2023-Moore-N4BI-AHM-Welcome.pdf</td>\n",
       "      <td>5</td>\n",
       "      <td>[0.8471789, -0.40933934, 0.85754687, -0.496454...</td>\n",
       "      <td>[-0.6693567037582397, -0.02434205450117588, -0...</td>\n",
       "      <td>[-0.22939722, -0.6752343, 0.57570195, 0.046955...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          key zenodo_record_id  \\\n",
       "0  record10008464_pdf1_slide1         10008464   \n",
       "1  record10008464_pdf1_slide2         10008464   \n",
       "2  record10008464_pdf1_slide3         10008464   \n",
       "3  record10008464_pdf1_slide4         10008464   \n",
       "4  record10008464_pdf1_slide5         10008464   \n",
       "\n",
       "                   zenodo_filename  page_number  \\\n",
       "0  2023-Moore-N4BI-AHM-Welcome.pdf            1   \n",
       "1  2023-Moore-N4BI-AHM-Welcome.pdf            2   \n",
       "2  2023-Moore-N4BI-AHM-Welcome.pdf            3   \n",
       "3  2023-Moore-N4BI-AHM-Welcome.pdf            4   \n",
       "4  2023-Moore-N4BI-AHM-Welcome.pdf            5   \n",
       "\n",
       "                                                text  \\\n",
       "0  [0.8054575, -0.4204579, 0.111390926, 0.2636367...   \n",
       "1  [1.1522328, 0.02467385, 0.23145455, 0.17258961...   \n",
       "2  [1.0057557, 0.18344171, 0.03796136, 0.23549518...   \n",
       "3  [1.2563236, 0.25261688, 0.0040982994, 0.202414...   \n",
       "4  [0.8471789, -0.40933934, 0.85754687, -0.496454...   \n",
       "\n",
       "                                              visual  \\\n",
       "0  [0.22024887800216675, 0.6564452648162842, 0.02...   \n",
       "1  [-0.3846272826194763, -0.01668522134423256, -0...   \n",
       "2  [-0.32376205921173096, 0.06897055357694626, 0....   \n",
       "3  [-0.03493745997548103, -0.11242333054542542, 0...   \n",
       "4  [-0.6693567037582397, -0.02434205450117588, -0...   \n",
       "\n",
       "                                               mixed  \n",
       "0  [0.22330284, -0.5643485, 0.32713842, -0.074667...  \n",
       "1  [0.34757608, -0.6562839, 0.5335755, -0.1906935...  \n",
       "2  [0.06456853, -0.79703176, 0.871621, -0.8768049...  \n",
       "3  [0.3643795, -0.40593308, 0.97395766, -0.495041...  \n",
       "4  [-0.22939722, -0.6752343, 0.57570195, 0.046955...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9cf8164-ed99-42b4-bfd1-172c5242bbb4",
   "metadata": {},
   "source": [
    "## Extract all unique IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6c961793-04c3-41a1-bdb3-2c044055d761",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['10008464' '10008465' '10083555' '10679054' '10687658' '10815329'\n",
      " '10816895' '10886749' '10939519' '10942559' '10970869' '10972692'\n",
      " '10990107' '11031746' '11066250' '11107798' '11265038' '11396199'\n",
      " '11472148' '11474407' '11548617' '12623730' '3778431' '4317149' '4328911'\n",
      " '4334697' '4461261' '4630788' '4748510' '4748534' '4778265' '8323588'\n",
      " '8329305' '8329306' '8414318']\n"
     ]
    }
   ],
   "source": [
    "unique_zenodo_ids = df[\"zenodo_record_id\"].unique()\n",
    "print(unique_zenodo_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4c698f5-2348-4a03-9852-5927b326d052",
   "metadata": {},
   "source": [
    "Create a Set to ensure that all entries have the same license"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "470c8981-268c-4a66-b78a-d6a735aa5a84",
   "metadata": {},
   "source": [
    "## Now create the ReadMe file with the following information:\n",
    "    - Links to the original zenodo records\n",
    "    - Authors\n",
    "    - Licenses of the original records."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06b0d910-8be1-4d03-95d1-35fb86ff7fff",
   "metadata": {},
   "source": [
    "First, gather information for each record:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c12bd317-309a-43c3-b959-4948cc8e3dd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "records_info = []\n",
    "\n",
    "for record in unique_zenodo_ids:\n",
    "    \n",
    "    url = f\"https://zenodo.org/api/records/{record}\"\n",
    "    response = requests.get(url)\n",
    "    if response.status_code == 200:\n",
    "        data = response.json()\n",
    "        authors = \", \".join([creator.get(\"name\", \"Unknown Author\") for creator in data.get(\"metadata\", {}).get(\"creators\", [])])           \n",
    "        record_url = data.get(\"links\", {}).get(\"html\", f\"https://zenodo.org/record/{record}\")\n",
    "    else:\n",
    "        print(f\"Error: Unable to fetch data (Status Code: {response.status_code})\")\n",
    "    license = get_zenodo_license(record)\n",
    "    \n",
    "    records_info.append(f\"- **[Zenodo Record {record}]({record_url})**  \\n  **Authors**: {authors}  \\n  **License**: {license}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddb302a1-8c95-4909-a906-735273ab79e3",
   "metadata": {},
   "source": [
    "Second, combine it with some more general information (in Markdown Style)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9d4cd7d9-f3fb-4078-a909-8f0fb1354824",
   "metadata": {},
   "outputs": [],
   "source": [
    "readme_content = (\n",
    "    \"# About this Dataset\\n\\n\"\n",
    "    \"This Dataset contains data from several Presentation Slides. For each Slide the following information is available:\\n\\n\"\n",
    "    \"- key: recordID_pdfNumber_slideNumber\\n\\n\"\n",
    "    \"- zenodo_record_id : record ID\\n\\n\"\n",
    "    \"- zenodo_filename: PDF Filename\\n\\n\"\n",
    "    \"- text: Text Embedding (using the mixedbread-ai/mxbai-embed-large-v1 model)\\n\\n\"\n",
    "    \"- visual: Vision Embedding (using the openai/clip-vit-base-patch32 model)\\n\\n\"\n",
    "    \"- mixed: Mixed Embedding (text embedding of a generated structured response that describes the Slide as an Image, using gpt-4o model)\\n\\n\"\n",
    "    \"\\n\"\n",
    "    \"# Zenodo Records Information\\n\\n\"\n",
    "    \"This repository contains data from Zenodo records.\\n\\n\"\n",
    "    \"## Records\\n\\n\" +\n",
    "    \"\\n\".join(records_info)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5db8ead9-ff5e-49e3-9538-626cb9739f1e",
   "metadata": {},
   "source": [
    "Third, upload the combined information as a valid Markdown file to the Huggingface Repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bc191fc1-c7d9-44f9-ae08-a4249c0f4929",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"HUGGINGFACE_README.md\", \"w\", encoding=\"utf-8\") as f:\n",
    "    f.write(readme_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e0da5224-cab3-4193-a198-327d486c6574",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error uploading README.md: (Request ID: Root=1-67c99b6c-767b0bc9165a594f487504ac;5d943691-3cd1-4428-92db-c5e426ed3885)\n",
      "\n",
      "403 Forbidden: Authorization error..\n",
      "Cannot access content at: https://huggingface.co/api/datasets/ScaDS-AI/SlightInsight_Cache/preupload/main?create_pr=1.\n",
      "Make sure your token has the correct permissions.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lea/.local/lib/python3.10/site-packages/huggingface_hub/hf_api.py:9234: UserWarning: Warnings while validating metadata in README.md:\n",
      "- empty or missing yaml metadata in repo card\n",
      "  warnings.warn(f\"Warnings while validating metadata in README.md:\\n{message}\")\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import HfApi\n",
    "import io\n",
    "\n",
    "api = HfApi()\n",
    "readme_file = io.BytesIO(readme_content.encode(\"utf-8\"))\n",
    "\n",
    "try:\n",
    "    api.upload_file(\n",
    "        path_or_fileobj=readme_file,\n",
    "        path_in_repo=\"README.md\",  \n",
    "        repo_id=repo_name,\n",
    "        repo_type=\"dataset\",\n",
    "        create_pr = True \n",
    "        )\n",
    "    print(f\"README.md successfully uploaded to {repo_id}!\")\n",
    "except Exception as e:\n",
    "    print(f\"Error uploading README.md: {e}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}