adding chunk id

franciscojavierarceo · franciscojavierarceo · commit fea2a5e6cbc1 · 2025-03-14T13:56:00.000-04:00
Signed-off-by: Francisco Javier Arceo &lt;farceo@redhat.com&gt;
diff --git a/examples/rag-docling/docling-demo.ipynb b/examples/rag-docling/docling-demo.ipynb
@@ -393,7 +393,9 @@
    "cell_type": "code",
    "execution_count": 15,
    "id": "d250b618-8320-4237-84e6-d46d9d932613",
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [
     {
      "data": {
@@ -507,16 +509,6 @@
     "finaldf.head()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "c51f5e3d-e894-4b78-9e81-2a4a3a3ea02e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "finaldf.to_parquet('feature_repo/data/docling_samples.parquet', index=False)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 17,
@@ -548,6 +540,41 @@
    "source": [
     "pdf_example"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "4006f71d-975b-4863-a133-681253d643f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import hashlib\n",
+    "\n",
+    "def generate_chunk_id(file_name: str, raw_chunk_markdown: str) -> str:\n",
+    "    \"\"\"Generate a unique chunk ID based on file_name and raw_chunk_markdown.\"\"\"\n",
+    "    unique_string = f\"{file_name}-{raw_chunk_markdown}\"\n",
+    "    return hashlib.sha256(unique_string.encode()).hexdigest()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "8449ad8a-c8d3-4da4-9bf1-45939f6647a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "finaldf[\"chunk_id\"] = finaldf.apply(lambda row: generate_chunk_id(row[\"file_name\"], row[\"raw_chunk_markdown\"]), axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "c51f5e3d-e894-4b78-9e81-2a4a3a3ea02e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "finaldf.to_parquet('feature_repo/data/docling_samples.parquet', index=False)"
+   ]
   }
  ],
  "metadata": {
