pdf_type_summary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3edbd136",
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "import fitz\n",
    "import pdfminer\n",
    "import shutil\n",
    "from pdfminer.layout import LAParams\n",
    "from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter\n",
    "from pdfminer.pdfpage import PDFPage\n",
    "from io import StringIO\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8067aca1",
   "metadata": {},
   "source": [
    "# PDF summary function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a0b05e1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pdf_type_summary(path):\n",
    "    Total_pdf = 0\n",
    "    count_mix = 0\n",
    "    count_text = 0\n",
    "    count_image = 0\n",
    "    count_unvalid = 0\n",
    "    for root, dirs, files in os.walk(path, topdown=False):\n",
    "        for name in dirs:\n",
    "            directory = (root+ '/' +name+ '/')\n",
    "            for filename in os.listdir(directory):\n",
    "                #print (filename)\n",
    "                if filename.endswith(\".pdf\"):\n",
    "                    \n",
    "                    from pdfminer.pdfpage import PDFPage\n",
    "                    #searchable_pages = []\n",
    "                    #non_searchable_pages = []\n",
    "                    #page_num = 0\n",
    "\n",
    "                    with open(directory+filename, 'rb') as infile:\n",
    "                        Total_pdf += 1\n",
    "                        searchable_pages = []\n",
    "                        non_searchable_pages = []\n",
    "                        page_num = 0\n",
    "                        for page in PDFPage.get_pages(infile):\n",
    "                            page_num += 1\n",
    "                            if 'Font' in page.resources.keys():\n",
    "                                searchable_pages.append(page_num)\n",
    "                            else:\n",
    "                                non_searchable_pages.append(page_num)\n",
    "                    if page_num > 0:\n",
    "                        if len(searchable_pages) == 0:\n",
    "                            count_image += 1\n",
    "                            #print(f\"Document '{fname}' has {page_num} page(s). \"\n",
    "                            #      f\"Complete document is non-searchable\")\n",
    "                        elif len(non_searchable_pages) == 0:\n",
    "                            count_text += 1\n",
    "                            #print(f\"Document '{fname}' has {page_num} page(s). \"\n",
    "                            #      f\"Complete document is searchable\")\n",
    "                        else:\n",
    "                            count_mix += 1\n",
    "                            #print (f\"Mixed Document\")\n",
    "                            #print(f\"searchable_pages : {searchable_pages}\")\n",
    "                            #print(f\"non_searchable_pages : {non_searchable_pages}\")\n",
    "                    else:\n",
    "                        count_unvalid += 1\n",
    "                        #print(f\"Not a valid document\")\n",
    "    print (f\"Total Number of Documents:\", Total_pdf)\n",
    "    print (f\"Number of Documents containing only Text:\", count_text)\n",
    "    print (f\"Number of Documents containing only Image:\", count_image)\n",
    "    print (f\"Number of Mixed Text/Image Documents:\", count_mix)\n",
    "    print (f\"Number of Unvalid Documents:\", count_unvalid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7d227eab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The PDF <_io.BufferedReader name='C:\\\\Users\\\\aamini\\\\Work\\\\NRCan\\\\Automatic data capture\\\\Analysis\\\\Pdf_classification\\\\Data\\\\NTGSref_2023February10 (5)\\\\034725_034725\\\\034725_Report\\\\034725_Appendices/034725_Appendix_G_Dril_Core_Assay_Certificates/COA_YW17034549_122387-38289151.pdf'> contains a metadata field indicating that it should not allow text extraction. Ignoring this field and proceeding. Use the check_extractable if you want to raise an error in this case\n",
      "The PDF <_io.BufferedReader name='C:\\\\Users\\\\aamini\\\\Work\\\\NRCan\\\\Automatic data capture\\\\Analysis\\\\Pdf_classification\\\\Data\\\\NTGSref_2023February10 (5)\\\\034725_034725\\\\034725_Report\\\\034725_Appendices/034725_Appendix_G_Dril_Core_Assay_Certificates/COA_YW17039051_122387-38357039.pdf'> contains a metadata field indicating that it should not allow text extraction. Ignoring this field and proceeding. Use the check_extractable if you want to raise an error in this case\n",
      "The PDF <_io.BufferedReader name='C:\\\\Users\\\\aamini\\\\Work\\\\NRCan\\\\Automatic data capture\\\\Analysis\\\\Pdf_classification\\\\Data\\\\NTGSref_2023February10 (5)\\\\034725_034725\\\\034725_Report\\\\034725_Appendices/034725_Appendix_G_Dril_Core_Assay_Certificates/COA_YW17041433_122387-38491986.pdf'> contains a metadata field indicating that it should not allow text extraction. Ignoring this field and proceeding. Use the check_extractable if you want to raise an error in this case\n",
      "The PDF <_io.BufferedReader name='C:\\\\Users\\\\aamini\\\\Work\\\\NRCan\\\\Automatic data capture\\\\Analysis\\\\Pdf_classification\\\\Data\\\\NTGSref_2023February12 (1)\\\\085474_085474\\\\085474_085474_Digital/085474_Volume 2/App III.pdf'> contains a metadata field indicating that it should not allow text extraction. Ignoring this field and proceeding. Use the check_extractable if you want to raise an error in this case\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Number of Documents: 636\n",
      "Number of Documents containing only Text: 338\n",
      "Number of Documents containing only Image: 275\n",
      "Number of Mixed Text/Image Documents: 23\n",
      "Number of Unvalid Documents: 0\n"
     ]
    }
   ],
   "source": [
    "# Insert raw data path below:\n",
    "path = r\"C:\\Users\\aamini\\Work\\NRCan\\Automatic data capture\\Analysis\\Pdf_classification\"\n",
    "get_pdf_type_summary(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "454a4943",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae39fff4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e83736b3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2bb3e82",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "745af3e4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36e97ef5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bc6ee26",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f52911d8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}