parse_thiele_sections.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a70af84b-8738-4e0a-89b9-e422f5403346",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "243fb469-ecd4-4077-9160-7c717c1c376e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%ls ./"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ad07a0f6-eb84-49b2-91f7-cf45b64ff210",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data successfully saved to ./danmarks_folkesagn_sections/thiele_legend_stories.csv\n",
      "Total number of stories: 901\n",
      "Highest story ID: 901\n"
     ]
    }
   ],
   "source": [
    "# Initialize variables\n",
    "data = []\n",
    "story_id = 1\n",
    "\n",
    "# Function to parse a single file\n",
    "def parse_file(filename, vol, sec, sub):\n",
    "    global story_id\n",
    "    with open(filename, 'r', encoding='utf-8') as file:\n",
    "        lines = file.readlines()\n",
    "        title, content, annot = '', '', ''\n",
    "        for line in lines:\n",
    "            if line.startswith('T '):\n",
    "                if title:  # If a new title is found and previous story data exists, save it\n",
    "                    data.append([story_id, vol, sec, sub, title.strip(), content.strip(), annot.strip()])\n",
    "                    story_id += 1\n",
    "                    content, annot = '', ''\n",
    "                title = line[2:].strip()\n",
    "            elif line.startswith('C '):\n",
    "                content += line[2:].strip() + ' '\n",
    "            elif line.startswith('A '):\n",
    "                annot += line[2:].strip() + ' '\n",
    "        \n",
    "        # Append the last story in the file\n",
    "        if title:\n",
    "            data.append([story_id, vol, sec, sub, title.strip(), content.strip(), annot.strip()])\n",
    "            story_id += 1\n",
    "\n",
    "# Directory containing text files\n",
    "directory = './danmarks_folkesagn_sections'  # Replace with the path to your text files\n",
    "\n",
    "# Process each file in the directory\n",
    "for filename in os.listdir(directory):\n",
    "    if filename.endswith('.txt'):\n",
    "        parts = filename.split('_')\n",
    "        vol = int(parts[0][3:])  # Extract volume number\n",
    "        sec = int(parts[1][3:])  # Extract section number\n",
    "        sub = int(parts[2][3:5])  # Extract subsection number\n",
    "        parse_file(os.path.join(directory, filename), vol, sec, sub)\n",
    "\n",
    "# Create DataFrame\n",
    "df = pd.DataFrame(data, columns=['id', 'vol', 'sec', 'sub', 'title', 'content', 'annot'])\n",
    "\n",
    "# Save to CSV\n",
    "output_file = './danmarks_folkesagn_sections/thiele_legend_stories.csv'\n",
    "df.to_csv(output_file, index=False)\n",
    "\n",
    "# Print the total number of stories and the highest story ID\n",
    "total_stories = df.shape[0]\n",
    "highest_story_id = df['id'].max()\n",
    "print(f\"Data successfully saved to {output_file}\")\n",
    "print(f\"Total number of stories: {total_stories}\")\n",
    "print(f\"Highest story ID: {highest_story_id}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e34284a4-d08d-492d-8f8e-6e040a9fffe8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (tess_image)",
   "language": "python",
   "name": "tess_image"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}