DataGeneration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from rdflib import Graph, Literal, RDF, RDFS, URIRef, Namespace, BNode\n",
    "from rdflib.namespace import XSD\n",
    "import urllib.parse\n",
    "from SPARQLWrapper import SPARQLWrapper, JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#CSV Files:\n",
    "coursecsv = 'CSVs/data.csv'\n",
    "studentcsv = 'CSVs/students.csv'\n",
    "lecturescsv = 'CSVs/lecture.csv'\n",
    "topicscsv = 'CSVs/topic.csv'\n",
    "\n",
    "# TTL Files: (Generated)\n",
    "uniPath = 'Data/universities.ttl'\n",
    "coursePath = 'Data/courses.ttl'\n",
    "studentPath = 'Data/students.ttl'\n",
    "lecturesPath = 'Data/lectures.ttl'\n",
    "topicsPath = 'Data/topics.ttl'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Universities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = Graph()\n",
    "ex = Namespace(\"http://example.org/vocab/\")\n",
    "foaf = Namespace(\"http://xmlns.com/foaf/0.1/\")\n",
    "rdf = Namespace(\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\")\n",
    "rdfs = Namespace(\"http://www.w3.org/2000/01/rdf-schema#\")\n",
    "xsd = Namespace(\"http://www.w3.org/2001/XMLSchema#\")\n",
    "dbo = Namespace(\"http://dbpedia.org/ontology/\")\n",
    "dbr = Namespace(\"http://dbpedia.org/resource/\")\n",
    "\n",
    "g.bind(\"ex\", ex)\n",
    "g.bind(\"foaf\", foaf)\n",
    "g.bind(\"rdf\", rdf)\n",
    "g.bind(\"rdfs\", rdfs)\n",
    "g.bind(\"xsd\", xsd)\n",
    "g.bind(\"dbo\", dbo)\n",
    "g.bind(\"dbr\", dbr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/aryansaxena/Desktop/Intelligent Systems/IS_AK_G_04\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N5e362dace8d544bda9cfc74150b62057 (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.parse(\"Data/courses.ttl\", format=\"turtle\")\n",
    "\n",
    "sparql = SPARQLWrapper(\"http://dbpedia.org/sparql\")\n",
    "\n",
    "sparql.setQuery(\"\"\"\n",
    "    SELECT ?university ?universityName WHERE {\n",
    "        ?university rdf:type dbo:University .\n",
    "        ?university dbo:country dbr:Canada .\n",
    "        ?university rdfs:label ?universityName .\n",
    "        FILTER (LANG(?universityName) = 'en')\n",
    "    } LIMIT 100\n",
    "\"\"\")\n",
    "sparql.setReturnFormat(JSON)\n",
    "results = sparql.query().convert()\n",
    "\n",
    "# Manually adding concordia\n",
    "concordia_uri = URIRef(\"http://dbpedia.org/resource/Concordia_University\")\n",
    "concordia_name = Literal(\"Concordia University\", lang=\"en\")\n",
    "# Check if Concordia is already in the graph, if not, add it\n",
    "if (concordia_uri, RDF.type, dbo.University) not in g:\n",
    "    g.add((concordia_uri, RDF.type, dbo.University))\n",
    "    g.add((concordia_uri, RDFS.label, concordia_name))\n",
    "\n",
    "for course_uri in g.subjects(RDF.type, ex.Course):\n",
    "    g.add((concordia_uri, dbo.offersCourse, course_uri))\n",
    "\n",
    "for result in results[\"results\"][\"bindings\"]:\n",
    "    university_uri = URIRef(result[\"university\"][\"value\"])\n",
    "    university_name = Literal(result[\"universityName\"][\"value\"], lang=\"en\")\n",
    "    g.add((university_uri, RDF.type, dbo.University))\n",
    "    g.add((university_uri, RDFS.label, university_name))\n",
    "\n",
    "g.serialize(destination=uniPath, format='turtle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Course Data (From Concordia Open Data):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = Graph()\n",
    "ex = Namespace(\"http://example.org/vocab/\")\n",
    "foaf = Namespace(\"http://xmlns.com/foaf/0.1/\")\n",
    "rdf = Namespace(\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\")\n",
    "rdfs = Namespace(\"http://www.w3.org/2000/01/rdf-schema#\")\n",
    "xsd = Namespace(\"http://www.w3.org/2001/XMLSchema#\")\n",
    "g.bind(\"ex\", ex)\n",
    "g.bind(\"foaf\", foaf)\n",
    "g.bind(\"rdf\", rdf)\n",
    "g.bind(\"rdfs\", rdfs)\n",
    "g.bind(\"xsd\", xsd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N5b878fbb1efc400ebb4eb3e480c17caa (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catalog_df = pd.read_csv(coursecsv, encoding='utf-16')\n",
    "for index, row in catalog_df.iterrows():\n",
    "    course_id_str = str(row['Course ID'])\n",
    "    course_uri = URIRef(f\"http://example.org/vocab/course/{urllib.parse.quote(course_id_str)}\")\n",
    "    g.add((course_uri, RDF.type, ex.Course))\n",
    "    g.add((course_uri, ex.subject, Literal(row['Subject'], datatype=XSD.string)))\n",
    "    g.add((course_uri, ex.number, Literal(row['Catalog'], datatype=XSD.string)))\n",
    "    g.add((course_uri, ex.description, Literal(row['Long Title'], datatype=XSD.string)))\n",
    "    g.add((course_uri, ex.credits, Literal(row['Class Units'], datatype=XSD.decimal)))\n",
    "    if not pd.isnull(row['Pre Requisite Description']):\n",
    "        g.add((course_uri, ex.preRequisiteDescription, Literal(row['Pre Requisite Description'], datatype=XSD.string)))\n",
    "    if not pd.isnull(row['Equivalent Courses']):\n",
    "        g.add((course_uri, ex.equivalentCourses, Literal(row['Equivalent Courses'], datatype=XSD.string)))\n",
    "\n",
    "\n",
    "g.serialize(destination=coursePath, format='turtle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Students"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = Graph()\n",
    "ex = Namespace(\"http://example.org/vocab/\")\n",
    "foaf = Namespace(\"http://xmlns.com/foaf/0.1/\")\n",
    "rdf = Namespace(\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\")\n",
    "rdfs = Namespace(\"http://www.w3.org/2000/01/rdf-schema#\")\n",
    "xsd = Namespace(\"http://www.w3.org/2001/XMLSchema#\")\n",
    "g.bind(\"ex\", ex)\n",
    "g.bind(\"foaf\", foaf)\n",
    "g.bind(\"rdf\", rdf)\n",
    "g.bind(\"rdfs\", rdfs)\n",
    "g.bind(\"xsd\", xsd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N7c58b839ee6c49c6953885173a3af39b (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "students_df = pd.read_csv(studentcsv)\n",
    "for index, row in students_df.iterrows():\n",
    "    student_uri = ex[f'student/{urllib.parse.quote(row[\"id_number\"])}']\n",
    "    g.add((student_uri, RDF.type, ex.Student))\n",
    "    g.add((student_uri, foaf.name, Literal(f\"{row['first_name']} {row['last_name']}\", datatype=XSD.string)))\n",
    "    g.add((student_uri, foaf.mbox, Literal(row['email'], datatype=XSD.string)))\n",
    "    g.add((student_uri, ex.studentID, Literal(row['id_number'], datatype=XSD.string)))\n",
    "\n",
    "    for i in range(1, 5):  \n",
    "        course_uri = ex[f'course/{row[f\"course_{i}_id\"]}']\n",
    "        grade = row[f\"course_{i}_grade\"]\n",
    "        course_completion = BNode()\n",
    "        g.add((student_uri, ex.completedCourse, course_completion))\n",
    "        g.add((course_completion, RDF.type, URIRef(\"http://example.org/vocab/CompletedCourse\")))\n",
    "        g.add((course_completion, ex.course, course_uri))\n",
    "        g.add((course_completion, ex.courseGrade, Literal(grade, datatype=XSD.string)))\n",
    "    \n",
    "    for comp_field in ['competent_in_course_id1', 'competent_in_course_id2']:\n",
    "        competency_course_uri = ex[f'course/{row[comp_field]}']\n",
    "        g.add((student_uri, ex.hasCompetency, competency_course_uri))\n",
    "\n",
    "\n",
    "g.serialize(destination=studentPath, format='turtle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = Graph()\n",
    "ex = Namespace(\"http://example.org/vocab/\")\n",
    "foaf = Namespace(\"http://xmlns.com/foaf/0.1/\")\n",
    "rdf = Namespace(\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\")\n",
    "rdfs = Namespace(\"http://www.w3.org/2000/01/rdf-schema#\")\n",
    "xsd = Namespace(\"http://www.w3.org/2001/XMLSchema#\")\n",
    "g.bind(\"ex\", ex)\n",
    "g.bind(\"foaf\", foaf)\n",
    "g.bind(\"rdf\", rdf)\n",
    "g.bind(\"rdfs\", rdfs)\n",
    "g.bind(\"xsd\", xsd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=Nb4803833562948b6a69f8c942b85a542 (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lectures_df = pd.read_csv(lecturescsv)  \n",
    "for index, row in lectures_df.iterrows():\n",
    "    # Create a URI for the lecture based on its Identifier\n",
    "    lecture_uri = URIRef(f\"http://example.org/vocab/lecture/{urllib.parse.quote(str(row['Identifier']))}\")\n",
    "    g.add((lecture_uri, RDF.type, ex.Lecture))\n",
    "    # Adding Course ID for linking with the Course with lecture of course property\n",
    "\n",
    "    course_uri = URIRef(f\"http://example.org/vocab/course/{urllib.parse.quote(str(row['CourseId']))}\")\n",
    "    g.add((lecture_uri, ex.lectureOfCourse, course_uri))\n",
    "\n",
    "    # Adding the lecture name\n",
    "    g.add((lecture_uri, ex.lectureName, Literal(row['Title'], datatype=XSD.string)))\n",
    "\n",
    "    # Adding the lecture number as integer\n",
    "    g.add((lecture_uri, ex.lectureNumber, Literal(row['Identifier'], datatype=XSD.integer)))\n",
    "\n",
    "    # Adding the DBPedia Link using seeAlso row, linking it to the topics covered\n",
    "    dbpedia_uri = URIRef(f\"{(row['seeAlso'])}\")\n",
    "    g.add((lecture_uri, ex.lectureTopic, dbpedia_uri))\n",
    "\n",
    "\n",
    "# Serialization path should be corrected as per your directory structure\n",
    "g.serialize(destination=lecturesPath, format='turtle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = Graph()\n",
    "ex = Namespace(\"http://example.org/vocab/\")\n",
    "foaf = Namespace(\"http://xmlns.com/foaf/0.1/\")\n",
    "rdf = Namespace(\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\")\n",
    "rdfs = Namespace(\"http://www.w3.org/2000/01/rdf-schema#\")\n",
    "xsd = Namespace(\"http://www.w3.org/2001/XMLSchema#\")\n",
    "g.bind(\"ex\", ex)\n",
    "g.bind(\"foaf\", foaf)\n",
    "g.bind(\"rdf\", rdf)\n",
    "g.bind(\"rdfs\", rdfs)\n",
    "g.bind(\"xsd\", xsd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N83a98d659c664301885c4f2cafa2bfc8 (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics_df = pd.read_csv(topicscsv)\n",
    "for index, row in topics_df.iterrows():\n",
    "    # Creating URI with Topic using TopicLink\n",
    "    topic_uri = URIRef(row['Topic Link'])\n",
    "    # Create URI for Course\n",
    "    course_uri = URIRef(f\"http://example.org/vocab/course/{row['CourseId']}\")\n",
    "    # Create URI for Lecture\n",
    "    lecture_uri = URIRef(f\"http://example.org/vocab/lecture/{row['Identifier']}\")\n",
    "    # Adding the Topic to the Graph\n",
    "    g.add((topic_uri, RDF.type, ex.Topic))\n",
    "    # Adding the Course to the Graph\n",
    "    g.add((topic_uri, ex.isTopicOfCourse, course_uri))\n",
    "    # Adding the Lecture to the Graph\n",
    "    g.add((topic_uri, ex.isTopicOfLecture, lecture_uri))\n",
    "    # Adding Entity Type\n",
    "    g.add((topic_uri, ex.entityType, Literal(row['Entity Type'], datatype=XSD.string)))\n",
    "    # Adding the topic name (Properly Formatting since its in between quotes, can be in multiple lines too)\n",
    "    g.add((topic_uri, ex.topicName, Literal(row['Topic Name'], datatype=XSD.string)))\n",
    "    # Adding what Type of Material it is using Type - Worksheet or lecture\n",
    "    g.add((topic_uri, ex.materialType, Literal(row['Type'], datatype=XSD.string)))\n",
    "\n",
    "g.serialize(destination=topicsPath, format='turtle')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}