Wikipedia Scraping

BeautifulSoup and Flask Framework

Author: Abhay-1552

Wikipdedia/flask_rendering.py

@@ -0,0 +1,27 @@
+from flask import Flask, render_template, request
+import practice_beautifulsoap as data
+
+app = Flask(__name__, template_folder='template')
+
+
+@app.route('/', methods=["GET", "POST"])
+def index():
+    languages = data.lang()
+    return render_template('index.html', languages=languages)
+
+
+@app.route("/display", methods=["POST"])
+def output():
+    if request.method == "POST":
+        entered_topic = request.form.get("topic")
+        selected_language = request.form.get("language")
+
+        soup_data = data.data(entered_topic, selected_language)
+        soup_image = data.get_image_urls(entered_topic)
+
+        return render_template('output.html', heading=entered_topic.upper(), data=soup_data,
+                               url=soup_image, language=selected_language)
+
+
+if __name__ == "__main__":
+    app.run(debug=True)

Wikipdedia/main.py

@@ -0,0 +1,16 @@
+# This is a sample Python script.
+
+# Press Shift+F10 to execute it or replace it with your code.
+# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
+
+
+def print_hi(name):
+    # Use a breakpoint in the code line below to debug your script.
+    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.
+
+
+# Press the green button in the gutter to run the script.
+if __name__ == '__main__':
+    print_hi('PyCharm')
+
+# See PyCharm help at https://www.jetbrains.com/help/pycharm/

Wikipdedia/practice_beautifulsoap.py

@@ -0,0 +1,69 @@
+from bs4 import BeautifulSoup
+import requests
+
+language_symbols = {}
+
+
+def lang():
+    try:
+        response = requests.get("https://www.wikipedia.org/")
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        for option in soup.find_all('option'):
+            language = option.text
+            symbol = option['lang']
+            language_symbols[language] = symbol
+
+        return list(language_symbols.keys())
+
+    except requests.exceptions.RequestException as e:
+        print("Error fetching language data:", e)
+        return []
+
+
+def data(selected_topic, selected_language):
+    symbol = language_symbols.get(selected_language)
+
+    try:
+        url = f"https://{symbol}.wikipedia.org/wiki/{selected_topic}"
+        data_response = requests.get(url)
+        data_response.raise_for_status()
+        data_soup = BeautifulSoup(data_response.content, 'html.parser')
+
+        main_content = data_soup.find('div', {'id': 'mw-content-text'})
+        filtered_content = ""
+
+        if main_content:
+            for element in main_content.descendants:
+                if element.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+                    filtered_content += "\n" + element.get_text(strip=True).upper() + "\n"
+
+                elif element.name == 'p':
+                    filtered_content += element.get_text(strip=True) + "\n"
+
+        return filtered_content
+
+    except requests.exceptions.RequestException as e:
+        print("Error fetching Wikipedia content:", e)
+        return "Error fetching data."
+
+
+def get_image_urls(query):
+    try:
+        search_url = f"https://www.google.com/search?q={query}&tbm=isch"
+        image_response = requests.get(search_url)
+        image_response.raise_for_status()
+        image_soup = BeautifulSoup(image_response.content, 'html.parser')
+
+        image_urls = []
+        for img in image_soup.find_all('img'):
+            image_url = img.get('src')
+            if image_url and image_url.startswith("http"):
+                image_urls.append(image_url)
+
+        return image_urls[0]
+
+    except requests.exceptions.RequestException as e:
+        print("Error fetching image URLs:", e)
+        return None

Wikipdedia/static/js/output.js

@@ -0,0 +1,9 @@
+function validateForm() {
+    var language = document.getElementById("language").value;
+
+    if (language === "Select") {
+        alert("Please select a language.");
+        return false;
+    }
+}
+

Wikipdedia/template/index.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Input Web Page</title>
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+    <script src="static/js/output.js"></script>
+</head>
+
+<body>
+    <div class="container text-center mt-4">
+        <img src="https://www.wikipedia.org/portal/wikipedia.org/assets/img/[email protected]" height="150px" width="150px">
+        <h1>Wikipedia</h1>
+    </div>
+
+    <form action="/display" method="post" class="container mt-4" onsubmit="return validateForm()">
+        <div class="form-group">
+            <label for="topic">Topic Name:</label>
+            <input type="text" class="form-control" name='topic' id="topic" placeholder="Enter Topic" required>
+        </div>
+        <div class="form-group">
+            <label for="language">Select Language:</label>
+            <select name="language" id="language" class="form-control" required>
+                <option value="Select">Select</option>
+                {% for language in languages %}
+                <option value="{{ language }}">{{ language }}</option>
+                {% endfor %}
+            </select>
+        </div>
+        <input type="submit" value="Submit" class="btn btn-primary">
+        <input type="reset" value="Clear" class="btn btn-danger">
+    </form>
+
+    <script src="https://code.jquery.com/jquery-3.5.1.slim.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@popperjs/[email protected]/dist/umd/popper.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+
+</body>
+
+</html>

Wikipdedia/template/output.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Output Web Page</title>
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+    <style>
+        .container pre {
+            white-space: pre-wrap;
+            font-family: Arial, sans-serif;
+            font-size: 17px;
+        }
+    </style>
+</head>
+
+<body>
+    <div class="container text-center mt-4">
+        <img src="{{url}}" height="150px" width="150px">
+        <h1> {{ heading }} </h1>
+        <h6> in {{ language }} language</h6>
+    </div>
+    <div class="container mt-4">
+        <div class="container border p-3">
+            <pre>{{ data }}</pre>
+        </div>
+    </div>
+
+    <script src="https://code.jquery.com/jquery-3.5.1.slim.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@popperjs/[email protected]/dist/umd/popper.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+</body>
+
+</html>