Finished robots training, added robots.txt

Acrylami · Oct 11, 2020 · cf4c21d · cf4c21d
1 parent a6822d6
commit cf4c21d
Show file tree

Hide file tree

Showing 19 changed files with 278 additions and 6 deletions.
diff --git a/cyber/__pycache__/forms.cpython-38.pyc b/cyber/__pycache__/forms.cpython-38.pyc
diff --git a/cyber/__pycache__/routes.cpython-38.pyc b/cyber/__pycache__/routes.cpython-38.pyc
diff --git a/cyber/forms.py b/cyber/forms.py
@@ -41,6 +41,10 @@ class InspectForm1(FlaskForm):
     submit = SubmitField('Submit')
 
 class InspectForm2(FlaskForm):
+    password = PasswordField('Password', validators=[DataRequired()])
+    submit = SubmitField('Submit')
+
+class BasicPassword(FlaskForm):
     password = PasswordField('Password', validators=[DataRequired()])
     submit = SubmitField('Submit')    
 

diff --git a/cyber/routes.py b/cyber/routes.py
@@ -204,6 +204,18 @@ def challenge_redirect(training_name):
                 return render_template(success_file_name, message=message) #this keeps the url at the same place!! very useful
             else:
                 flash('Invalid password. Note to self: I hid the password in an image')
+
+    if (training_name == "robots"):
+        form = BasicPassword()
+        if form.validate_on_submit():
+            if (form.password.data == 'notverywellhidden'):
+                #Set points in database and return success
+                message = set_points(training_name)
+                success_file_name = "/training/"+training_name+ "/" + training_name + "-success.html"
+                return render_template(success_file_name, message=message) #this keeps the url at the same place!! very useful
+            else:
+                flash('That is not the flag... You can find the secret page by looking on robots.txt')
+
 
     elif (training_name == "manual-cracking"):
         form = PasswordCracking1()
@@ -215,9 +227,9 @@ def challenge_redirect(training_name):
                 return render_template(success_file_name, message=message) #this keeps the url at the same place!! very useful
             else:
                 flash('Invalid username or password.')     
-    else:
+    #else:
         #should not run...
-        return render_template('500.html'), 500
+        #return render_template('500.html'), 500
 
     return render_template(file_name, form=form)
 ####

diff --git a/cyber/static/img/badges/cody.mdp b/cyber/static/img/badges/cody.mdp
diff --git a/cyber/static/img/badges/cody.png b/cyber/static/img/badges/cody.png
diff --git a/cyber/static/img/badges/spider.mdp b/cyber/static/img/badges/spider.mdp
diff --git a/cyber/static/img/badges/spider.png b/cyber/static/img/badges/spider.png
diff --git a/cyber/static/img/web-hacking/evil-spider.png b/cyber/static/img/web-hacking/evil-spider.png
diff --git a/cyber/static/img/web-hacking/evil-spiders.png b/cyber/static/img/web-hacking/evil-spiders.png
diff --git a/cyber/static/img/web-hacking/robots-example-1.png b/cyber/static/img/web-hacking/robots-example-1.png
diff --git a/cyber/static/img/web-hacking/robots-example-2.png b/cyber/static/img/web-hacking/robots-example-2.png
diff --git a/cyber/static/img/web-hacking/spider.png b/cyber/static/img/web-hacking/spider.png
diff --git a/cyber/templates/profile.html b/cyber/templates/profile.html
@@ -52,8 +52,8 @@ <h4>GAME</h4>
                         <h4>TRAINING</h4>
 
 
-                        <img src="{{ url_for('static', filename='img/placeholder_badge.png') }}" data-toggle="tooltip" title="???">
-                        <img src="{{ url_for('static', filename='img/placeholder_badge.png') }}" data-toggle="tooltip" title="???">
+                        <img src="{{ url_for('static', filename='img/badges/spider.png') }}" data-toggle="tooltip" title="You found the secret page from robots.txt!">
+                        <img src="{{ url_for('static', filename='img/badges/cody.png') }}" data-toggle="tooltip" title="You completed the source code training series!">
                         <img src="{{ url_for('static', filename='img/placeholder_badge.png') }}" data-toggle="tooltip" title="???">
                         <img src="{{ url_for('static', filename='img/placeholder_badge.png') }}" data-toggle="tooltip" title="???">
                         <img src="{{ url_for('static', filename='img/placeholder_badge.png') }}" data-toggle="tooltip" title="???">

diff --git a/cyber/templates/training/robots/robots-2.html b/cyber/templates/training/robots/robots-2.html
@@ -0,0 +1,86 @@
+{% extends "layout.html" %}
+<!-- this layout just goes back ../ for all the links so it doesn't break since it's in the /training/ directory -->
+<!-- may be needed for other pages too -->
+
+{% block content %}
+
+<div class="content">
+    <div class="container-fluid">
+        <div class="row">
+            <div class="card training_background">
+
+                <!-- TRAINING HEADER (Args: Category, training name, points, t_name for image, background class=default "new-colour" so can be left empty) -->
+                {% call training_header("Web Hacking", training.display_name, training.points, training.training_name, "sky-blue") %}
+                {% endcall %}
+                <!-- TRAINING HEADER -->
+
+                <div class="">
+                    <div class="card-body training_card">
+
+                        <h2>Search Engines</h2>
+                        <p>Before we can get into the details of robots.txt, what it does, and how it works it's going to be necessary to understand how search engines like Google work</p>
+                        <p>Google doesn't get its results by magic, it has to search through the internet for every website it can access, and when it finds a website, it analyses every page of it to decide how it should index it</p>
+                        <p>Say you publish a website about cooking pumpkin pies- Google might see that some of your keywords you've included in the metadata is "food", "cooking", and "pumpkins. Google will analyse this and other information and make your result more likely to be found if someone searches those terms</p>
+
+                        <h3>How does it do that?</h3>
+                        <img class="img-fluid img-thumbnail mx-auto d-block" src="{{ url_for('static', filename='img/web-hacking/spider.png') }}">
+                        <p>Search engines use something called <u>web crawlers</u>, also commonly called <u>spiders</u>.</p>
+                        <p>These creepy crawlies are a type of <b>bot</b> that go from website to website downloading each one and 'indexing' them, which means organising them to make it easy to access each website by its unique properties.</p>
+
+                        <h3>So back to Robots.txt</h3>
+                        <p>The first thing a web crawler does is look for a robots.txt file on each website it visits. This file tells the web crawlers which parts of the website it is allowed to access and index, and also which web crawlers are allowed to.</p>
+                        <img class="img-fluid img-thumbnail mx-auto d-block" src="{{ url_for('static', filename='img/web-hacking/robots-example-1.png') }}">
+                        <caption>This example tells <i>every</i> web crawler to ignore the 'junk' directory. This means the junk directory will not be indexed and crawlers are not allowed to visit it</caption>
+
+                        <img class="img-fluid img-thumbnail mx-auto d-block" src="{{ url_for('static', filename='img/web-hacking/robots-example-2.png') }}">
+                        <caption>This example tells <i>just the Googlebot-Image</i> crawler to ignore a particular image, <code>dogs.jpg</code>. This means that Google Images will not be allowed to look at this image and index it on Google Images for anyone to find</caption>
+                        <p>In a robots.txt file, * means all crawlers, and / means all files on a website</p>
+
+                        <h2>Quick Quiz!</h2>
+                        <p>Which of these will disallow <u>Google's News</u> web crawler from indexing the <u>private_file.html</u> web page?</p>
+                        <div class="table-responsive">
+                            <table class="table">
+                                <tr>
+                                    <td class="text-center"><button type="button" class="btn btn-default quiz" onclick="wrong()">User-agent: Googlebot-news<br>
+                                            Disallow: /</button></td>
+                                    <td class="text-center"><button type="button" class="btn btn-default quiz" onclick="wrong()">User-agent: *<br>
+                                            Disallow: /private_file.html</button></td>
+                                </tr>
+                                <tr>
+                                    <td class="text-center"><button type="button" class="btn btn-default quiz" onclick="wrong()">User-agent: Googlebot-Image<br>
+                                            Allow: /private/</button></td>
+                                    <td class="text-center"><button type="button" class="btn btn-default quiz" onclick="correct()">User-agent: Googlebot-news<br>
+                                            Disallow: /private_file.html</button></td>
+                                </tr>
+                            </table>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <!-- PAGE NAV (Args: Current page passed from routes.py, training name passed from routes.py, total number of pages) -->
+        {% call pagination(current_page, training.training_name, 3) %}
+        {% endcall %}
+        <!-- END PAGE NAV -->
+
+    </div>
+</div>
+
+
+<script>
+    function wrong() {
+        swal("Oops", "Not quite! Have another look at the examples and read the details of the question again", "error")
+    }
+
+    function correct() {
+        swal("Nice!", "You've got it!", "success")
+    }
+
+</script>
+
+
+
+
+
+{% endblock content %}
diff --git a/cyber/templates/training/robots/robots-3.html b/cyber/templates/training/robots/robots-3.html
@@ -0,0 +1,91 @@
+{% extends "layout.html" %}
+<!-- this layout just goes back ../ for all the links so it doesn't break since it's in the /training/ directory -->
+<!-- may be needed for other pages too -->
+
+{% block content %}
+
+<div class="content">
+    <div class="container-fluid">
+        <div class="row">
+            <div class="card training_background">
+
+                <!-- TRAINING HEADER (Args: Category, training name, points, t_name for image, background class=default "new-colour" so can be left empty) -->
+                {% call training_header("Web Hacking", training.display_name, training.points, training.training_name, "sky-blue") %}
+                {% endcall %}
+                <!-- TRAINING HEADER -->
+
+                <div class="">
+                    <div class="card-body training_card">
+
+                        <h2>Benefits of robots.txt</h2>
+                        <p>So now we know what robots.txt does, let's find out why it's used.</p>
+                        <p>The first reason would be to hide all or parts of your website from certain search engines. Say you have a particular part of your website you don't want people to find using a search engine- you can hide that away using robots.txt. Or perhaps you really don't want your images to show up in Google Images specifically, you can do that too.</p>
+                        <p>The other use could be to reduce the traffic your website gets. All the web crawlers visiting your site is traffic that your server has to handle. You can disallow crawlers on some or all of your website to reduce the traffic coming in via different bots.</p>
+
+                        <h3>How it shouldn't be used</h3>
+                        <img class="img-fluid img-thumbnail mx-auto d-block" src="{{ url_for('static', filename='img/web-hacking/evil-spiders.png') }}">
+                        <p>A bad web developer might use robots.txt as an attempt to hide a secret part of their website from unathorised people. But although this might make it less likely that someone will stumble upon their secret page, it doesn't prevent someone from accessing it if they do find it</p>
+                        <p>The challenge for this training is to navigate to the robots.txt page of this website, and find the URL of the secret page we're hiding.</p>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <!-- CHALLENGE (Args: Points, text, hint, training name) -->
+        {% call challenge(training.points, "Find a secret page on the CyberQuest website by looking at our robots.txt file", "Look for a suspicious page that's disallowed on robots.txt, then try to visit it by typing: 'this websites URL'/'secret_file' into  your browser", training.training_name) %}
+        {% endcall %}
+        <!-- END CHALLENGE -->
+        <!-- PAGE NAV (Args: Current page passed from routes.py, training name passed from routes.py, total number of pages) -->
+        {% call pagination(current_page, training.training_name, 3) %}
+        {% endcall %}
+        <!-- END PAGE NAV -->
+
+    </div>
+</div>
+
+
+<script>
+    function prompt_login() {
+
+
+        swal("Oh no, you're not logged in! Making an account lets you save your progress, AND earn points and hacker ranks.", {
+                icon: "warning",
+                buttons: {
+                    continue: {
+                        text: "Continue anyway",
+                        value: "continue",
+                        className: "grey-bg",
+                    },
+                    register: {
+                        text: "Register",
+                        value: "register",
+                    },
+                    login: {
+                        text: "Login",
+                        value: "login",
+                    },
+                },
+            })
+            .then((value) => {
+                switch (value) {
+                    case "login":
+                        window.location.href = "../../login";
+                        break;
+
+                    case "register":
+                        window.location.href = "../../register";
+                        break;
+
+                    case "continue":
+                        window.open('../robots/challenge', '_blank');
+                        break;
+                }
+            });
+    }
+
+</script>
+
+
+
+
+
+{% endblock content %}
diff --git a/cyber/templates/training/robots/robots-challenge.html b/cyber/templates/training/robots/robots-challenge.html
@@ -0,0 +1,46 @@
+{% extends "layout.html" %}
+{% block content %}
+
+<div class="content">
+    <div class="container-fluid">
+        <div class="col-md-6">
+            <div class="card">
+                <div class="card-header card-header-primary">
+                    <h4 class="card-title">Enter the 'flag' you find on the secret page</h4>
+                    <p class="card-category"></p>
+                </div>
+                <div class="card-body">
+                    <form method="POST" action="" autocomplete="off">
+                        {{ form.hidden_tag() }}
+                        <div class="row">
+                            <div class="col-md-11">
+                                <div class="form-group">
+                                    <label class="bmd-label-floating"> {{ form.password.label }} </label>
+                                    {{ form.password(class="form-control") }}
+
+                                    {% for error in form.password.errors %}
+                                    <span class="error"> {{ error }} </span>
+                                    {% endfor %}
+
+                                </div>
+
+                            </div>
+                        </div>
+                        <div class="row">
+                            <div class="col-md-12">
+                                <button type="submit" class="btn btn-info pull-right btn-lg" value=Login>Login now</button>
+                                <div class="clearfix">
+                                </div>
+                            </div>
+
+                        </div>
+                    </form>
+
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+
+{% endblock content %}
diff --git a/cyber/templates/training/robots/robots-success.html b/cyber/templates/training/robots/robots-success.html
@@ -0,0 +1,32 @@
+{% extends "layout.html" %}
+{% block content %}
+
+<div class="content">
+    <div class="container-fluid">
+        <div class="col-md-6">
+            <div class="card">
+                <div class="card-header card-header-primary">
+                    <h4 class="card-title">Enter the 'flag' you find on the secret page</h4>
+                    <p class="card-category"></p>
+                </div>
+                <div class="card-body">
+                    {% if message == "No Login" %}
+                    <p>Congrats on completing the training! You could have earned points for this by <a href='{{ url_for("login") }}'>logging in</a> or <a href='{{ url_for("register") }}'>registering an account</a>.</p>
+                    {% elif message == "Already Completed" %}
+                    <p>You did it!</p>
+                    <p>You have already completed this challenge before, so you didn't earn any points this time!</p>
+                    {% elif message == "Points" %}
+                    <p>Congrats on completing the training! 100 points have been added to your account</p>
+
+                    {% else %}
+                    <p>You sneaky thing! You can't get points by coming to this link without completing the challenge, but nice try.</p>
+                    {% endif %}
+
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+
+{% endblock content %}
diff --git a/cyber/templates/training/robots/secret.html b/cyber/templates/training/robots/secret.html
@@ -12,9 +12,10 @@
 
                 <div class="">
                     <div class="card-body training_card">
-                        <p>Woah! A secret page!</p>
+                        <h1>Woah! A secret page!</h1>
+                        <img class="img-fluid img-thumbnail mx-auto d-block" src="{{ url_for('static', filename='img/web-hacking/evil-spider.png') }}">
                         <p>I can't believe you found this... Did you come here from robots.txt?</p>
-                        <p>If you made it here because of a particular training, the code you need to enter to get your points is: <code>notverywellhidden</code></p>
+                        <p>If you made it here because of a particular training, the flag you need to enter to get your points is: <code>notverywellhidden</code></p>
                     </div>
                 </div>
             </div>