changes and comments

.gitignore

@@ -3,3 +3,4 @@
 __pycache__/
 /scripts/
 *.sql
+*.ipynb

indicatorlibrary/quickstart/assets.py

@@ -0,0 +1,61 @@
+# Code developed during SCT 18 hackathon
+import re
+import urllib.request
+
+import pandas as pd
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+
+res = "C:\\Users\\Don\\PycharmProjects\\Zalando_Hackathon\\Res\\"
+
+# download if needed
+# nltk.download('stopwords', res + "nltk")
+# nltk.download('punkt', res + "nltk")
+
+input_text = "I'm working on a WASH project in a refugee camp and we are improving hygiene practices and access to safe, clean water."
+# input_text = "I'm working on a USG funded program in education and I would like to track students enrolled in courses"
+# input_text = "I am looking for an output indicator that let's me track who is knowledgable about the gender practices"
+
+stop_words = set(stopwords.words('english'))
+
+word_tokens_input = word_tokenize(input_text.lower())
+filtered_sentence_input = [w for w in word_tokens_input if w not in stop_words]
+
+correlations = []
+
+csv_data = pd.read_csv("indicator_new.csv")
+# print(csv_data.columns)
+for i in range(1, 2055):  # TODO:
+# for i in range(460, 470):  # TODO:
+    data = csv_data[csv_data["id"] == i]
+    # print(i)
+    indicator = ""
+    if not pd.isna(csv_data["Indicator"].iloc[i]):
+        indicator += csv_data["Indicator"].iloc[i]
+    if not pd.isna(csv_data["Sector"].iloc[i]):
+        indicator += csv_data["Sector"].iloc[i]
+    if not pd.isna(csv_data["Definition"].iloc[i]):
+        indicator += csv_data["Definition"].iloc[i]
+    indicator = re.sub(r'[^0-9a-z ]+', '', indicator.lower())
+    word_tokens_indicator = word_tokenize(indicator)
+    filtered_sentence_indicator = [w for w in word_tokens_indicator if w not in stop_words]
+
+    correlation = [(w, filtered_sentence_input.count(w)) for w in filtered_sentence_input if w in filtered_sentence_indicator]
+    correlation = set(correlation)
+    # print(correlation)
+    score = 0
+    for w, count in correlation:
+        count *= filtered_sentence_indicator.count(w)
+        score += count
+    # score = len(correlation)  # Perhaps a better score?
+    correlations.append({"id": i, "score": score})
+
+
+print(correlations)
+sorted_correlations = sorted(correlations, key=lambda x: x["score"], reverse=True)
+print(sorted_correlations)
+print(input_text)
+
+for i in range(min(10, len(sorted_correlations))):
+    id = sorted_correlations[i]["id"]
+    print(i+1, "(%d)" % sorted_correlations[i]["score"], "-", csv_data["Indicator"].iloc[id])

indicatorlibrary/quickstart/filters.py

@@ -1,7 +1,7 @@
 from django_filters import FilterSet, ModelChoiceFilter, ChoiceFilter
 from .models import Indicator
 
-
+# Using django filters for easy filtering than ORM add field in level or sector for more
 STATUS_CHOICES_LEVEL = (
     ('Goal','Goal'),
     ('Outcome','Outcome'),

indicatorlibrary/quickstart/forms.py

@@ -5,7 +5,7 @@
 from django.contrib.auth.forms import UserCreationForm
 
 
-# UserForm = mmf(User, fields = ('first_name', 'last_name', 'email'))
+
 IndicatorAddForm = mmf(Indicator, fields = ('name','level','sector','subsector',
                                             'number','definition',
                                             'justification','unit_of_measure',
@@ -48,26 +48,3 @@ class Meta:
         model = Profile
         fields = ('organization',)
 
-# class UserProfileForm(forms.ModelForm):
-#     def __init__(self, instance=None, *args, **kwargs):
-#         # Add these fields from the user object
-#         _fields = ('first_name', 'last_name', 'email',)
-#         # Retrieve initial (current) data from the user object
-#         _initial = model_to_dict(instance.user, _fields) if instance is not None else {}
-#         # Pass the initial data to the base
-#         super(UserProfileForm, self).__init__(initial=_initial, instance=instance, *args, **kwargs)
-#         # Retrieve the fields from the user model and update the fields with it
-#         self.fields.update(fields_for_model(User, _fields))
-#
-#     class Meta:
-#         model = UserProfile
-#         exclude = ('user',)
-#
-#     def save(self, *args, **kwargs):
-#         u = self.instance.user
-#         u.first_name = self.cleaned_data['first_name']
-#         u.last_name = self.cleaned_data['last_name']
-#         u.email = self.cleaned_data['email']
-#         u.save()
-#         profile = super(UserProfileForm, self).save(*args,**kwargs)
-#         return profile

indicatorlibrary/quickstart/models.py

@@ -6,7 +6,8 @@
 from django.dispatch import receiver
 # Create your models here.    created_by = models.ForeignKey('auth.User', related_name='indicators', null=True, blank=True, on_delete=models.SET_NULL)
 
-
+# The first one is user extended model
+# TODO: make many to many relations in indicator frequqncy source and sector
 
 class Profile(models.Model):
     user = models.OneToOneField(User, on_delete=models.CASCADE)
@@ -70,7 +71,7 @@ def __str__(self):
     disaggregation = models.TextField(max_length=500, blank=True)
     direction_of_change = models.TextField(max_length=500, null=True, blank=True, verbose_name="Direction of Change")
     baseline = models.TextField(max_length=500, null=True, blank=True)
-    lop_target = models.IntegerField("LOP Target",default=0, blank=True)
+    lop_target = models.IntegerField("LOP Target", default=0, blank=True)
     rationale_for_target = models.TextField(max_length=500, null=True, blank=True)
     means_of_verification = models.TextField(max_length=500, null=True, blank=True, verbose_name="Means of Verification")
     question_format = models.TextField(max_length=500, null=True, blank=True, verbose_name="Question Format")

indicatorlibrary/quickstart/models_dbparsed.py

@@ -5,6 +5,7 @@
 #   * Make sure each ForeignKey has `on_delete` set to the desired behavior.
 #   * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table
 # Feel free to rename the models, but don't rename db_table values or field names.
+# It was just to try out the parsing do not try to change this with main models file
 from django.db import models
 
 

indicatorlibrary/quickstart/templates/base.html

@@ -15,54 +15,54 @@
 
     <link href="{%static '/css/simple-sidebar.css' %}" rel="stylesheet">
 </head>
-        <body>
-            <a href="#menu-toggle" class="btn btn-secondary" id="menu-toggle">Menu</a>
+<body>
+<a href="#menu-toggle" class="btn btn-secondary" id="menu-toggle">Menu</a>
 
-              <div id="wrapper">
+<div id="wrapper">
 
-        <!-- Sidebar -->
-        <div id="sidebar-wrapper">
-            <ul class="sidebar-nav">
-                <li class="sidebar-brand">
-                    <a href="#">
-                        Menu
-                    </a>
-                </li>
-                <li>
-                    <a href="/catalog/">Indicators</a>
-                </li>
-                <li>
-                    <a href="/profile/">Profile</a>
-                </li>
-                <li>
-                    <a href="/add/">Add Indicator</a>
-                </li>
-                <li>
-                    <a href="/about/">About us</a>
-                </li>
-            </ul>
-        </div>
-                  <div id="page-content-wrapper">
+    <!-- Sidebar -->
+    <div id="sidebar-wrapper">
+        <ul class="sidebar-nav">
+            <li class="sidebar-brand">
+                <a href="#">
+                    Menu
+                </a>
+            </li>
+            <li>
+                <a href="/catalog/">Indicators</a>
+            </li>
+            <li>
+                <a href="/profile/">Profile</a>
+            </li>
+            <li>
+                <a href="/add/">Add Indicator</a>
+            </li>
+            <li>
+                <a href="/about/">About us</a>
+            </li>
+        </ul>
+    </div>
+    <div id="page-content-wrapper">
 
-            <div class="container-fluid">
+        <div class="container-fluid">
             <h1 style="colour: #9cff9e; font-size:32px">Indicator Library</h1>
             {% block content %}
             {% endblock %}
-            </div>
         </div>
     </div>
-            <script src="{%static '/vendor/jquery/jquery.min.js'%}"></script>
+</div>
+<script src="{%static '/vendor/jquery/jquery.min.js'%}"></script>
 
 
-    <script src="{%static '/vendor/bootstrap/js/bootstrap.bundle.min.js'%}"></script>
+<script src="{%static '/vendor/bootstrap/js/bootstrap.bundle.min.js'%}"></script>
 
-    <!-- Menu Toggle Script -->
-    <script>
+<!-- Menu Toggle Script -->
+<script>
     $("#menu-toggle").click(function(e) {
         e.preventDefault();
         $("#wrapper").toggleClass("toggled");
     });
-    </script>
+</script>
 </body>
 
 </html>

indicatorlibrary/quickstart/templates/base_generic.html

@@ -1,18 +1,18 @@
 {% block content %}{% endblock %}
 {% block pagination %}
-  {% if is_paginated %}
-    <div class="pagination">
+    {% if is_paginated %}
+        <div class="pagination">
       <span class="page-links">
         {% if page_obj.has_previous %}
-          <a href="{{ request.path }}?page={{ page_obj.previous_page_number }}">previous</a>
+            <a href="{{ request.path }}?page={{ page_obj.previous_page_number }}">previous</a>
         {% endif %}
-        <span class="page-current">
+          <span class="page-current">
           <p>Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.</p>
         </span>
-        {% if page_obj.has_next %}
-          <a href="{{ request.path }}?page={{ page_obj.next_page_number }}">next</a>
-        {% endif %}
+          {% if page_obj.has_next %}
+              <a href="{{ request.path }}?page={{ page_obj.next_page_number }}">next</a>
+          {% endif %}
       </span>
-    </div>
-  {% endif %}
+        </div>
+    {% endif %}
 {% endblock %}

indicatorlibrary/quickstart/templates/quickstart/Profile.html

@@ -1,10 +1,10 @@
 {% extends 'base.html' %}
 {% block content %}
     <div class="container-fluid">
-    <h2>User Profile</h2>
-    <p>Username: {{ user }}</p>
-    <p>Email : {{ user.email }}</p>
-    <p>First name: {{ user.first_name }}</p>
-    <p>Org: {{ user.profile.organization }}</p>
+        <h2>User Profile</h2>
+        <p>Username: {{ user }}</p>
+        <p>Email : {{ user.email }}</p>
+        <p>First name: {{ user.first_name }}</p>
+        <p>Org: {{ user.profile.organization }}</p>
     </div>
 {% endblock %}

indicatorlibrary/quickstart/templates/quickstart/about.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 {% block content %}
     <p><label href="https://toladata.com">TolaData</label> is supporting an open source project called the Indicator Library, which started as an exercise to collect and compile nearly 2,500 commonly used indicators used by NGOs and donors in the international development and humanitarian sector as well as to include indicators used for corporate reporting on the SDGs.
-<br>
-The project’s goal is to improve the use of indicators by promoting standard, reusable indicators alongside guidance to help users effectively plan and collect useful data for their projects.</p>
+        <br>
+        The project’s goal is to improve the use of indicators by promoting standard, reusable indicators alongside guidance to help users effectively plan and collect useful data for their projects.</p>
 {% endblock %}

indicatorlibrary/quickstart/templates/quickstart/add_indicator.html

@@ -1,20 +1,20 @@
 {% extends 'base.html' %}
 {% block content %}
-  <h2>Add your own indicator</h2>
-  <form method="post">
-    {% csrf_token %}
-    {% for field in form %}
-      <p>
-        {{ field.label_tag }}<br>
-        {{ field }}
-        {% if field.help_text %}
-          <small style="color: grey">{{ field.help_text }}</small>
-        {% endif %}
-        {% for error in field.errors %}
-          <p style="color: red">{{ error }}</p>
+    <h2>Add your own indicator</h2>
+    <form method="post">
+        {% csrf_token %}
+        {% for field in form %}
+            <p>
+                {{ field.label_tag }}<br>
+                {{ field }}
+                {% if field.help_text %}
+                    <small style="color: grey">{{ field.help_text }}</small>
+                {% endif %}
+                {% for error in field.errors %}
+                    <p style="color: red">{{ error }}</p>
+                {% endfor %}
+            </p>
         {% endfor %}
-      </p>
-    {% endfor %}
-    <button type="submit">Submit</button>
-  </form>
+        <button type="submit">Submit</button>
+    </form>
 {% endblock %}

indicatorlibrary/quickstart/templates/quickstart/indicator_detail.html

@@ -2,7 +2,7 @@
 
 {% block content %}
 
-  <h1 style="colour: #9cff9e; font-size:32px">{{ indicator.name }}</h1>
+    <h1 style="colour: #9cff9e; font-size:32px">{{ indicator.name }}</h1>
 
     <p><strong>Level:</strong> <a href="">{{ indicator.level }}</a></p> <!-- author detail link not yet defined -->
     <p><strong>Sector:</strong> {{ indicator.sector }}</p>

indicatorlibrary/quickstart/templates/quickstart/indilist.html

@@ -1,36 +1,36 @@
 {% extends 'base.html' %}
 
-      {% block content %}
+{% block content %}
 
 
-  <form method="get">
-    {{ filter.form.as_p }}
-    <button type="submit" class="btn btn-secondary">Search</button>
-  </form>
-          <table class="table">
-  <thead>
-    <tr>
-        <th scope="col">#</th>
-      <th scope="col">Indicator ID</th>
-      <th scope="col">Name</th>
-      <th scope="col">Level</th>
-      <th scope="col">Sector</th>
-        <th scope="col">JSON API</th>
-    </tr>
-  </thead>
-  <tbody>
+    <form method="get">
+        {{ filter.form.as_p }}
+        <button type="submit" class="btn btn-secondary">Search</button>
+    </form>
+    <table class="table">
+        <thead>
+        <tr>
+            <th scope="col">#</th>
+            <th scope="col">Indicator ID</th>
+            <th scope="col">Name</th>
+            <th scope="col">Level</th>
+            <th scope="col">Sector</th>
+            <th scope="col">JSON API</th>
+        </tr>
+        </thead>
+        <tbody>
 
-  {% for indicator in filter.qs %}
-    <td>
-      <th scope="row">{{ indicator.id }}</th>
-      <td> <a href="{{indicator.id}} ">{{indicator.name}}</a></td>
-      <td>{{indicator.level}}</td>
-      <td>{{indicator.sector}}</td>
-      <td><a href="http://indilib.herokuapp.com/indicator/{{ indicator.id}}/?format=json"><strong>Json API:</strong> http://indilib.herokuapp.com/indicator/{{ indicator.id}}/?format=json</a></td>
-    </tr>
+        {% for indicator in filter.qs %}
+            <td>
+            <th scope="row">{{ indicator.id }}</th>
+            <td> <a href="{{indicator.id}} ">{{indicator.name}}</a></td>
+            <td>{{indicator.level}}</td>
+            <td>{{indicator.sector}}</td>
+            <td><a href="http://indilib.herokuapp.com/indicator/{{ indicator.id}}/?format=json"><strong>Json API:</strong> http://indilib.herokuapp.com/indicator/{{ indicator.id}}/?format=json</a></td>
+            </tr>
 
-{% endfor %}
-      </tbody>
-</table>
+        {% endfor %}
+        </tbody>
+    </table>
 
 {% endblock %}