pagination added, block dev bw added

tacc_stats/analysis/exam/__init__.py

@@ -33,5 +33,6 @@
 
 from tacc_stats.analysis.exam.lliteopenclose import LLiteOpenClose
 from tacc_stats.analysis.exam.mcdrambw import MCDRAMBW
+from tacc_stats.analysis.exam.block_avebw import BlockAveBW
 from tacc_stats.analysis.exam.exams import Auditor
 

tacc_stats/analysis/exam/block_avebw.py

@@ -0,0 +1,12 @@
+from exams import Test
+import numpy
+
+class BlockAveBW(Test):
+    k1 = ['block', 'block']     
+    k2 = ['rd_sectors', 'wr_sectors']
+
+    comp_operator='>'
+
+    def compute_metric(self):
+        self.metric = (self.arc(self.ts.data[0]) + self.arc(self.ts.data[1]))/(1024*1024)
+        return

tacc_stats/site/machine/migrations/0009_job_blockavebw.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.6 on 2017-03-20 16:27
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('machine', '0008_job_mcdrambw'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='job',
+            name='BlockAveBW',
+            field=models.FloatField(null=True),
+        ),
+    ]

tacc_stats/site/machine/models.py

@@ -61,6 +61,7 @@ class Job(models.Model):
 
     LLiteOpenClose   =  models.FloatField(null=True)
     MCDRAMBW   =  models.FloatField(null=True)
+    BlockAveBW = models.FloatField(null=True)
 
     def __unicode__(self):
         return str(self.id)

tacc_stats/site/machine/templates/machine/index.html

@@ -95,8 +95,51 @@ <h4>Flagged Jobs:</h4>
   </table>
 </div>
 </div>
+{% if date %}
+<h3>List of Jobs run on {{ date }}</h3>
+{% endif %}
+<h4> #Jobs = {{nj}} </h4>
+
+    {% if job_list.has_other_pages %}
+    <ul class="pagination">
+      {% if job_list.has_previous %}
+
+      {% if current_path %}
+      <li><a href="{{ current_path }}&page={{ job_list.previous_page_number }}">&laquo;</a></li>
+      {% else %}
+      <li><a href="?page={{ job_list.previous_page_number }}">&laquo;</a></li>
+      {% endif %}
+
+      {% else %}
+      <li class="disabled"><span>&laquo;</span></li>
+      {% endif %}
+      {% for i in job_list.paginator.page_range %}
+      {% if job_list.number == i %}
+      <li class="active"><span>{{ i }} <span class="sr-only">(current)</span></span></li>
+      {% else %}
+
+      {% if current_path %}
+      <li><a href="{{ current_path }}&page={{ i }}">{{ i }}</a></li>
+      {% else %}
+      <li><a href="?page={{ i }}">{{ i }}</a></li>
+      {% endif %}
+
+      {% endif %}
+      {% endfor %}
+      {% if job_list.has_next %}
+
+      {% if current_path %}
+      <li><a href="{{ current_path }}&page={{ job_list.next_page_number }}">&raquo;</a></li>
+      {% else %}
+      <li><a href="?page={{ job_list.next_page_number }}">&raquo;</a></li>
+      {% endif %}
+
+      {% else %}
+      <li class="disabled"><span>&raquo;</span></li>
+      {% endif %}
+    </ul>
+    {% endif %}
 
-<h3># Jobs over 1 mn in run time = {{nj}}</h3>
     <table class="table table-condensed table-bordered">
       <thead>
         <tr>
@@ -140,4 +183,46 @@ <h3># Jobs over 1 mn in run time = {{nj}}</h3>
 	{% endfor %}
       <tbody> 
     </table>
+
+    {% if job_list.has_other_pages %}
+    <ul class="pagination">
+      {% if job_list.has_previous %}
+
+      {% if current_path %}
+      <li><a href="{{ current_path }}&page={{ job_list.previous_page_number }}">&laquo;</a></li>
+      {% else %}
+      <li><a href="?page={{ job_list.previous_page_number }}">&laquo;</a></li>
+      {% endif %}
+
+      {% else %}
+      <li class="disabled"><span>&laquo;</span></li>
+      {% endif %}
+      {% for i in job_list.paginator.page_range %}
+      {% if job_list.number == i %}
+      <li class="active"><span>{{ i }} <span class="sr-only">(current)</span></span></li>
+      {% else %}
+
+      {% if current_path %}
+      <li><a href="{{ current_path }}&page={{ i }}">{{ i }}</a></li>
+      {% else %}
+      <li><a href="?page={{ i }}">{{ i }}</a></li>
+      {% endif %}
+
+      {% endif %}
+      {% endfor %}
+      {% if job_list.has_next %}
+
+      {% if current_path %}
+      <li><a href="{{ current_path }}&page={{ job_list.next_page_number }}">&raquo;</a></li>
+      {% else %}
+      <li><a href="?page={{ job_list.next_page_number }}">&raquo;</a></li>
+      {% endif %}
+
+      {% else %}
+      <li class="disabled"><span>&raquo;</span></li>
+      {% endif %}
+    </ul>
+    {% endif %}
+
+
 {% endblock %}

tacc_stats/site/machine/thresholds.cfg

@@ -23,4 +23,5 @@ LnetAveBW	 >	1000
 InternodeIBMaxBW >	1000
 LnetMaxBW	 > 	1000
 LLiteOpenClose	 >	1000	
-OSCReqs		 >	1000
+OSCReqs		 >	1000
+BlockAveBW	 >	500

tacc_stats/site/machine/update_db.py

@@ -48,6 +48,7 @@ def update_comp_info(thresholds = None):
                   'MDCWait'  : ['MDCWait', '>', 10000],
                   'LLiteOpenClose'  : ['LLiteOpenClose', '>', 10000],
                   'MCDRAMBW'  : ['MCDRAMBW', '>', 400],
+                  'BlockAveBW'  : ['BlockAveBW', '>', 400],
                   }
     if thresholds:
         for key,val in thresholds.iteritems():
@@ -155,7 +156,8 @@ def update(date,rerun=False):
     ctr = 0
     for root, directory, pickle_files in os.walk(pickle_dir):
         num_files = len(pickle_files)
-        print "Number of pickle files in",root,'=',num_files
+        print "Number of pickle files in", root,'=',num_files
+        print "Number of database records", Job.objects.filter(date = date).count()
         for pickle_file in sorted(pickle_files):
 
             ctr += 1
@@ -279,6 +281,7 @@ def update_metric_fields(date, rerun = False):
     aud.stage(exam.OSCWait, ignore_qs=[], min_time = min_time)
     aud.stage(exam.LLiteOpenClose, ignore_qs=[], min_time = min_time)
     aud.stage(exam.MCDRAMBW, ignore_qs=[], min_time = min_time)
+    aud.stage(exam.BlockAveBW, ignore_qs=[], min_time = min_time)
 
     print 'Run the following tests for:',date
     for name, test in aud.measures.iteritems():

tacc_stats/site/machine/views.py

@@ -3,6 +3,7 @@
 from django.views.generic import DetailView, ListView
 from django.db.models import Q, F, FloatField, ExpressionWrapper
 from django.core.cache import cache 
+from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 
 import os,sys,pwd
 import cPickle as pickle 
@@ -93,81 +94,96 @@ def search(request):
             return HttpResponseRedirect("/machine/job/"+str(job.id)+"/")
         except: pass
     try:
-        fields = request.GET.dict()
-        new_fields = {k:v for k,v in fields.items() if v}
-        fields = new_fields
-
-        if 'opt_field1' in fields.keys() and 'value1' in fields.keys():
-            fields[fields['opt_field1']] = fields['value1']
-            del fields['opt_field1'], fields['value1']
-        if 'opt_field2' in fields.keys() and 'value2' in fields.keys():
-            fields[fields['opt_field2']] = fields['value2']
-            del fields['opt_field2'], fields['value2']
-        if 'opt_field3' in fields.keys() and 'value3' in fields.keys():
-            fields[fields['opt_field3']] = fields['value3']
-            del fields['opt_field3'], fields['value3']
-
-        return index(request, **fields)
+        return index(request)
     except: pass
 
     return dates(request, error = True)
 
 
-def index(request, **field):
+def index(request, **kwargs):
+
+    fields = request.GET.dict()
+    fields = {k:v for k,v in fields.items() if v}
+    fields.update(kwargs)
+
+    if 'page' in fields: del fields['page']
+    if 'opt_field1' in fields.keys() and 'value1' in fields.keys():
+        fields[fields['opt_field1']] = fields['value1']
+        del fields['opt_field1'], fields['value1']
+    if 'opt_field2' in fields.keys() and 'value2' in fields.keys():
+        fields[fields['opt_field2']] = fields['value2']
+        del fields['opt_field2'], fields['value2']
+    if 'opt_field3' in fields.keys() and 'value3' in fields.keys():
+        fields[fields['opt_field3']] = fields['value3']
+        del fields['opt_field3'], fields['value3']
 
     name = ''
-    for key, val in field.iteritems():
-        name += '['+key+'='+val+']-'
+    for key, val in fields.iteritems():
+        name += key+'='+val+'\n'
 
     order_key = '-id'
-    if 'order_key' in field: 
-        order_key = field['order_key']
-        del field['order_key']
+    if 'order_key' in fields: 
+        order_key = fields['order_key']
+        del fields['order_key']
 
-    if field.has_key('date'): 
-        date = field['date'].split('-')
+    if fields.has_key('date'): 
+        date = fields['date'].split('-')
         if len(date) == 2:
-            field['date__year'] = date[0]
-            field['date__month'] = date[1]
-            del field['date']
+            fields['date__year'] = date[0]
+            fields['date__month'] = date[1]
+            del fields['date']
+
+
+    job_list = Job.objects.filter(**fields).distinct().order_by(order_key)
 
-    print field
-    job_list = Job.objects.filter(**field).distinct().order_by(order_key)
+    fields['name'] =  'Query [fields=values] ' + name.rstrip('-')    
 
-    field['name'] = name + 'search'
-    field['histograms'] = hist_summary(job_list)
+    paginator = Paginator(job_list,100)
+    page = request.GET.get('page')
+    try:
+        jobs = paginator.page(page)
+    except PageNotAnInteger:
+        jobs = paginator.page(1)
+    except EmptyPage:
+        jobs = paginator.page(paginator.num_pages)
+
+    fields['histograms'] = hist_summary(job_list)
 
-    field['job_list'] = job_list
-    field['nj'] = job_list.count()
+    #fields['job_list'] = job_list
+    fields['job_list'] = jobs
+
+    fields['nj'] = job_list.count()
 
     # Computed Metrics
-    field['cat_job_list']  = job_list.filter(Q(cat__lte = 0.001) | Q(cat__gte = 1000)).exclude(cat = float('nan'))
+    fields['cat_job_list']  = job_list.filter(Q(cat__lte = 0.001) | Q(cat__gte = 1000)).exclude(cat = float('nan'))
 
     completed_list = job_list.exclude(status__in=['CANCELLED','FAILED']).order_by('-id')
     if len(completed_list) > 0:
-        field['md_job_list'] = job_list.exclude(LLiteOpenClose__isnull = True ).order_by('-LLiteOpenClose')
+        fields['md_job_list'] = job_list.exclude(LLiteOpenClose__isnull = True ).order_by('-LLiteOpenClose')
         try:
-            field['md_job_list'] = field['md_job_list'][0:10]
+            fields['md_job_list'] = fields['md_job_list'][0:10]
         except: pass
 
-        field['idle_job_list'] = completed_list.filter(idle__gte = 0.99)
-        field['mem_job_list'] = completed_list.filter(mem__lte = 30, queue = 'largemem')
+        fields['idle_job_list'] = completed_list.filter(idle__gte = 0.99)
+        fields['mem_job_list'] = completed_list.filter(mem__lte = 30, queue = 'largemem')
 
-        field['cpi_thresh'] = 1.5
-        field['cpi_job_list']  = completed_list.exclude(cpi = float('nan')).filter(cpi__gte = field['cpi_thresh'])
-        field['cpi_per'] = 100*field['cpi_job_list'].count()/float(completed_list.count())
+        fields['cpi_thresh'] = 1.5
+        fields['cpi_job_list']  = completed_list.exclude(cpi = float('nan')).filter(cpi__gte = fields['cpi_thresh'])
+        fields['cpi_per'] = 100*fields['cpi_job_list'].count()/float(completed_list.count())
 
-        field['gigebw_thresh'] = 2**20
-        field['gigebw_job_list']  = completed_list.exclude(GigEBW = float('nan')).filter(GigEBW__gte = field['gigebw_thresh'])
+        fields['gigebw_thresh'] = 2**20
+        fields['gigebw_job_list']  = completed_list.exclude(GigEBW = float('nan')).filter(GigEBW__gte = fields['gigebw_thresh'])
 
-        field['md_job_list'] = list_to_dict(field['md_job_list'],'LLiteOpenClose')
-        field['idle_job_list'] = list_to_dict(field['idle_job_list'],'idle')
-        field['cat_job_list'] = list_to_dict(field['cat_job_list'],'cat')
-        field['cpi_job_list'] = list_to_dict(field['cpi_job_list'],'cpi')
-        field['mem_job_list'] = list_to_dict(field['mem_job_list'],'mem')
-        field['gigebw_job_list'] = list_to_dict(field['gigebw_job_list'],'GigEBW')
+        fields['md_job_list'] = list_to_dict(fields['md_job_list'],'LLiteOpenClose')
+        fields['idle_job_list'] = list_to_dict(fields['idle_job_list'],'idle')
+        fields['cat_job_list'] = list_to_dict(fields['cat_job_list'],'cat')
+        fields['cpi_job_list'] = list_to_dict(fields['cpi_job_list'],'cpi')
+        fields['mem_job_list'] = list_to_dict(fields['mem_job_list'],'mem')
+        fields['gigebw_job_list'] = list_to_dict(fields['gigebw_job_list'],'GigEBW')
 
-    return render_to_response("machine/index.html", field)
+    if '?' in request.get_full_path():
+        fields['current_path'] = request.get_full_path()
+    return render_to_response("machine/index.html", fields)
 
 def list_to_dict(job_list,metric):
     job_dict={}

tacc_stats/site/tacc_stats_site/settings.py

@@ -107,8 +107,9 @@
             'context_processors': [
                 # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
                 # list if you haven't customized them:
-                'django.contrib.auth.context_processors.auth',
                 'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
                 'django.template.context_processors.i18n',
                 'django.template.context_processors.media',
                 'django.template.context_processors.static',
@@ -125,8 +126,8 @@
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
-    #'django.middleware.cache.UpdateCacheMiddleware',
-    #'django.middleware.cache.FetchFromCacheMiddleware',
+    'django.middleware.cache.UpdateCacheMiddleware',
+    'django.middleware.cache.FetchFromCacheMiddleware',
 )
 
 ROOT_URLCONF = 'tacc_stats.site.tacc_stats_site.urls'
@@ -178,17 +179,15 @@
         },
     }
 }
-"""
+
 CACHES = {
     'normal': {
         'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
         'LOCATION': '127.0.0.1:11211',
         },
-
-    'default': { 
-        'BACKEND':'tacc_stats.site.tacc_stats_site.cache.LargeMemcachedCache',
+    'default': {
+        'BACKEND': 'tacc_stats.site.tacc_stats_site.cache.LargeMemcachedCache',
         'LOCATION': '127.0.0.1:11211',
-        'TIMEOUT': None,
-        }
-    }
-"""
+        'TIMEOUT' : None,
+        },
+}