[Commits] act-server branch master updated.

C. Scott Ananian cscott at laptop.org
Fri Dec 5 20:28:45 EST 2008


This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "/home/cscott/public_git/act-server".

The branch, master has been updated
       via  af008d0e95995c41bc3a13a84088ab78341a1bde (commit)
       via  678ae23fbc97438f732fb2081db410db98ceb4ac (commit)
       via  1485bc134656350d7edb9f778437548a9727dcb1 (commit)
       via  1a551891267d93d4be147d34ea1a8796cdd32b15 (commit)
      from  3168d19cc86ca733826bc11e086dd182de64ff62 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

 oats/request/views.py                 |   22 ++++++-
 oats/stats/models.py                  |   12 ++++
 oats/stats/views.py                   |  114 ++++++++++++++++++++++++---------
 templates/request/request_detail.html |   28 ++++++++-
 templates/request/request_list.html   |   25 +++++++
 templates/stats/serialnums.html       |   16 +++--
 6 files changed, 178 insertions(+), 39 deletions(-)

- Log -----------------------------------------------------------------
commit af008d0e95995c41bc3a13a84088ab78341a1bde
Author: C. Scott Ananian <cscott at laptop.org>
Date:   Fri Dec 5 20:28:26 2008 -0500

    Fix dates in updates statistics graph; tweak build sorting and x axis steps.

diff --git a/oats/stats/views.py b/oats/stats/views.py
index 5d0021c..052e432 100644
--- a/oats/stats/views.py
+++ b/oats/stats/views.py
@@ -180,7 +180,7 @@ def _ule_uniq_requests(dates, build_hashes, groups=(None,None), one_day=False):
             if groups[1] is not None:
                 ule = ule.filter(groups[1])
             ule = ule.values('req_serialnum').distinct()
-            counts[d] = len(ule) # ugh: len(x). django bug.
+            counts[d] = ule.count()
             StatsCache.objects.get_or_create(key=ckey, date=d,
                                              defaults={'value':counts[d]})
     return counts
@@ -235,6 +235,7 @@ def updates(request, html=False):
     # pretty names
     for build in Build.objects.filter(hash__in=hashes):
         if build.name.startswith('joyride'): continue # HACK!
+        if '(' not in build.name: continue # only show 'special' builds
         mapping[build.name] = build.hash
     # group all unnamed builds together.
     unnamed = hashes - set(mapping.values())
@@ -253,9 +254,13 @@ def updates(request, html=False):
         ofc_set(resp, 'x_label_style', 13, '#000000', 2, 3)
     else:
         # don't include last date, which is incomplete.
-        dates = list(x['date'] for x in dus.values('date').distinct())[-100:-1]
+        # XXX len(dus.values('date').distinct()) >>
+        #         dus.values('date').distinct().count()
+        # this seems to be related to http://code.djangoproject.com/ticket/2939
+        dates = sorted(set(x['date'] for x in dus.values('date').distinct()))[:-1]
+        # XXX should really thin down 'dates'
         ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
-        ofc_set(resp, 'x_axis_steps', 7)
+        ofc_set(resp, 'x_axis_steps', 14)
         ofc_set(resp, 'x_label_style', 13, '#000000', 2, 7)
 
     def suf(tag, n):
@@ -273,7 +278,16 @@ def updates(request, html=False):
         count_requests = _ule_uniq_requests
     else:
         count_requests = _dus_requests
-    for build_name, build_hash in sorted(mapping.items()):
+    def sort_build((bname, bhash)):
+        # sort by the part in parens first.
+        if '(' in bname:
+            s = (bname.split('(',1)[-1]).split(')',1)[:1][0]
+            # if there's not an 'rc' part, add 'z' so release comes after rc's
+            if 'rc' not in s: s += ' z'
+        else:
+            s = 'xyzzy' # sort after the numbers
+        return (s, bname, bhash)
+    for build_name, build_hash in sorted(mapping.items(), key=sort_build):
         counts = {}
         build_hashes = [ build_hash ]
         if build_name == 'other':

commit 678ae23fbc97438f732fb2081db410db98ceb4ac
Author: C. Scott Ananian <cscott at laptop.org>
Date:   Fri Dec 5 19:21:52 2008 -0500

    Limit unique SN graph to 180 points; add "last 6 months" selection checkbox.
    
    Unique SN graph becomes too dense if we include all the points, but
    now we let you select whether you want to see a point a day for the
    last 6 months, or decimated points for "all time".

diff --git a/oats/stats/views.py b/oats/stats/views.py
index ff2812a..5d0021c 100644
--- a/oats/stats/views.py
+++ b/oats/stats/views.py
@@ -337,6 +337,7 @@ def serialnums(request, html=False):
         return render_to_response(request, 'stats/serialnums.html', graph=graph)
     # generate data file.
     ACCUM = 'cumulative' in request.GET # stacked, or not.
+    RECENT = 'recent' in request.GET # show full range, or just last 6 months
     resp = HttpResponse(mimetype='text/plain')
     #resp['Content-Disposition'] = 'attachment; filename=graph.dat'
     resp['Refresh'] = str(60*60*12) # every 12 hours.
@@ -349,10 +350,12 @@ def serialnums(request, html=False):
 
     # don't include last date, which is incomplete.
     dates = list(UpdateLogEntry.objects.dates('req_when', 'day'))[:-1]
-    # halve list of dates until there are only ~250 labels
-    while len(dates) > 250:
-        dates = dates[-250:]
-        #dates = dates[::2]
+    # halve list of dates until there are only 6 months worth of labels.
+    while len(dates) > 180:
+        if RECENT:
+            dates = dates[-180:]
+        else:
+            dates = dates[::2]
     
     ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
 
diff --git a/templates/stats/serialnums.html b/templates/stats/serialnums.html
index bc901ef..f3b8d70 100644
--- a/templates/stats/serialnums.html
+++ b/templates/stats/serialnums.html
@@ -3,13 +3,16 @@
 {% block content %}
 <h1>Unique serial numbers seen</h1>
 <script type="text/javascript">
-function update(elem)
+function update()
 {
-    if (elem.checked) {
-       reload('cumulative')
-    } else {
-       reload('')
+    params = 'x';
+    if (document.getElementById('cumulative').checked) {
+       params += ';cumulative';
     }
+    if (document.getElementById('recent').checked) {
+       params += ';recent';
+    }
+    reload(params);
 }
 function reload(params)
 {
@@ -27,6 +30,7 @@ function findSWF(movieName) {
 <script type="text/javascript" src="/static/swfobject.js"></script>
 {{ graph|safe }}
 <form>
-<input type="checkbox" id="cumulative" onchange="update(this)" /> Cumulative
+<input type="checkbox" id="cumulative" onchange="update()" /> Cumulative
+<input type="checkbox" id="recent" onchange="update()" /> Show last 6 months
 </form>
 {% endblock %}

commit 1485bc134656350d7edb9f778437548a9727dcb1
Author: C. Scott Ananian <cscott at laptop.org>
Date:   Fri Dec 5 19:14:40 2008 -0500

    Add a statistics memoization cache to speed up graph generation.

diff --git a/oats/stats/models.py b/oats/stats/models.py
index 3fa0c24..34ec34d 100644
--- a/oats/stats/models.py
+++ b/oats/stats/models.py
@@ -47,3 +47,15 @@ class UpdateLogEntry(models.Model):
     class Meta:
         verbose_name_plural = 'Update Log Entries'
         permissions = (('can_download_raw', 'Can download raw update statistics'),)
+
+class StatsCache(models.Model):
+    """Simple table to cache computed statistics."""
+    date = models.DateTimeField('date key for memoization cache',
+                                editable=False)
+    key = models.CharField('freetext key for memoization cache',
+                           max_length=255, editable=False)
+    value = models.IntegerField('cached numerical value')
+    def __unicode__(self):
+        return '%s %s: %d' % (str(self.date), repr(self.key), self.value)
+    class Meta:
+        unique_together = (('key','date'))
diff --git a/oats/stats/views.py b/oats/stats/views.py
index 8e47e66..ff2812a 100644
--- a/oats/stats/views.py
+++ b/oats/stats/views.py
@@ -6,7 +6,7 @@ from django import forms
 from oats.request.views import render_to_response # our tweaked version
 from oats.streams.models import Build
 from oats.gts.models import UpdateGroup, LaptopGroup
-from models import UpdateLogEntry, DailyUpdateStats
+from models import UpdateLogEntry, DailyUpdateStats, StatsCache
 from binascii import hexlify
 from openflashchart import ofc_html, ofc_set
 from math import log, ceil
@@ -121,11 +121,22 @@ def _ofc_y_axis(resp, y_max):
     ofc_set(resp, 'y_max', rounded)
     ofc_set(resp, 'y_ticks', 5, 10, y_ticks) # don't understand the 5, 10
 
-def _dus_requests(dates, build_hashes, groups=None, one_day=False):
+def _sc_key(stat_type, build_hashes, groups, one_day):
+    import sha
+    # munge the build hashes together.
+    bh = ':'.join(sorted(build_hashes))
+    ckey = '%s%s|%s[' % \
+           (stat_type, ('H' if one_day else 'D'), sha.new(bh).hexdigest())
+    if groups[1] is not None:
+        ckey +=  (':'.join(groups[0]))
+    ckey += ']'
+    return ckey
+
+def _dus_requests(dates, build_hashes, groups=(None,None), one_day=False):
     """Return a mapping from each of the given `dates` to an integer
     count of update requests for any of the given `build_hashes`."""
     counts = {}
-    if groups is None and not one_day:
+    if groups[1] is None and not one_day:
         dus = DailyUpdateStats.objects
         for x in dus.filter(hash__in=build_hashes).values('date','count'):
             counts[x['date']] = counts.get(x['date'], 0) + x['count']
@@ -134,16 +145,22 @@ def _dus_requests(dates, build_hashes, groups=None, one_day=False):
             delta = timedelta(hours=1)
         else:
             delta = timedelta(days=1)
+        ckey = _sc_key('DUS', build_hashes, groups, one_day)
         for d in dates:
-            c = UpdateLogEntry.objects
-            if groups is not None:
-                c = c.filter(groups)
-            c = c.filter(req_when__gte=d, req_when__lt=(d+delta),
-                         req_hash__in=build_hashes).count()
-            counts[d] = c
+            try:
+                counts[d] = StatsCache.objects.get(key=ckey,date=d).value
+            except StatsCache.DoesNotExist:
+                c = UpdateLogEntry.objects
+                if groups[1] is not None:
+                    c = c.filter(groups[1])
+                c = c.filter(req_when__gte=d, req_when__lt=(d+delta),
+                             req_hash__in=build_hashes).count()
+                counts[d] = c
+                StatsCache.objects.get_or_create\
+                    (key=ckey,date=d, defaults={'value':counts[d]})
     return counts
 
-def _ule_uniq_requests(dates, build_hashes, groups=None, one_day=False):
+def _ule_uniq_requests(dates, build_hashes, groups=(None,None), one_day=False):
     """Return a mapping from each of the given `dates` to an integer
     count of update requests from unique serial numbers for any of the
     given `build_hashes`."""
@@ -152,30 +169,38 @@ def _ule_uniq_requests(dates, build_hashes, groups=None, one_day=False):
         delta = timedelta(hours=1)
     else:
         delta = timedelta(days=1)
+    ckey = _sc_key('ULE', build_hashes, groups, one_day)
     for d in dates:
-        ule = UpdateLogEntry.objects\
-              .filter(req_when__gte=d, req_when__lt=(d+delta),
-                      req_hash__in=build_hashes)
-        if groups is not None:
-            ule = ule.filter(groups)
-        ule = ule.values('req_serialnum').distinct()
-        counts[d] = counts.get(d, 0) + len(ule) # ugh: len(x). django bug.
+        try:
+            counts[d] = StatsCache.objects.get(key=ckey,date=d).value
+        except StatsCache.DoesNotExist:
+            ule = UpdateLogEntry.objects\
+                  .filter(req_when__gte=d, req_when__lt=(d+delta),
+                          req_hash__in=build_hashes)
+            if groups[1] is not None:
+                ule = ule.filter(groups[1])
+            ule = ule.values('req_serialnum').distinct()
+            counts[d] = len(ule) # ugh: len(x). django bug.
+            StatsCache.objects.get_or_create(key=ckey, date=d,
+                                             defaults={'value':counts[d]})
     return counts
 
 def _parse_groups(request):
-    if 'groups' not in request.GET: return None
+    """Return key for groups as well as the database queryset."""
+    if 'groups' not in request.GET: return None, None
     # look up these update groups
     groups = [ x.strip() for x in request.GET['groups'].split(',')]
+    groups.sort()
     ug = [int(x[1:]) for x in groups if x.startswith('u')]
     lg = [int(x[1:]) for x in groups if x.startswith('l')]
     ugQ = Q(laptop__update_group__pk__in=ug)
     lgQ = Q(laptop__other_groups__pk__in=lg)
     # optimization: remove redundant part of query if will never match.
     if not lg:
-        return ugQ
+        return groups, ugQ
     if not ug:
-        return lgQ
-    return ugQ | lgQ
+        return groups, lgQ
+    return groups, (ugQ | lgQ)
 
 @login_required
 def updates(request, html=False):
@@ -228,7 +253,7 @@ def updates(request, html=False):
         ofc_set(resp, 'x_label_style', 13, '#000000', 2, 3)
     else:
         # don't include last date, which is incomplete.
-        dates = list(x['date'] for x in dus.values('date').distinct())[:-1]
+        dates = list(x['date'] for x in dus.values('date').distinct())[-100:-1]
         ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
         ofc_set(resp, 'x_axis_steps', 7)
         ofc_set(resp, 'x_label_style', 13, '#000000', 2, 7)
@@ -324,6 +349,10 @@ def serialnums(request, html=False):
 
     # don't include last date, which is incomplete.
     dates = list(UpdateLogEntry.objects.dates('req_when', 'day'))[:-1]
+    # halve list of dates until there are only ~250 labels
+    while len(dates) > 250:
+        dates = dates[-250:]
+        #dates = dates[::2]
     
     ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
 
@@ -331,26 +360,32 @@ def serialnums(request, html=False):
     from django.db import connection
     cursor = connection.cursor()
     for d in dates:
-        if False: # when http://code.djangoproject.com/ticket/2939 is fixed
+        ckey = 'USN'
+        try:
+            count = StatsCache.objects.get(key=ckey,date=d).value
+        except StatsCache.DoesNotExist:
+            # this query didn't used to work, but
+            #   http://code.djangoproject.com/ticket/2939
+            # has been fixed in Django 1.0.
             count = UpdateLogEntry.objects.filter(req_when__lt=d).values('req_serialnum').distinct().count()
-        elif False: # a really slow workaround
-            count = len(UpdateLogEntry.objects.filter(req_when__lt=d).values('req_serialnum').distinct())
-        else:
-            cursor.execute('SELECT %s, COUNT(DISTINCT req_serialnum) FROM stats_updatelogentry WHERE req_when < %s', [d, d+timedelta(1)])
-            count = cursor.fetchone()[1]
+            StatsCache.objects.get_or_create(key=ckey,date=d,
+                                             defaults={'value':count})
         v.append(count)
     cursor.close()
     # if not cumulative, compute diffs.
     if ACCUM:
         ofc_set(resp, 'y_legend', 'Unique SNs seen to date', 12)
+        maxv = max(v)
     else:
         ofc_set(resp, 'y_legend', 'New SNs seen on date', 12)
         for i in reversed(xrange(1, len(v))):
             v[i] = v[i] - v[i-1]
+        v[0] = 'null' # bogus diff if range doesn't begin at epoch
+        maxv = max(v[1:])
     # now write data value arrays
     ofc_set(resp, 'area_hollow', 2, 3, 25, colors[0], 'unique SNs', 10)
     ofc_set(resp, 'values', *v)
-    _ofc_y_axis(resp, max(v))
+    _ofc_y_axis(resp, maxv)
     #ofc_set(resp, 'tool_tip', '#tip#/day<br>#key#<br>#x_label#')
 
     return resp

commit 1a551891267d93d4be147d34ea1a8796cdd32b15
Author: C. Scott Ananian <cscott at laptop.org>
Date:   Fri Dec 5 17:50:20 2008 -0500

    Paginate the requested lease status pages, to deal with Uruguay's massive requests.

diff --git a/oats/request/views.py b/oats/request/views.py
index 43a001b..1ae813e 100644
--- a/oats/request/views.py
+++ b/oats/request/views.py
@@ -4,6 +4,7 @@ from django.views.generic.list_detail import object_list, object_detail
 from django.contrib.auth.decorators import login_required, permission_required
 from django.template import RequestContext, loader
 from django.contrib.humanize.templatetags.humanize import naturalday
+from django.core.paginator import Paginator, InvalidPage, EmptyPage
 from django.db import transaction
 from django import forms
 from django.utils.translation import ugettext as _
@@ -27,6 +28,7 @@ def render_to_response(request, template, **vardict):
 def index(request, user):
     queryset = Request.objects.filter(user=user)
     return object_list(request, allow_empty=True, queryset=queryset,
+                       paginate_by=50,
                        extra_context={'user':user,
                                       'title':_('Lease and key request history'),
                                       'is_manager':(user!=request.user)})
@@ -48,12 +50,30 @@ def status(request, request_id):
         percent_done = 100. * ready_leases / total_leases
     else:
         percent_done = 0
+    # only show 250 leases per page, so that we don't completely choke on
+    # uruguay's requests with 15k leases!
+    paginator = Paginator(r.requestedlease_set.all(), 250)
+    # Make sure page request is an int or 'last'
+    try:
+        page = int(request.GET.get('page', '1'))
+    except ValueError:
+        if request.GET.get('page', '1') == 'last':
+            page = paginator.num_pages
+        else:
+            raise Http404
+    # If page request is out of range, deliver last page of results.
+    try:
+        leases = paginator.page(page)
+    except (EmptyPage, InvalidPage):
+        leases = paginator.page(paginator.num_pages)
+
     resp = object_detail(request, object_id=request_id,
                          queryset=Request.objects,
                          extra_context={'title':title,
                                         'percent_done': percent_done,
                                         'total_leases': total_leases,
-                                        'ready_leases': ready_leases})
+                                        'ready_leases': ready_leases,
+                                        'leases': leases})
     if total_leases != ready_leases:
         resp['Refresh'] = str(120) # refresh every two minutes
     return resp
diff --git a/templates/request/request_detail.html b/templates/request/request_detail.html
index 1a44084..060680a 100644
--- a/templates/request/request_detail.html
+++ b/templates/request/request_detail.html
@@ -55,7 +55,7 @@ Requested leases ({{ percent }}% ready):
 </h3>
 <table>
   <tr><th>{% trans "Serial Number" %}</th><th>{% trans "Type" %}</th><th>{% trans "Duration" %}</th><th>{% trans "Ready?" %}</th></tr>
-{% for reqlease in object.requestedlease_set.all %}
+{% for reqlease in leases.object_list %}
   <tr>
     <td>{{ reqlease.laptop.serial_number }}</td>
     <td>{{ reqlease.disposition }}</td>
@@ -70,4 +70,30 @@ Requested leases ({{ percent }}% ready):
 {% endfor %}
 </table>
 
+{% if leases.has_other_pages %}
+<div class="pagination">
+  <span class="step-links">
+  {% if leases.has_previous %}
+  &lt;-
+  <a href="?page=1">{% trans "first" %}</a>
+  <a href="?page={{ leases.previous_page_number }}">{% trans "previous" %}</a>
+  {% endif %}
+  <span class="current">
+  {% with leases.number as page_number %}
+  {% with leases.paginator.num_pages as total_pages %}
+  {% blocktrans %}
+  Page {{ page_number }} of {{ total_pages }}
+  {% endblocktrans %}
+  {% endwith %}
+  {% endwith %}
+  </span>
+  {% if leases.has_next %}
+  <a href="?page={{ leases.next_page_number }}">{% trans "next"%}</a>
+  <a href="?page=last">{% trans "last"%}</a>
+  -&gt;
+  {% endif %}
+  </span>
+</div>
+{% endif %}
+
 {% endblock %}
diff --git a/templates/request/request_list.html b/templates/request/request_list.html
index c8a88b4..c9324a7 100644
--- a/templates/request/request_list.html
+++ b/templates/request/request_list.html
@@ -41,6 +41,31 @@
   </li>
   {% endfor %}
 </ul>
+{% if is_paginated %}
+<div class="pagination">
+  <span class="step-links">
+  {% if page_obj.has_previous %}
+  &lt;-
+  <a href="?page=1">{% trans "first" %}</a>
+  <a href="?page={{ page_obj.previous_page_number }}">{% trans "previous" %}</a>
+  {% endif %}
+  <span class="current">
+  {% with page_obj.number as page_number %}
+  {% with paginator.num_pages as total_pages %}
+  {% blocktrans %}
+  Page {{ page_number }} of {{ total_pages }}
+  {% endblocktrans %}
+  {% endwith %}
+  {% endwith %}
+  </span>
+  {% if page_obj.has_next %}
+  <a href="?page={{ page_obj.next_page_number }}">{% trans "next"%}</a>
+  <a href="?page=last">{% trans "last"%}</a>
+  -&gt;
+  {% endif %}
+  </span>
+</div>
+{% endif %}
 {% else %}
   <p>
 {% if is_manager %}
-----------------------------------------------------------------------


--
/home/cscott/public_git/act-server


More information about the Commits mailing list