[Commits] act-server branch master updated.
C. Scott Ananian
cscott at laptop.org
Fri Dec 5 20:28:45 EST 2008
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "/home/cscott/public_git/act-server".
The branch, master has been updated
via af008d0e95995c41bc3a13a84088ab78341a1bde (commit)
via 678ae23fbc97438f732fb2081db410db98ceb4ac (commit)
via 1485bc134656350d7edb9f778437548a9727dcb1 (commit)
via 1a551891267d93d4be147d34ea1a8796cdd32b15 (commit)
from 3168d19cc86ca733826bc11e086dd182de64ff62 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
oats/request/views.py | 22 ++++++-
oats/stats/models.py | 12 ++++
oats/stats/views.py | 114 ++++++++++++++++++++++++---------
templates/request/request_detail.html | 28 ++++++++-
templates/request/request_list.html | 25 +++++++
templates/stats/serialnums.html | 16 +++--
6 files changed, 178 insertions(+), 39 deletions(-)
- Log -----------------------------------------------------------------
commit af008d0e95995c41bc3a13a84088ab78341a1bde
Author: C. Scott Ananian <cscott at laptop.org>
Date: Fri Dec 5 20:28:26 2008 -0500
Fix dates in updates statistics graph; tweak build sorting and x axis steps.
diff --git a/oats/stats/views.py b/oats/stats/views.py
index 5d0021c..052e432 100644
--- a/oats/stats/views.py
+++ b/oats/stats/views.py
@@ -180,7 +180,7 @@ def _ule_uniq_requests(dates, build_hashes, groups=(None,None), one_day=False):
if groups[1] is not None:
ule = ule.filter(groups[1])
ule = ule.values('req_serialnum').distinct()
- counts[d] = len(ule) # ugh: len(x). django bug.
+ counts[d] = ule.count()
StatsCache.objects.get_or_create(key=ckey, date=d,
defaults={'value':counts[d]})
return counts
@@ -235,6 +235,7 @@ def updates(request, html=False):
# pretty names
for build in Build.objects.filter(hash__in=hashes):
if build.name.startswith('joyride'): continue # HACK!
+ if '(' not in build.name: continue # only show 'special' builds
mapping[build.name] = build.hash
# group all unnamed builds together.
unnamed = hashes - set(mapping.values())
@@ -253,9 +254,13 @@ def updates(request, html=False):
ofc_set(resp, 'x_label_style', 13, '#000000', 2, 3)
else:
# don't include last date, which is incomplete.
- dates = list(x['date'] for x in dus.values('date').distinct())[-100:-1]
+ # XXX len(dus.values('date').distinct()) >>
+ # dus.values('date').distinct().count()
+ # this seems to be related to http://code.djangoproject.com/ticket/2939
+ dates = sorted(set(x['date'] for x in dus.values('date').distinct()))[:-1]
+ # XXX should really thin down 'dates'
ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
- ofc_set(resp, 'x_axis_steps', 7)
+ ofc_set(resp, 'x_axis_steps', 14)
ofc_set(resp, 'x_label_style', 13, '#000000', 2, 7)
def suf(tag, n):
@@ -273,7 +278,16 @@ def updates(request, html=False):
count_requests = _ule_uniq_requests
else:
count_requests = _dus_requests
- for build_name, build_hash in sorted(mapping.items()):
+ def sort_build((bname, bhash)):
+ # sort by the part in parens first.
+ if '(' in bname:
+ s = (bname.split('(',1)[-1]).split(')',1)[:1][0]
+ # if there's not an 'rc' part, add 'z' so release comes after rc's
+ if 'rc' not in s: s += ' z'
+ else:
+ s = 'xyzzy' # sort after the numbers
+ return (s, bname, bhash)
+ for build_name, build_hash in sorted(mapping.items(), key=sort_build):
counts = {}
build_hashes = [ build_hash ]
if build_name == 'other':
commit 678ae23fbc97438f732fb2081db410db98ceb4ac
Author: C. Scott Ananian <cscott at laptop.org>
Date: Fri Dec 5 19:21:52 2008 -0500
Limit unique SN graph to 180 points; add "last 6 months" selection checkbox.
Unique SN graph becomes too dense if we include all the points, but
now we let you select whether you want to see a point a day for the
last 6 months, or decimated points for "all time".
diff --git a/oats/stats/views.py b/oats/stats/views.py
index ff2812a..5d0021c 100644
--- a/oats/stats/views.py
+++ b/oats/stats/views.py
@@ -337,6 +337,7 @@ def serialnums(request, html=False):
return render_to_response(request, 'stats/serialnums.html', graph=graph)
# generate data file.
ACCUM = 'cumulative' in request.GET # stacked, or not.
+ RECENT = 'recent' in request.GET # show full range, or just last 6 months
resp = HttpResponse(mimetype='text/plain')
#resp['Content-Disposition'] = 'attachment; filename=graph.dat'
resp['Refresh'] = str(60*60*12) # every 12 hours.
@@ -349,10 +350,12 @@ def serialnums(request, html=False):
# don't include last date, which is incomplete.
dates = list(UpdateLogEntry.objects.dates('req_when', 'day'))[:-1]
- # halve list of dates until there are only ~250 labels
- while len(dates) > 250:
- dates = dates[-250:]
- #dates = dates[::2]
+ # halve list of dates until there are only 6 months worth of labels.
+ while len(dates) > 180:
+ if RECENT:
+ dates = dates[-180:]
+ else:
+ dates = dates[::2]
ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
diff --git a/templates/stats/serialnums.html b/templates/stats/serialnums.html
index bc901ef..f3b8d70 100644
--- a/templates/stats/serialnums.html
+++ b/templates/stats/serialnums.html
@@ -3,13 +3,16 @@
{% block content %}
<h1>Unique serial numbers seen</h1>
<script type="text/javascript">
-function update(elem)
+function update()
{
- if (elem.checked) {
- reload('cumulative')
- } else {
- reload('')
+ params = 'x';
+ if (document.getElementById('cumulative').checked) {
+ params += ';cumulative';
}
+ if (document.getElementById('recent').checked) {
+ params += ';recent';
+ }
+ reload(params);
}
function reload(params)
{
@@ -27,6 +30,7 @@ function findSWF(movieName) {
<script type="text/javascript" src="/static/swfobject.js"></script>
{{ graph|safe }}
<form>
-<input type="checkbox" id="cumulative" onchange="update(this)" /> Cumulative
+<input type="checkbox" id="cumulative" onchange="update()" /> Cumulative
+<input type="checkbox" id="recent" onchange="update()" /> Show last 6 months
</form>
{% endblock %}
commit 1485bc134656350d7edb9f778437548a9727dcb1
Author: C. Scott Ananian <cscott at laptop.org>
Date: Fri Dec 5 19:14:40 2008 -0500
Add a statistics memoization cache to speed up graph generation.
diff --git a/oats/stats/models.py b/oats/stats/models.py
index 3fa0c24..34ec34d 100644
--- a/oats/stats/models.py
+++ b/oats/stats/models.py
@@ -47,3 +47,15 @@ class UpdateLogEntry(models.Model):
class Meta:
verbose_name_plural = 'Update Log Entries'
permissions = (('can_download_raw', 'Can download raw update statistics'),)
+
+class StatsCache(models.Model):
+ """Simple table to cache computed statistics."""
+ date = models.DateTimeField('date key for memoization cache',
+ editable=False)
+ key = models.CharField('freetext key for memoization cache',
+ max_length=255, editable=False)
+ value = models.IntegerField('cached numerical value')
+ def __unicode__(self):
+ return '%s %s: %d' % (str(self.date), repr(self.key), self.value)
+ class Meta:
+ unique_together = (('key','date'))
diff --git a/oats/stats/views.py b/oats/stats/views.py
index 8e47e66..ff2812a 100644
--- a/oats/stats/views.py
+++ b/oats/stats/views.py
@@ -6,7 +6,7 @@ from django import forms
from oats.request.views import render_to_response # our tweaked version
from oats.streams.models import Build
from oats.gts.models import UpdateGroup, LaptopGroup
-from models import UpdateLogEntry, DailyUpdateStats
+from models import UpdateLogEntry, DailyUpdateStats, StatsCache
from binascii import hexlify
from openflashchart import ofc_html, ofc_set
from math import log, ceil
@@ -121,11 +121,22 @@ def _ofc_y_axis(resp, y_max):
ofc_set(resp, 'y_max', rounded)
ofc_set(resp, 'y_ticks', 5, 10, y_ticks) # don't understand the 5, 10
-def _dus_requests(dates, build_hashes, groups=None, one_day=False):
+def _sc_key(stat_type, build_hashes, groups, one_day):
+ import sha
+ # munge the build hashes together.
+ bh = ':'.join(sorted(build_hashes))
+ ckey = '%s%s|%s[' % \
+ (stat_type, ('H' if one_day else 'D'), sha.new(bh).hexdigest())
+ if groups[1] is not None:
+ ckey += (':'.join(groups[0]))
+ ckey += ']'
+ return ckey
+
+def _dus_requests(dates, build_hashes, groups=(None,None), one_day=False):
"""Return a mapping from each of the given `dates` to an integer
count of update requests for any of the given `build_hashes`."""
counts = {}
- if groups is None and not one_day:
+ if groups[1] is None and not one_day:
dus = DailyUpdateStats.objects
for x in dus.filter(hash__in=build_hashes).values('date','count'):
counts[x['date']] = counts.get(x['date'], 0) + x['count']
@@ -134,16 +145,22 @@ def _dus_requests(dates, build_hashes, groups=None, one_day=False):
delta = timedelta(hours=1)
else:
delta = timedelta(days=1)
+ ckey = _sc_key('DUS', build_hashes, groups, one_day)
for d in dates:
- c = UpdateLogEntry.objects
- if groups is not None:
- c = c.filter(groups)
- c = c.filter(req_when__gte=d, req_when__lt=(d+delta),
- req_hash__in=build_hashes).count()
- counts[d] = c
+ try:
+ counts[d] = StatsCache.objects.get(key=ckey,date=d).value
+ except StatsCache.DoesNotExist:
+ c = UpdateLogEntry.objects
+ if groups[1] is not None:
+ c = c.filter(groups[1])
+ c = c.filter(req_when__gte=d, req_when__lt=(d+delta),
+ req_hash__in=build_hashes).count()
+ counts[d] = c
+ StatsCache.objects.get_or_create\
+ (key=ckey,date=d, defaults={'value':counts[d]})
return counts
-def _ule_uniq_requests(dates, build_hashes, groups=None, one_day=False):
+def _ule_uniq_requests(dates, build_hashes, groups=(None,None), one_day=False):
"""Return a mapping from each of the given `dates` to an integer
count of update requests from unique serial numbers for any of the
given `build_hashes`."""
@@ -152,30 +169,38 @@ def _ule_uniq_requests(dates, build_hashes, groups=None, one_day=False):
delta = timedelta(hours=1)
else:
delta = timedelta(days=1)
+ ckey = _sc_key('ULE', build_hashes, groups, one_day)
for d in dates:
- ule = UpdateLogEntry.objects\
- .filter(req_when__gte=d, req_when__lt=(d+delta),
- req_hash__in=build_hashes)
- if groups is not None:
- ule = ule.filter(groups)
- ule = ule.values('req_serialnum').distinct()
- counts[d] = counts.get(d, 0) + len(ule) # ugh: len(x). django bug.
+ try:
+ counts[d] = StatsCache.objects.get(key=ckey,date=d).value
+ except StatsCache.DoesNotExist:
+ ule = UpdateLogEntry.objects\
+ .filter(req_when__gte=d, req_when__lt=(d+delta),
+ req_hash__in=build_hashes)
+ if groups[1] is not None:
+ ule = ule.filter(groups[1])
+ ule = ule.values('req_serialnum').distinct()
+ counts[d] = len(ule) # ugh: len(x). django bug.
+ StatsCache.objects.get_or_create(key=ckey, date=d,
+ defaults={'value':counts[d]})
return counts
def _parse_groups(request):
- if 'groups' not in request.GET: return None
+ """Return key for groups as well as the database queryset."""
+ if 'groups' not in request.GET: return None, None
# look up these update groups
groups = [ x.strip() for x in request.GET['groups'].split(',')]
+ groups.sort()
ug = [int(x[1:]) for x in groups if x.startswith('u')]
lg = [int(x[1:]) for x in groups if x.startswith('l')]
ugQ = Q(laptop__update_group__pk__in=ug)
lgQ = Q(laptop__other_groups__pk__in=lg)
# optimization: remove redundant part of query if will never match.
if not lg:
- return ugQ
+ return groups, ugQ
if not ug:
- return lgQ
- return ugQ | lgQ
+ return groups, lgQ
+ return groups, (ugQ | lgQ)
@login_required
def updates(request, html=False):
@@ -228,7 +253,7 @@ def updates(request, html=False):
ofc_set(resp, 'x_label_style', 13, '#000000', 2, 3)
else:
# don't include last date, which is incomplete.
- dates = list(x['date'] for x in dus.values('date').distinct())[:-1]
+ dates = list(x['date'] for x in dus.values('date').distinct())[-100:-1]
ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
ofc_set(resp, 'x_axis_steps', 7)
ofc_set(resp, 'x_label_style', 13, '#000000', 2, 7)
@@ -324,6 +349,10 @@ def serialnums(request, html=False):
# don't include last date, which is incomplete.
dates = list(UpdateLogEntry.objects.dates('req_when', 'day'))[:-1]
+ # halve list of dates until there are only ~250 labels
+ while len(dates) > 250:
+ dates = dates[-250:]
+ #dates = dates[::2]
ofc_set(resp, 'x_labels', *[x.strftime('%d %b %y') for x in dates])
@@ -331,26 +360,32 @@ def serialnums(request, html=False):
from django.db import connection
cursor = connection.cursor()
for d in dates:
- if False: # when http://code.djangoproject.com/ticket/2939 is fixed
+ ckey = 'USN'
+ try:
+ count = StatsCache.objects.get(key=ckey,date=d).value
+ except StatsCache.DoesNotExist:
+ # this query didn't used to work, but
+ # http://code.djangoproject.com/ticket/2939
+ # has been fixed in Django 1.0.
count = UpdateLogEntry.objects.filter(req_when__lt=d).values('req_serialnum').distinct().count()
- elif False: # a really slow workaround
- count = len(UpdateLogEntry.objects.filter(req_when__lt=d).values('req_serialnum').distinct())
- else:
- cursor.execute('SELECT %s, COUNT(DISTINCT req_serialnum) FROM stats_updatelogentry WHERE req_when < %s', [d, d+timedelta(1)])
- count = cursor.fetchone()[1]
+ StatsCache.objects.get_or_create(key=ckey,date=d,
+ defaults={'value':count})
v.append(count)
cursor.close()
# if not cumulative, compute diffs.
if ACCUM:
ofc_set(resp, 'y_legend', 'Unique SNs seen to date', 12)
+ maxv = max(v)
else:
ofc_set(resp, 'y_legend', 'New SNs seen on date', 12)
for i in reversed(xrange(1, len(v))):
v[i] = v[i] - v[i-1]
+ v[0] = 'null' # bogus diff if range doesn't begin at epoch
+ maxv = max(v[1:])
# now write data value arrays
ofc_set(resp, 'area_hollow', 2, 3, 25, colors[0], 'unique SNs', 10)
ofc_set(resp, 'values', *v)
- _ofc_y_axis(resp, max(v))
+ _ofc_y_axis(resp, maxv)
#ofc_set(resp, 'tool_tip', '#tip#/day<br>#key#<br>#x_label#')
return resp
commit 1a551891267d93d4be147d34ea1a8796cdd32b15
Author: C. Scott Ananian <cscott at laptop.org>
Date: Fri Dec 5 17:50:20 2008 -0500
Paginate the requested lease status pages, to deal with Uruguay's massive requests.
diff --git a/oats/request/views.py b/oats/request/views.py
index 43a001b..1ae813e 100644
--- a/oats/request/views.py
+++ b/oats/request/views.py
@@ -4,6 +4,7 @@ from django.views.generic.list_detail import object_list, object_detail
from django.contrib.auth.decorators import login_required, permission_required
from django.template import RequestContext, loader
from django.contrib.humanize.templatetags.humanize import naturalday
+from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.db import transaction
from django import forms
from django.utils.translation import ugettext as _
@@ -27,6 +28,7 @@ def render_to_response(request, template, **vardict):
def index(request, user):
queryset = Request.objects.filter(user=user)
return object_list(request, allow_empty=True, queryset=queryset,
+ paginate_by=50,
extra_context={'user':user,
'title':_('Lease and key request history'),
'is_manager':(user!=request.user)})
@@ -48,12 +50,30 @@ def status(request, request_id):
percent_done = 100. * ready_leases / total_leases
else:
percent_done = 0
+ # only show 250 leases per page, so that we don't completely choke on
+ # uruguay's requests with 15k leases!
+ paginator = Paginator(r.requestedlease_set.all(), 250)
+ # Make sure page request is an int or 'last'
+ try:
+ page = int(request.GET.get('page', '1'))
+ except ValueError:
+ if request.GET.get('page', '1') == 'last':
+ page = paginator.num_pages
+ else:
+ raise Http404
+ # If page request is out of range, deliver last page of results.
+ try:
+ leases = paginator.page(page)
+ except (EmptyPage, InvalidPage):
+ leases = paginator.page(paginator.num_pages)
+
resp = object_detail(request, object_id=request_id,
queryset=Request.objects,
extra_context={'title':title,
'percent_done': percent_done,
'total_leases': total_leases,
- 'ready_leases': ready_leases})
+ 'ready_leases': ready_leases,
+ 'leases': leases})
if total_leases != ready_leases:
resp['Refresh'] = str(120) # refresh every two minutes
return resp
diff --git a/templates/request/request_detail.html b/templates/request/request_detail.html
index 1a44084..060680a 100644
--- a/templates/request/request_detail.html
+++ b/templates/request/request_detail.html
@@ -55,7 +55,7 @@ Requested leases ({{ percent }}% ready):
</h3>
<table>
<tr><th>{% trans "Serial Number" %}</th><th>{% trans "Type" %}</th><th>{% trans "Duration" %}</th><th>{% trans "Ready?" %}</th></tr>
-{% for reqlease in object.requestedlease_set.all %}
+{% for reqlease in leases.object_list %}
<tr>
<td>{{ reqlease.laptop.serial_number }}</td>
<td>{{ reqlease.disposition }}</td>
@@ -70,4 +70,30 @@ Requested leases ({{ percent }}% ready):
{% endfor %}
</table>
+{% if leases.has_other_pages %}
+<div class="pagination">
+ <span class="step-links">
+ {% if leases.has_previous %}
+ <-
+ <a href="?page=1">{% trans "first" %}</a>
+ <a href="?page={{ leases.previous_page_number }}">{% trans "previous" %}</a>
+ {% endif %}
+ <span class="current">
+ {% with leases.number as page_number %}
+ {% with leases.paginator.num_pages as total_pages %}
+ {% blocktrans %}
+ Page {{ page_number }} of {{ total_pages }}
+ {% endblocktrans %}
+ {% endwith %}
+ {% endwith %}
+ </span>
+ {% if leases.has_next %}
+ <a href="?page={{ leases.next_page_number }}">{% trans "next"%}</a>
+ <a href="?page=last">{% trans "last"%}</a>
+ ->
+ {% endif %}
+ </span>
+</div>
+{% endif %}
+
{% endblock %}
diff --git a/templates/request/request_list.html b/templates/request/request_list.html
index c8a88b4..c9324a7 100644
--- a/templates/request/request_list.html
+++ b/templates/request/request_list.html
@@ -41,6 +41,31 @@
</li>
{% endfor %}
</ul>
+{% if is_paginated %}
+<div class="pagination">
+ <span class="step-links">
+ {% if page_obj.has_previous %}
+ <-
+ <a href="?page=1">{% trans "first" %}</a>
+ <a href="?page={{ page_obj.previous_page_number }}">{% trans "previous" %}</a>
+ {% endif %}
+ <span class="current">
+ {% with page_obj.number as page_number %}
+ {% with paginator.num_pages as total_pages %}
+ {% blocktrans %}
+ Page {{ page_number }} of {{ total_pages }}
+ {% endblocktrans %}
+ {% endwith %}
+ {% endwith %}
+ </span>
+ {% if page_obj.has_next %}
+ <a href="?page={{ page_obj.next_page_number }}">{% trans "next"%}</a>
+ <a href="?page=last">{% trans "last"%}</a>
+ ->
+ {% endif %}
+ </span>
+</div>
+{% endif %}
{% else %}
<p>
{% if is_manager %}
-----------------------------------------------------------------------
--
/home/cscott/public_git/act-server
More information about the Commits
mailing list