From patchwork Tue Jan 17 01:32:12 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael-Doyle Hudson X-Patchwork-Id: 6246 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id D69ED23E13 for ; Tue, 17 Jan 2012 01:32:14 +0000 (UTC) Received: from mail-bk0-f52.google.com (mail-bk0-f52.google.com [209.85.214.52]) by fiordland.canonical.com (Postfix) with ESMTP id A9B18A18275 for ; Tue, 17 Jan 2012 01:32:14 +0000 (UTC) Received: by bkbzt4 with SMTP id zt4so1187508bkb.11 for ; Mon, 16 Jan 2012 17:32:14 -0800 (PST) Received: by 10.204.10.73 with SMTP id o9mr438572bko.99.1326763934332; Mon, 16 Jan 2012 17:32:14 -0800 (PST) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.205.82.144 with SMTP id ac16cs105575bkc; Mon, 16 Jan 2012 17:32:14 -0800 (PST) Received: by 10.180.91.42 with SMTP id cb10mr18996750wib.15.1326763932732; Mon, 16 Jan 2012 17:32:12 -0800 (PST) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id k66si11331634weq.23.2012.01.16.17.32.12 (version=TLSv1/SSLv3 cipher=OTHER); Mon, 16 Jan 2012 17:32:12 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1RmxuG-0005JK-H4 for ; Tue, 17 Jan 2012 01:32:12 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id 71E59E04DB for ; Tue, 17 Jan 2012 01:32:12 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-scheduler X-Launchpad-Branch: ~linaro-validation/lava-scheduler/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 113 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-scheduler/trunk] Rev 113: allow results to be submitted to non-anonymous bundle streams Message-Id: <20120117013212.10253.8596.launchpad@ackee.canonical.com> Date: Tue, 17 Jan 2012 01:32:12 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="14664"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: e91aab811425d8019a635b03951f2d279e8ce901 Merge authors: Michael Hudson-Doyle (mwhudson) Related merge proposals: https://code.launchpad.net/~mwhudson/lava-scheduler/submit-to-private-stream/+merge/88011 proposed by: Michael Hudson-Doyle (mwhudson) review: Approve - Zygmunt Krynicki (zkrynicki) ------------------------------------------------------------ revno: 113 [merge] committer: Michael Hudson-Doyle branch nick: trunk timestamp: Tue 2012-01-17 14:30:18 +1300 message: allow results to be submitted to non-anonymous bundle streams added: lava_scheduler_app/migrations/0012_auto__add_field_testjob_submit_token.py modified: lava_scheduler_app/models.py lava_scheduler_app/tests.py lava_scheduler_daemon/dbjobsource.py --- lp:lava-scheduler https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk You are subscribed to branch lp:lava-scheduler. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk/+edit-subscription === added file 'lava_scheduler_app/migrations/0012_auto__add_field_testjob_submit_token.py' --- lava_scheduler_app/migrations/0012_auto__add_field_testjob_submit_token.py 1970-01-01 00:00:00 +0000 +++ lava_scheduler_app/migrations/0012_auto__add_field_testjob_submit_token.py 2012-01-10 01:11:00 +0000 @@ -0,0 +1,105 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'TestJob.submit_token' + db.add_column('lava_scheduler_app_testjob', 'submit_token', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['linaro_django_xmlrpc.AuthToken'], null=True), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'TestJob.submit_token' + db.delete_column('lava_scheduler_app_testjob', 'submit_token_id') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'lava_scheduler_app.device': { + 'Meta': {'object_name': 'Device'}, + 'current_job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.TestJob']", 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'device_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.DeviceType']"}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '200', 'primary_key': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'lava_scheduler_app.devicetype': { + 'Meta': {'object_name': 'DeviceType'}, + 'name': ('django.db.models.fields.SlugField', [], {'max_length': '50', 'primary_key': 'True', 'db_index': 'True'}) + }, + 'lava_scheduler_app.tag': { + 'Meta': {'object_name': 'Tag'}, + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}) + }, + 'lava_scheduler_app.testjob': { + 'Meta': {'object_name': 'TestJob'}, + 'actual_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}), + 'definition': ('django.db.models.fields.TextField', [], {}), + 'description': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'log_file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'requested_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}), + 'requested_device_type': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.DeviceType']"}), + 'results_link': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '400', 'null': 'True', 'blank': 'True'}), + 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'submit_time': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'submit_token': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['linaro_django_xmlrpc.AuthToken']", 'null': 'True'}), + 'submitter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'linaro_django_xmlrpc.authtoken': { + 'Meta': {'object_name': 'AuthToken'}, + 'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_used_on': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'secret': ('django.db.models.fields.CharField', [], {'default': "'z53d9om32nyy04rnuhy09xi5emz00f02igc497gjpz8lkjd02uijemou95fyyjfgrsttfzbg4hm0y10u5jbpcxyfowq669b6q6ud6z6rjkbzgatobh066exd45rx88q7'", 'unique': 'True', 'max_length': '128'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'auth_tokens'", 'to': "orm['auth.User']"}) + } + } + + complete_apps = ['lava_scheduler_app'] === modified file 'lava_scheduler_app/models.py' --- lava_scheduler_app/models.py 2011-12-15 03:16:40 +0000 +++ lava_scheduler_app/models.py 2012-01-10 01:11:00 +0000 @@ -4,6 +4,7 @@ from django.db import models from django.utils.translation import ugettext as _ +from linaro_django_xmlrpc.models import AuthToken class JSONDataError(ValueError): """Error raised when JSON is syntactically valid but ill-formed.""" @@ -135,6 +136,8 @@ verbose_name = _(u"Submitter"), ) + submit_token = models.ForeignKey(AuthToken, null=True) + description = models.CharField( verbose_name = _(u"Description"), max_length = 200, === modified file 'lava_scheduler_app/tests.py' --- lava_scheduler_app/tests.py 2011-12-14 05:02:36 +0000 +++ lava_scheduler_app/tests.py 2012-01-10 03:09:19 +0000 @@ -9,6 +9,8 @@ from django_testscenarios.ubertest import TestCase +from linaro_django_xmlrpc.models import AuthToken + from lava_scheduler_app.models import Device, DeviceType, Tag, TestJob @@ -413,12 +415,86 @@ device = Device.objects.get(pk=device.pk) self.assertEqual(job, device.current_job) + def test_getJobForBoard_creates_token(self): + device = self.factory.make_device(hostname='panda01') + job = self.factory.make_testjob(requested_device=device) + transaction.commit() + DatabaseJobSource().getJobForBoard_impl('panda01') + # reload from the database + job = TestJob.objects.get(pk=job.pk) + device = Device.objects.get(pk=device.pk) + self.assertIsNotNone(job.submit_token) + self.assertEqual(job.submitter, job.submit_token.user) + + def test_getJobForBoard_inserts_target_into_json(self): + panda_type = self.factory.ensure_device_type(name='panda') + self.factory.make_device(hostname='panda01', device_type=panda_type) + definition = {'foo': 'bar'} + self.factory.make_testjob( + requested_device_type=panda_type, + definition=json.dumps(definition)) + transaction.commit() + json_data = DatabaseJobSource().getJobForBoard_impl('panda01') + self.assertIn('target', json_data) + self.assertEqual('panda01', json_data['target']) + + def test_getJobForBoard_inserts_submit_token_into_json(self): + panda_type = self.factory.ensure_device_type(name='panda') + self.factory.make_device(hostname='panda01', device_type=panda_type) + definition = { + 'actions': [ + { + "command": "submit_results", + "parameters": + { + "server": "http://test-server/RPC2/", + "stream": "/private/personal/test/test/", + } + } + ] + } + job = self.factory.make_testjob( + requested_device_type=panda_type, + definition=json.dumps(definition)) + transaction.commit() + json_data = DatabaseJobSource().getJobForBoard_impl('panda01') + job = TestJob.objects.get(pk=job.pk) + submit_job_params = json_data['actions'][0]['parameters'] + self.assertIn('token', submit_job_params) + self.assertEqual(job.submit_token.secret, submit_job_params['token']) + + def test_getJobForBoard_adds_user_to_url(self): + panda_type = self.factory.ensure_device_type(name='panda') + self.factory.make_device(hostname='panda01', device_type=panda_type) + user = User.objects.create_user('test', 'e@mail.invalid', 'test') + user.save() + definition = { + 'actions': [ + { + "command": "submit_results", + "parameters": + { + "server": "http://test-server/RPC2/", + "stream": "/private/personal/test/test/", + } + } + ] + } + job = self.factory.make_testjob( + requested_device_type=panda_type, submitter=user, + definition=json.dumps(definition)) + transaction.commit() + json_data = DatabaseJobSource().getJobForBoard_impl('panda01') + job = TestJob.objects.get(pk=job.pk) + submit_job_params = json_data['actions'][0]['parameters'] + self.assertEqual("http://test@test-server/RPC2/", submit_job_params['server']) + def get_device_and_running_job(self): device = self.factory.make_device(hostname='panda01') job = self.factory.make_testjob(requested_device=device) transaction.commit() DatabaseJobSource().getJobForBoard_impl('panda01') - return device, job + return device, TestJob.objects.get(pk=job.pk) def test_jobCompleted_set_statuses_success(self): device, job = self.get_device_and_running_job() @@ -469,6 +545,15 @@ device = Device.objects.get(pk=device.pk) self.assertEquals(None, device.current_job) + def test_jobCompleted_deletes_token(self): + device, job = self.get_device_and_running_job() + token = job.submit_token + transaction.commit() + DatabaseJobSource().jobCompleted_impl('panda01', 0) + self.assertRaises( + AuthToken.DoesNotExist, + AuthToken.objects.get, pk=token.pk) + def test_getLogFileForJobOnBoard_returns_writable_file(self): device, job = self.get_device_and_running_job() definition = {'foo': 'bar'} === modified file 'lava_scheduler_daemon/dbjobsource.py' --- lava_scheduler_daemon/dbjobsource.py 2012-01-16 23:13:54 +0000 +++ lava_scheduler_daemon/dbjobsource.py 2012-01-17 01:30:18 +0000 @@ -1,6 +1,7 @@ import datetime import json import logging +import urlparse from django.core.files.base import ContentFile from django.db import connection @@ -8,6 +9,8 @@ from django.db.models import Q from django.db.utils import DatabaseError +from linaro_django_xmlrpc.models import AuthToken + from twisted.internet.threads import deferToThread from zope.interface import implements @@ -15,6 +18,7 @@ from lava_scheduler_app.models import Device, TestJob from lava_scheduler_daemon.jobsource import IJobSource + try: from psycopg2 import InterfaceError, OperationalError except ImportError: @@ -77,6 +81,35 @@ def getBoardList(self): return self.deferForDB(self.getBoardList_impl) + def _get_json_data(self, job): + json_data = json.loads(job.definition) + json_data['target'] = job.actual_device.hostname + # The rather extreme paranoia in what follows could be much reduced if + # we thoroughly validated job data in submit_job. We don't (yet?) + # and there is no sane way to report errors at this stage, so, + # paranoia (the dispatcher will choke on bogus input in a more + # informative way). + if 'actions' not in json_data: + return json_data + actions = json_data['actions'] + for action in actions: + if not isinstance(action, dict): + continue + if action.get('command') != 'submit_results': + continue + params = action.get('parameters') + if not isinstance(params, dict): + continue + params['token'] = job.submit_token.secret + if not 'server' in params or not isinstance(params['server'], unicode): + continue + parsed = urlparse.urlsplit(params['server']) + netloc = job.submitter.username + '@' + parsed.hostname + parsed = list(parsed) + parsed[1] = netloc + params['server'] = urlparse.urlunsplit(parsed) + return json_data + def getJobForBoard_impl(self, board_name): while True: device = Device.objects.get(hostname=board_name) @@ -125,9 +158,9 @@ else: job.log_file.save( 'job-%s.log' % job.id, ContentFile(''), save=False) + job.submit_token = AuthToken.objects.create(user=job.submitter) job.save() - json_data = json.loads(job.definition) - json_data['target'] = device.hostname + json_data = self._get_json_data(job) transaction.commit() return json_data else: @@ -172,6 +205,7 @@ "Unexpected job state in jobCompleted: %s" % job.status) job.status = TestJob.COMPLETE job.end_time = datetime.datetime.utcnow() + job.submit_token.delete() device.save() job.save()