From patchwork Tue Jul 12 03:03:14 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Zygmunt Krynicki X-Patchwork-Id: 2655 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 7B4A524348 for ; Tue, 12 Jul 2011 03:03:17 +0000 (UTC) Received: from mail-qw0-f52.google.com (mail-qw0-f52.google.com [209.85.216.52]) by fiordland.canonical.com (Postfix) with ESMTP id E7B18A1851D for ; Tue, 12 Jul 2011 03:03:16 +0000 (UTC) Received: by qwb8 with SMTP id 8so3109652qwb.11 for ; Mon, 11 Jul 2011 20:03:16 -0700 (PDT) Received: by 10.229.217.3 with SMTP id hk3mr2825496qcb.38.1310439796338; Mon, 11 Jul 2011 20:03:16 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.229.217.78 with SMTP id hl14cs227141qcb; Mon, 11 Jul 2011 20:03:15 -0700 (PDT) Received: by 10.227.173.202 with SMTP id q10mr4935498wbz.91.1310439794962; Mon, 11 Jul 2011 20:03:14 -0700 (PDT) Received: from adelie.canonical.com (adelie.canonical.com [91.189.90.139]) by mx.google.com with ESMTP id fw18si27185232wbb.105.2011.07.11.20.03.14; Mon, 11 Jul 2011 20:03:14 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) client-ip=91.189.90.139; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) smtp.mail=bounces@canonical.com Received: from loganberry.canonical.com ([91.189.90.37]) by adelie.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1QgTFi-0000G9-2O for ; Tue, 12 Jul 2011 03:03:14 +0000 Received: from loganberry.canonical.com (localhost [127.0.0.1]) by loganberry.canonical.com (Postfix) with ESMTP id 072562E84F6 for ; Tue, 12 Jul 2011 03:03:14 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dashboard X-Launchpad-Branch: ~linaro-validation/lava-dashboard/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 241 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dashboard/trunk] Rev 241: Merge performance improvements to Bundle.deserialize() Message-Id: <20110712030314.18329.92163.launchpad@loganberry.canonical.com> Date: Tue, 12 Jul 2011 03:03:14 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13388"; Instance="initZopeless config overlay" X-Launchpad-Hash: a766cc7a530c4079faef77334a9fb880e252e040 Merge authors: Michael Hudson-Doyle (mwhudson) Related merge proposals: https://code.launchpad.net/~mwhudson/lava-dashboard/deserialize-performance/+merge/65934 proposed by: Michael Hudson-Doyle (mwhudson) review: Approve - Zygmunt Krynicki (zkrynicki) ------------------------------------------------------------ revno: 241 [merge] committer: Zygmunt Krynicki branch nick: test_browser timestamp: Tue 2011-07-12 05:00:26 +0200 message: Merge performance improvements to Bundle.deserialize() added: dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py modified: dashboard_app/helpers.py dashboard_app/models.py --- lp:lava-dashboard https://code.launchpad.net/~linaro-validation/lava-dashboard/trunk You are subscribed to branch lp:lava-dashboard. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dashboard/trunk/+edit-subscription === modified file 'dashboard_app/helpers.py' --- dashboard_app/helpers.py 2011-06-17 01:33:55 +0000 +++ dashboard_app/helpers.py 2011-07-04 06:04:22 +0000 @@ -5,15 +5,24 @@ from uuid import UUID import base64 import logging +import time from django.core.files.base import ContentFile -from django.db import transaction, IntegrityError +from django.db import connection, transaction, IntegrityError from linaro_dashboard_bundle.errors import DocumentFormatError from linaro_dashboard_bundle.evolution import DocumentEvolution from linaro_dashboard_bundle.io import DocumentIO from linaro_json.extensions import datetime_extension, timedelta_extension +PROFILE_LOGGING = False + + +def is_postgres(): + return (connection.settings_dict['ENGINE'] + == 'django.db.backends.postgresql_psycopg2') + + class IBundleFormatImporter(object): """ Interface for bundle format importers. @@ -73,7 +82,7 @@ This prevents InternalError (but is racy with other transactions). Still it's a little bit better to report the exception raised below rather than the IntegrityError that would have been raised otherwise. - + The code copes with both (using transactions around _import_document() and _remove_created_files() that gets called if something is wrong) """ @@ -100,6 +109,23 @@ for c_test_run in doc.get("test_runs", []): self._import_test_run(c_test_run, s_bundle) + def __init__(self): + self._qc = 0 + self._time = time.time() + + def _log(self, method_name): + if PROFILE_LOGGING: + t = time.time() - self._time + if t > 0.1: + p = '**' + else: + p = ' ' + logging.warning( + '%s %s %.2f %d', p, method_name, t, + len(connection.queries) - self._qc) + self._qc = len(connection.queries) + self._time = time.time() + def _import_test_run(self, c_test_run, s_bundle): """ Import TestRun @@ -122,11 +148,17 @@ # needed for foreign key models below s_test_run.save() # import all the bits and pieces + self._log('starting') self._import_test_results(c_test_run, s_test_run) + self._log('results') self._import_attachments(c_test_run, s_test_run) + self._log('attachments') self._import_hardware_context(c_test_run, s_test_run) + self._log('hardware') self._import_software_context(c_test_run, s_test_run) + self._log('software') self._import_attributes(c_test_run, s_test_run) + self._log('attributes') # collect all the changes that happen before the previous save s_test_run.save() return s_test_run @@ -134,7 +166,7 @@ def _import_software_context(self, c_test_run, s_test_run): """ Import software context. - + In format 1.0 that's just a list of packages and software image description """ @@ -163,66 +195,337 @@ s_test.save() return s_test - def _import_test_results(self, c_test_run, s_test_run): - """ - Import TestRun.test_results - """ - from dashboard_app.models import TestResult - - for index, c_test_result in enumerate(c_test_run.get("test_results", []), 1): - s_test_case = self._import_test_case( - c_test_result, s_test_run.test) + def _import_test_results_sqlite(self, c_test_results, s_test_run): + cursor = connection.cursor() + + # XXX I don't understand how the _order column that Django adds is + # supposed to work. I just set it to 0 here. + + data = [] + + for index, c_test_result in enumerate(c_test_results, 1): + timestamp = c_test_result.get("timestamp") if timestamp: timestamp = datetime_extension.from_json(timestamp) duration = c_test_result.get("duration", None) if duration: duration = timedelta_extension.from_json(duration) + duration = (duration.microseconds + + (duration.seconds * 10 ** 6) + + (duration.days * 24 * 60 * 60 * 10 ** 6)) result = self._translate_result_string(c_test_result["result"]) - s_test_result = TestResult.objects.create( - test_run = s_test_run, - test_case = s_test_case, - result = result, - measurement = c_test_result.get("measurement", None), - filename = c_test_result.get("log_filename", None), - lineno = c_test_result.get("log_lineno", None), - message = c_test_result.get("message", None), - relative_index = index, - timestamp = timestamp, - duration = duration, - ) - s_test_result.save() # needed for foreign key models below - self._import_attributes(c_test_result, s_test_result) - - def _import_test_case(self, c_test_result, s_test): - """ - Import TestCase - """ - if "test_case_id" not in c_test_result: + + data.append(( + s_test_run.id, + index, + timestamp, + duration, + c_test_result.get("log_filename", None), + result, + c_test_result.get("measurement", None), + c_test_result.get("message", None), + c_test_result.get("log_lineno", None), + s_test_run.test.id, + c_test_result.get("test_case_id", None), + )) + + cursor.executemany( + """ + INSERT INTO dashboard_app_testresult ( + test_run_id, + relative_index, + timestamp, + microseconds, + filename, + result, + measurement, + message, + lineno, + _order, + test_case_id + ) select + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + 0, + dashboard_app_testcase.id + FROM dashboard_app_testcase + WHERE dashboard_app_testcase.test_id = %s + AND dashboard_app_testcase.test_case_id + = %s + """, data) + + cursor.close() + + def _import_test_results_pgsql(self, c_test_results, s_test_run): + cursor = connection.cursor() + + # XXX I don't understand how the _order column that Django adds is + # supposed to work. I just let it default to 0 here. + + data = [] + + for i in range(0, len(c_test_results), 1000): + + cursor.execute( + """ + CREATE TEMPORARY TABLE newtestresults ( + relative_index INTEGER, + timestamp TIMESTAMP WITH TIME ZONE, + microseconds BIGINT, + filename TEXT, + result SMALLINT, + measurement NUMERIC(20,10), + message TEXT, + test_case_id TEXT, + lineno INTEGER + ) + """) + + data = [] + + for index, c_test_result in enumerate(c_test_results[i:i+1000], i+1): + + timestamp = c_test_result.get("timestamp") + if timestamp: + timestamp = datetime_extension.from_json(timestamp) + duration = c_test_result.get("duration", None) + if duration: + duration = timedelta_extension.from_json(duration) + duration = (duration.microseconds + + (duration.seconds * 10 ** 6) + + (duration.days * 24 * 60 * 60 * 10 ** 6)) + result = self._translate_result_string(c_test_result["result"]) + + data.extend([ + index, + timestamp, + duration, + c_test_result.get("log_filename", None), + result, + c_test_result.get("measurement", None), + c_test_result.get("message", None), + c_test_result.get("test_case_id", None), + c_test_result.get("log_lineno", None), + ]) + + sequel = ',\n'.join( + ["(" + "%s" % (', '.join(['%s']*9),) + ")"] * (len(data) // 9)) + + cursor.execute( + """ + INSERT INTO newtestresults ( + relative_index, + timestamp, + microseconds, + filename, + result, + measurement, + message, + test_case_id, + lineno + ) VALUES """ + sequel, data) + + cursor.execute( + """ + INSERT INTO dashboard_app_testresult ( + test_run_id, + relative_index, + timestamp, + microseconds, + filename, + result, + measurement, + message, + test_case_id, + lineno + ) SELECT + %s, + relative_index, + timestamp, + microseconds, + filename, + result, + measurement, + message, + dashboard_app_testcase.id, + lineno + FROM newtestresults, dashboard_app_testcase + WHERE dashboard_app_testcase.test_id = %s + AND dashboard_app_testcase.test_case_id + = newtestresults.test_case_id + """ % (s_test_run.id, s_test_run.test.id)) + + cursor.execute( + """ + DROP TABLE newtestresults + """) + + cursor.close() + + def _import_test_results(self, c_test_run, s_test_run): + """ + Import TestRun.test_results + """ + from dashboard_app.models import TestResult + + c_test_results = c_test_run.get("test_results", []) + + if not c_test_results: return - from dashboard_app.models import TestCase - s_test_case, test_case_created = TestCase.objects.get_or_create( - test = s_test, - test_case_id = c_test_result["test_case_id"], - defaults = {'units': c_test_result.get("units", "")}) - if test_case_created: - s_test_case.save() - return s_test_case + + if is_postgres(): + self._import_test_cases_pgsql(c_test_results, s_test_run.test) + else: + self._import_test_cases_sqlite(c_test_results, s_test_run.test) + + if is_postgres(): + self._import_test_results_pgsql(c_test_results, s_test_run) + else: + self._import_test_results_sqlite(c_test_results, s_test_run) + + for index, c_test_result in enumerate(c_test_run.get("test_results", []), 1): + if c_test_result.get("attributes", {}): + s_test_result = TestResult.objects.get( + relative_index=index, test_run=s_test_run) + self._import_attributes(c_test_result, s_test_result) + self._log('test result attributes') + + def _import_test_cases_sqlite(self, c_test_results, s_test): + """ + Import TestCase + """ + id_units = [] + for c_test_result in c_test_results: + if "test_case_id" not in c_test_result: + continue + id_units.append( + (c_test_result["test_case_id"], + c_test_result.get("units", ""))) + + cursor = connection.cursor() + + data = [] + for (test_case_id, units) in id_units: + data.append((s_test.id, units, test_case_id)) + cursor.executemany( + """ + INSERT OR IGNORE INTO + dashboard_app_testcase (test_id, units, name, test_case_id) + VALUES (%s, %s, '', %s) + """, data) + cursor.close() + + def _import_test_cases_pgsql(self, c_test_results, s_test): + """ + Import TestCase + """ + id_units = [] + for c_test_result in c_test_results: + if "test_case_id" not in c_test_result: + continue + id_units.append( + (c_test_result["test_case_id"], + c_test_result.get("units", ""))) + + cursor = connection.cursor() + for i in range(0, len(id_units), 1000): + + cursor.execute( + """ + CREATE TEMPORARY TABLE + newtestcases (test_case_id text, units text) + """) + data = [] + for (id, units) in id_units[i:i+1000]: + data.append(id) + data.append(units) + sequel = ',\n'.join(["(%s, %s)"] * (len(data) // 2)) + cursor.execute( + """ + INSERT INTO newtestcases (test_case_id, units) VALUES + """ + sequel, data) + + cursor.execute( + """ + INSERT INTO + dashboard_app_testcase (test_id, units, name, test_case_id) + SELECT %s, units, E'', test_case_id FROM newtestcases + WHERE NOT EXISTS (SELECT 1 FROM dashboard_app_testcase + WHERE test_id = %s + AND newtestcases.test_case_id + = dashboard_app_testcase.test_case_id) + """ % (s_test.id, s_test.id)) + cursor.execute( + """ + drop table newtestcases + """) + cursor.close() + + def _import_packages_scratch_sqlite(self, cursor, packages): + data = [] + for c_package in packages: + data.append((c_package['name'], c_package['version'])) + cursor.executemany( + """ + INSERT INTO dashboard_app_softwarepackagescratch + (name, version) VALUES (%s, %s) + """, data) + + def _import_packages_scratch_pgsql(self, cursor, packages): + for i in range(0, len(packages), 1000): + data = [] + for c_package in packages[i:i+1000]: + data.append(c_package['name']) + data.append(c_package['version']) + sequel = ', '.join(["(%s, %s)"] * (len(data) // 2)) + cursor.execute( + "INSERT INTO dashboard_app_softwarepackagescratch (name, version) VALUES " + sequel, data) def _import_packages(self, c_test_run, s_test_run): """ Import TestRun.pacakges """ - from dashboard_app.models import SoftwarePackage - - for c_package in self._get_sw_context(c_test_run).get("packages", []): - s_package, package_created = SoftwarePackage.objects.get_or_create( - name=c_package["name"], # required by schema - version=c_package["version"] # required by schema - ) - if package_created: - s_package.save() - s_test_run.packages.add(s_package) + packages = self._get_sw_context(c_test_run).get("packages", []) + if not packages: + return + cursor = connection.cursor() + + if is_postgres(): + self._import_packages_scratch_pgsql(cursor, packages) + else: + self._import_packages_scratch_sqlite(cursor, packages) + + cursor.execute( + """ + INSERT INTO dashboard_app_softwarepackage (name, version) + SELECT name, version FROM dashboard_app_softwarepackagescratch + EXCEPT SELECT name, version FROM dashboard_app_softwarepackage + """) + cursor.execute( + """ + INSERT INTO + dashboard_app_testrun_packages (testrun_id, softwarepackage_id) + SELECT %s, id FROM dashboard_app_softwarepackage + WHERE EXISTS ( + SELECT * FROM dashboard_app_softwarepackagescratch + WHERE dashboard_app_softwarepackage.name + = dashboard_app_softwarepackagescratch.name + AND dashboard_app_softwarepackage.version + = dashboard_app_softwarepackagescratch.version) + """ % s_test_run.id) + cursor.execute( + """ + delete from dashboard_app_softwarepackagescratch + """) + cursor.close() def _import_devices(self, c_test_run, s_test_run): """ === added file 'dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py' --- dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py 1970-01-01 00:00:00 +0000 +++ dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py 2011-07-04 02:09:01 +0000 @@ -0,0 +1,181 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'SoftwarePackageScratch' + db.create_table('dashboard_app_softwarepackagescratch', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('version', self.gf('django.db.models.fields.CharField')(max_length=64)), + )) + db.send_create_signal('dashboard_app', ['SoftwarePackageScratch']) + + + def backwards(self, orm): + + # Deleting model 'SoftwarePackageScratch' + db.delete_table('dashboard_app_softwarepackagescratch') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'dashboard_app.attachment': { + 'Meta': {'object_name': 'Attachment'}, + 'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'public_url': ('django.db.models.fields.URLField', [], {'max_length': '512', 'blank': 'True'}) + }, + 'dashboard_app.bundle': { + 'Meta': {'ordering': "['-uploaded_on']", 'object_name': 'Bundle'}, + 'bundle_stream': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bundles'", 'to': "orm['dashboard_app.BundleStream']"}), + 'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'content_sha1': ('django.db.models.fields.CharField', [], {'max_length': '40', 'unique': 'True', 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_deserialized': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'uploaded_by': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'uploaded_bundles'", 'null': 'True', 'to': "orm['auth.User']"}), + 'uploaded_on': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}) + }, + 'dashboard_app.bundledeserializationerror': { + 'Meta': {'object_name': 'BundleDeserializationError'}, + 'bundle': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'deserialization_error'", 'unique': 'True', 'primary_key': 'True', 'to': "orm['dashboard_app.Bundle']"}), + 'error_message': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'traceback': ('django.db.models.fields.TextField', [], {'max_length': '32768'}) + }, + 'dashboard_app.bundlestream': { + 'Meta': {'object_name': 'BundleStream'}, + 'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_anonymous': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'pathname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'slug': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}) + }, + 'dashboard_app.hardwaredevice': { + 'Meta': {'object_name': 'HardwareDevice'}, + 'description': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'dashboard_app.namedattribute': { + 'Meta': {'unique_together': "(('object_id', 'name'),)", 'object_name': 'NamedAttribute'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'value': ('django.db.models.fields.CharField', [], {'max_length': '512'}) + }, + 'dashboard_app.softwarepackage': { + 'Meta': {'unique_together': "(('name', 'version'),)", 'object_name': 'SoftwarePackage'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '64'}) + }, + 'dashboard_app.softwarepackagescratch': { + 'Meta': {'object_name': 'SoftwarePackageScratch'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '64'}) + }, + 'dashboard_app.softwaresource': { + 'Meta': {'object_name': 'SoftwareSource'}, + 'branch_revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'branch_url': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'branch_vcs': ('django.db.models.fields.CharField', [], {'max_length': '10'}), + 'commit_timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'project_name': ('django.db.models.fields.CharField', [], {'max_length': '32'}) + }, + 'dashboard_app.test': { + 'Meta': {'object_name': 'Test'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'test_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'}) + }, + 'dashboard_app.testcase': { + 'Meta': {'unique_together': "(('test', 'test_case_id'),)", 'object_name': 'TestCase'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_cases'", 'to': "orm['dashboard_app.Test']"}), + 'test_case_id': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'units': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}) + }, + 'dashboard_app.testresult': { + 'Meta': {'ordering': "('_order',)", 'object_name': 'TestResult'}, + '_order': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lineno': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'measurement': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '10', 'blank': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'microseconds': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'relative_index': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'result': ('django.db.models.fields.PositiveSmallIntegerField', [], {}), + 'test_case': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'test_results'", 'null': 'True', 'to': "orm['dashboard_app.TestCase']"}), + 'test_run': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_results'", 'to': "orm['dashboard_app.TestRun']"}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}) + }, + 'dashboard_app.testrun': { + 'Meta': {'ordering': "['-import_assigned_date']", 'object_name': 'TestRun'}, + 'analyzer_assigned_date': ('django.db.models.fields.DateTimeField', [], {}), + 'analyzer_assigned_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}), + 'bundle': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Bundle']"}), + 'devices': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.HardwareDevice']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_assigned_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwarePackage']"}), + 'sources': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwareSource']"}), + 'sw_image_desc': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Test']"}), + 'time_check_performed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['dashboard_app'] === modified file 'dashboard_app/models.py' --- dashboard_app/models.py 2011-07-12 02:30:17 +0000 +++ dashboard_app/models.py 2011-07-12 03:00:26 +0000 @@ -82,6 +82,22 @@ def link_to_packages_ubuntu_com(self): return u"http://packages.ubuntu.com/{name}".format(name=self.name) + +class SoftwarePackageScratch(models.Model): + """ + Staging area for SoftwarePackage data. + + The code that keeps SoftwarePackage dumps data into here before more + carefully inserting it into the real SoftwarePackage table. + + No data should ever be committed in this table. It would be a temporary + table, but oddities in how the sqlite DB-API wrapper handles transactions + makes this impossible. + """ + name = models.CharField(max_length=64) + version = models.CharField(max_length=64) + + class NamedAttribute(models.Model): """ Model for adding generic named attributes