diff mbox

[Branch,~linaro-validation/lava-dashboard/trunk] Rev 241: Merge performance improvements to Bundle.deserialize()

Message ID 20110712030314.18329.92163.launchpad@loganberry.canonical.com
State Accepted
Headers show

Commit Message

Zygmunt Krynicki July 12, 2011, 3:03 a.m. UTC
Merge authors:
  Michael Hudson-Doyle (mwhudson)
Related merge proposals:
  https://code.launchpad.net/~mwhudson/lava-dashboard/deserialize-performance/+merge/65934
  proposed by: Michael Hudson-Doyle (mwhudson)
  review: Approve - Zygmunt Krynicki (zkrynicki)
------------------------------------------------------------
revno: 241 [merge]
committer: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
branch nick: test_browser
timestamp: Tue 2011-07-12 05:00:26 +0200
message:
  Merge performance improvements to Bundle.deserialize()
added:
  dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py
modified:
  dashboard_app/helpers.py
  dashboard_app/models.py


--
lp:lava-dashboard
https://code.launchpad.net/~linaro-validation/lava-dashboard/trunk

You are subscribed to branch lp:lava-dashboard.
To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dashboard/trunk/+edit-subscription
diff mbox

Patch

=== modified file 'dashboard_app/helpers.py'
--- dashboard_app/helpers.py	2011-06-17 01:33:55 +0000
+++ dashboard_app/helpers.py	2011-07-04 06:04:22 +0000
@@ -5,15 +5,24 @@ 
 from uuid import UUID
 import base64
 import logging
+import time
 
 from django.core.files.base import ContentFile
-from django.db import transaction, IntegrityError
+from django.db import connection, transaction, IntegrityError
 from linaro_dashboard_bundle.errors import DocumentFormatError
 from linaro_dashboard_bundle.evolution import DocumentEvolution
 from linaro_dashboard_bundle.io import DocumentIO
 from linaro_json.extensions import datetime_extension, timedelta_extension
 
 
+PROFILE_LOGGING = False
+
+
+def is_postgres():
+    return (connection.settings_dict['ENGINE']
+            == 'django.db.backends.postgresql_psycopg2')
+
+
 class IBundleFormatImporter(object):
     """
     Interface for bundle format importers.
@@ -73,7 +82,7 @@ 
         This prevents InternalError (but is racy with other transactions).
         Still it's a little bit better to report the exception raised below
         rather than the IntegrityError that would have been raised otherwise.
-        
+
         The code copes with both (using transactions around _import_document()
         and _remove_created_files() that gets called if something is wrong)
         """
@@ -100,6 +109,23 @@ 
         for c_test_run in doc.get("test_runs", []):
             self._import_test_run(c_test_run, s_bundle)
 
+    def __init__(self):
+        self._qc = 0
+        self._time = time.time()
+
+    def _log(self, method_name):
+        if PROFILE_LOGGING:
+            t = time.time() - self._time
+            if t > 0.1:
+                p = '**'
+            else:
+                p = '  '
+            logging.warning(
+                '%s %s %.2f %d', p, method_name, t,
+                len(connection.queries) - self._qc)
+            self._qc = len(connection.queries)
+            self._time = time.time()
+
     def _import_test_run(self, c_test_run, s_bundle):
         """
         Import TestRun
@@ -122,11 +148,17 @@ 
         # needed for foreign key models below
         s_test_run.save()
         # import all the bits and pieces
+        self._log('starting')
         self._import_test_results(c_test_run, s_test_run)
+        self._log('results')
         self._import_attachments(c_test_run, s_test_run)
+        self._log('attachments')
         self._import_hardware_context(c_test_run, s_test_run)
+        self._log('hardware')
         self._import_software_context(c_test_run, s_test_run)
+        self._log('software')
         self._import_attributes(c_test_run, s_test_run)
+        self._log('attributes')
         # collect all the changes that happen before the previous save
         s_test_run.save()
         return s_test_run
@@ -134,7 +166,7 @@ 
     def _import_software_context(self, c_test_run, s_test_run):
         """
         Import software context.
-        
+
         In format 1.0 that's just a list of packages and software image
         description
         """
@@ -163,66 +195,337 @@ 
             s_test.save()
         return s_test
 
-    def _import_test_results(self, c_test_run, s_test_run):
-        """
-        Import TestRun.test_results
-        """
-        from dashboard_app.models import TestResult
-
-        for index, c_test_result in enumerate(c_test_run.get("test_results", []), 1):
-            s_test_case = self._import_test_case(
-                c_test_result, s_test_run.test)
+    def _import_test_results_sqlite(self, c_test_results, s_test_run):
+        cursor = connection.cursor()
+
+        # XXX I don't understand how the _order column that Django adds is
+        # supposed to work.  I just set it to 0 here.
+
+        data = []
+
+        for index, c_test_result in enumerate(c_test_results, 1):
+
             timestamp = c_test_result.get("timestamp")
             if timestamp:
                 timestamp = datetime_extension.from_json(timestamp)
             duration = c_test_result.get("duration", None)
             if duration:
                 duration = timedelta_extension.from_json(duration)
+                duration = (duration.microseconds +
+                            (duration.seconds * 10 ** 6) +
+                            (duration.days * 24 * 60 * 60 * 10 ** 6))
             result = self._translate_result_string(c_test_result["result"])
-            s_test_result = TestResult.objects.create(
-                test_run = s_test_run,
-                test_case = s_test_case,
-                result = result,
-                measurement = c_test_result.get("measurement", None),
-                filename = c_test_result.get("log_filename", None),
-                lineno = c_test_result.get("log_lineno", None),
-                message = c_test_result.get("message", None),
-                relative_index = index,
-                timestamp = timestamp,
-                duration = duration,
-            )
-            s_test_result.save() # needed for foreign key models below
-            self._import_attributes(c_test_result, s_test_result)
-
-    def _import_test_case(self, c_test_result, s_test):
-        """
-        Import TestCase
-        """
-        if "test_case_id" not in c_test_result:
+
+            data.append((
+                s_test_run.id,
+                index,
+                timestamp,
+                duration,
+                c_test_result.get("log_filename", None),
+                result,
+                c_test_result.get("measurement", None),
+                c_test_result.get("message", None),
+                c_test_result.get("log_lineno", None),
+                s_test_run.test.id,
+                c_test_result.get("test_case_id", None),
+                ))
+
+        cursor.executemany(
+            """
+            INSERT INTO dashboard_app_testresult (
+                test_run_id,
+                relative_index,
+                timestamp,
+                microseconds,
+                filename,
+                result,
+                measurement,
+                message,
+                lineno,
+                _order,
+                test_case_id
+            ) select
+                %s,
+                %s,
+                %s,
+                %s,
+                %s,
+                %s,
+                %s,
+                %s,
+                %s,
+                0,
+                dashboard_app_testcase.id
+                FROM dashboard_app_testcase
+                  WHERE dashboard_app_testcase.test_id = %s
+                    AND dashboard_app_testcase.test_case_id
+                      = %s
+            """, data)
+
+        cursor.close()
+
+    def _import_test_results_pgsql(self, c_test_results, s_test_run):
+        cursor = connection.cursor()
+
+        # XXX I don't understand how the _order column that Django adds is
+        # supposed to work.  I just let it default to 0 here.
+
+        data = []
+
+        for i in range(0, len(c_test_results), 1000):
+
+            cursor.execute(
+                """
+                CREATE TEMPORARY TABLE newtestresults (
+                    relative_index INTEGER,
+                    timestamp      TIMESTAMP WITH TIME ZONE,
+                    microseconds   BIGINT,
+                    filename       TEXT,
+                    result         SMALLINT,
+                    measurement    NUMERIC(20,10),
+                    message        TEXT,
+                    test_case_id   TEXT,
+                    lineno         INTEGER
+                    )
+                """)
+
+            data = []
+
+            for index, c_test_result in enumerate(c_test_results[i:i+1000], i+1):
+
+                timestamp = c_test_result.get("timestamp")
+                if timestamp:
+                    timestamp = datetime_extension.from_json(timestamp)
+                duration = c_test_result.get("duration", None)
+                if duration:
+                    duration = timedelta_extension.from_json(duration)
+                    duration = (duration.microseconds +
+                                (duration.seconds * 10 ** 6) +
+                                (duration.days * 24 * 60 * 60 * 10 ** 6))
+                result = self._translate_result_string(c_test_result["result"])
+
+                data.extend([
+                    index,
+                    timestamp,
+                    duration,
+                    c_test_result.get("log_filename", None),
+                    result,
+                    c_test_result.get("measurement", None),
+                    c_test_result.get("message", None),
+                    c_test_result.get("test_case_id", None),
+                    c_test_result.get("log_lineno", None),
+                    ])
+
+            sequel = ',\n'.join(
+                ["(" + "%s" % (', '.join(['%s']*9),) + ")"] * (len(data) // 9))
+
+            cursor.execute(
+                """
+                INSERT INTO newtestresults (
+                    relative_index,
+                    timestamp,
+                    microseconds,
+                    filename,
+                    result,
+                    measurement,
+                    message,
+                    test_case_id,
+                    lineno
+                ) VALUES """ + sequel, data)
+
+            cursor.execute(
+                """
+                INSERT INTO dashboard_app_testresult (
+                    test_run_id,
+                    relative_index,
+                    timestamp,
+                    microseconds,
+                    filename,
+                    result,
+                    measurement,
+                    message,
+                    test_case_id,
+                    lineno
+                ) SELECT
+                    %s,
+                    relative_index,
+                    timestamp,
+                    microseconds,
+                    filename,
+                    result,
+                    measurement,
+                    message,
+                    dashboard_app_testcase.id,
+                    lineno
+                    FROM newtestresults, dashboard_app_testcase
+                      WHERE dashboard_app_testcase.test_id = %s
+                        AND dashboard_app_testcase.test_case_id
+                          = newtestresults.test_case_id
+                """ % (s_test_run.id, s_test_run.test.id))
+
+            cursor.execute(
+                """
+                DROP TABLE newtestresults
+                """)
+
+        cursor.close()
+
+    def _import_test_results(self, c_test_run, s_test_run):
+        """
+        Import TestRun.test_results
+        """
+        from dashboard_app.models import TestResult
+
+        c_test_results = c_test_run.get("test_results", [])
+
+        if not c_test_results:
             return
-        from dashboard_app.models import TestCase
-        s_test_case, test_case_created = TestCase.objects.get_or_create(
-            test = s_test,
-            test_case_id = c_test_result["test_case_id"],
-            defaults = {'units': c_test_result.get("units", "")})
-        if test_case_created:
-            s_test_case.save()
-        return s_test_case
+
+        if is_postgres():
+            self._import_test_cases_pgsql(c_test_results, s_test_run.test)
+        else:
+            self._import_test_cases_sqlite(c_test_results, s_test_run.test)
+
+        if is_postgres():
+            self._import_test_results_pgsql(c_test_results, s_test_run)
+        else:
+            self._import_test_results_sqlite(c_test_results, s_test_run)
+
+        for index, c_test_result in enumerate(c_test_run.get("test_results", []), 1):
+            if c_test_result.get("attributes", {}):
+                s_test_result = TestResult.objects.get(
+                    relative_index=index, test_run=s_test_run)
+                self._import_attributes(c_test_result, s_test_result)
+        self._log('test result attributes')
+
+    def _import_test_cases_sqlite(self, c_test_results, s_test):
+        """
+        Import TestCase
+        """
+        id_units = []
+        for c_test_result in c_test_results:
+            if "test_case_id" not in c_test_result:
+                continue
+            id_units.append(
+                (c_test_result["test_case_id"],
+                 c_test_result.get("units", "")))
+
+        cursor = connection.cursor()
+
+        data = []
+        for (test_case_id, units) in id_units:
+            data.append((s_test.id, units, test_case_id))
+        cursor.executemany(
+            """
+            INSERT OR IGNORE INTO
+                dashboard_app_testcase (test_id, units, name, test_case_id)
+            VALUES (%s, %s, '', %s)
+            """, data)
+        cursor.close()
+
+    def _import_test_cases_pgsql(self, c_test_results, s_test):
+        """
+        Import TestCase
+        """
+        id_units = []
+        for c_test_result in c_test_results:
+            if "test_case_id" not in c_test_result:
+                continue
+            id_units.append(
+                (c_test_result["test_case_id"],
+                 c_test_result.get("units", "")))
+
+        cursor = connection.cursor()
+        for i in range(0, len(id_units), 1000):
+
+            cursor.execute(
+                """
+                CREATE TEMPORARY TABLE
+                    newtestcases (test_case_id text, units text)
+                """)
+            data = []
+            for (id, units) in id_units[i:i+1000]:
+                data.append(id)
+                data.append(units)
+            sequel = ',\n'.join(["(%s, %s)"] * (len(data) // 2))
+            cursor.execute(
+                """
+                INSERT INTO newtestcases (test_case_id, units) VALUES
+                """ + sequel, data)
+
+            cursor.execute(
+                """
+                INSERT INTO
+                    dashboard_app_testcase (test_id, units, name, test_case_id)
+                SELECT %s, units, E'', test_case_id FROM newtestcases
+                WHERE NOT EXISTS (SELECT 1 FROM dashboard_app_testcase
+                                  WHERE test_id = %s
+                                    AND newtestcases.test_case_id
+                                      = dashboard_app_testcase.test_case_id)
+                """ % (s_test.id, s_test.id))
+            cursor.execute(
+                """
+                drop table newtestcases
+                """)
+        cursor.close()
+
+    def _import_packages_scratch_sqlite(self, cursor, packages):
+        data = []
+        for c_package in packages:
+            data.append((c_package['name'], c_package['version']))
+        cursor.executemany(
+            """
+            INSERT INTO dashboard_app_softwarepackagescratch
+                   (name, version) VALUES (%s, %s)
+            """, data)
+
+    def _import_packages_scratch_pgsql(self, cursor, packages):
+        for i in range(0, len(packages), 1000):
+            data = []
+            for c_package in packages[i:i+1000]:
+                data.append(c_package['name'])
+                data.append(c_package['version'])
+            sequel = ', '.join(["(%s, %s)"] * (len(data) // 2))
+            cursor.execute(
+                "INSERT INTO dashboard_app_softwarepackagescratch (name, version) VALUES " + sequel, data)
 
     def _import_packages(self, c_test_run, s_test_run):
         """
         Import TestRun.pacakges
         """
-        from dashboard_app.models import SoftwarePackage
-
-        for c_package in self._get_sw_context(c_test_run).get("packages", []):
-            s_package, package_created = SoftwarePackage.objects.get_or_create(
-                name=c_package["name"], # required by schema
-                version=c_package["version"] # required by schema
-            )
-            if package_created:
-                s_package.save()
-            s_test_run.packages.add(s_package)
+        packages = self._get_sw_context(c_test_run).get("packages", [])
+        if not packages:
+            return
+        cursor = connection.cursor()
+
+        if is_postgres():
+            self._import_packages_scratch_pgsql(cursor, packages)
+        else:
+            self._import_packages_scratch_sqlite(cursor, packages)
+
+        cursor.execute(
+            """
+            INSERT INTO dashboard_app_softwarepackage (name, version)
+            SELECT name, version FROM dashboard_app_softwarepackagescratch
+            EXCEPT SELECT name, version FROM dashboard_app_softwarepackage
+            """)
+        cursor.execute(
+            """
+            INSERT INTO
+                dashboard_app_testrun_packages (testrun_id, softwarepackage_id)
+            SELECT %s, id FROM dashboard_app_softwarepackage
+                WHERE EXISTS (
+                    SELECT * FROM dashboard_app_softwarepackagescratch
+                        WHERE dashboard_app_softwarepackage.name
+                            = dashboard_app_softwarepackagescratch.name
+                          AND dashboard_app_softwarepackage.version
+                            = dashboard_app_softwarepackagescratch.version)
+            """ % s_test_run.id)
+        cursor.execute(
+            """
+            delete from dashboard_app_softwarepackagescratch
+            """)
+        cursor.close()
 
     def _import_devices(self, c_test_run, s_test_run):
         """

=== added file 'dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py'
--- dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py	1970-01-01 00:00:00 +0000
+++ dashboard_app/migrations/0004_auto__add_softwarepackagescratch.py	2011-07-04 02:09:01 +0000
@@ -0,0 +1,181 @@ 
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'SoftwarePackageScratch'
+        db.create_table('dashboard_app_softwarepackagescratch', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('version', self.gf('django.db.models.fields.CharField')(max_length=64)),
+        ))
+        db.send_create_signal('dashboard_app', ['SoftwarePackageScratch'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'SoftwarePackageScratch'
+        db.delete_table('dashboard_app_softwarepackagescratch')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'dashboard_app.attachment': {
+            'Meta': {'object_name': 'Attachment'},
+            'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}),
+            'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'public_url': ('django.db.models.fields.URLField', [], {'max_length': '512', 'blank': 'True'})
+        },
+        'dashboard_app.bundle': {
+            'Meta': {'ordering': "['-uploaded_on']", 'object_name': 'Bundle'},
+            'bundle_stream': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bundles'", 'to': "orm['dashboard_app.BundleStream']"}),
+            'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}),
+            'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'content_sha1': ('django.db.models.fields.CharField', [], {'max_length': '40', 'unique': 'True', 'null': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_deserialized': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'uploaded_by': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'uploaded_bundles'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'uploaded_on': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'})
+        },
+        'dashboard_app.bundledeserializationerror': {
+            'Meta': {'object_name': 'BundleDeserializationError'},
+            'bundle': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'deserialization_error'", 'unique': 'True', 'primary_key': 'True', 'to': "orm['dashboard_app.Bundle']"}),
+            'error_message': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
+            'traceback': ('django.db.models.fields.TextField', [], {'max_length': '32768'})
+        },
+        'dashboard_app.bundlestream': {
+            'Meta': {'object_name': 'BundleStream'},
+            'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_anonymous': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'pathname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'slug': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        },
+        'dashboard_app.hardwaredevice': {
+            'Meta': {'object_name': 'HardwareDevice'},
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'device_type': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'dashboard_app.namedattribute': {
+            'Meta': {'unique_together': "(('object_id', 'name'),)", 'object_name': 'NamedAttribute'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'value': ('django.db.models.fields.CharField', [], {'max_length': '512'})
+        },
+        'dashboard_app.softwarepackage': {
+            'Meta': {'unique_together': "(('name', 'version'),)", 'object_name': 'SoftwarePackage'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '64'})
+        },
+        'dashboard_app.softwarepackagescratch': {
+            'Meta': {'object_name': 'SoftwarePackageScratch'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '64'})
+        },
+        'dashboard_app.softwaresource': {
+            'Meta': {'object_name': 'SoftwareSource'},
+            'branch_revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'branch_url': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'branch_vcs': ('django.db.models.fields.CharField', [], {'max_length': '10'}),
+            'commit_timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project_name': ('django.db.models.fields.CharField', [], {'max_length': '32'})
+        },
+        'dashboard_app.test': {
+            'Meta': {'object_name': 'Test'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'test_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
+        },
+        'dashboard_app.testcase': {
+            'Meta': {'unique_together': "(('test', 'test_case_id'),)", 'object_name': 'TestCase'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_cases'", 'to': "orm['dashboard_app.Test']"}),
+            'test_case_id': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'units': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'})
+        },
+        'dashboard_app.testresult': {
+            'Meta': {'ordering': "('_order',)", 'object_name': 'TestResult'},
+            '_order': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lineno': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'measurement': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '10', 'blank': 'True'}),
+            'message': ('django.db.models.fields.TextField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'microseconds': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'relative_index': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'result': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'test_case': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'test_results'", 'null': 'True', 'to': "orm['dashboard_app.TestCase']"}),
+            'test_run': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_results'", 'to': "orm['dashboard_app.TestRun']"}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'dashboard_app.testrun': {
+            'Meta': {'ordering': "['-import_assigned_date']", 'object_name': 'TestRun'},
+            'analyzer_assigned_date': ('django.db.models.fields.DateTimeField', [], {}),
+            'analyzer_assigned_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}),
+            'bundle': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Bundle']"}),
+            'devices': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.HardwareDevice']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_assigned_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'packages': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwarePackage']"}),
+            'sources': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwareSource']"}),
+            'sw_image_desc': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Test']"}),
+            'time_check_performed': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        }
+    }
+
+    complete_apps = ['dashboard_app']

=== modified file 'dashboard_app/models.py'
--- dashboard_app/models.py	2011-07-12 02:30:17 +0000
+++ dashboard_app/models.py	2011-07-12 03:00:26 +0000
@@ -82,6 +82,22 @@ 
     def link_to_packages_ubuntu_com(self):
         return u"http://packages.ubuntu.com/{name}".format(name=self.name)
 
+
+class SoftwarePackageScratch(models.Model):
+    """
+    Staging area for SoftwarePackage data.
+
+    The code that keeps SoftwarePackage dumps data into here before more
+    carefully inserting it into the real SoftwarePackage table.
+
+    No data should ever be committed in this table.  It would be a temporary
+    table, but oddities in how the sqlite DB-API wrapper handles transactions
+    makes this impossible.
+    """
+    name = models.CharField(max_length=64)
+    version = models.CharField(max_length=64)
+
+
 class NamedAttribute(models.Model):
     """
     Model for adding generic named attributes