=== modified file 'README'
@@ -6,6 +6,45 @@
External dependency
-------------------
The following debian packages are needed:
+* python-setuptools
* python-apt
* usbutils
-* python-testrepository - for running unit tests
\ No newline at end of file
+* python-testrepository - for running unit tests
+
+How to install from the source code
+===================================
+
+1. Run: ./setup.py install
+
+
+How to setup from the source code for development
+=================================================
+
+1. Run: ./setup.py develop --user
+
+NOTE: You will get an error regarding "ImportError: No module named versiontools". It is a know error. The workaround is
+run the setup.py again
+
+2. Run: ./setup.py develop --user
+
+This will put your development branch is in the python path and the scripts in <home>/.local/bin
+
+3. Add <home>/.local.bin in your PATH
+
+
+To install build-in tests
+=========================
+1. Run: lava-test list-tests
+2. Run: lava-test install <test>
+
+To install tests inside a python package
+========================================
+1. Get the test code from bzr branch lp:~linaro-graphics-wg/+junk/linaro-graphics-wg-tests
+2. Install the tests into python packages manager. ./setup.py install
+3. Run: lava-test list-tests
+
+
+To install test define with a json file
+=======================================
+1. Run: lava-test register-test file://localhost/<..>/examples/stream.json
+2. Run: lava-test list-tests
\ No newline at end of file
=== modified file 'abrek/api.py'
@@ -3,6 +3,38 @@
"""
from abc import abstractmethod, abstractproperty
+class ITestProvider(object):
+ """
+ Abrek test provider.
+
+ Abstract source of abrek tests.
+ """
+
+ @abstractmethod
+ def __init__(self, config):
+ """
+ Initialize test provider with the specified configuration object. The
+ configuration object is obtained from the abrek providers registry.
+ """
+
+ @abstractmethod
+ def __iter__(self):
+ """
+ Iterates over instances of ITest exposed by this provider
+ """
+
+ @abstractmethod
+ def __getitem__(self, test_name):
+ """
+ Return an instance of ITest with the specified name
+ """
+
+ @abstractproperty
+ def description(self):
+ """
+ The description string used by abrek list-tests
+ """
+
class ITest(object):
"""
=== modified file 'abrek/builtins.py'
@@ -149,9 +149,27 @@
List all known tests
"""
def run(self):
- from abrek import test_definitions
- from pkgutil import walk_packages
- print "Known tests:"
- for importer, mod, ispkg in walk_packages(test_definitions.__path__):
- print mod
-
+ from abrek.testdef import TestLoader
+ for provider in TestLoader().get_providers():
+ print provider.description
+ for test in provider:
+ print " - %s" % test
+
+
+class cmd_register_test(abrek.command.AbrekCmd):
+ """
+ Register declarative tests
+ """
+
+ arglist = ['test_url']
+
+ def run(self):
+ if len(self.args) != 1:
+ self.parser.error("You need to provide an URL to a test definition file")
+ test_url = self.args[0]
+ from abrek.providers import RegistryProvider
+ try:
+ RegistryProvider.register_remote_test(test_url)
+ except ValueError as exc:
+ print "Unable to register test: %s" % exc
+ sys.exit(1)
=== added file 'abrek/cache.py'
@@ -0,0 +1,69 @@
+"""
+Cache module for Abrek
+"""
+import contextlib
+import hashlib
+import os
+import urllib2
+
+
+class AbrekCache(object):
+ """
+ Cache class for Abrek
+ """
+
+ _instance = None
+
+ def __init__(self):
+ home = os.environ.get('HOME', '/')
+ basecache = os.environ.get('XDG_CACHE_HOME',
+ os.path.join(home, '.cache'))
+ self.cache_dir = os.path.join(basecache, 'abrek')
+
+ @classmethod
+ def get_instance(cls):
+ if cls._instance is None:
+ cls._instance = cls()
+ return cls._instance
+
+ def open_cached(self, key, mode="r"):
+ """
+ Acts like open() but the pathname is relative to the
+ abrek-specific cache directory.
+ """
+ if "w" in mode and not os.path.exists(self.cache_dir):
+ os.makedirs(self.cache_dir)
+ if os.path.isabs(key):
+ raise ValueError("key cannot be an absolute path")
+ try:
+ stream = open(os.path.join(self.cache_dir, key), mode)
+ yield stream
+ finally:
+ stream.close()
+
+ def _key_for_url(self, url):
+ return hashlib.sha1(url).hexdigest()
+
+ def _refresh_url_cache(self, key, url):
+ with contextlib.nested(
+ contextlib.closing(urllib2.urlopen(url)),
+ self.open_cached(key, "wb")) as (in_stream, out_stream):
+ out_stream.write(in_stream.read())
+
+ @contextlib.contextmanager
+ def open_cached_url(self, url):
+ """
+ Like urlopen.open() but the content may be cached.
+ """
+ # Do not cache local files, this is not what users would expect
+ if url.startswith("file://"):
+ stream = urllib2.urlopen(url)
+ else:
+ key = self._key_for_url(url)
+ try:
+ stream = self.open_cached(key, "rb")
+ except IOError as exc:
+ self._refresh_url_cache(key, url)
+ stream = self.open_cached(key, "rb")
+ yield stream
+ stream.close()
=== modified file 'abrek/command.py'
@@ -75,7 +75,7 @@
self.name())
def _usage(self):
- usagestr = "Usage: abrek %s" % self.name()
+ usagestr = "Usage: lava-test %s" % self.name()
for arg in self.arglist:
if arg[0] == '*':
usagestr += " %s" % arg[1:].upper()
=== modified file 'abrek/config.py'
@@ -14,9 +14,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
+import json
class AbrekConfig(object):
+
def __init__(self):
home = os.environ.get('HOME', '/')
baseconfig = os.environ.get('XDG_CONFIG_HOME',
@@ -26,6 +28,57 @@
self.configdir = os.path.join(baseconfig, 'abrek')
self.installdir = os.path.join(basedata, 'abrek', 'installed-tests')
self.resultsdir = os.path.join(basedata, 'abrek', 'results')
+ self.registry = self._load_registry()
+
+ @property
+ def _registry_pathname(self):
+ return os.path.join(self.configdir, "registry.json")
+
+ def _load_registry(self):
+ try:
+ with open(self._registry_pathname) as stream:
+ return json.load(stream)
+ except (IOError, ValueError) as exc:
+ return self._get_default_registry()
+
+ def _save_registry(self):
+ if not os.path.exists(self.configdir):
+ os.makedirs(self.configdir)
+ with open(self._registry_pathname, "wt") as stream:
+ json.dump(self.registry, stream, indent=2)
+
+ def _get_default_registry(self):
+ return {
+ "format": "Abrek Test Registry 1.0 Experimental",
+ "providers": [
+ {
+ "entry_point": "abrek.providers:BuiltInProvider",
+ },
+ {
+ "entry_point": "abrek.providers:PkgResourcesProvider",
+ },
+ {
+ "entry_point": "abrek.providers:RegistryProvider",
+ "config": {
+ "entries": []
+ }
+ }
+ ]
+ }
+
+ def get_provider_config(self, entry_point_name):
+ if "providers" not in self.registry:
+ self.registry["providers"] = []
+ for provider_info in self.registry["providers"]:
+ if provider_info.get("entry_point") == entry_point_name:
+ break
+ else:
+ provider_info = {"entry_point": entry_point_name}
+ self.registry["providers"].append(provider_info)
+ if "config" not in provider_info:
+ provider_info["config"] = {}
+ return provider_info["config"]
+
_config = None
@@ -38,4 +91,3 @@
def set_config(config):
global _config
_config = config
-
=== added file 'abrek/providers.py'
@@ -0,0 +1,139 @@
+"""
+Test providers.
+
+Allow listing and loading of tests in a generic way.
+"""
+
+from pkg_resources import working_set
+
+from abrek.api import ITestProvider
+from abrek.cache import AbrekCache
+from abrek.config import get_config
+from abrek.testdef import AbrekDeclarativeTest
+
+
+class BuiltInProvider(ITestProvider):
+ """
+ Test provider that provides tests shipped in the Abrek source tree
+ """
+
+ _builtin_tests = [
+ 'glmemperf',
+ 'gmpbench',
+ 'gtkperf',
+ 'ltp',
+ 'posixtestsuite',
+ 'pwrmgmt',
+ 'stream',
+ 'tiobench',
+ 'x11perf',
+ ]
+
+ def __init__(self, config):
+ pass
+
+ @property
+ def description(self):
+ return "Tests built directly into Abrek:"
+
+ def __iter__(self):
+ return iter(self._builtin_tests)
+
+ def __getitem__(self, test_name):
+ try:
+ module = __import__("abrek.test_definitions.%s" % test_name, fromlist=[''])
+ except ImportError:
+ raise KeyError(test_name)
+ else:
+ return module.testobj
+
+
+class PkgResourcesProvider(ITestProvider):
+ """
+ Test provider that provides tests declared in pkg_resources working_set
+
+ By default it looks at the 'abrek.test_definitions' name space but it can
+ be changed with custom 'namespace' configuration entry.
+ """
+
+ def __init__(self, config):
+ self._config = config
+
+ @property
+ def namespace(self):
+ return self._config.get("namespace", "abrek.test_definitions")
+
+ @property
+ def description(self):
+ return "Tests provided by installed python packages:"
+
+ def __iter__(self):
+ for entry_point in working_set.iter_entry_points(self.namespace):
+ yield entry_point.name
+
+ def __getitem__(self, test_name):
+ for entry_point in working_set.iter_entry_points(self.namespace, test_name):
+ return entry_point.load().testobj
+ raise KeyError(test_name)
+
+
+class RegistryProvider(ITestProvider):
+ """
+ Test provider that provides declarative tests listed in the test registry.
+ """
+ def __init__(self, config):
+ self._config = config
+ self._cache = None
+
+ @property
+ def entries(self):
+ """
+ List of URLs to AbrekDeclarativeTest description files
+ """
+ return self._config.get("entries", [])
+
+ @property
+ def description(self):
+ return "Tests provided by Abrek registry:"
+
+ @classmethod
+ def register_remote_test(self, test_url):
+ config = get_config() # This is a different config object from self._config
+ provider_config = config.get_provider_config("abrek.providers:RegistryProvider")
+ if "entries" not in provider_config:
+ provider_config["entries"] = []
+ if test_url not in provider_config["entries"]:
+ provider_config["entries"].append(test_url)
+ config._save_registry()
+ else:
+ raise ValueError("This test is already registered")
+
+ def _load_remote_test(self, test_url):
+ """
+ Load AbrekDeclarativeTest from a (possibly cached copy of) test_url
+ """
+ cache = AbrekCache.get_instance()
+ with cache.open_cached_url(test_url) as stream:
+ return AbrekDeclarativeTest.load_from_stream(stream)
+
+ def _fill_cache(self):
+ """
+ Fill the cache of all remote tests
+ """
+ if self._cache is not None:
+ return
+ self._cache = {}
+ for test_url in self.entries:
+ test = self._load_remote_test(test_url)
+ if test.testname in self._cache:
+ raise ValueError("Duplicate test %s declared" % test.testname)
+ self._cache[test.testname] = test
+
+ def __iter__(self):
+ self._fill_cache()
+ for test_name in self._cache.iterkeys():
+ yield test_name
+
+ def __getitem__(self, test_name):
+ self._fill_cache()
+ return self._cache[test_name]
=== modified file 'abrek/testdef.py'
@@ -14,9 +14,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import hashlib
+import json
import os
import re
import shutil
+import decimal
import sys
import time
from commands import getstatusoutput
@@ -143,6 +145,30 @@
os.chdir(self.origdir)
+class AbrekDeclarativeTest(AbrekTest):
+ """
+ Declarative test is like AbrekTest but cannot contain any python code and
+ is completely encapsulated in .json file.
+ """
+
+ @classmethod
+ def load_from_stream(cls, stream):
+ return cls(json.load(stream))
+
+ def save_to_stream(self, stream):
+ json.dumps(self.about, stream, indent="2")
+
+ def __init__(self, about):
+ self.about = about
+ super(AbrekDeclarativeTest, self).__init__(self.about.get('test_id'))
+ self.installer = AbrekTestInstaller(**self.about.get('install', {}))
+ self.runner = AbrekTestRunner(**self.about.get('run', {}))
+ if self.about.get('parse', {}).get('native', False) is True:
+ self.parser = AbrekNativeTestParser(self)
+ else:
+ self.parser = AbrekForeignTestParser(**self.about.get('parse', {}))
+
+
class AbrekTestInstaller(object):
"""Base class for defining an installer object.
@@ -232,7 +258,7 @@
self.endtime = datetime.utcnow()
-class AbrekTestParser(object):
+class AbrekForeignTestParser(object):
"""Base class for defining a test parser
This class can be used as-is for simple results parsers, but will
@@ -290,6 +316,8 @@
if not match:
continue
data = match.groupdict()
+ if 'measurement' in data:
+ data['measurement'] = decimal.Decimal(data['measurement'])
data["log_filename"] = filename
data["log_lineno"] = lineno
self.results['test_results'].append(data)
@@ -348,23 +376,60 @@
id['test_case_id'] = re.sub(badchars, "", id['test_case_id'])
-def testloader(testname):
- """
- Load the test definition, which can be either an individual
- file, or a directory with an __init__.py
- """
- importpath = "abrek.test_definitions.%s" % testname
- try:
- mod = __import__(importpath)
- except ImportError:
- print "unknown test '%s'" % testname
- sys.exit(1)
- for i in importpath.split('.')[1:]:
- mod = getattr(mod,i)
- try:
- base = mod.testdir.testobj
- except AttributeError:
- base = mod.testobj
-
- return base
-
+AbrekTestParser = AbrekForeignTestParser
+
+
+class AbrekNativeTestParser(object):
+
+ def __init__(self, test_def):
+ self.test_def = test_def
+
+ def parse(self):
+ raise NotImplementedError(self.parse)
+
+
+class TestLoader(object):
+ """
+ Test loader.
+
+ Encapsulates Abrek's knowledge of available tests.
+
+ Test can be loaded by name with TetsLoader.get_test_by_name. Test can also
+ be listed by TestLoader.get_providers() and then iterating over tests
+ returned by each provider.
+ """
+
+ def __init__(self):
+ self._config = get_config()
+
+ def get_providers(self):
+ """
+ Return a generator of available providers
+ """
+ import pkg_resources
+ for provider_info in self._config.registry.get("providers", []):
+ entry_point_name = provider_info.get("entry_point")
+ module_name, attrs = entry_point_name.split(':', 1)
+ attrs = attrs.split('.')
+ try:
+ entry_point = pkg_resources.EntryPoint(entry_point_name, module_name, attrs,
+ dist=pkg_resources.get_distribution("lava-test"))
+ provider_cls = entry_point.load()
+ provider = provider_cls(provider_info.get("config", {}))
+ yield provider
+ except pkg_resources.DistributionNotFound as ex:
+ raise RuntimeError("lava-test is not properly set up. Please read the REAMME file")
+
+ def get_test_by_name(self, test_name):
+ """
+ Lookup a test with the specified name
+ """
+ for provider in self.get_providers():
+ try:
+ return provider[test_name]
+ except KeyError:
+ pass
+ raise ValueError("No such test %r" % test)
+
+
+testloader = TestLoader().get_test_by_name
=== added directory 'examples'
=== added file 'examples/power-management-tests.json'
@@ -0,0 +1,14 @@
+{
+ "format": "Abrek Test Definition 1.0 Experimental",
+ "test_id": "linaro.pmwg",
+ "install": {
+ "steps": ["bzr get lp:~zkrynicki/+junk/linaro-pm-qa-tests"],
+ "deps": ["bzr"]
+ },
+ "run": {
+ "steps": ["cd linaro-pm-qa-tests && bash testcases/cpufreq/avail_freq01.sh"]
+ },
+ "parse": {
+ "native": true
+ }
+}
=== added file 'examples/stream.json'
@@ -0,0 +1,18 @@
+{
+ "format": "Abrek Test Definition Format 1.0 Experimental",
+ "test_id": "stream-json",
+ "install": {
+ "url": "http://www.cs.virginia.edu/stream/FTP/Code/stream.c",
+ "steps": ["cc stream.c -O2 -fopenmp -o stream"]
+ },
+ "run": {
+ "steps": ["./stream"]
+ },
+ "parse": {
+ "pattern": "^(?P<test_case_id>\\w+):\\W+(?P<measurement>\\d+\\.\\d+)",
+ "appendall": {
+ "units": "MB/s",
+ "result": "pass"
+ }
+ }
+}
=== modified file 'tests/imposters.py'
@@ -42,6 +42,24 @@
self.configdir = os.path.join(basedir, "config")
self.installdir = os.path.join(basedir, "install")
self.resultsdir = os.path.join(basedir, "results")
+ self.registry = {
+ "format": "Abrek Test Registry 1.0 Experimental",
+ "providers": [
+ {
+ "entry_point": "abrek.providers:BuiltInProvider",
+ },
+ {
+ "entry_point": "abrek.providers:PkgResourcesProvider",
+ },
+ {
+ "entry_point": "abrek.providers:RegistryProvider",
+ "config": {
+ "entries": []
+ }
+ }
+ ]
+ }
+
self.tmpdir = tempfile.mkdtemp()
self.config = fakeconfig(self.tmpdir)
set_config(self.config)
@@ -60,3 +78,4 @@
@property
def resultsdir(self):
return self.config.resultsdir
+
=== modified file 'tests/test_abrekcmd.py'
@@ -38,7 +38,7 @@
class cmd_test_help(AbrekCmd):
"""Test Help"""
pass
- expected_str = 'Usage: abrek test-help\n\nOptions:\n -h, ' + \
+ expected_str = 'Usage: lava-test test-help\n\nOptions:\n -h, ' + \
'--help show this help message and exit\n\n' + \
'Description:\nTest Help'
cmd = cmd_test_help()
@@ -47,7 +47,7 @@
def test_no_help(self):
class cmd_test_no_help(AbrekCmd):
pass
- expected_str = 'Usage: abrek test-no-help\n\nOptions:\n -h, ' + \
+ expected_str = 'Usage: lava-test test-no-help\n\nOptions:\n -h, ' + \
'--help show this help message and exit'
cmd = cmd_test_no_help()
self.assertEqual(expected_str, cmd.help())
@@ -61,7 +61,7 @@
self.assertTrue("install" in cmds)
def test_arglist(self):
- expected_str = 'Usage: abrek arglist FOO'
+ expected_str = 'Usage: lava-test arglist FOO'
class cmd_arglist(AbrekCmd):
arglist = ['*foo']
pass