"""Test generation helpers
Intended to functionalize common tasks when working with the pytest_generate_tests hook.
When running a test, it is quite often the case that multiple parameters need to be passed
to a single test. An example of this would be the need to run a Provider Add test against
multiple providers. We will assume that the providers are stored in the yaml under a common
structure like so:
.. code-block:: yaml
providers:
prov_1:
name: test
ip: 10.0.0.1
test_vm: abc1
prov_2:
name: test2
ip: 10.0.0.2
test_vm: abc2
Our test requires that we have a Provider Object and as an example, the 'test_vm' field of the
object. Let's assume a test prototype like so::
test_provider_add(provider_obj, test_vm):
In this case we require the test to be run twice, once for prov_1 and then again for prov_2.
We are going to use the generate function to help us provide parameters to pass to
``pytest_generate_tests()``. ``pytest_generate_tests()`` requires three pieces of
information, ``argnames``, ``argvalues`` and an ``idlist``. ``argnames`` turns into the
names we use for fixtures. In this case, ``provider_obj`` and ``provider_mgmt_sys``.
``argvalues`` becomes the place where the ``provider_obj`` and ``provider_mgmt_sys``
items are stored. Each element of ``argvalues`` is a list containing a value for both
``provider_obj`` and ``provider_mgmt_sys``. Thus, taking an element from ``argvalues``
gives us the values to unpack to make up one test. An example is below, where we assume
that a provider object is obtained via the ``Provider`` class, and the ``mgmt_sys object``
is obtained via a ``Wrapanapi`` class.
===== =============== =================
~ provider_obj test_vm
===== =============== =================
prov1 Provider(prov1) abc1
prov2 Provider(prov2) abc2
===== =============== =================
This is analogous to the following layout:
========= =============== ===============
~ argnames[0] argnames[1]
========= =============== ===============
idlist[0] argvalues[0][0] argvalues[0][1]
idlist[1] argvalues[1][0] argvalues[1][1]
========= =============== ===============
This could be generated like so:
.. code-block:: python
def gen_providers:
argnames = ['provider_obj', 'test_vm']
argvalues = []
idlist = []
for provider in yaml['providers']:
idlist.append(provider)
argvalues.append([
Provider(yaml['providers'][provider]['name']),
yaml['providers'][provider]['test_vm'])
])
return argnames, argvalues, idlist
This is then used with pytest_generate_tests like so::
pytest_generate_tests(gen_providers)
Additionally, py.test joins the values of ``idlist`` with dashes to generate a unique id for this
test, falling back to joining ``argnames`` with dashes if ``idlist`` is not set. This is the value
seen in square brackets in a test report on parametrized tests.
More information on ``parametrize`` can be found in pytest's documentation:
* https://pytest.org/latest/parametrize.html#_pytest.python.Metafunc.parametrize
"""
import pytest
from cfme.common.provider import BaseProvider
from cfme.infrastructure.config_management import get_config_manager_from_config
from cfme.roles import group_data
from cfme.utils.conf import cfme_data
from cfme.utils.log import logger
from cfme.utils.providers import ProviderFilter, list_providers
def _param_check(metafunc, argnames, argvalues):
"""Helper function to check if parametrizing is necessary
* If no argnames were specified, parametrization is unnecessary.
* If argvalues were generated, parametrization is necessary.
* If argnames were specified, but no values were generated, the test cannot run successfully,
and will be uncollected using the :py:mod:`markers.uncollect` mark.
See usage in :py:func:`parametrize`
Args:
metafunc: metafunc objects from pytest_generate_tests
argnames: argnames list for use in metafunc.parametrize
argvalues: argvalues list for use in metafunc.parametrize
Returns:
* ``True`` if this test should be parametrized
* ``False`` if it shouldn't be parametrized
* ``None`` if the test will be uncollected
"""
# If no parametrized args were named, don't parametrize
if not argnames:
return False
# If parametrized args were named and values were generated, parametrize
elif any(argvalues):
return True
# If parametrized args were named, but no values were generated, mark this test to be
# removed from the test collection. Otherwise, py.test will try to find values for the
# items in argnames by looking in its fixture pool, which will almost certainly fail.
else:
# module and class are optional, but function isn't
modname = getattr(metafunc.module, '__name__', None)
classname = getattr(metafunc.cls, '__name__', None)
funcname = metafunc.function.__name__
test_name = '.'.join(filter(None, (modname, classname, funcname)))
uncollect_msg = 'Parametrization for {} yielded no values,'\
' marked for uncollection'.format(test_name)
logger.warning(uncollect_msg)
# apply the mark
pytest.mark.uncollect(reason=uncollect_msg)(metafunc.function)
[docs]def parametrize(metafunc, argnames, argvalues, *args, **kwargs):
"""parametrize wrapper that calls :py:func:`_param_check`, and only parametrizes when needed
This can be used in any place where conditional parametrization is used.
"""
if _param_check(metafunc, argnames, argvalues):
metafunc.parametrize(argnames, argvalues, *args, **kwargs)
# if param check failed and the test was supposed to be parametrized around a provider
elif 'provider' in metafunc.fixturenames:
try:
# hack to pass trough in case of a failed param_check
# where it sets a custom message
metafunc.function.uncollect
except AttributeError:
pytest.mark.uncollect(
reason="provider was not parametrized did you forget --use-provider?"
)(metafunc.function)
[docs]def generate(*args, **kwargs):
"""Functional handler for inline pytest_generate_tests definition
Args:
gen_func: Test generator function, expected to return argnames, argvalues, and an idlist
suitable for use with pytest's parametrize method in pytest_generate_tests hooks
indirect: Optional keyword argument. If seen, it will be removed from the kwargs
passed to gen_func and used in the wrapped pytest parametrize call
scope: Optional keyword argument. If seen, it will be removed from the kwargs
passed to gen_func and used in the wrapped pytest parametrize call
filter_unused: Optional keyword argument. If True (the default), parametrized tests will
be inspected, and only argnames matching fixturenames will be used to parametrize the
test. If seen, it will be removed from the kwargs passed to gen_func.
*args: Additional positional arguments which will be passed to ``gen_func``
**kwargs: Additional keyword arguments whill be passed to ``gen_func``
Usage:
# Abstract example:
pytest_generate_tests = testgen.generate(arg1, arg2, kwarg1='a')
# Concrete example using all infrastructure providers and module scope
pytest_generate_tests = testgen.generate([InfraProvider], scope="module")
# Another concrete example using only VMware and SCVMM providers with 'retire' flag
pf = ProviderFilter(
classes=[WMwareProvider, SCVMMProvider]), required_flags=['retire'])
pytest_generate_tests = testgen.generate(
gen_func=testgen.providers, filters=[pf], scope="module")
Note:
``filter_unused`` is helpful, in that you don't have to accept all of the args in argnames
in every test in the module. However, if all tests don't share one common parametrized
argname, py.test may not have enough information to properly organize tests beyond the
'function' scope. Thus, when parametrizing in the module scope, it's a good idea to include
at least one common argname in every test signature to give pytest a clue in sorting tests.
"""
# Pull out/default kwargs for this function and parametrize; any args and kwargs that are not
# pulled out here will be passed into gen_func within pytest_generate_tests below
scope = kwargs.pop('scope', 'function')
indirect = kwargs.pop('indirect', False)
filter_unused = kwargs.pop('filter_unused', True)
gen_func = kwargs.pop('gen_func', providers_by_class)
from cfme.utils.pytest_shortcuts import fixture_filter
# If parametrize doesn't get you what you need, steal this and modify as needed
def pytest_generate_tests(metafunc):
# Pass through of args and kwargs
argnames, argvalues, idlist = gen_func(metafunc, *args, **kwargs)
# Filter out argnames that aren't requested on the metafunc test item, so not all tests
# need all fixtures to run, and tests not using gen_func's fixtures aren't parametrized.
if filter_unused:
argnames, argvalues = fixture_filter(metafunc, argnames, argvalues)
# See if we have to parametrize at all after filtering
parametrize(metafunc, argnames, argvalues, indirect=indirect, ids=idlist, scope=scope)
return pytest_generate_tests
[docs]def providers(metafunc, filters=None):
""" Gets providers based on given (+ global) filters
Note:
Using the default 'function' scope, each test will be run individually for each provider
before moving on to the next test. To group all tests related to single provider together,
parametrize tests in the 'module' scope.
Note:
testgen for providers now requires the usage of test_flags for collection to work.
Please visit http://cfme-tests.readthedocs.org/guides/documenting.html#documenting-tests
for more details.
"""
filters = filters or []
argnames = []
argvalues = []
idlist = []
# Obtains the test's flags in form of a ProviderFilter
meta = getattr(metafunc.function, 'meta', None)
test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag')
if test_flag_str:
test_flags = test_flag_str.split(',')
flags_filter = ProviderFilter(required_flags=test_flags)
filters = filters + [flags_filter]
for provider in list_providers(filters):
argvalues.append([provider])
# Use the provider key for idlist, helps with readable parametrized test output
idlist.append(provider.key)
# Add provider to argnames if missing
if 'provider' in metafunc.fixturenames and 'provider' not in argnames:
metafunc.function = pytest.mark.uses_testgen()(metafunc.function)
argnames.append('provider')
if metafunc.config.getoption('sauce'):
break
return argnames, argvalues, idlist
[docs]def providers_by_class(metafunc, classes, required_fields=None):
""" Gets providers by their class
Args:
metafunc: Passed in by pytest
classes: List of classes to fetch
required_fields: See :py:class:`cfme.utils.provider.ProviderFilter`
Usage:
# In the function itself
def pytest_generate_tests(metafunc):
argnames, argvalues, idlist = testgen.providers_by_class(
[GCEProvider, AzureProvider], required_fields=['provisioning']
)
metafunc.parametrize(argnames, argvalues, ids=idlist, scope='module')
# Using the parametrize wrapper
pytest_generate_tests = testgen.parametrize([GCEProvider], scope='module')
"""
pf = ProviderFilter(classes=classes, required_fields=required_fields)
return providers(metafunc, filters=[pf])
[docs]def all_providers(metafunc, **options):
""" Returns providers of all types """
return providers_by_class(metafunc, [BaseProvider], **options)
[docs]def auth_groups(metafunc, auth_mode):
"""Provides two test params based on the 'auth_modes' and 'group_roles' in cfme_data:
``group_name``:
expected group name in provided by the backend specified in ``auth_mode``
``group_data``:
list of nav destinations that should be visible as a member of ``group_name``
Args:
auth_mode: One of the auth_modes specified in ``cfme_data.get('auth_modes', {})``
"""
argnames = ['group_name', 'group_data']
argvalues = []
idlist = []
if auth_mode in cfme_data.get('auth_modes', {}):
# If auth_modes exists, group_roles is assumed to exist as well
for group in group_data:
argvalues.append([group, sorted(group_data[group])])
idlist.append(group)
return argnames, argvalues, idlist
[docs]def config_managers(metafunc):
"""Provides config managers
"""
argnames = ['config_manager_obj']
argvalues = []
idlist = []
data = cfme_data.get('configuration_managers', {})
for cfg_mgr_key in data:
argvalues.append([get_config_manager_from_config(cfg_mgr_key)])
idlist.append(cfg_mgr_key)
return argnames, argvalues, idlist
[docs]def pxe_servers(metafunc):
"""Provides pxe data
"""
argnames = ['pxe_name']
items = cfme_data.get('pxe_servers', {}).keys()
return argnames, [items], items