Source code for cfme.utils.testgen

"""Test generation helpers

Intended to functionalize common tasks when working with the pytest_generate_tests hook.

When running a test, it is quite often the case that multiple parameters need to be passed
to a single test. An example of this would be the need to run a Provider Add test against
multiple providers. We will assume that the providers are stored in the yaml under a common
structure like so:

.. code-block:: yaml

            name: test
            test_vm: abc1
            name: test2
            test_vm: abc2

Our test requires that we have a Provider Object and as an example, the 'test_vm' field of the
object. Let's assume a test prototype like so::

    test_provider_add(provider_obj, test_vm):

In this case we require the test to be run twice, once for prov_1 and then again for prov_2.
We are going to use the generate function to help us provide parameters to pass to
``pytest_generate_tests()``. ``pytest_generate_tests()`` requires three pieces of
information, ``argnames``, ``argvalues`` and an ``idlist``. ``argnames`` turns into the
names we use for fixtures. In this case, ``provider_obj`` and ``provider_mgmt_sys``.
``argvalues`` becomes the place where the ``provider_obj`` and ``provider_mgmt_sys``
items are stored. Each element of ``argvalues`` is a list containing a value for both
``provider_obj`` and ``provider_mgmt_sys``. Thus, taking an element from ``argvalues``
gives us the values to unpack to make up one test. An example is below, where we assume
that a provider object is obtained via the ``Provider`` class, and the ``mgmt_sys object``
is obtained via a ``Wrapanapi`` class.

===== =============== =================
~     provider_obj    test_vm
===== =============== =================
prov1 Provider(prov1) abc1
prov2 Provider(prov2) abc2
===== =============== =================

This is analogous to the following layout:

========= =============== ===============
~         argnames[0]     argnames[1]
========= =============== ===============
idlist[0] argvalues[0][0] argvalues[0][1]
idlist[1] argvalues[1][0] argvalues[1][1]
========= =============== ===============

This could be generated like so:

.. code-block:: python

    def gen_providers:

        argnames = ['provider_obj', 'test_vm']
        argvalues = []
        idlist = []

        for provider in yaml['providers']:

        return argnames, argvalues, idlist

This is then used with pytest_generate_tests like so::


Additionally, py.test joins the values of ``idlist`` with dashes to generate a unique id for this
test, falling back to joining ``argnames`` with dashes if ``idlist`` is not set. This is the value
seen in square brackets in a test report on parametrized tests.

More information on ``parametrize`` can be found in pytest's documentation:


import pytest

from cfme.common.provider import BaseProvider
from cfme.infrastructure.config_management import get_config_manager_from_config
from cfme.roles import group_data
from cfme.utils.conf import cfme_data, auth_data
from cfme.utils.log import logger
from cfme.utils.providers import ProviderFilter, list_providers

def _param_check(metafunc, argnames, argvalues):
    """Helper function to check if parametrizing is necessary

    * If no argnames were specified, parametrization is unnecessary.
    * If argvalues were generated, parametrization is necessary.
    * If argnames were specified, but no values were generated, the test cannot run successfully,
      and will be uncollected using the :py:mod:`markers.uncollect` mark.

    See usage in :py:func:`parametrize`

        metafunc: metafunc objects from pytest_generate_tests
        argnames: argnames list for use in metafunc.parametrize
        argvalues: argvalues list for use in metafunc.parametrize

        * ``True`` if this test should be parametrized
        * ``False`` if it shouldn't be parametrized
        * ``None`` if the test will be uncollected

    # If no parametrized args were named, don't parametrize
    if not argnames:
        return False
    # If parametrized args were named and values were generated, parametrize
    elif any(argvalues):
        return True
    # If parametrized args were named, but no values were generated, mark this test to be
    # removed from the test collection. Otherwise, py.test will try to find values for the
    # items in argnames by looking in its fixture pool, which will almost certainly fail.
        # module and class are optional, but function isn't
        modname = getattr(metafunc.module, '__name__', None)
        classname = getattr(metafunc.cls, '__name__', None)
        funcname = metafunc.function.__name__

        test_name = '.'.join(filter(None, (modname, classname, funcname)))
        uncollect_msg = 'Parametrization for {} yielded no values,'\
            ' marked for uncollection'.format(test_name)

        # apply the mark

[docs]def parametrize(metafunc, argnames, argvalues, *args, **kwargs): """parametrize wrapper that calls :py:func:`_param_check`, and only parametrizes when needed This can be used in any place where conditional parametrization is used. """ if _param_check(metafunc, argnames, argvalues): metafunc.parametrize(argnames, argvalues, *args, **kwargs) # if param check failed and the test was supposed to be parametrized around a provider elif 'provider' in metafunc.fixturenames: try: # hack to pass trough in case of a failed param_check # where it sets a custom message metafunc.function.uncollect except AttributeError: pytest.mark.uncollect( reason="provider was not parametrized did you forget --use-provider?" )(metafunc.function)
[docs]def generate(*args, **kwargs): """Functional handler for inline pytest_generate_tests definition Args: gen_func: Test generator function, expected to return argnames, argvalues, and an idlist suitable for use with pytest's parametrize method in pytest_generate_tests hooks indirect: Optional keyword argument. If seen, it will be removed from the kwargs passed to gen_func and used in the wrapped pytest parametrize call scope: Optional keyword argument. If seen, it will be removed from the kwargs passed to gen_func and used in the wrapped pytest parametrize call filter_unused: Optional keyword argument. If True (the default), parametrized tests will be inspected, and only argnames matching fixturenames will be used to parametrize the test. If seen, it will be removed from the kwargs passed to gen_func. *args: Additional positional arguments which will be passed to ``gen_func`` **kwargs: Additional keyword arguments whill be passed to ``gen_func`` Usage: # Abstract example: pytest_generate_tests = testgen.generate(arg1, arg2, kwarg1='a') # Concrete example using all infrastructure providers and module scope pytest_generate_tests = testgen.generate([InfraProvider], scope="module") # Another concrete example using only VMware and SCVMM providers with 'retire' flag pf = ProviderFilter( classes=[WMwareProvider, SCVMMProvider]), required_flags=['retire']) pytest_generate_tests = testgen.generate( gen_func=testgen.providers, filters=[pf], scope="module") Note: ``filter_unused`` is helpful, in that you don't have to accept all of the args in argnames in every test in the module. However, if all tests don't share one common parametrized argname, py.test may not have enough information to properly organize tests beyond the 'function' scope. Thus, when parametrizing in the module scope, it's a good idea to include at least one common argname in every test signature to give pytest a clue in sorting tests. """ # Pull out/default kwargs for this function and parametrize; any args and kwargs that are not # pulled out here will be passed into gen_func within pytest_generate_tests below scope = kwargs.pop('scope', 'function') indirect = kwargs.pop('indirect', False) filter_unused = kwargs.pop('filter_unused', True) gen_func = kwargs.pop('gen_func', providers_by_class) from cfme.utils.pytest_shortcuts import fixture_filter # If parametrize doesn't get you what you need, steal this and modify as needed def pytest_generate_tests(metafunc): # Pass through of args and kwargs argnames, argvalues, idlist = gen_func(metafunc, *args, **kwargs) # Filter out argnames that aren't requested on the metafunc test item, so not all tests # need all fixtures to run, and tests not using gen_func's fixtures aren't parametrized. if filter_unused: argnames, argvalues = fixture_filter(metafunc, argnames, argvalues) # See if we have to parametrize at all after filtering parametrize(metafunc, argnames, argvalues, indirect=indirect, ids=idlist, scope=scope) return pytest_generate_tests
[docs]def providers(metafunc, filters=None): """ Gets providers based on given (+ global) filters Note: Using the default 'function' scope, each test will be run individually for each provider before moving on to the next test. To group all tests related to single provider together, parametrize tests in the 'module' scope. Note: testgen for providers now requires the usage of test_flags for collection to work. Please visit for more details. """ filters = filters or [] argnames = [] argvalues = [] idlist = [] # Obtains the test's flags in form of a ProviderFilter meta = getattr(metafunc.function, 'meta', None) test_flag_str = getattr(meta, 'kwargs', {}).get('from_docs', {}).get('test_flag') if test_flag_str: test_flags = test_flag_str.split(',') flags_filter = ProviderFilter(required_flags=test_flags) filters = filters + [flags_filter] for provider in list_providers(filters): argvalues.append([provider]) # Use the provider key for idlist, helps with readable parametrized test output idlist.append(provider.key) # Add provider to argnames if missing if 'provider' in metafunc.fixturenames and 'provider' not in argnames: metafunc.function = pytest.mark.uses_testgen()(metafunc.function) argnames.append('provider') if metafunc.config.getoption('sauce'): break return argnames, argvalues, idlist
[docs]def providers_by_class(metafunc, classes, required_fields=None): """ Gets providers by their class Args: metafunc: Passed in by pytest classes: List of classes to fetch required_fields: See :py:class:`cfme.utils.provider.ProviderFilter` Usage: # In the function itself def pytest_generate_tests(metafunc): argnames, argvalues, idlist = testgen.providers_by_class( [GCEProvider, AzureProvider], required_fields=['provisioning'] ) metafunc.parametrize(argnames, argvalues, ids=idlist, scope='module') # Using the parametrize wrapper pytest_generate_tests = testgen.parametrize([GCEProvider], scope='module') """ pf = ProviderFilter(classes=classes, required_fields=required_fields) return providers(metafunc, filters=[pf])
[docs]def all_providers(metafunc, **options): """ Returns providers of all types """ return providers_by_class(metafunc, [BaseProvider], **options)
[docs]def auth_groups(metafunc, auth_mode): """Provides two test params based on the 'auth_modes' and 'group_roles' in cfme_data: ``group_name``: expected group name in provided by the backend specified in ``auth_mode`` ``group_data``: list of nav destinations that should be visible as a member of ``group_name`` Args: auth_mode: One of the auth_modes specified in ``cfme_data.get('auth_modes', {})`` """ argnames = ['group_name', 'group_data'] argvalues = [] idlist = [] if auth_mode in auth_data.get('auth_providers', {}): # If auth_modes exists, group_roles is assumed to exist as well for group in group_data: argvalues.append([group, sorted(group_data[group])]) idlist.append(group) return argnames, argvalues, idlist
[docs]def config_managers(metafunc): """Provides config managers """ argnames = ['config_manager_obj'] argvalues = [] idlist = [] data = cfme_data.get('configuration_managers', {}) for cfg_mgr_key in data: argvalues.append([get_config_manager_from_config(cfg_mgr_key)]) idlist.append(cfg_mgr_key) return argnames, argvalues, idlist
[docs]def pxe_servers(metafunc): """Provides pxe data """ argnames = ['pxe_name'] argvalues = [[k] for k in sorted(cfme_data.get('pxe_servers', {}).keys())] idlist = [k[0] for k in argvalues] return argnames, argvalues, idlist