Source code for cfme.scripting.quickstart

from __future__ import print_function

import os
import sys
import argparse
import subprocess
import json
import hashlib

    import distro
except ImportError:
    sys.path.insert(0, os.path.join(
        'requirements', 'quickstart',
    import distro

parser = argparse.ArgumentParser()
parser.add_argument("--mk-virtualenv", default="../cfme_venv")
parser.add_argument("--system-site-packages", action="store_true")
parser.add_argument("--config-path", default="../cfme-qe-yamls/complete/")

DISTRO_DATA = distro.linux_distribution()[:2]
IS_SCRIPT = sys.argv[0] == __file__
CWD = os.getcwd()  # we expect to be in the workdir
IS_ROOT = os.getuid() == 0
REDHAT_BASED = os.path.isfile('/etc/redhat-release')
CREATED = object()
REQUIREMENT_FILE = 'requirements/frozen.txt'
HAS_DNF = os.path.exists('/usr/bin/dnf')
IN_VIRTUALENV = getattr(sys, 'real_prefix', None) is not None

PRISTINE_ENV = dict(os.environ)

    " python-virtualenv gcc postgresql-devel libxml2-devel"
    " libxslt-devel zeromq3-devel libcurl-devel"
    " redhat-rpm-config gcc-c++ openssl-devel"
    " libffi-devel python-devel tesseract"
    " freetype-devel")

    " python2-virtualenv gcc postgresql-devel libxml2-devel"
    " libxslt-devel zeromq3-devel libcurl-devel"
    " redhat-rpm-config gcc-c++ openssl-devel"
    " libffi-devel python2-devel tesseract"
    " freetype-devel")

    " python2-virtualenv gcc postgresql-devel libxml2-devel"
    " libxslt-devel zeromq-devel libcurl-devel"
    " redhat-rpm-config gcc-c++ openssl-devel"
    " libffi-devel python2-devel tesseract"
    " freetype-devel")

    os.environ['PYCURL_SSL_LIBRARY'] = 'nss'
    if DISTRO_DATA == ("Fedora", "25"):
    elif DISTRO_DATA == ("Fedora", "26"):

    if HAS_DNF:
        INSTALL_COMMAND = 'dnf install -y'
        INSTALL_COMMAND = 'yum install -y'
    if not IS_ROOT:

[docs]def command_text(command, shell): if shell: return command else: command = [repr(x) if ' ' in x else x for x in command] return ' '.join(command)
[docs]def call_or_exit(command, shell=False, **kw): try: print('QS $', command_text(command, shell)) res =, shell=shell, **kw) except Exception as e: print(repr(e)) sys.exit(1) else: if res: print("call failed with", res) sys.exit(res)
[docs]def pip_json_list(venv): os.environ.pop('PYTHONHOME', None) proc = subprocess.Popen([ os.path.join(venv, 'bin/pip'), 'list', '--format=json', ], stdout=subprocess.PIPE) return json.load(proc.stdout)
[docs]def install_system_packages(): if REDHAT_BASED: call_or_exit(INSTALL_COMMAND + REDHAT_PACKAGES, shell=True) else: print("WARNING: unknown distribution,", "please ensure you have the required packages installed") print("INFO: on redhat based systems this is the equivalend of:") print("$ dnf install -y", REDHAT_PACKAGES_OLD)
[docs]def setup_virtualenv(target, use_site): if os.path.isdir(target): print("INFO: Virtualenv", target, "already exists, skipping creation") return CREATED add = ['--system-site-packages'] if use_site else [] call_or_exit(['virtualenv', target] + add) venv_call(target, 'pip', 'install', '-U', # setuptools_scm and docutils installation prevents # missbehaved packages from failing 'pip', 'wheel', 'setuptools_scm', 'docutils')
[docs]def venv_call(venv_path, command, *args, **kwargs): # pop PYTHONHOME to avoid nested environments os.environ.pop('PYTHONHOME', None) call_or_exit([ os.path.join(venv_path, 'bin', command), ] + list(args), **kwargs)
[docs]def hash_file(path): content_hash = hashlib.sha1() with open(path, 'rb') as fp: content_hash.update( return content_hash.hexdigest()
[docs]def install_requirements(venv_path, quiet=False): remember_file = os.path.join(venv_path, '.cfme_requirements_hash') current_hash = hash_file(REQUIREMENT_FILE) if os.path.isfile(remember_file): with open(remember_file, 'r') as fp: last_hash = elif os.path.exists(remember_file): sys.exit("ERROR: {} is required to be a file".format(remember_file)) else: last_hash = None if last_hash == current_hash: print("INFO: skipping requirement installation as frozen ones didn't change") print(" to enforce please invoke pip manually") return elif last_hash is not None: current_packages = pip_json_list(venv_path) print("INFO:", REQUIREMENT_FILE, 'changed, updating virtualenv') venv_call( venv_path, 'pip', 'install', '-r', REQUIREMENT_FILE, '--no-binary', 'pycurl', *(['-q'] if quiet else [])) with open(remember_file, 'w') as fp: fp.write(current_hash) if last_hash is not None: updated_packages = pip_json_list(venv_path) print_packages_diff(old=current_packages, new=updated_packages)
[docs]def pip_version_list_to_map(version_list): res = {} for item in version_list: try: res[item['name']] = item['version'] except KeyError: pass return res
[docs]def version_changes(old, new): names = sorted(set(old) & set(new)) for name in names: initial = old.get(name, 'missing') afterwards = new.get(name, 'removed') if initial != afterwards: yield name, initial, afterwards
[docs]def self_install(venv_path): venv_call(venv_path, 'pip', 'install', '-q', '-e', '.')
[docs]def disable_bytecode(venv_path): venv_call(venv_path, 'python', '-m', 'cfme.scripting.disable_bytecode')
[docs]def ensure_pycurl_works(venv_path): venv_call(venv_path, 'python', '-c', 'import curl', env=PRISTINE_ENV)
[docs]def main(args): if not IN_VIRTUALENV: # invoked from outside, its ok to be slow install_system_packages() else: print("INFO: skipping installation of system packages from inside of virtualenv") venv_state = setup_virtualenv( args.mk_virtualenv, args.system_site_packages) install_requirements(args.mk_virtualenv, quiet=(venv_state is CREATED)) disable_bytecode(args.mk_virtualenv) self_install(args.mk_virtualenv) link_config_files(args.config_path, 'conf') ensure_pycurl_works(args.mk_virtualenv) if not IN_VIRTUALENV: print("INFO: please remember to activate the virtualenv via") print(" .", os.path.join(args.mk_virtualenv, 'bin/activate'))
if IS_SCRIPT: main(parser.parse_args())