Source code for cfme.utils.appliance.db

import attr
from cached_property import cached_property
import fauxfactory
from textwrap import dedent

from cfme.utils import db, conf, clear_property_cache, datafile
from cfme.utils.conf import credentials
from cfme.utils.path import scripts_path
from cfme.utils.wait import wait_for

from .plugin import AppliancePlugin, AppliancePluginException


[docs]class ApplianceDBException(AppliancePluginException): """Basic Exception for Appliance DB object""" pass
[docs]@attr.s class ApplianceDB(AppliancePlugin): """Holder for appliance DB related methods and functions""" _ssh_client = attr.ib(default=None) # Until this needs a version pick, make it an attr postgres_version = 'rh-postgresql95' service_name = '{}-postgresql'.format(postgres_version) @cached_property def client(self): # slightly crappy: anything that changes self.address should also del(self.client) return db.Db(self.address) @cached_property def address(self): # pulls the db address from the appliance by default, falling back to the appliance # ip address (and issuing a warning) if that fails. methods that set up the internal # db should set db_address to something else when they do that if self.appliance.db_host: return self.appliance.db_host try: db_addr = self.appliance.wait_for_host_address() if db_addr is None: return self.appliance.hostname db_addr = db_addr.strip() ip_addr = self.appliance.ssh_client.run_command('ip address show') if db_addr in ip_addr.output or db_addr.startswith('127') or 'localhost' in db_addr: # address is local, use the appliance address return self.appliance.hostname else: return db_addr except (IOError, KeyError) as exc: self.logger.error('Unable to pull database address from appliance') self.logger.error(exc) return self.appliance.hostname @property def is_partition_extended(self): return self.appliance.ssh_client.run_command( "ls /var/www/miq/vmdb/.db_partition_extended") == 0
[docs] def extend_partition(self): """Extends the /var partition with DB while shrinking the unused /repo partition""" if self.is_partition_extended: return with self.appliance.ssh_client as ssh: result = ssh.run_command("df -h") self.logger.info("File systems before extending the DB partition:\n{}" .format(result.output)) ssh.run_command("umount /repo") ssh.run_command("lvreduce --force --size -9GB /dev/mapper/VG--CFME-lv_repo") ssh.run_command("mkfs.xfs -f /dev/mapper/VG--CFME-lv_repo") ssh.run_command("lvextend --resizefs --size +9GB /dev/mapper/VG--CFME-lv_var") ssh.run_command("mount -a") result = ssh.run_command("df -h") self.logger.info("File systems after extending the DB partition:\n{}" .format(result.output)) ssh.run_command("touch /var/www/miq/vmdb/.db_partition_extended")
[docs] def drop(self): """ Drops the vmdb_production database Note: EVM service has to be stopped for this to work. """ def _db_dropped(): self.appliance.db.restart_db_service self.appliance.ssh_client.run_command('dropdb vmdb_production', timeout=15) result = self.appliance.ssh_client.run_command( "psql -l | grep vmdb_production | wc -l", timeout=15) return result.success wait_for(_db_dropped, delay=5, timeout=60, message="drop the vmdb_production DB")
[docs] def create(self): """ Creates new vmdb_production database Note: EVM service has to be stopped for this to work. """ result = self.appliance.ssh_client.run_command('createdb vmdb_production', timeout=30) assert result.success, "Failed to create clean database: {}".format(result.output)
[docs] def migrate(self): """migrates a given database and updates REGION/GUID files""" ssh = self.ssh_client result = ssh.run_rake_command("db:migrate", timeout=300) assert result.success, "Failed to migrate new database: {}".format(result.output) result = ssh.run_rake_command( 'db:migrate:status 2>/dev/null | grep "^\s*down"', timeout=30) assert result.failed, ("Migration failed; migrations in 'down' state found: {}" .format(result.output)) # fetch GUID and REGION from the DB and use it to replace data in /var/www/miq/vmdb/GUID # and /var/www/miq/vmdb/REGION respectively data_query = { 'guid': 'select guid from miq_servers', 'region': 'select region from miq_regions' } for data_type, db_query in data_query.items(): data_filepath = '/var/www/miq/vmdb/{}'.format(data_type.upper()) result = ssh.run_command( 'psql -d vmdb_production -t -c "{}"'.format(db_query), timeout=15) assert result.success, "Failed to fetch {}: {}".format(data_type, result.output) db_data = result.output.strip() assert db_data, "No {} found in database; query '{}' returned no records".format( data_type, db_query) result = ssh.run_command( "echo -n '{}' > {}".format(db_data, data_filepath), timeout=15) assert result.success, "Failed to replace data in {} with '{}': {}".format( data_filepath, db_data, result.output)
[docs] def automate_reset(self): result = self.ssh_client.run_rake_command("evm:automate:reset", timeout=300) assert result.success, "Failed to reset automate: {}".format(result.output)
[docs] def fix_auth_key(self): result = self.ssh_client.run_command("fix_auth -i invalid", timeout=45) assert result.success, "Failed to change invalid passwords: {}".format(result.output)
# fix db password
[docs] def fix_auth_dbyml(self): result = self.ssh_client.run_command("fix_auth --databaseyml -i {}".format( credentials['database']['password']), timeout=45) assert result.success, "Failed to change invalid password: {}".format(result.output)
[docs] def reset_user_pass(self): result = self.ssh_client.run_rails_command( '"u = User.find_by_userid(\'admin\'); u.password = \'{}\'; u.save!"' .format(self.appliance.user.credential.secret)) assert result.success, "Failed to change UI password of {} to {}:" \ .format(self.appliance.user.credential.principal, self.appliance.user.credential.secret, result.output)
@property def ssh_client(self, **connect_kwargs): # Not lazycached to allow for the db address changing if self.is_internal: return self.appliance.ssh_client else: if self._ssh_client is None: self._ssh_client = self.appliance.ssh_client(hostname=self.address) return self._ssh_client
[docs] def backup(self, database_path="/tmp/evm_db.backup"): """Backup VMDB database Changed from Rake task due to a bug in 5.9 """ from . import ApplianceException self.logger.info('Backing up database') result = self.appliance.ssh_client.run_command( 'pg_dump --format custom --file {} vmdb_production'.format( database_path)) if result.failed: msg = 'Failed to backup database' self.logger.error(msg) raise ApplianceException(msg)
[docs] def restore(self, database_path="/tmp/evm_db.backup"): """Restore VMDB database """ from . import ApplianceException self.logger.info('Restoring database') result = self.appliance.ssh_client.run_rake_command( 'evm:db:restore:local --trace -- --local-file "{}"'.format(database_path)) if result.failed: msg = 'Failed to restore database on appl {}, output is {}'.format(self.address, result.output) self.logger.error(msg) raise ApplianceException(msg) if self.appliance.version > '5.8': result = self.ssh_client.run_command("fix_auth --databaseyml -i {}".format( conf.credentials['database'].password), timeout=45) if result.failed: self.logger.error("Failed to change invalid db password: {}" .format(result.output))
[docs] def setup(self, **kwargs): """Configure database On downstream appliances, invokes the internal database setup. On all appliances waits for database to be ready. """ key_address = kwargs.pop('key_address', None) db_address = kwargs.pop('db_address', None) self.logger.info('Starting DB setup') is_pod = kwargs.pop('is_pod', False) if self.appliance.is_downstream: # We only execute this on downstream appliances. if is_pod: self.enable_external(db_address=db_address, **kwargs) else: self.enable_internal(key_address=key_address, **kwargs) elif not self.appliance.evmserverd.running: self.appliance.evmserverd.start() self.appliance.evmserverd.enable() # just to be sure here. self.appliance.wait_for_web_ui() # Make sure the database is ready wait_for(func=lambda: self.is_ready, message='appliance db ready', delay=20, num_sec=1200) self.logger.info('DB setup complete')
[docs] def loosen_pgssl(self, with_ssl=False): """Loosens postgres connections""" self.logger.info('Loosening postgres permissions') # Init SSH client client = self.appliance.ssh_client # set root password cmd = "psql -d vmdb_production -c \"alter user {} with password '{}'\"".format( conf.credentials['database']['username'], conf.credentials['database']['password'] ) client.run_command(cmd) # back up pg_hba.conf scl = self.postgres_version client.run_command('mv /opt/rh/{scl}/root/var/lib/pgsql/data/pg_hba.conf ' '/opt/rh/{scl}/root/var/lib/pgsql/data/pg_hba.conf.sav'.format(scl=scl)) if with_ssl: ssl = 'hostssl all all all cert map=sslmap' else: ssl = '' # rewrite pg_hba.conf write_pg_hba = dedent("""\ cat > /opt/rh/{scl}/root/var/lib/pgsql/data/pg_hba.conf <<EOF local all postgres,root trust host all all 0.0.0.0/0 md5 {ssl} EOF """.format(ssl=ssl, scl=scl)) client.run_command(write_pg_hba) client.run_command("chown postgres:postgres " "/opt/rh/{scl}/root/var/lib/pgsql/data/pg_hba.conf".format(scl=scl)) # restart postgres result = client.run_command("systemctl restart {scl}-postgresql".format(scl=scl)) return result.rc
[docs] def enable_internal(self, region=0, key_address=None, db_password=None, ssh_password=None, db_disk=None): """Enables internal database Args: region: Region number of the CFME appliance. key_address: Address of CFME appliance where key can be fetched. db_disk: Path of the db disk for --dbdisk appliance_console_cli. If not specified it tries to load it from the appliance. Note: If key_address is None, a new encryption key is generated for the appliance. """ # self.logger.info('Enabling internal DB (region {}) on {}.'.format(region, self.address)) self.address = self.appliance.hostname clear_property_cache(self, 'client') client = self.ssh_client # Defaults db_password = db_password or conf.credentials['database']['password'] ssh_password = ssh_password or conf.credentials['ssh']['password'] if not db_disk: try: db_disk = self.appliance.unpartitioned_disks[0] except IndexError: db_disk = None self.logger.warning( 'Failed to set --dbdisk from the appliance. On 5.9.0.3+ it will fail.') # make sure the dbdisk is unmounted, RHOS ephemeral disks come up mounted result = client.run_command('umount {}'.format(db_disk)) if not result.success: self.logger.warning('umount non-zero return, output was: '.format(result)) if self.appliance.has_cli: base_command = 'appliance_console_cli --region {}'.format(region) # use the cli if key_address: command_options = ('--internal --fetch-key {key} -p {db_pass} -a {ssh_pass}' .format(key=key_address, db_pass=db_password, ssh_pass=ssh_password)) else: command_options = '--internal --force-key -p {db_pass}'.format(db_pass=db_password) if db_disk: command_options = ' '.join([command_options, '--dbdisk {}'.format(db_disk)]) result = client.run_command(' '.join([base_command, command_options])) if result.failed or 'failed' in result.output.lower(): raise Exception('Could not set up the database:\n{}'.format(result.output)) else: # no cli, use the enable internal db script rbt_repl = { 'miq_lib': '/var/www/miq/lib', 'region': region, 'postgres_version': self.postgres_version } # Find and load our rb template with replacements rbt = datafile.data_path_for_filename('enable-internal-db.rbt', scripts_path.strpath) rb = datafile.load_data_file(rbt, rbt_repl) # sent rb file over to /tmp remote_file = '/tmp/{}'.format(fauxfactory.gen_alphanumeric()) client.put_file(rb.name, remote_file) # Run the rb script, clean it up when done result = client.run_command('ruby {}'.format(remote_file)) client.run_command('rm {}'.format(remote_file)) self.logger.info('Output from appliance db configuration: %s', result.output) return result.rc, result.output
[docs] def enable_external(self, db_address, region=0, db_name=None, db_username=None, db_password=None): """Enables external database Args: db_address: Address of the external database region: Number of region to join db_name: Name of the external DB db_username: Username to access the external DB db_password: Password to access the external DB Returns a tuple of (exitstatus, script_output) for reporting, if desired """ self.logger.info('Enabling external DB (db_address {}, region {}) on {}.' .format(db_address, region, self.address)) # reset the db address and clear the cached db object if we have one self.address = db_address clear_property_cache(self, 'client') # default db_name = db_name or 'vmdb_production' db_username = db_username or conf.credentials['database']['username'] db_password = db_password or conf.credentials['database']['password'] client = self.ssh_client if self.appliance.has_cli: if not client.is_pod: # copy v2 key master_client = client(hostname=self.address) rand_filename = "/tmp/v2_key_{}".format(fauxfactory.gen_alphanumeric()) master_client.get_file("/var/www/miq/vmdb/certs/v2_key", rand_filename) client.put_file(rand_filename, "/var/www/miq/vmdb/certs/v2_key") # enable external DB with cli result = client.run_command( 'appliance_console_cli ' '--hostname {0} --region {1} --dbname {2} --username {3} --password {4}'.format( self.address, region, db_name, db_username, db_password ) ) else: # no cli, use the enable external db script rbt_repl = { 'miq_lib': '/var/www/miq/lib', 'host': self.address, 'region': region, 'database': db_name, 'username': db_username, 'password': db_password } # Find and load our rb template with replacements rbt = datafile.data_path_for_filename('enable-internal-db.rbt', scripts_path.strpath) rb = datafile.load_data_file(rbt, rbt_repl) # Init SSH client and sent rb file over to /tmp remote_file = '/tmp/{}'.format(fauxfactory.gen_alphanumeric()) client.put_file(rb.name, remote_file) # Run the rb script, clean it up when done result = client.run_command('ruby {}'.format(remote_file)) client.run_command('rm {}'.format(remote_file)) if result.failed: self.logger.error('error enabling external db') self.logger.error(result.output) msg = ('Appliance {} failed to enable external DB running on {}' .format(self.appliance.hostname, db_address)) self.logger.error(msg) from . import ApplianceException raise ApplianceException(msg) return result.rc, result.output
@property def is_dedicated_active(self): result = self.appliance.ssh_client.run_command( "systemctl status {}-postgresql.service | grep running".format( self.postgres_version)) return result.success
[docs] def wait_for(self, timeout=600): """Waits for appliance database to be ready Args: timeout: Number of seconds to wait until timeout (default ``180``) """ wait_for(func=lambda: self.is_ready, message='appliance.db.is_ready', delay=20, num_sec=timeout)
@property def is_enabled(self): """Is database enabled""" return self.address is not None @property def is_internal(self): """Is database internal""" return self.address == self.appliance.hostname @property def is_ready(self): """Is database ready""" # Using 'and' chain instead of all(...) to # prevent calling more things after a step fails return self.is_online and self.has_database and self.has_tables @property def is_online(self): """Is database online""" db_check_command = ('env PGPASSWORD={pwd} psql -U {user} ' '-h {ip} -t -c "select now()" postgres') ensure_host = True if self.ssh_client.is_pod else False db_check_command = db_check_command.format(ip=self.address, user=conf.credentials['database']['username'], pwd=conf.credentials['database']['password']) result = self.ssh_client.run_command(db_check_command, ensure_host=ensure_host) return result.success @property def has_database(self): """Does database have a database defined""" db_check_command = ('env PGPASSWORD={pwd} psql -U {user} -t -h {ip} -c ' '"SELECT datname FROM pg_database ' 'WHERE datname LIKE \'vmdb_%\';" postgres | grep -q vmdb_production') ensure_host = True if self.ssh_client.is_pod else False db_check_command = db_check_command.format(ip=self.address, user=conf.credentials['database']['username'], pwd=conf.credentials['database']['password']) result = self.ssh_client.run_command(db_check_command, ensure_host=ensure_host) return result.success @property def has_tables(self): """Does database have tables defined""" db_check_command = ('env PGPASSWORD={pwd} psql -U {user} -t -h {ip} -c "SELECT * FROM ' 'information_schema.tables WHERE table_schema = \'public\';" ' 'vmdb_production | grep -q vmdb_production') ensure_host = True if self.ssh_client.is_pod else False db_check_command = db_check_command.format(ip=self.address, user=conf.credentials['database']['username'], pwd=conf.credentials['database']['password']) result = self.ssh_client.run_command(db_check_command, ensure_host=ensure_host) return result.success
[docs] def start_db_service(self): """Starts the postgresql service via systemctl""" self.logger.info('Starting service: {}'.format(self.service_name)) with self.ssh_client as ssh: result = ssh.run_command('systemctl start {}'.format(self.service_name)) assert result.success, 'Failed to start {}'.format(self.service_name) self.logger.info('Started service: {}'.format(self.service_name))
[docs] def stop_db_service(self): """Starts the postgresql service via systemctl""" service = '{}-postgresql'.format(self.postgres_version) self.logger.info('Stopping {}'.format(service)) with self.ssh_client as ssh: result = ssh.run_command('systemctl stop {}'.format(self.service_name)) assert result.success, 'Failed to stop {}'.format(service) self.logger.info('Stopped {}'.format(service))
[docs] def restart_db_service(self): """restarts the postgresql service via systemctl""" service = '{}-postgresql'.format(self.postgres_version) self.logger.info('Restarting {}'.format(service)) with self.ssh_client as ssh: result = ssh.run_command('systemctl restart {}'.format(self.service_name)) assert result.success, 'Failed to restart {}'.format(service) self.logger.info('Restarted {}'.format(service))