File neutron-evacuate-lbaasv2-agent.py of Package openstack-neutron

Overview Repositories Revisions Requests Users Attributes Meta

File neutron-evacuate-lbaasv2-agent.py of Package openstack-neutron (Revision 865f276abc636fd37c37ae6e8bd87fba)

Currently displaying revision 865f276abc636fd37c37ae6e8bd87fba , Show latest

#!/usr/bin/python
#
# Copyright 2017, SUSE LINUX GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import importlib
import socket
import sqlalchemy
import sys
from oslo_utils import timeutils
from oslo_config import cfg
from oslo_log import log as logging

LOG = logging.getLogger(__name__)

# Try to fall back loading the neutron-ha-tool from an absolute path.
# As on the installed system we're installing neutron-ha-tool without
# the ".py" extension, which causes importlib.import_module to fail.
try:
    hatool = importlib.import_module("neutron-ha-tool")
except ImportError:
    import imp
    import os
    dirname = os.path.dirname(os.path.abspath(__file__))
    hatool = imp.load_source(
        '', os.path.join(dirname, '/usr/bin/neutron-ha-tool'))

class EvacuateLbaasV2Agent(object):

def __init__(self):
        _default_opts = [
            cfg.StrOpt('host',
                       required=True,
                       help='name of the host to evacuate'),
            cfg.IntOpt('agent_down_time', default=75),
            cfg.BoolOpt('use_crm',
                        default=False,
                        help='When restarting an agent, assume a pacemaker '
                        'setup and use crm to switch the node into '
                        'maintenance mode before restarting the agent.'),
            cfg.BoolOpt('source_agent_restart',
                        default=True,
                        help='Restart the neutron-lbaasv2-agent '
                        'on the node that is being evacuated.'),
            cfg.BoolOpt('delete_namespaces',
                        default=False,
                        help='After evacuating an agent, ssh to it, '
                        'delete all related loadbalancer namespaces and '
                        'terminate all processes still running in those '
                        'namespaces.')
        ]

_db_opts = [
            cfg.StrOpt('connection',
                       deprecated_name='sql_connection',
                       default='',
                       secret=True,
                       help='URL to database'),
        ]

cfg.CONF.register_cli_opts(_db_opts, 'database')
        cfg.CONF.register_cli_opts(_default_opts)
        logging.register_options(cfg.CONF)
        logging.set_defaults()

self.host_to_evacuate = ""

# Find all possible destination agents for our migration. Ignore
    # agent that have set admin_state_down or did not send  their
    # heartbeat recently
    def available_destination_agents(self):
        ret = self.connection.execute(
            "SELECT id,host,heartbeat_timestamp "
            "FROM agents WHERE "
            "agents.host != %s "
            "AND agents.binary='neutron-lbaasv2-agent' "
            "AND agents.admin_state_up=true;",
            self.host_to_evacuate)

return [{'id': id, 'host': host}
                for id, host, timestamp in ret.fetchall()
                if not timeutils.is_older_than(timestamp,
                                               cfg.CONF.agent_down_time)]

def loadbalancers_on_agent(self, agent_hostname):
        ret = self.connection.execute(
            "SELECT loadbalancer_id "
            "FROM lbaas_loadbalanceragentbindings,agents "
            "WHERE agents.host = %s "
            "AND agents.binary='neutron-lbaasv2-agent' "
            "AND agents.id=lbaas_loadbalanceragentbindings.agent_id;",
            agent_hostname)
        return [id[0] for id in ret.fetchall()]

def reassign_loadbalancers(self, load_balancer_ids, target_agents):
        for agent in target_agents:
            agent['load'] = len(self.loadbalancers_on_agent(agent['host']))

agents_to_restart = set()
        for loadbalancer in load_balancer_ids:
            agent = min(target_agents, key=lambda x: x['load'])
            LOG.info("Reassigning loadbalancer %s to agent %s" %
                     (loadbalancer, agent))

self.connection.execute(
                "UPDATE lbaas_loadbalanceragentbindings "
                "SET agent_id = %s "
                "WHERE loadbalancer_id = %s;",
                agent['id'], loadbalancer
            )
            agents_to_restart.add(agent['host'])
            agent['load'] = agent['load'] + 1
        return agents_to_restart

def setup(self):
        cfg.CONF(project='neutron')
        # Always log to stderr, ignore setting in neutron.conf
        cfg.CONF.set_override("use_stderr", True)
        logging.setup(cfg.CONF, "neutron")
        self.host_to_evacuate = cfg.CONF.host

db_uri = cfg.CONF.database.connection
        db = sqlalchemy.create_engine(db_uri)
        try:
            self.connection = db.connect()
        except sqlalchemy.exc.SQLAlchemyError as e:
            LOG.error('Cannot connect to database: %s' % e)
            raise

def run(self):
        load_balancers = self.loadbalancers_on_agent(self.host_to_evacuate)
        if load_balancers:
            LOG.info("Evacuating loadbalancers: %s", load_balancers)
            target_agents = self.available_destination_agents()

if not target_agents:
                LOG.error("No agents available as a destination for "
                          "the loadbalancer evacuation.")
                return(1)

LOG.info("Available destination agents %s", target_agents)
            agents_to_restart = list(
                self.reassign_loadbalancers(load_balancers, target_agents)
            )

if (cfg.CONF.source_agent_restart or cfg.CONF.delete_namespaces):
                # Make sure the source agent is handled first
                agents_to_restart.insert(0, self.host_to_evacuate)

LOG.info("agents to restart: %s" % agents_to_restart)

for host in agents_to_restart:
                LOG.info("restarting agent on %s" % host)
                cleanup = RemoteLbaasV2Cleanup(host, timeout=30)
                if (host != self.host_to_evacuate or cfg.CONF.source_agent_restart):  # noqa
                    if cfg.CONF.use_crm:
                        cleanup.restart_lbaasv2_agent_crm()
                    else:
                        cleanup.restart_lbaasv2_agent_systemd()

if (host == self.host_to_evacuate and cfg.CONF.delete_namespaces):  # noqa
                    cleanup.delete_lbaasv2_namespaces(load_balancers)
        else:
            LOG.info("The agent on %s is not hosting any loadbalancers.",
                     self.host_to_evacuate)
        return(0)

class RemoteLbaasV2Cleanup(hatool.RemoteNodeCleanup):

def restart_lbaasv2_agent_crm(self):
        self._ssh_connect()
        with self.ssh_client:
            try:
                self._simple_ssh_command("crm --wait node maintenance")
                self._simple_ssh_command(
                    "systemctl restart openstack-neutron-lbaasv2-agent")
                self._simple_ssh_command("crm --wait node ready")
            except socket.timeout:
                LOG.warn("SSH timeout exceeded. Failed to restart "
                         "openstack-neutron-lbaasv2-agent on %s",
                         self.target_host)

def restart_lbaasv2_agent_systemd(self):
        self._ssh_connect()
        with self.ssh_client:
            try:
                self._simple_ssh_command(
                    "systemctl restart openstack-neutron-lbaasv2-agent")
            except socket.timeout:
                LOG.warn("SSH timeout exceeded. Failed to restart "
                         "openstack-neutron-lbaasv2-agent on %s",
                         self.target_host)

def delete_lbaasv2_namespaces(self, loadbalancer_ids):
        self._ssh_connect()
        with self.ssh_client:
            for lb in loadbalancer_ids:
                namespace = "qlbaas-" + lb
                LOG.info("deleting namespace %s on %s",
                         namespace, self.target_host)
                try:
                    if self._namespace_exists(namespace):
                        self._kill_pids_in_namespace(namespace)
                        self._simple_ssh_command(self.netns_del + namespace)
                except socket.timeout:
                    LOG.warn("SSH timeout exceeded. Failed to delete "
                             "namespace %s on %s", namespace,
                             self.target_host)

if __name__ == '__main__':
    evacuate = EvacuateLbaasV2Agent()
    evacuate.setup()
    ret = evacuate.run()

sys.exit(ret)

Places

File neutron-evacuate-lbaasv2-agent.py of Package openstack-neutron (Revision 865f276abc636fd37c37ae6e8bd87fba)

Places