Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
SUSE:SLE-15-SP1:GA
salt.10902
python-3.7-support.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File python-3.7-support.patch of Package salt.10902
From 1c18b65e6cee9f84843f99f9f30b6d088f687930 Mon Sep 17 00:00:00 2001 From: Dmitry Kuzmenko <dmitry.kuzmenko@dsr-corporation.com> Date: Mon, 16 Apr 2018 22:41:44 +0300 Subject: [PATCH] Python 3.7 support Tornado 5.0 compatibility fixes Allow running pytest>=3.5.0 Run off of a temporary config use salt.utils.zeromq add io_loop handling to runtests engine fix pylint fix _create_stream and tornado 5.0 This should be the last fix for tornado 5.0 allow using tornado 5.0 Fix last test for tornado Thank you so much @bdarnell for helping me get this last fix. With this, the test suite passes with tornado 5. fix pylint Rename module to full wording Fix imports Fix docstring typo Fix CLI config Fix comments Fix docstrings Rename async function to asynchronous Change internal function signatures to avoid reserved word Remove internal variables/properties with the reserved words Fix local opts from CLI Fix log error/info/warning and exception messages Cleanup docstrings at module level Fix function signatures in Cassandra module Lintfix: PEP8 requires two empty lines Deprecate 'async' parameter in Mandrill API Revert api call: it is about "functionname_async" suffix. Add 'async' backward compatibility Update docstring Use kwargs instead of directly named parameters Support original API Fix nag-message Keep runner API unchanged fix unicode literals Remove async keyword, moving it into the kwargs. Fix configuration setting --- requirements/base.txt | 2 +- requirements/dev_python27.txt | 4 +- requirements/dev_python34.txt | 4 +- requirements/pytest.txt | 2 +- salt/client/api.py | 6 +- salt/client/mixins.py | 4 +- salt/cloud/clouds/msazure.py | 2 +- salt/cloud/clouds/profitbricks.py | 2 +- salt/cloud/clouds/xen.py | 2 +- salt/daemons/masterapi.py | 6 +- salt/engines/ircbot.py | 5 +- salt/engines/slack.py | 4 +- salt/engines/webhook.py | 3 +- salt/master.py | 6 +- salt/minion.py | 19 +-- salt/modules/cassandra_cql.py | 22 +-- salt/modules/mandrill.py | 21 ++- salt/modules/saltutil.py | 6 +- salt/netapi/rest_cherrypy/app.py | 4 +- salt/netapi/rest_cherrypy/event_processor.py | 2 +- salt/netapi/rest_tornado/__init__.py | 2 +- salt/netapi/rest_tornado/event_processor.py | 2 +- salt/netapi/rest_tornado/saltnado.py | 16 +-- .../rest_tornado/saltnado_websockets.py | 2 +- salt/returners/cassandra_cql_return.py | 8 +- salt/runner.py | 10 +- salt/thorium/runner.py | 6 +- salt/thorium/wheel.py | 4 +- salt/transport/client.py | 2 +- salt/transport/ipc.py | 47 +++---- salt/transport/server.py | 2 +- salt/transport/tcp.py | 127 +++++++++--------- salt/utils/{async.py => asynchronous.py} | 22 +-- salt/utils/event.py | 18 +-- salt/utils/process.py | 4 +- salt/utils/thin.py | 2 +- salt/wheel/__init__.py | 4 +- tests/conftest.py | 25 +--- .../files/engines/runtests_engine.py | 11 +- .../netapi/rest_tornado/test_app.py | 2 +- tests/support/case.py | 27 +++- tests/unit/test_minion.py | 7 +- tests/unit/utils/test_async.py | 20 +-- 43 files changed, 254 insertions(+), 242 deletions(-) rename salt/utils/{async.py => asynchronous.py} (81%) diff --git a/requirements/base.txt b/requirements/base.txt index de490ed07f..08d836574f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,6 +3,6 @@ msgpack-python>0.3,!=0.5.5 PyYAML MarkupSafe requests>=1.0.0 -tornado>=4.2.1,<5.0 +tornado>=4.2.1,<6.0 # Required by Tornado to handle threads stuff. futures>=2.0 diff --git a/requirements/dev_python27.txt b/requirements/dev_python27.txt index 8834842ecd..42ba18893a 100644 --- a/requirements/dev_python27.txt +++ b/requirements/dev_python27.txt @@ -1,4 +1,4 @@ --r base.txt +-r base-py2.txt mock apache-libcloud>=0.14.0 @@ -6,7 +6,7 @@ boto>=2.32.1 boto3>=1.2.1 moto>=0.3.6 SaltPyLint>=v2017.3.6 -pytest +pytest>=3.5.0 git+https://github.com/eisensheng/pytest-catchlog.git@develop#egg=Pytest-catchlog git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt testinfra>=1.7.0 diff --git a/requirements/dev_python34.txt b/requirements/dev_python34.txt index 19a2574d62..25b9cf45cf 100644 --- a/requirements/dev_python34.txt +++ b/requirements/dev_python34.txt @@ -1,4 +1,4 @@ --r base.txt +-r base-py3.txt mock apache-libcloud>=0.14.0 @@ -11,7 +11,7 @@ moto>=0.3.6 # prevent it from being successfully installed (at least on Python 3.4). httpretty SaltPyLint>=v2017.2.29 -pytest +pytest>=3.5.0 git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt git+https://github.com/eisensheng/pytest-catchlog.git@develop#egg=Pytest-catchlog testinfra>=1.7.0 diff --git a/requirements/pytest.txt b/requirements/pytest.txt index 60e4e44e0e..e5ec2acff3 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -1,3 +1,3 @@ -pytest +pytest>=3.5.0 pytest-helpers-namespace pytest-tempdir diff --git a/salt/client/api.py b/salt/client/api.py index ac6f6de24a..b2aab460fa 100644 --- a/salt/client/api.py +++ b/salt/client/api.py @@ -93,7 +93,7 @@ class APIClient(object): The cmd dict items are as follows: - mode: either 'sync' or 'async'. Defaults to 'async' if missing + mode: either 'sync' or 'asynchronous'. Defaults to 'asynchronous' if missing fun: required. If the function is to be run on the master using either a wheel or runner client then the fun: includes either 'wheel.' or 'runner.' as a prefix and has three parts separated by '.'. @@ -120,7 +120,7 @@ class APIClient(object): ''' cmd = dict(cmd) # make copy client = 'minion' # default to local minion client - mode = cmd.get('mode', 'async') # default to 'async' + mode = cmd.get('mode', 'async') # check for wheel or runner prefix to fun name to use wheel or runner client funparts = cmd.get('fun', '').split('.') @@ -162,7 +162,7 @@ class APIClient(object): ''' return self.runnerClient.master_call(**kwargs) - runner_sync = runner_async # always runner async, so works in either mode + runner_sync = runner_async # always runner asynchronous, so works in either mode def wheel_sync(self, **kwargs): ''' diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 29b6077661..4182fa5b81 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -458,7 +458,7 @@ class SyncClientMixin(object): class AsyncClientMixin(object): ''' - A mixin for *Client interfaces to enable easy async function execution + A mixin for *Client interfaces to enable easy asynchronous function execution ''' client = None tag_prefix = None @@ -510,7 +510,7 @@ class AsyncClientMixin(object): tag = salt.utils.event.tagify(jid, prefix=self.tag_prefix) return {'tag': tag, 'jid': jid} - def async(self, fun, low, user='UNKNOWN', pub=None): + def asynchronous(self, fun, low, user='UNKNOWN', pub=None): ''' Execute the function in a multiprocess and return the event tag to use to watch for the return diff --git a/salt/cloud/clouds/msazure.py b/salt/cloud/clouds/msazure.py index 9d48616db6..0994b0c3a4 100644 --- a/salt/cloud/clouds/msazure.py +++ b/salt/cloud/clouds/msazure.py @@ -885,7 +885,7 @@ def _wait_for_async(conn, request_id): while result.status == 'InProgress': count = count + 1 if count > 120: - raise ValueError('Timed out waiting for async operation to complete.') + raise ValueError('Timed out waiting for asynchronous operation to complete.') time.sleep(5) result = conn.get_operation_status(request_id) diff --git a/salt/cloud/clouds/profitbricks.py b/salt/cloud/clouds/profitbricks.py index 1ce0a162f0..8d13bf7b70 100644 --- a/salt/cloud/clouds/profitbricks.py +++ b/salt/cloud/clouds/profitbricks.py @@ -1098,7 +1098,7 @@ def _wait_for_completion(conn, promise, wait_timeout, msg): ) raise Exception( - 'Timed out waiting for async operation {0} "{1}" to complete.'.format( + 'Timed out waiting for asynchronous operation {0} "{1}" to complete.'.format( msg, six.text_type(promise['requestId']) ) ) diff --git a/salt/cloud/clouds/xen.py b/salt/cloud/clouds/xen.py index 5823fd7cc9..e29b6a98f1 100644 --- a/salt/cloud/clouds/xen.py +++ b/salt/cloud/clouds/xen.py @@ -724,7 +724,7 @@ def _wait_for_ip(name, session): def _run_async_task(task=None, session=None): ''' - Run XenAPI task in async mode to prevent timeouts + Run XenAPI task in asynchronous mode to prevent timeouts ''' if task is None or session is None: return None diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 4fe00934cd..e23b370b63 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -1068,9 +1068,9 @@ class LocalFuncs(object): try: fun = load.pop('fun') runner_client = salt.runner.RunnerClient(self.opts) - return runner_client.async(fun, - load.get('kwarg', {}), - username) + return runner_client.asynchronous(fun, + load.get('kwarg', {}), + username) except Exception as exc: log.exception('Exception occurred while introspecting %s') return {'error': {'name': exc.__class__.__name__, diff --git a/salt/engines/ircbot.py b/salt/engines/ircbot.py index e3b8778cd2..c93eb5fb95 100644 --- a/salt/engines/ircbot.py +++ b/salt/engines/ircbot.py @@ -97,14 +97,15 @@ class IRCClient(object): self.allow_nicks = allow_nicks self.disable_query = disable_query self.io_loop = tornado.ioloop.IOLoop(make_current=False) + self.io_loop.make_current() self._connect() def _connect(self): _sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) if self.ssl is True: - self._stream = tornado.iostream.SSLIOStream(_sock, ssl_options={'cert_reqs': ssl.CERT_NONE}, io_loop=self.io_loop) + self._stream = tornado.iostream.SSLIOStream(_sock, ssl_options={'cert_reqs': ssl.CERT_NONE}) else: - self._stream = tornado.iostream.IOStream(_sock, io_loop=self.io_loop) + self._stream = tornado.iostream.IOStream(_sock) self._stream.set_close_callback(self.on_closed) self._stream.connect((self.host, self.port), self.on_connect) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 6537c29412..b761d388db 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -717,7 +717,7 @@ class SlackClient(object): :param interval: time to wait between ending a loop and beginning the next ''' - log.debug('Going to run a command async') + log.debug('Going to run a command asynchronous') runner_functions = sorted(salt.runner.Runner(__opts__).functions) # Parse args and kwargs cmd = msg['cmdline'][0] @@ -739,7 +739,7 @@ class SlackClient(object): log.debug('Command %s will run via runner_functions', cmd) # pylint is tripping # pylint: disable=missing-whitespace-after-comma - job_id_dict = runner.async(cmd, {'args': args, 'kwargs': kwargs}) + job_id_dict = runner.asynchronous(cmd, {'args': args, 'kwargs': kwargs}) job_id = job_id_dict['jid'] # Default to trying to run as a client module. diff --git a/salt/engines/webhook.py b/salt/engines/webhook.py index a00eb813a1..6c92113fb7 100644 --- a/salt/engines/webhook.py +++ b/salt/engines/webhook.py @@ -81,6 +81,7 @@ def start(address=None, port=5000, ssl_crt=None, ssl_key=None): if all([ssl_crt, ssl_key]): ssl_options = {"certfile": ssl_crt, "keyfile": ssl_key} io_loop = tornado.ioloop.IOLoop(make_current=False) - http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options, io_loop=io_loop) + io_loop.make_current() + http_server = tornado.httpserver.HTTPServer(application, ssl_options=ssl_options) http_server.listen(port, address=address) io_loop.start() diff --git a/salt/master.py b/salt/master.py index 6fb37ece1a..6d34d13de4 100644 --- a/salt/master.py +++ b/salt/master.py @@ -1878,9 +1878,9 @@ class ClearFuncs(object): try: fun = clear_load.pop('fun') runner_client = salt.runner.RunnerClient(self.opts) - return runner_client.async(fun, - clear_load.get('kwarg', {}), - username) + return runner_client.asynchronous(fun, + clear_load.get('kwarg', {}), + username) except Exception as exc: log.error('Exception occurred while introspecting %s: %s', fun, exc) return {'error': {'name': exc.__class__.__name__, diff --git a/salt/minion.py b/salt/minion.py index 4a30e70be5..173a43d06a 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -920,7 +920,7 @@ class MinionManager(MinionBase): install_zmq() self.io_loop = ZMQDefaultLoop.current() self.process_manager = ProcessManager(name='MultiMinionProcessManager') - self.io_loop.spawn_callback(self.process_manager.run, async=True) + self.io_loop.spawn_callback(self.process_manager.run, **{'async': True}) # Tornado backward compat def __del__(self): self.destroy() @@ -1117,7 +1117,7 @@ class Minion(MinionBase): time.sleep(sleep_time) self.process_manager = ProcessManager(name='MinionProcessManager') - self.io_loop.spawn_callback(self.process_manager.run, async=True) + self.io_loop.spawn_callback(self.process_manager.run, **{'async': True}) # We don't have the proxy setup yet, so we can't start engines # Engines need to be able to access __proxy__ if not salt.utils.platform.is_proxy(): @@ -2487,13 +2487,15 @@ class Minion(MinionBase): if beacons and self.connected: self._fire_master(events=beacons) - new_periodic_callbacks['beacons'] = tornado.ioloop.PeriodicCallback(handle_beacons, loop_interval * 1000, io_loop=self.io_loop) + new_periodic_callbacks['beacons'] = tornado.ioloop.PeriodicCallback( + handle_beacons, loop_interval * 1000) if before_connect: # Make sure there is a chance for one iteration to occur before connect handle_beacons() if 'cleanup' not in self.periodic_callbacks: - new_periodic_callbacks['cleanup'] = tornado.ioloop.PeriodicCallback(self._fallback_cleanups, loop_interval * 1000, io_loop=self.io_loop) + new_periodic_callbacks['cleanup'] = tornado.ioloop.PeriodicCallback( + self._fallback_cleanups, loop_interval * 1000) # start all the other callbacks for periodic_cb in six.itervalues(new_periodic_callbacks): @@ -2545,14 +2547,15 @@ class Minion(MinionBase): # TODO: actually listen to the return and change period def handle_schedule(): self.process_schedule(self, loop_interval) - new_periodic_callbacks['schedule'] = tornado.ioloop.PeriodicCallback(handle_schedule, 1000, io_loop=self.io_loop) + new_periodic_callbacks['schedule'] = tornado.ioloop.PeriodicCallback(handle_schedule, 1000) if before_connect: # Make sure there is a chance for one iteration to occur before connect handle_schedule() if 'cleanup' not in self.periodic_callbacks: - new_periodic_callbacks['cleanup'] = tornado.ioloop.PeriodicCallback(self._fallback_cleanups, loop_interval * 1000, io_loop=self.io_loop) + new_periodic_callbacks['cleanup'] = tornado.ioloop.PeriodicCallback( + self._fallback_cleanups, loop_interval * 1000) # start all the other callbacks for periodic_cb in six.itervalues(new_periodic_callbacks): @@ -2609,7 +2612,7 @@ class Minion(MinionBase): self._fire_master('ping', 'minion_ping', sync=False, timeout_handler=ping_timeout_handler) except Exception: log.warning('Attempt to ping master failed.', exc_on_loglevel=logging.DEBUG) - self.periodic_callbacks['ping'] = tornado.ioloop.PeriodicCallback(ping_master, ping_interval * 1000, io_loop=self.io_loop) + self.periodic_callbacks['ping'] = tornado.ioloop.PeriodicCallback(ping_master, ping_interval * 1000) self.periodic_callbacks['ping'].start() # add handler to subscriber @@ -3078,7 +3081,7 @@ class SyndicManager(MinionBase): # forward events every syndic_event_forward_timeout self.forward_events = tornado.ioloop.PeriodicCallback(self._forward_events, self.opts['syndic_event_forward_timeout'] * 1000, - io_loop=self.io_loop) + ) self.forward_events.start() # Make sure to gracefully handle SIGUSR1 diff --git a/salt/modules/cassandra_cql.py b/salt/modules/cassandra_cql.py index 82b211bddf..30db93dccc 100644 --- a/salt/modules/cassandra_cql.py +++ b/salt/modules/cassandra_cql.py @@ -93,6 +93,7 @@ from salt.exceptions import CommandExecutionError # Import 3rd-party libs from salt.ext import six from salt.ext.six.moves import range +import salt.utils.versions SSL_VERSION = 'ssl_version' @@ -128,7 +129,7 @@ def __virtual__(): def _async_log_errors(errors): - log.error('Cassandra_cql async call returned: %s', errors) + log.error('Cassandra_cql asynchronous call returned: %s', errors) def _load_properties(property_name, config_option, set_default=False, default=None): @@ -361,9 +362,8 @@ def cql_query(query, contact_points=None, port=None, cql_user=None, cql_pass=Non return ret -def cql_query_with_prepare(query, statement_name, statement_arguments, async=False, - callback_errors=None, - contact_points=None, port=None, cql_user=None, cql_pass=None): +def cql_query_with_prepare(query, statement_name, statement_arguments, callback_errors=None, contact_points=None, + port=None, cql_user=None, cql_pass=None, **kwargs): ''' Run a query on a Cassandra cluster and return a dictionary. @@ -377,8 +377,8 @@ def cql_query_with_prepare(query, statement_name, statement_arguments, async=Fal :type statement_name: str :param statement_arguments: Bind parameters for the SQL statement :type statement_arguments: list[str] - :param async: Run this query in asynchronous mode - :type async: bool + :param async: Run this query in asynchronous mode + :type async: bool :param callback_errors: Function to call after query runs if there is an error :type callback_errors: Function callable :param contact_points: The Cassandra cluster addresses, can either be a string or a list of IPs. @@ -401,12 +401,14 @@ def cql_query_with_prepare(query, statement_name, statement_arguments, async=Fal # Insert data asynchronously salt this-node cassandra_cql.cql_query_with_prepare "name_insert" "INSERT INTO USERS (first_name, last_name) VALUES (?, ?)" \ - statement_arguments=['John','Doe'], async=True + statement_arguments=['John','Doe'], asynchronous=True # Select data, should not be asynchronous because there is not currently a facility to return data from a future salt this-node cassandra_cql.cql_query_with_prepare "name_select" "SELECT * FROM USERS WHERE first_name=?" \ statement_arguments=['John'] ''' + # Backward-compatibility with Python 3.7: "async" is a reserved word + asynchronous = kwargs.get('async', False) try: cluster, session = _connect(contact_points=contact_points, port=port, cql_user=cql_user, cql_pass=cql_pass) @@ -431,7 +433,7 @@ def cql_query_with_prepare(query, statement_name, statement_arguments, async=Fal ret = [] try: - if async: + if asynchronous: future_results = session.execute_async(bound_statement.bind(statement_arguments)) # future_results.add_callbacks(_async_log_errors) else: @@ -441,7 +443,7 @@ def cql_query_with_prepare(query, statement_name, statement_arguments, async=Fal msg = "ERROR: Cassandra query failed: {0} reason: {1}".format(query, e) raise CommandExecutionError(msg) - if not async and results: + if not asynchronous and results: for result in results: values = {} for key, value in six.iteritems(result): @@ -456,7 +458,7 @@ def cql_query_with_prepare(query, statement_name, statement_arguments, async=Fal # If this was a synchronous call, then we either have an empty list # because there was no return, or we have a return - # If this was an async call we only return the empty list + # If this was an asynchronous call we only return the empty list return ret diff --git a/salt/modules/mandrill.py b/salt/modules/mandrill.py index 248939d09c..7044060154 100644 --- a/salt/modules/mandrill.py +++ b/salt/modules/mandrill.py @@ -24,6 +24,7 @@ import logging # Import Salt libs import salt.utils.json +import salt.utils.versions # import third party try: @@ -137,12 +138,13 @@ def _http_request(url, def send(message, - async=False, + asynchronous=False, ip_pool=None, send_at=None, api_url=None, api_version=None, - api_key=None): + api_key=None, + **kwargs): ''' Send out the email using the details from the ``message`` argument. @@ -151,14 +153,14 @@ def send(message, sent as dictionary with at fields as specified in the Mandrill API documentation. - async: ``False`` + asynchronous: ``False`` Enable a background sending mode that is optimized for bulk sending. - In async mode, messages/send will immediately return a status of - "queued" for every recipient. To handle rejections when sending in async + In asynchronous mode, messages/send will immediately return a status of + "queued" for every recipient. To handle rejections when sending in asynchronous mode, set up a webhook for the 'reject' event. Defaults to false for messages with no more than 10 recipients; messages with more than 10 recipients are always sent asynchronously, regardless of the value of - async. + asynchronous. ip_pool The name of the dedicated ip pool that should be used to send the @@ -229,6 +231,11 @@ def send(message, result: True ''' + if 'async' in kwargs: # Remove this in Sodium + salt.utils.versions.warn_until('Sodium', 'Parameter "async" is renamed to "asynchronous" ' + 'and will be removed in version {version}.') + asynchronous = bool(kwargs['async']) + params = _get_api_params(api_url=api_url, api_version=api_version, api_key=api_key) @@ -238,7 +245,7 @@ def send(message, data = { 'key': params['api_key'], 'message': message, - 'async': async, + 'async': asynchronous, 'ip_pool': ip_pool, 'send_at': send_at } diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 50d1f14ed9..b951983f5c 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -947,10 +947,11 @@ def refresh_pillar(): ret = False # Effectively a no-op, since we can't really return without an event system return ret + pillar_refresh = salt.utils.functools.alias_function(refresh_pillar, 'pillar_refresh') -def refresh_modules(async=True): +def refresh_modules(**kwargs): ''' Signal the minion to refresh the module and grain data @@ -964,8 +965,9 @@ def refresh_modules(async=True): salt '*' saltutil.refresh_modules ''' + asynchronous = bool(kwargs.get('async', True)) try: - if async: + if asynchronous: # If we're going to block, first setup a listener ret = __salt__['event.fire']({}, 'module_refresh') else: diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index d4b6c93f28..c1d333adcd 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -529,7 +529,7 @@ described above, the most effective and most scalable way to use both Salt and salt-api is to run commands asynchronously using the ``local_async``, ``runner_async``, and ``wheel_async`` clients. -Running async jobs results in being able to process 3x more commands per second +Running asynchronous jobs results in being able to process 3x more commands per second for ``LocalClient`` and 17x more commands per second for ``RunnerClient``, in addition to much less network traffic and memory requirements. Job returns can be fetched from Salt's job cache via the ``/jobs/<jid>`` endpoint, or they can @@ -2534,7 +2534,7 @@ class WebsocketEndpoint(object): parent_pipe, child_pipe = Pipe() handler.pipe = parent_pipe handler.opts = self.opts - # Process to handle async push to a client. + # Process to handle asynchronous push to a client. # Each GET request causes a process to be kicked off. proc = Process(target=event_stream, args=(handler, child_pipe)) proc.start() diff --git a/salt/netapi/rest_cherrypy/event_processor.py b/salt/netapi/rest_cherrypy/event_processor.py index e409a00180..f0cf6d361a 100644 --- a/salt/netapi/rest_cherrypy/event_processor.py +++ b/salt/netapi/rest_cherrypy/event_processor.py @@ -180,7 +180,7 @@ class SaltInfo(object): 'expr_type': 'list', 'mode': 'client', 'client': 'local', - 'async': 'local_async', + 'asynchronous': 'local_async', 'token': token, }) diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py index 5ed2769068..5b4dc03bdb 100644 --- a/salt/netapi/rest_tornado/__init__.py +++ b/salt/netapi/rest_tornado/__init__.py @@ -128,6 +128,6 @@ def start(): raise SystemExit(1) try: - tornado.ioloop.IOLoop.instance().start() + tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt: raise SystemExit(0) diff --git a/salt/netapi/rest_tornado/event_processor.py b/salt/netapi/rest_tornado/event_processor.py index d8c338836e..70a379e2c5 100644 --- a/salt/netapi/rest_tornado/event_processor.py +++ b/salt/netapi/rest_tornado/event_processor.py @@ -194,7 +194,7 @@ class SaltInfo(object): 'expr_type': 'list', 'mode': 'client', 'client': 'local', - 'async': 'local_async', + 'asynchronous': 'local_async', 'token': token, }) diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 7f927e9473..c69b5e0976 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -202,14 +202,12 @@ import tornado.ioloop import tornado.web import tornado.gen from tornado.concurrent import Future -from zmq.eventloop import ioloop -from salt.ext import six # pylint: enable=import-error - -# instantiate the zmq IOLoop (specialized poller) -ioloop.install() +import salt.utils +salt.utils.zeromq.install_zmq() # salt imports +import salt.ext.six as six import salt.netapi import salt.utils.args import salt.utils.event @@ -245,7 +243,7 @@ def _json_dumps(obj, **kwargs): # # master side # - "runner" (done) -# - "wheel" (need async api...) +# - "wheel" (need asynchronous api...) AUTH_TOKEN_HEADER = 'X-Auth-Token' @@ -274,7 +272,7 @@ class Any(Future): class EventListener(object): ''' Class responsible for listening to the salt master event bus and updating - futures. This is the core of what makes this async, this allows us to do + futures. This is the core of what makes this asynchronous, this allows us to do non-blocking work in the main processes and "wait" for an event to happen ''' @@ -324,7 +322,7 @@ class EventListener(object): timeout=None ): ''' - Get an event (async of course) return a future that will get it later + Get an event (asynchronous of course) return a future that will get it later ''' # if the request finished, no reason to allow event fetching, since we # can't send back to the client @@ -629,7 +627,7 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 self.write(self.serialize(ret)) - # TODO: make async? Underlying library isn't... and we ARE making disk calls :( + # TODO: make asynchronous? Underlying library isn't... and we ARE making disk calls :( def post(self): ''' :ref:`Authenticate <rest_tornado-auth>` against Salt's eauth system diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index 89cdfd039a..cf6d51852f 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -411,7 +411,7 @@ class FormattedEventsHandler(AllEventsHandler): # pylint: disable=W0223,W0232 'tgt': '*', 'token': self.token, 'mode': 'client', - 'async': 'local_async', + 'asynchronous': 'local_async', 'client': 'local' }) while True: diff --git a/salt/returners/cassandra_cql_return.py b/salt/returners/cassandra_cql_return.py index 8e92e32147..0ec8c2db27 100644 --- a/salt/returners/cassandra_cql_return.py +++ b/salt/returners/cassandra_cql_return.py @@ -204,7 +204,7 @@ def returner(ret): __salt__['cassandra_cql.cql_query_with_prepare'](query, 'returner_return', tuple(statement_arguments), - async=True) + asynchronous=True) except CommandExecutionError: log.critical('Could not insert into salt_returns with Cassandra returner.') raise @@ -228,7 +228,7 @@ def returner(ret): __salt__['cassandra_cql.cql_query_with_prepare'](query, 'returner_minion', tuple(statement_arguments), - async=True) + asynchronous=True) except CommandExecutionError: log.critical('Could not store minion ID with Cassandra returner.') raise @@ -270,7 +270,7 @@ def event_return(events): try: __salt__['cassandra_cql.cql_query_with_prepare'](query, 'salt_events', statement_arguments, - async=True) + asynchronous=True) except CommandExecutionError: log.critical('Could not store events with Cassandra returner.') raise @@ -300,7 +300,7 @@ def save_load(jid, load, minions=None): try: __salt__['cassandra_cql.cql_query_with_prepare'](query, 'save_load', statement_arguments, - async=True) + asynchronous=True) except CommandExecutionError: log.critical('Could not save load in jids table.') raise diff --git a/salt/runner.py b/salt/runner.py index d96a8af38c..fea1031abb 100644 --- a/salt/runner.py +++ b/salt/runner.py @@ -240,13 +240,13 @@ class Runner(RunnerClient): if self.opts.get('eauth'): async_pub = self.cmd_async(low) else: - async_pub = self.async(self.opts['fun'], - low, - user=user, - pub=async_pub) + async_pub = self.asynchronous(self.opts['fun'], + low, + user=user, + pub=async_pub) # by default: info will be not enougth to be printed out ! log.warning( - 'Running in async mode. Results of this execution may ' + 'Running in asynchronous mode. Results of this execution may ' 'be collected by attaching to the master event bus or ' 'by examing the master job cache, if configured. ' 'This execution is running under tag %s', async_pub['tag'] diff --git a/salt/thorium/runner.py b/salt/thorium/runner.py index d6235d40e7..9545eac35c 100644 --- a/salt/thorium/runner.py +++ b/salt/thorium/runner.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -React by calling async runners +React by calling asynchronous runners ''' # Import python libs from __future__ import absolute_import, print_function, unicode_literals @@ -14,7 +14,7 @@ def cmd( arg=(), **kwargs): ''' - Execute a runner async: + Execute a runner asynchronous: USAGE: @@ -42,7 +42,7 @@ def cmd( func = name local_opts = {} local_opts.update(__opts__) - local_opts['async'] = True # ensure this will be run async + local_opts['async'] = True # ensure this will be run asynchronous local_opts.update({ 'fun': func, 'arg': arg, diff --git a/salt/thorium/wheel.py b/salt/thorium/wheel.py index 7c98eff4bd..e3c4bf1701 100644 --- a/salt/thorium/wheel.py +++ b/salt/thorium/wheel.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -React by calling async runners +React by calling asynchronous runners ''' # Import python libs from __future__ import absolute_import, print_function, unicode_literals @@ -14,7 +14,7 @@ def cmd( arg=(), **kwargs): ''' - Execute a runner async: + Execute a runner asynchronous: USAGE: diff --git a/salt/transport/client.py b/salt/transport/client.py index 86c4962f94..ca83ac9376 100644 --- a/salt/transport/client.py +++ b/salt/transport/client.py @@ -10,7 +10,7 @@ from __future__ import absolute_import, print_function, unicode_literals import logging # Import Salt Libs -from salt.utils.async import SyncWrapper +from salt.utils.asynchronous import SyncWrapper log = logging.getLogger(__name__) diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index 7ea9bf8c87..84aa781ef0 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -130,11 +130,11 @@ class IPCServer(object): else: self.sock = tornado.netutil.bind_unix_socket(self.socket_path) - tornado.netutil.add_accept_handler( - self.sock, - self.handle_connection, - io_loop=self.io_loop, - ) + with salt.utils.asynchronous.current_ioloop(self.io_loop): + tornado.netutil.add_accept_handler( + self.sock, + self.handle_connection, + ) self._started = True @tornado.gen.coroutine @@ -196,10 +196,10 @@ class IPCServer(object): log.trace('IPCServer: Handling connection ' 'to address: %s', address) try: - stream = IOStream( - connection, - io_loop=self.io_loop, - ) + with salt.utils.asynchronous.current_ioloop(self.io_loop): + stream = IOStream( + connection, + ) self.io_loop.spawn_callback(self.handle_stream, stream) except Exception as exc: log.error('IPC streaming error: %s', exc) @@ -329,10 +329,10 @@ class IPCClient(object): break if self.stream is None: - self.stream = IOStream( - socket.socket(sock_type, socket.SOCK_STREAM), - io_loop=self.io_loop, - ) + with salt.utils.asynchronous.current_ioloop(self.io_loop): + self.stream = IOStream( + socket.socket(sock_type, socket.SOCK_STREAM), + ) try: log.trace('IPCClient: Connecting to socket: %s', self.socket_path) @@ -510,11 +510,11 @@ class IPCMessagePublisher(object): else: self.sock = tornado.netutil.bind_unix_socket(self.socket_path) - tornado.netutil.add_accept_handler( - self.sock, - self.handle_connection, - io_loop=self.io_loop, - ) + with salt.utils.asynchronous.current_ioloop(self.io_loop): + tornado.netutil.add_accept_handler( + self.sock, + self.handle_connection, + ) self._started = True @tornado.gen.coroutine @@ -545,17 +545,14 @@ class IPCMessagePublisher(object): def handle_connection(self, connection, address): log.trace('IPCServer: Handling connection to address: %s', address) try: + kwargs = {} if self.opts['ipc_write_buffer'] > 0: + kwargs['max_write_buffer_size'] = self.opts['ipc_write_buffer'] log.trace('Setting IPC connection write buffer: %s', (self.opts['ipc_write_buffer'])) + with salt.utils.asynchronous.current_ioloop(self.io_loop): stream = IOStream( connection, - io_loop=self.io_loop, - max_write_buffer_size=self.opts['ipc_write_buffer'] - ) - else: - stream = IOStream( - connection, - io_loop=self.io_loop + **kwargs ) self.streams.add(stream) diff --git a/salt/transport/server.py b/salt/transport/server.py index 46c14bdb39..1d67dc98af 100644 --- a/salt/transport/server.py +++ b/salt/transport/server.py @@ -55,7 +55,7 @@ class ReqServerChannel(object): ''' Do anything you need post-fork. This should handle all incoming payloads and call payload_handler. You will also be passed io_loop, for all of your - async needs + asynchronous needs ''' pass diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 334ed0a3ad..6d39ed877b 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -19,7 +19,7 @@ import errno # Import Salt Libs import salt.crypt -import salt.utils.async +import salt.utils.asynchronous import salt.utils.event import salt.utils.platform import salt.utils.process @@ -475,7 +475,7 @@ class AsyncTCPPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.tran 'tok': self.tok, 'data': data, 'tag': tag} - req_channel = salt.utils.async.SyncWrapper( + req_channel = salt.utils.asynchronous.SyncWrapper( AsyncTCPReqChannel, (self.opts,) ) try: @@ -597,23 +597,22 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra self.payload_handler = payload_handler self.io_loop = io_loop self.serial = salt.payload.Serial(self.opts) - if USE_LOAD_BALANCER: - self.req_server = LoadBalancerWorker(self.socket_queue, - self.handle_message, - io_loop=self.io_loop, - ssl_options=self.opts.get('ssl')) - else: - if salt.utils.platform.is_windows(): - self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - _set_tcp_keepalive(self._socket, self.opts) - self._socket.setblocking(0) - self._socket.bind((self.opts['interface'], int(self.opts['ret_port']))) - self.req_server = SaltMessageServer(self.handle_message, - io_loop=self.io_loop, - ssl_options=self.opts.get('ssl')) - self.req_server.add_socket(self._socket) - self._socket.listen(self.backlog) + with salt.utils.asynchronous.current_ioloop(self.io_loop): + if USE_LOAD_BALANCER: + self.req_server = LoadBalancerWorker(self.socket_queue, + self.handle_message, + ssl_options=self.opts.get('ssl')) + else: + if salt.utils.platform.is_windows(): + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + _set_tcp_keepalive(self._socket, self.opts) + self._socket.setblocking(0) + self._socket.bind((self.opts['interface'], int(self.opts['ret_port']))) + self.req_server = SaltMessageServer(self.handle_message, + ssl_options=self.opts.get('ssl')) + self.req_server.add_socket(self._socket) + self._socket.listen(self.backlog) salt.transport.mixins.auth.AESReqServerMixin.post_fork(self, payload_handler, io_loop) @tornado.gen.coroutine @@ -698,6 +697,7 @@ class SaltMessageServer(tornado.tcpserver.TCPServer, object): ''' def __init__(self, message_handler, *args, **kwargs): super(SaltMessageServer, self).__init__(*args, **kwargs) + self.io_loop = tornado.ioloop.IOLoop.current() self.clients = [] self.message_handler = message_handler @@ -774,10 +774,9 @@ class TCPClientKeepAlive(tornado.tcpclient.TCPClient): ''' Override _create_stream() in TCPClient to enable keep alive support. ''' - def __init__(self, opts, resolver=None, io_loop=None): + def __init__(self, opts, resolver=None): self.opts = opts - super(TCPClientKeepAlive, self).__init__( - resolver=resolver, io_loop=io_loop) + super(TCPClientKeepAlive, self).__init__(resolver=resolver) def _create_stream(self, max_buffer_size, af, addr, **kwargs): # pylint: disable=unused-argument ''' @@ -793,9 +792,10 @@ class TCPClientKeepAlive(tornado.tcpclient.TCPClient): _set_tcp_keepalive(sock, self.opts) stream = tornado.iostream.IOStream( sock, - io_loop=self.io_loop, max_buffer_size=max_buffer_size) - return stream.connect(addr) + if tornado.version_info < (5,): + return stream.connect(addr) + return stream, stream.connect(addr) class SaltMessageClientPool(salt.transport.MessageClientPool): @@ -855,8 +855,8 @@ class SaltMessageClient(object): self.io_loop = io_loop or tornado.ioloop.IOLoop.current() - self._tcp_client = TCPClientKeepAlive( - opts, io_loop=self.io_loop, resolver=resolver) + with salt.utils.asynchronous.current_ioloop(self.io_loop): + self._tcp_client = TCPClientKeepAlive(opts, resolver=resolver) self._mid = 1 self._max_messages = int((1 << 31) - 2) # number of IDs before we wrap @@ -879,33 +879,33 @@ class SaltMessageClient(object): return self._closing = True if hasattr(self, '_stream') and not self._stream.closed(): - self._stream.close() - if self._read_until_future is not None: - # This will prevent this message from showing up: - # '[ERROR ] Future exception was never retrieved: - # StreamClosedError' - # This happens because the logic is always waiting to read - # the next message and the associated read future is marked - # 'StreamClosedError' when the stream is closed. - self._read_until_future.exc_info() - if (not self._stream_return_future.done() and - self.io_loop != tornado.ioloop.IOLoop.current( - instance=False)): - # If _stream_return() hasn't completed, it means the IO - # Loop is stopped (such as when using - # 'salt.utils.async.SyncWrapper'). Ensure that - # _stream_return() completes by restarting the IO Loop. - # This will prevent potential errors on shutdown. - orig_loop = tornado.ioloop.IOLoop.current() - self.io_loop.make_current() - try: + # If _stream_return() hasn't completed, it means the IO + # Loop is stopped (such as when using + # 'salt.utils.asynchronous.SyncWrapper'). Ensure that + # _stream_return() completes by restarting the IO Loop. + # This will prevent potential errors on shutdown. + try: + orig_loop = tornado.ioloop.IOLoop.current() + self.io_loop.make_current() + self._stream.close() + if self._read_until_future is not None: + # This will prevent this message from showing up: + # '[ERROR ] Future exception was never retrieved: + # StreamClosedError' + # This happens because the logic is always waiting to read + # the next message and the associated read future is marked + # 'StreamClosedError' when the stream is closed. + self._read_until_future.exception() + if (not self._stream_return_future.done() and + self.io_loop != tornado.ioloop.IOLoop.current( + instance=False)): self.io_loop.add_future( self._stream_return_future, lambda future: self.io_loop.stop() ) self.io_loop.start() - finally: - orig_loop.make_current() + finally: + orig_loop.make_current() self._tcp_client.close() # Clear callback references to allow the object that they belong to # to be deleted. @@ -945,21 +945,21 @@ class SaltMessageClient(object): if self._closing: break try: - if (self.source_ip or self.source_port) and tornado.version_info >= (4, 5): - ### source_ip and source_port are supported only in Tornado >= 4.5 - # See http://www.tornadoweb.org/en/stable/releases/v4.5.0.html - # Otherwise will just ignore these args - self._stream = yield self._tcp_client.connect(self.host, - self.port, - ssl_options=self.opts.get('ssl'), - source_ip=self.source_ip, - source_port=self.source_port) - else: - if self.source_ip or self.source_port: + kwargs = {} + if self.source_ip or self.source_port: + if tornado.version_info >= (4, 5): + ### source_ip and source_port are supported only in Tornado >= 4.5 + # See http://www.tornadoweb.org/en/stable/releases/v4.5.0.html + # Otherwise will just ignore these args + kwargs = {'source_ip': self.source_ip, + 'source_port': self.source_port} + else: log.warning('If you need a certain source IP/port, consider upgrading Tornado >= 4.5') + with salt.utils.asynchronous.current_ioloop(self.io_loop): self._stream = yield self._tcp_client.connect(self.host, self.port, - ssl_options=self.opts.get('ssl')) + ssl_options=self.opts.get('ssl'), + **kwargs) self._connecting_future.set_result(True) break except Exception as e: @@ -1162,7 +1162,8 @@ class PubServer(tornado.tcpserver.TCPServer, object): TCP publisher ''' def __init__(self, opts, io_loop=None): - super(PubServer, self).__init__(io_loop=io_loop, ssl_options=opts.get('ssl')) + super(PubServer, self).__init__(ssl_options=opts.get('ssl')) + self.io_loop = io_loop self.opts = opts self._closing = False self.clients = set() @@ -1429,9 +1430,9 @@ class TCPPubServerChannel(salt.transport.server.PubServerChannel): pull_uri = int(self.opts.get('tcp_master_publish_pull', 4514)) else: pull_uri = os.path.join(self.opts['sock_dir'], 'publish_pull.ipc') - # TODO: switch to the actual async interface + # TODO: switch to the actual asynchronous interface #pub_sock = salt.transport.ipc.IPCMessageClient(self.opts, io_loop=self.io_loop) - pub_sock = salt.utils.async.SyncWrapper( + pub_sock = salt.utils.asynchronous.SyncWrapper( salt.transport.ipc.IPCMessageClient, (pull_uri,) ) diff --git a/salt/utils/async.py b/salt/utils/asynchronous.py similarity index 81% rename from salt/utils/async.py rename to salt/utils/asynchronous.py index 55d21d0ccc..16a7088360 100644 --- a/salt/utils/async.py +++ b/salt/utils/asynchronous.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Helpers/utils for working with tornado async stuff +Helpers/utils for working with tornado asynchronous stuff ''' from __future__ import absolute_import, print_function, unicode_literals @@ -30,9 +30,9 @@ class SyncWrapper(object): This is uses as a simple wrapper, for example: - async = AsyncClass() + asynchronous = AsyncClass() # this method would reguarly return a future - future = async.async_method() + future = asynchronous.async_method() sync = SyncWrapper(async_factory_method, (arg1, arg2), {'kwarg1': 'val'}) # the sync wrapper will automatically wait on the future @@ -46,15 +46,15 @@ class SyncWrapper(object): kwargs['io_loop'] = self.io_loop with current_ioloop(self.io_loop): - self.async = method(*args, **kwargs) + self.asynchronous = method(*args, **kwargs) def __getattribute__(self, key): try: return object.__getattribute__(self, key) except AttributeError as ex: - if key == 'async': + if key == 'asynchronous': raise ex - attr = getattr(self.async, key) + attr = getattr(self.asynchronous, key) if hasattr(attr, '__call__'): def wrap(*args, **kwargs): # Overload the ioloop for the func call-- since it might call .current() @@ -75,15 +75,15 @@ class SyncWrapper(object): def __del__(self): ''' - On deletion of the async wrapper, make sure to clean up the async stuff + On deletion of the asynchronous wrapper, make sure to clean up the asynchronous stuff ''' - if hasattr(self, 'async'): - if hasattr(self.async, 'close'): + if hasattr(self, 'asynchronous'): + if hasattr(self.asynchronous, 'close'): # Certain things such as streams should be closed before # their associated io_loop is closed to allow for proper # cleanup. - self.async.close() - del self.async + self.asynchronous.close() + del self.asynchronous self.io_loop.close() del self.io_loop elif hasattr(self, 'io_loop'): diff --git a/salt/utils/event.py b/salt/utils/event.py index cb9f5018e3..19fbfc7a78 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -72,7 +72,7 @@ import tornado.iostream # Import salt libs import salt.config import salt.payload -import salt.utils.async +import salt.utils.asynchronous import salt.utils.cache import salt.utils.dicttrim import salt.utils.platform @@ -221,7 +221,7 @@ class SaltEvent(object): :param Bool keep_loop: Pass a boolean to determine if we want to keep the io loop or destroy it when the event handle is destroyed. This is useful when using event - loops from within third party async code + loops from within third party asynchronous code ''' self.serial = salt.payload.Serial({'serial': 'msgpack'}) self.keep_loop = keep_loop @@ -357,7 +357,7 @@ class SaltEvent(object): return True if self._run_io_loop_sync: - with salt.utils.async.current_ioloop(self.io_loop): + with salt.utils.asynchronous.current_ioloop(self.io_loop): if self.subscriber is None: self.subscriber = salt.transport.ipc.IPCMessageSubscriber( self.puburi, @@ -376,7 +376,7 @@ class SaltEvent(object): io_loop=self.io_loop ) - # For the async case, the connect will be defered to when + # For the asynchronous case, the connect will be defered to when # set_event_handler() is invoked. self.cpub = True return self.cpub @@ -402,7 +402,7 @@ class SaltEvent(object): return True if self._run_io_loop_sync: - with salt.utils.async.current_ioloop(self.io_loop): + with salt.utils.asynchronous.current_ioloop(self.io_loop): if self.pusher is None: self.pusher = salt.transport.ipc.IPCMessageClient( self.pulluri, @@ -420,7 +420,7 @@ class SaltEvent(object): self.pulluri, io_loop=self.io_loop ) - # For the async case, the connect will be deferred to when + # For the asynchronous case, the connect will be deferred to when # fire_event() is invoked. self.cpush = True return self.cpush @@ -625,7 +625,7 @@ class SaltEvent(object): ret = self._check_pending(tag, match_func) if ret is None: - with salt.utils.async.current_ioloop(self.io_loop): + with salt.utils.asynchronous.current_ioloop(self.io_loop): if auto_reconnect: raise_errors = self.raise_errors self.raise_errors = True @@ -736,7 +736,7 @@ class SaltEvent(object): serialized_data]) msg = salt.utils.stringutils.to_bytes(event, 'utf-8') if self._run_io_loop_sync: - with salt.utils.async.current_ioloop(self.io_loop): + with salt.utils.asynchronous.current_ioloop(self.io_loop): try: self.io_loop.run_sync(lambda: self.pusher.send(msg)) except Exception as ex: @@ -1079,7 +1079,7 @@ class EventPublisher(salt.utils.process.SignalHandlingMultiprocessingProcess): ''' salt.utils.process.appendproctitle(self.__class__.__name__) self.io_loop = tornado.ioloop.IOLoop() - with salt.utils.async.current_ioloop(self.io_loop): + with salt.utils.asynchronous.current_ioloop(self.io_loop): if self.opts['ipc_mode'] == 'tcp': epub_uri = int(self.opts['tcp_master_pub_port']) epull_uri = int(self.opts['tcp_master_pull_port']) diff --git a/salt/utils/process.py b/salt/utils/process.py index 6dafaf0441..82719b9034 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -472,7 +472,7 @@ class ProcessManager(object): del self._process_map[pid] @gen.coroutine - def run(self, async=False): + def run(self, asynchronous=False): ''' Load and start all available api modules ''' @@ -495,7 +495,7 @@ class ProcessManager(object): # The event-based subprocesses management code was removed from here # because os.wait() conflicts with the subprocesses management logic # implemented in `multiprocessing` package. See #35480 for details. - if async: + if asynchronous: yield gen.sleep(10) else: time.sleep(10) diff --git a/salt/utils/thin.py b/salt/utils/thin.py index b99e407583..9a74b8d7d6 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -701,7 +701,7 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', 'salt/utils/openstack', 'salt/utils/openstack/__init__.py', 'salt/utils/openstack/swift.py', - 'salt/utils/async.py', + 'salt/utils/asynchronous.py', 'salt/utils/process.py', 'salt/utils/jinja.py', 'salt/utils/rsax931.py', diff --git a/salt/wheel/__init__.py b/salt/wheel/__init__.py index abfd776342..65092ef974 100644 --- a/salt/wheel/__init__.py +++ b/salt/wheel/__init__.py @@ -57,7 +57,7 @@ class WheelClient(salt.client.mixins.SyncClientMixin, return self.low(fun, kwargs, print_event=kwargs.get('print_event', True), full_return=kwargs.get('full_return', False)) # TODO: Inconsistent with runner client-- the runner client's master_call gives - # an async return, unlike this + # an asynchronous return, unlike this def master_call(self, **kwargs): ''' Execute a wheel function through the master network interface (eauth). @@ -120,7 +120,7 @@ class WheelClient(salt.client.mixins.SyncClientMixin, {'jid': '20131219224744416681', 'tag': 'salt/wheel/20131219224744416681'} ''' fun = low.pop('fun') - return self.async(fun, low) + return self.asynchronous(fun, low) def cmd(self, fun, arg=None, pub_data=None, kwarg=None, print_event=True, full_return=False): ''' diff --git a/tests/conftest.py b/tests/conftest.py index 7591cd46df..d2add41d41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,13 +55,17 @@ import salt.log.setup from salt.utils.odict import OrderedDict # Define the pytest plugins we rely on -pytest_plugins = ['pytest_catchlog', 'tempdir', 'helpers_namespace'] # pylint: disable=invalid-name +pytest_plugins = ['tempdir', 'helpers_namespace'] # pylint: disable=invalid-name # Define where not to collect tests from collect_ignore = ['setup.py'] log = logging.getLogger('salt.testsuite') +# Reset logging root handlers +for handler in logging.root.handlers: + logging.root.removeHandler(handler) + def pytest_tempdir_basename(): ''' @@ -197,25 +201,6 @@ def pytest_configure(config): called after command line options have been parsed and all plugins and initial conftest files been loaded. ''' - # Configure the console logger based on the catch_log settings. - # Most importantly, shutdown Salt's null, store and temporary logging queue handlers - catch_log = config.pluginmanager.getplugin('_catch_log') - cli_logging_handler = catch_log.log_cli_handler - # Add the pytest_catchlog CLI log handler to the logging root - logging.root.addHandler(cli_logging_handler) - cli_level = cli_logging_handler.level - cli_level = config._catchlog_log_cli_level - cli_format = cli_logging_handler.formatter._fmt - cli_date_format = cli_logging_handler.formatter.datefmt - # Setup the console logger which shuts down the null and the temporary queue handlers - salt.log.setup_console_logger( - log_level=salt.log.setup.LOG_VALUES_TO_LEVELS.get(cli_level, 'error'), - log_format=cli_format, - date_format=cli_date_format - ) - # Disable the store logging queue handler - salt.log.setup.setup_extended_logging({'extension_modules': ''}) - config.addinivalue_line('norecursedirs', os.path.join(CODE_DIR, 'templates')) config.addinivalue_line( 'markers', diff --git a/tests/integration/files/engines/runtests_engine.py b/tests/integration/files/engines/runtests_engine.py index 9247a3dfb4..426ab2a5b2 100644 --- a/tests/integration/files/engines/runtests_engine.py +++ b/tests/integration/files/engines/runtests_engine.py @@ -21,6 +21,7 @@ import logging # Import salt libs import salt.utils.event +import salt.utils.asynchronous # Import 3rd-party libs from tornado import gen @@ -69,11 +70,11 @@ class PyTestEngine(object): self.sock.bind(('localhost', port)) # become a server socket self.sock.listen(5) - netutil.add_accept_handler( - self.sock, - self.handle_connection, - io_loop=self.io_loop, - ) + with salt.utils.asynchronous.current_ioloop(self.io_loop): + netutil.add_accept_handler( + self.sock, + self.handle_connection, + ) def handle_connection(self, connection, address): log.warning('Accepted connection from %s. Role: %s', address, self.opts['__role']) diff --git a/tests/integration/netapi/rest_tornado/test_app.py b/tests/integration/netapi/rest_tornado/test_app.py index 96bb42f0f9..b6a572a0b5 100644 --- a/tests/integration/netapi/rest_tornado/test_app.py +++ b/tests/integration/netapi/rest_tornado/test_app.py @@ -372,7 +372,7 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase): def test_post_with_incorrect_client(self): ''' - The /minions endpoint is async only, so if you try something else + The /minions endpoint is asynchronous only, so if you try something else make sure you get an error ''' # get a token for this test diff --git a/tests/support/case.py b/tests/support/case.py index 8dbf733972..bf568a26e3 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -13,7 +13,7 @@ # pylint: disable=repr-flag-used-in-string # Import python libs -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import os import re import sys @@ -139,15 +139,28 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): ) return self.run_script('salt-ssh', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr, raw=True) - def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None): + def run_run(self, + arg_str, + with_retcode=False, + catch_stderr=False, + asynchronous=False, + timeout=60, + config_dir=None, + **kwargs): ''' Execute salt-run ''' - arg_str = '-c {0}{async_flag} -t {timeout} {1}'.format(config_dir or self.get_config_dir(), - arg_str, - timeout=timeout, - async_flag=' --async' if async else '') - return self.run_script('salt-run', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr) + asynchronous = kwargs.get('async', asynchronous) + arg_str = '-c {0}{async_flag} -t {timeout} {1}'.format( + config_dir or self.get_config_dir(), + arg_str, + timeout=timeout, + async_flag=' --async' if asynchronous else '') + return self.run_script('salt-run', + arg_str, + with_retcode=with_retcode, + catch_stderr=catch_stderr, + timeout=timeout) def run_run_plus(self, fun, *arg, **kwargs): ''' diff --git a/tests/unit/test_minion.py b/tests/unit/test_minion.py index 8ccd214012..01046d3bc7 100644 --- a/tests/unit/test_minion.py +++ b/tests/unit/test_minion.py @@ -11,6 +11,7 @@ import os # Import Salt Testing libs from tests.support.unit import TestCase, skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import AdaptedConfigurationTestCaseMixin from tests.support.helpers import skip_if_not_root # Import salt libs import salt.minion @@ -24,7 +25,7 @@ __opts__ = {} @skipIf(NO_MOCK, NO_MOCK_REASON) -class MinionTestCase(TestCase): +class MinionTestCase(TestCase, AdaptedConfigurationTestCaseMixin): def test_invalid_master_address(self): with patch.dict(__opts__, {'ipv6': False, 'master': float('127.0'), 'master_port': '4555', 'retry_dns': False}): self.assertRaises(SaltSystemExit, salt.minion.resolve_dns, __opts__) @@ -187,7 +188,7 @@ class MinionTestCase(TestCase): patch('salt.minion.Minion.sync_connect_master', MagicMock(side_effect=RuntimeError('stop execution'))), \ patch('salt.utils.process.SignalHandlingMultiprocessingProcess.start', MagicMock(return_value=True)), \ patch('salt.utils.process.SignalHandlingMultiprocessingProcess.join', MagicMock(return_value=True)): - mock_opts = copy.copy(salt.config.DEFAULT_MINION_OPTS) + mock_opts = self.get_config('minion', from_scratch=True) mock_opts['beacons_before_connect'] = True minion = salt.minion.Minion(mock_opts, io_loop=tornado.ioloop.IOLoop()) try: @@ -211,7 +212,7 @@ class MinionTestCase(TestCase): patch('salt.minion.Minion.sync_connect_master', MagicMock(side_effect=RuntimeError('stop execution'))), \ patch('salt.utils.process.SignalHandlingMultiprocessingProcess.start', MagicMock(return_value=True)), \ patch('salt.utils.process.SignalHandlingMultiprocessingProcess.join', MagicMock(return_value=True)): - mock_opts = copy.copy(salt.config.DEFAULT_MINION_OPTS) + mock_opts = self.get_config('minion', from_scratch=True) mock_opts['scheduler_before_connect'] = True minion = salt.minion.Minion(mock_opts, io_loop=tornado.ioloop.IOLoop()) try: diff --git a/tests/unit/utils/test_async.py b/tests/unit/utils/test_async.py index c93538f0dd..694a7aebfe 100644 --- a/tests/unit/utils/test_async.py +++ b/tests/unit/utils/test_async.py @@ -8,7 +8,7 @@ import tornado.testing import tornado.gen from tornado.testing import AsyncTestCase -import salt.utils.async as async +import salt.utils.asynchronous as asynchronous class HelperA(object): @@ -24,7 +24,7 @@ class HelperA(object): class HelperB(object): def __init__(self, a=None, io_loop=None): if a is None: - a = async.SyncWrapper(HelperA) + a = asynchronous.SyncWrapper(HelperA) self.a = a @tornado.gen.coroutine @@ -38,7 +38,7 @@ class TestSyncWrapper(AsyncTestCase): @tornado.testing.gen_test def test_helpers(self): ''' - Test that the helper classes do what we expect within a regular async env + Test that the helper classes do what we expect within a regular asynchronous env ''' ha = HelperA() ret = yield ha.sleep() @@ -50,29 +50,29 @@ class TestSyncWrapper(AsyncTestCase): def test_basic_wrap(self): ''' - Test that we can wrap an async caller. + Test that we can wrap an asynchronous caller. ''' - sync = async.SyncWrapper(HelperA) + sync = asynchronous.SyncWrapper(HelperA) ret = sync.sleep() self.assertTrue(ret) def test_double(self): ''' - Test when the async wrapper object itself creates a wrap of another thing + Test when the asynchronous wrapper object itself creates a wrap of another thing This works fine since the second wrap is based on the first's IOLoop so we don't have to worry about complex start/stop mechanics ''' - sync = async.SyncWrapper(HelperB) + sync = asynchronous.SyncWrapper(HelperB) ret = sync.sleep() self.assertFalse(ret) def test_double_sameloop(self): ''' - Test async wrappers initiated from the same IOLoop, to ensure that + Test asynchronous wrappers initiated from the same IOLoop, to ensure that we don't wire up both to the same IOLoop (since it causes MANY problems). ''' - a = async.SyncWrapper(HelperA) - sync = async.SyncWrapper(HelperB, (a,)) + a = asynchronous.SyncWrapper(HelperA) + sync = asynchronous.SyncWrapper(HelperB, (a,)) ret = sync.sleep() self.assertFalse(ret) -- 2.17.1
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
