From 8f4d7c4f7219444fc5674404b5488bb5d0e81535 Mon Sep 17 00:00:00 2001 From: Panos Date: Sat, 26 Dec 2020 22:23:46 +0000 Subject: [PATCH] =?UTF-8?q?Fixed=20encoding=20not=20used=20for=20cmd=20str?= =?UTF-8?q?ing.=20Updated=20changelog,=20documentat=E2=80=A6=20(#258)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fixed encoding not used for cmd string, updated tests. * Updated changelog, documentation, docstrings. --- Changelog.rst | 1 + doc/advanced.rst | 33 ++++++++++++++++------ pssh/clients/base/parallel.py | 6 ++-- pssh/clients/base/single.py | 13 +++++---- pssh/clients/native/parallel.py | 2 +- pssh/clients/ssh/parallel.py | 2 +- tests/native/test_parallel_client.py | 42 +++++++++++++++------------- 7 files changed, 62 insertions(+), 37 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 0ad99aa9..7f994c31 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -17,6 +17,7 @@ Fixes ----- * ``SSHClient`` with proxy enabled could not be used without setting port - #248 +* Encoding would not be applied to command string on ``run_command`` and interactive shells, `utf-8` used instead - #174. 2.3.2 diff --git a/doc/advanced.rst b/doc/advanced.rst index c5a0e9d7..a6586b23 100644 --- a/doc/advanced.rst +++ b/doc/advanced.rst @@ -448,19 +448,36 @@ Shell to use is configurable: Commands will be run under the ``zsh`` shell in the above example. The command string syntax of the shell must be used, typically `` -c``. -Output encoding -=============== +Output And Command Encoding +=========================== -By default, output is encoded as ``UTF-8``. This can be configured with the ``encoding`` keyword argument. +By default, command string and output are encoded as ``UTF-8``. This can be configured with the ``encoding`` keyword argument to ``run_command`` and ``open_shell``. .. code-block:: python - client = <..> + client = ParallelSSHClient(<..>) - client.run_command(<..>, encoding='utf-16') + cmd = b"echo \xbc".decode('latin-1') + output = client.run_command(cmd, encoding='latin-1') stdout = list(output[0].stdout) -Contents of ``stdout`` are `UTF-16` encoded. + +Contents of ``stdout`` are `latin-1` decoded. + +``cmd`` string is also `latin-1` encoded when running command or writing to interactive shell. + +Output encoding can also be changed by adjusting ``HostOutput.encoding``. + +.. code-block:: python + + client = ParallelSSHClient(<..>) + + output = client.run_command('echo me') + output[0].encoding = 'utf-16' + stdout = list(output[0].stdout) + +Contents of ``stdout`` are `utf-16` decoded. + .. note:: @@ -480,12 +497,12 @@ All output, including stderr, is sent to the ``stdout`` channel with PTY enabled client = <..> - client.run_command("echo 'asdf' >&2", use_pty=True) + output = client.run_command("echo 'asdf' >&2", use_pty=True) for line in output[0].stdout: print(line) -Note output is from the ``stdout`` channel while it was writeen to ``stderr``. +Note output is from the ``stdout`` channel while it was written to ``stderr``. :Output: .. code-block:: shell diff --git a/pssh/clients/base/parallel.py b/pssh/clients/base/parallel.py index 9a7b3b49..62db0081 100644 --- a/pssh/clients/base/parallel.py +++ b/pssh/clients/base/parallel.py @@ -115,7 +115,7 @@ def _open_shell(self, host_i, host, def open_shell(self, encoding='utf-8', read_timeout=None): """Open interactive shells on all hosts. - :param encoding: Encoding to use for shell output. + :param encoding: Encoding to use for command string and shell output. :type encoding: str :param read_timeout: Seconds before reading from output times out. :type read_timeout: float @@ -331,8 +331,8 @@ def join(self, output=None, consume_output=False, timeout=None, Since self.timeout is passed onto each individual SSH session it is **not** used for any parallel functions like `run_command` or `join`. :type timeout: int - :param encoding: Encoding to use for output. Must be valid - `Python codec `_ + :param encoding: Unused - encoding from each ``HostOutput`` is used instead. + To be removed in future releases. :type encoding: str :raises: :py:class:`pssh.exceptions.Timeout` on timeout requested and diff --git a/pssh/clients/base/single.py b/pssh/clients/base/single.py index 2850493f..46d83f5c 100644 --- a/pssh/clients/base/single.py +++ b/pssh/clients/base/single.py @@ -85,8 +85,8 @@ class InteractiveShell(object): ``InteractiveShell.output`` is a :py:class:`pssh.output.HostOutput` object. """ - __slots__ = ('_chan', '_client', 'output') - _EOL = '\n' + __slots__ = ('_chan', '_client', 'output', '_encoding') + _EOL = b'\n' def __init__(self, channel, client, encoding='utf-8', read_timeout=None): """ @@ -94,10 +94,13 @@ def __init__(self, channel, client, encoding='utf-8', read_timeout=None): :type channel: ``ssh2.channel.Channel`` or similar. :param client: The SSHClient that opened the channel. :type client: :py:class:`BaseSSHClient` + :param encoding: Encoding to use for command string when calling ``run`` and shell output. + :type encoding: str """ self._chan = channel self._client = client self._client._shell(self._chan) + self._encoding = encoding self.output = self._client._make_host_output( self._chan, encoding=encoding, read_timeout=read_timeout) @@ -142,7 +145,7 @@ def run(self, cmd): Note that ``\\n`` is appended to every string. :type cmd: str """ - cmd += self._EOL + cmd = cmd.encode(self._encoding) + self._EOL self._client._eagain_write(self._chan.write, cmd) @@ -227,7 +230,7 @@ def open_shell(self, encoding='utf-8', read_timeout=None): Can be used as context manager - ``with open_shell() as shell``. - :param encoding: Encoding to use for output from shell. + :param encoding: Encoding to use for command string and shell output. :type encoding: str :param read_timeout: Timeout in seconds for reading from output. :type read_timeout: float @@ -473,10 +476,10 @@ def run_command(self, command, sudo=False, user=None, _command = 'sudo -u %s -S ' % (user,) _shell = shell if shell else '$SHELL -c' _command += "%s '%s'" % (_shell, command,) + _command = _command.encode(encoding) with GTimeout(seconds=self.timeout): channel = self.execute(_command, use_pty=use_pty) _timeout = read_timeout if read_timeout else timeout - channel = self.execute(_command, use_pty=use_pty) host_out = self._make_host_output(channel, encoding, _timeout) return host_out diff --git a/pssh/clients/native/parallel.py b/pssh/clients/native/parallel.py index 8174f357..f27a0fec 100644 --- a/pssh/clients/native/parallel.py +++ b/pssh/clients/native/parallel.py @@ -173,7 +173,7 @@ def run_command(self, command, sudo=False, user=None, stop_on_errors=True, host list - :py:class:`pssh.exceptions.HostArgumentError` is raised otherwise :type host_args: tuple or list - :param encoding: Encoding to use for output. Must be valid + :param encoding: Encoding to use for command string and output. Must be valid `Python codec `_ :type encoding: str :param read_timeout: (Optional) Timeout in seconds for reading from stdout diff --git a/pssh/clients/ssh/parallel.py b/pssh/clients/ssh/parallel.py index cfccd218..7cdf009c 100644 --- a/pssh/clients/ssh/parallel.py +++ b/pssh/clients/ssh/parallel.py @@ -185,7 +185,7 @@ def run_command(self, command, sudo=False, user=None, stop_on_errors=True, host list - :py:class:`pssh.exceptions.HostArgumentError` is raised otherwise :type host_args: tuple or list - :param encoding: Encoding to use for output. Must be valid + :param encoding: Encoding to use for command string and output. Must be valid `Python codec `_ :type encoding: str :param read_timeout: (Optional) Timeout in seconds for reading from stdout diff --git a/tests/native/test_parallel_client.py b/tests/native/test_parallel_client.py index f53e3890..35d307fb 100644 --- a/tests/native/test_parallel_client.py +++ b/tests/native/test_parallel_client.py @@ -1151,27 +1151,31 @@ def test_per_host_dict_args_invalid(self): self.assertRaises( KeyError, self.client.run_command, cmd, host_args=host_args) - def test_ssh_client_utf_encoding(self): - """Test that unicode output works""" - expected = [u'é'] - _utf16 = u'é'.encode('utf-8').decode('utf-16') - cmd = u"echo 'é'" - output = self.client.run_command(cmd) + def test_run_command_encoding(self): + """Test that unicode command works""" + exp = b"\xbc" + _cmd = b"echo " + exp + cmd = _cmd.decode('latin-1') + expected = [exp.decode('latin-1')] + output = self.client.run_command(cmd, encoding='latin-1') stdout = list(output[0].stdout) - self.assertEqual(expected, stdout, - msg="Got unexpected unicode output %s - expected %s" % ( - stdout, expected,)) - output = self.client.run_command(cmd, encoding='utf-16') - _stdout = list(output[0].stdout) - self.assertEqual([_utf16], _stdout) - - def test_ssh_client_utf_encoding_join(self): - _utf16 = u'é'.encode('utf-8').decode('utf-16') - cmd = u"echo 'é'" - output = self.client.run_command(cmd, encoding='utf-16') - self.client.join(output, encoding='utf-16') + self.assertEqual(expected, stdout) + # With join + output = self.client.run_command(cmd, encoding='latin-1') + self.client.join(output) stdout = list(output[0].stdout) - self.assertEqual([_utf16], stdout) + self.assertEqual(expected, stdout) + + def test_shell_encoding(self): + exp = b"\xbc" + _cmd = b"echo " + exp + cmd = _cmd.decode('latin-1') + expected = [exp.decode('latin-1')] + shells = self.client.open_shell(encoding='latin-1') + self.client.run_shell_commands(shells, cmd) + self.client.join_shells(shells) + stdout = list(shells[0].stdout) + self.assertEqual(expected, stdout) def test_pty(self): cmd = "echo 'asdf' >&2"