Re-implement sh_string to use embedded literals, add lots of tests (#717

)

* Re-implement sh_string to use embedded literals, add lots of tests
* Add docs for sh_string so that unit tests get picked up
* Mark the docstrings as raw
* Rewrite doctests to not "print" since this breaks with invisible characters (issue with doctest itself)
* Add test which uses 16KB of random non-NUL data

.travis.yml

@@ -6,6 +6,12 @@ addons:
     - gcc-multilib
     - gcc-4.6-arm-linux-gnueabihf
     - lib32stdc++6
+    - ash
+    - bash
+    - dash
+    - ksh
+    - mksh
+    - zsh
 cache:
     - pip
     - directories:

docs/source/util/sh_string.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   test = pwnlib.util.sh_string.test
+
+:mod:`pwnlib.util.sh_string` --- Shell Expansion is Hard
+===============================================================
+
+.. automodule:: pwnlib.util.sh_string
+   :members:

pwn/toplevel.py

@@ -56,6 +56,7 @@
 from pwnlib.util.misc import *
 from pwnlib.util.packing import *
 from pwnlib.util.proc import pidof
+from pwnlib.util.sh_string import sh_string, sh_prepare, sh_command_with
 from pwnlib.util.splash import *
 from pwnlib.util.web import *
 

pwnlib/elf/elf.py

@@ -24,6 +24,7 @@
 from ..tubes.process import process
 from ..util import misc
 from ..util import packing
+from ..util import sh_string
 
 log = getLogger(__name__)
 
@@ -312,7 +313,7 @@ def _populate_libraries(self):
             return
 
         try:
-            cmd = misc.sh_command_with('ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null', self.path)
+            cmd = sh_string.sh_command_with('ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null', self.path)
 
             data = subprocess.check_output(cmd, shell = True, stderr = subprocess.STDOUT)
             libs = misc.parse_ldd_output(data)

pwnlib/tubes/ssh.py

@@ -19,6 +19,7 @@
 from ..util import hashes
 from ..util import misc
 from ..util import safeeval
+from ..util import sh_string
 from .process import process
 from .sock import sock
 
@@ -84,16 +85,16 @@ def __init__(self, parent, process = None, tty = False, wd = None, env = None, t
 
         if isinstance(process, (list, tuple)):
             fmt = ' '.join('%s' for s in process)
-            process = misc.sh_command_with(fmt, *process)
+            process = sh_string.sh_command_with(fmt, *process)
 
         if process and wd:
-            process = misc.sh_command_with('cd %s >/dev/null 2>&1;', wd) + process
+            process = sh_string.sh_command_with('cd %s >/dev/null 2>&1;', wd) + process
 
         if process and env:
             for name, value in env.items():
                 if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', name):
                     self.error('run(): Invalid environment key $r' % name)
-                process = '%s;%s' % (misc.sh_prepare(name, value, export=True), process)
+                process = '%s;%s' % (sh_string.sh_prepare(name, value, export=True), process)
 
         if process and tty:
             if raw:
@@ -876,7 +877,7 @@ def is_exe(path):
             if not aslr:
                 self.warn_once("ASLR is disabled!")
 
-            script = misc.sh_command_with('for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2', script)
+            script = sh_string.sh_command_with('for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2', script)
             with context.local(log_level='error'):
                 python = self.run(script, raw=raw)
             result = safeeval.const(python.recvline())
@@ -1147,7 +1148,7 @@ def close(self):
     def _libs_remote(self, remote):
         """Return a dictionary of the libraries used by a remote file."""
         cmd = '(ulimit -s unlimited; ldd %s > /dev/null && (LD_TRACE_LOADED_OBJECTS=1 %s || ldd %s)) 2>/dev/null'
-        cmd = misc.sh_command_with(lambda arg: cmd % (arg, arg, arg), remote)
+        cmd = sh_string.sh_command_with(lambda arg: cmd % (arg, arg, arg), remote)
         data, status = self.run_to_end(cmd)
         if status != 0:
             self.error('Unable to find libraries for %r' % remote)
@@ -1157,7 +1158,7 @@ def _libs_remote(self, remote):
 
     def _get_fingerprint(self, remote):
         cmd = '(openssl sha256 || sha256 || sha256sum) 2>/dev/null < %s'
-        cmd = misc.sh_command_with(cmd, remote)
+        cmd = sh_string.sh_command_with(cmd, remote)
         data, status = self.run_to_end(cmd)
 
         if status != 0:
@@ -1200,7 +1201,7 @@ def update(has, total):
             self.sftp.get(remote, local, update)
             return
 
-        cmd = misc.sh_command_with('wc -c < %s', remote)
+        cmd = sh_string.sh_command_with('wc -c < %s', remote)
         total, exitcode = self.run_to_end(cmd)
 
         if exitcode != 0:
@@ -1210,7 +1211,7 @@ def update(has, total):
         total = int(total)
 
         with context.local(log_level = 'ERROR'):
-            cmd = misc.sh_command_with('cat < %s', remote)
+            cmd = sh_string.sh_command_with('cat < %s', remote)
             c = self.run(cmd)
         data = ''
 
@@ -1321,7 +1322,7 @@ def download_dir(self, remote=None, local=None):
             remote = str(self.sftp.normalize(remote))
         else:
             with context.local(log_level='error'):
-                remote = self.system(misc.sh_command_with('readlink -f %s', remote))
+                remote = self.system(sh_string.sh_command_with('readlink -f %s', remote))
 
         dirname  = os.path.dirname(remote)
         basename = os.path.basename(remote)
@@ -1333,7 +1334,7 @@ def download_dir(self, remote=None, local=None):
 
         with context.local(log_level='error'):
             remote_tar = self.mktemp()
-            tar = self.system(misc.sh_command_with('tar -C %s -czf %s %s', dirname, remote_tar, basename))
+            tar = self.system(sh_string.sh_command_with('tar -C %s -czf %s %s', dirname, remote_tar, basename))
 
             if 0 != tar.wait():
                 self.error("Could not create remote tar")
@@ -1377,7 +1378,7 @@ def upload_data(self, data, remote):
                 return
 
         with context.local(log_level = 'ERROR'):
-            cmd = misc.sh_command_with('cat>%s', remote)
+            cmd = sh_string.sh_command_with('cat>%s', remote)
             s = self.run(cmd, tty=False)
             s.send(data)
             s.shutdown('send')
@@ -1461,7 +1462,7 @@ def download(self, file_or_directory, remote=None):
         if not self.sftp:
             self.error("Cannot determine remote file type without SFTP")
 
-        if 0 == self.system(misc.sh_command_with('test -d %s', file_or_directory)).wait():
+        if 0 == self.system(sh_string.sh_command_with('test -d %s', file_or_directory)).wait():
             self.download_dir(file_or_directory, remote)
         else:
             self.download_file(file_or_directory, remote)
@@ -1517,7 +1518,7 @@ def interactive(self, shell=None):
         s = self.shell(shell)
 
         if self.cwd != '.':
-            cmd = misc.sh_command_with('cd %s', self.cwd)
+            cmd = sh_string.sh_command_with('cd %s', self.cwd)
             s.sendline(cmd)
 
         s.interactive()
@@ -1559,7 +1560,7 @@ def set_working_directory(self, wd = None):
                 self.error("Could not generate a temporary directory (%i)\n%s" % (status, wd))
 
         else:
-            cmd = misc.sh_command_with('ls %s', wd)
+            cmd = sh_string.sh_command_with('ls %s', wd)
             _, status = self.run_to_end(cmd, wd = '.')
 
             if status:

pwnlib/util/__init__.py

@@ -15,4 +15,5 @@
 from . import packing
 from . import proc
 from . import safeeval
+from . import sh_string
 from . import web

pwnlib/util/misc.py

@@ -274,163 +274,6 @@ def mkdir_p(path):
         else:
             raise
 
-def sh_string(s):
-    """Outputs a string in a format that will be understood by /bin/sh.
-
-    If the string does not contain any bad characters, it will simply be
-    returned, possibly with quotes. If it contains bad characters, it will
-    be escaped in a way which is compatible with most known systems.
-
-    Examples:
-
-        >>> print sh_string('foobar')
-        foobar
-        >>> print sh_string('foo bar')
-        'foo bar'
-        >>> print sh_string("foo'bar")
-        "foo'bar"
-        >>> print sh_string("foo\\\\bar")
-        'foo\\bar'
-        >>> print sh_string("foo\\\\'bar")
-        "foo\\\\'bar"
-        >>> print sh_string("foo\\x01'bar")
-        "$(printf 'foo\\001\\047bar')"
-        >>> print `subprocess.check_output("echo -n " + sh_string("foo\\x01'bar"), shell = True)`
-        "foo\\x01'bar"
-    """
-
-    very_good = set(string.ascii_letters + string.digits)
-    good      = (very_good | set(string.punctuation + ' ')) - set("'")
-    alt_good  = (very_good | set(string.punctuation + ' ')) - set('!')
-
-    if '\x00' in s:
-        log.error("sh_string(): Cannot create a null-byte")
-    if s.endswith('\n'):
-        log.error("sh_string(): Cannot create a newline-terminated string")
-
-    if all(c in very_good for c in s):
-        return s
-    elif all(c in good for c in s):
-        return "'%s'" % s
-    elif all(c in alt_good for c in s):
-        fixed = ''
-        for c in s:
-            if c in '"\\$`':
-                fixed += '\\' + c
-            else:
-                fixed += c
-        return '"%s"' % fixed
-    else:
-        fixed = ''
-        for c in s:
-            if c == '\\':
-                fixed += '\\\\'
-            elif c == '\n':
-                fixed += '\\n'
-            elif c in good:
-                fixed += c
-            else:
-                fixed += '\\%03o' % ord(c)
-        return '"$(printf \'%s\')"' % fixed
-
-def sh_prepare(variables, export = False):
-    """Outputs a posix compliant shell command that will put the data specified
-    by the dictionary into the environment.
-
-    It is assumed that the keys in the dictionary are valid variable names that
-    does not need any escaping.
-
-    Arguments:
-      variables(dict): The variables to set.
-      export(bool): Should the variables be exported or only stored in the shell environment?
-      output(str): A valid posix shell command that will set the given variables.
-
-    It is assumed that `var` is a valid name for a variable in the shell.
-
-    Examples:
-
-        >>> print sh_prepare({'X': 'foobar'})
-        X=foobar
-        >>> r = sh_prepare({'X': 'foobar', 'Y': 'cookies'})
-        >>> r == 'X=foobar;Y=cookies' or r == 'Y=cookies;X=foobar'
-        True
-        >>> print sh_prepare({'X': 'foo bar'})
-        X='foo bar'
-        >>> print sh_prepare({'X': "foo'bar"})
-        X="foo'bar"
-        >>> print sh_prepare({'X': "foo\\\\bar"})
-        X='foo\\bar'
-        >>> print sh_prepare({'X': "foo\\\\'bar"})
-        X="foo\\\\'bar"
-        >>> print sh_prepare({'X': "foo\\x01'bar"})
-        X="$(printf 'foo\\001\\047bar')"
-        >>> print sh_prepare({'X': "foo\\x01'bar"}, export = True)
-        export X="$(printf 'foo\\001\\047bar')"
-        >>> print sh_prepare({'X': "foo\\x01'bar\\n"})
-        X="$(printf 'foo\\001\\047bar\\nx')";X=${X%x}
-        >>> print sh_prepare({'X': "foo\\x01'bar\\n"}, export = True)
-        X="$(printf 'foo\\001\\047bar\\nx')";export X=${X%x}
-        >>> print `subprocess.check_output('%s;echo -n "$X"' % sh_prepare({'X': "foo\\x01'bar"}), shell = True)`
-        "foo\\x01'bar"
-    """
-
-    out = []
-    export = 'export ' if export else ''
-
-    for k, v in variables.items():
-        if v.endswith('\n'):
-            out.append('%s=%s;%s%s=${%s%%x}' % (k, sh_string(v + "x"), export, k, k))
-        else:
-            out.append('%s%s=%s' % (export, k, sh_string(v)))
-    return ';'.join(out)
-
-def sh_command_with(f, *args):
-    """sh_command_with(f, arg0, ..., argN) -> command
-
-    Returns a command create by evaluating `f(new_arg0, ..., new_argN)`
-    whenever `f` is a function and `f % (new_arg0, ..., new_argN)` otherwise.
-
-    If the arguments are purely alphanumeric, then they are simply passed to
-    function. If they are simple to escape, they will be escaped and passed to
-    the function.
-
-    If the arguments contain trailing newlines, then it is hard to use them
-    directly because of a limitation in the posix shell. In this case the
-    output from `f` is prepended with a bit of code to create the variables.
-
-    Examples:
-
-        >>> print sh_command_with(lambda: "echo hello")
-        echo hello
-        >>> print sh_command_with(lambda x: "echo " + x, "hello")
-        echo hello
-        >>> print sh_command_with(lambda x: "echo " + x, "\\x01")
-        echo "$(printf '\\001')"
-        >>> import random
-        >>> random.seed(1)
-        >>> print sh_command_with(lambda x: "echo " + x, "\\x01\\n")
-        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
-        >>> random.seed(1)
-        >>> print sh_command_with("echo %s", "\\x01\\n")
-        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
-    """
-
-    args = list(args)
-    out = []
-
-    for n in range(len(args)):
-        if args[n].endswith('\n'):
-            v = fiddling.randoms(8)
-            out.append(sh_prepare({v: args[n]}))
-            args[n] = '"$%s"' % v
-        else:
-            args[n] = sh_string(args[n])
-    if hasattr(f, '__call__'):
-        out.append(f(*args))
-    else:
-        out.append(f % tuple(args))
-    return ';'.join(out)
-
 def dealarm_shell(tube):
     """Given a tube which is a shell, dealarm it.
     """