Allow libcdb search offline database

pwnlib/args.py

@@ -159,6 +159,10 @@ def STDERR(v):
     """Sends logging to ``stderr`` by default, instead of ``stdout``"""
     context.log_console = sys.stderr
 
+def LOCAL_LIBCDB(v):
+    """Sets local libcdb path"""
+    context.defaults['local_libcdb'] = v
+
 hooks = {
     'LOG_LEVEL': LOG_LEVEL,
     'LOG_FILE': LOG_FILE,
@@ -170,6 +174,7 @@ def STDERR(v):
     'NOASLR': NOASLR,
     'NOPTRACE': NOPTRACE,
     'STDERR': STDERR,
+    'LOCAL_LIBCDB': LOCAL_LIBCDB,
 }
 
 def initialize():

pwnlib/commandline/libcdb.py

@@ -3,8 +3,10 @@
 from __future__ import division
 from __future__ import print_function
 
+import os
 import re
 import shutil
+import subprocess
 import sys
 
 import pwnlib.args
@@ -44,17 +46,17 @@
 )
 
 lookup_parser.add_argument(
-    '--unstrip',
+    '--no-unstrip',
     action = 'store_true',
-    default = True,
+    default = False,
     help = 'Attempt to unstrip the libc binary with debug symbols from a debuginfod server'
 )
 
 lookup_parser.add_argument(
-    '--no-unstrip',
-    action = 'store_false',
-    dest = 'unstrip',
-    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+    '--offline',
+    action = 'store_true',
+    default = False,
+    help = 'Disable offline libcdb search mode'
 )
 
 hash_parser = libc_commands.add_parser(
@@ -87,17 +89,17 @@
 )
 
 hash_parser.add_argument(
-    '--unstrip',
+    '--no-unstrip',
     action = 'store_true',
-    default = True,
+    default = False,
     help = 'Attempt to unstrip the libc binary with debug symbols from a debuginfod server'
 )
 
 hash_parser.add_argument(
-    '--no-unstrip',
-    action = 'store_false',
-    dest = 'unstrip',
-    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+    '--offline',
+    action = 'store_true',
+    default = False,
+    help = 'Disable offline libcdb search mode'
 )
 
 file_parser = libc_commands.add_parser(
@@ -128,25 +130,33 @@
 )
 
 file_parser.add_argument(
-    '--unstrip',
+    '--no-unstrip',
     action = 'store_true',
     default = False,
     help = 'Attempt to unstrip the libc binary inplace with debug symbols from a debuginfod server'
 )
 
-common_symbols = ['dup2', 'printf', 'puts', 'read', 'system', 'write']
+offline_parser = libc_commands.add_parser(
+    'offline',
+    help = 'Offline the libc-database repository',
+    description = 'Lookup a libc version by function offsets'
+)
 
-def find_libc(params):
-    import requests
-    url    = "https://libc.rip/api/find"
-    result = requests.post(url, json=params, timeout=20)
-    log.debug('Request: %s', params)
-    log.debug('Result: %s', result.json())
-    if result.status_code != 200 or len(result.json()) == 0:
-        log.failure("Could not find libc for %s on libc.rip", params)
-        return []
+offline_parser.add_argument(
+    'categories',
+    metavar = 'categories',
+    nargs = '+',
+    help = 'Fetch the required libc category and symbol offsets. The parameter is the same as https://github.com/niklasb/libc-database/blob/master/get',
+)
+
+offline_parser.add_argument(
+    '--save-path',
+    type = str,
+    help = 'Set the save path for libc-database.',
+)
+
+common_symbols = ['dup2', 'printf', 'puts', 'read', 'system', 'write']
 
-    return result.json()
 
 def print_libc(libc):
     log.info('%s', text.red(libc['id']))
@@ -158,13 +168,13 @@ def print_libc(libc):
     for symbol in libc['symbols'].items():
         log.indented('\t%25s = %s', symbol[0], symbol[1])
 
-def handle_remote_libc(args, libc):
-    print_libc(libc)
+def fetch_libc(args, libc, hash_type="buildid"):
+    hash_value = libc[hash_type.replace("_", "")]
+
     if args.download_libc:
-        path = libcdb.search_by_build_id(libc['buildid'], args.unstrip)
+        path = libcdb.search_by_hash(hash_value, hash_type, not args.no_unstrip, args.offline)
+
         if path:
-            if args.unstrip:
-                libcdb.unstrip_libc(path)
             shutil.copy(path, './{}.so'.format(libc['id']))
 
 def translate_offset(offs, args, exe):
@@ -186,6 +196,14 @@ def collect_synthetic_symbols(exe):
 
     return available_symbols
 
+def dump_basic_symbols(exe, custom_symbols):
+    synthetic_symbols = collect_synthetic_symbols(exe)
+
+    symbols = common_symbols + (custom_symbols or []) + synthetic_symbols
+    symbols.sort()
+
+    return symbols
+
 def main(args):
     if len(sys.argv) < 3:
         parser.print_usage()
@@ -196,26 +214,50 @@ def main(args):
         if len(pairs) % 2 != 0:
             log.failure('Uneven number of arguments. Please provide "symbol offset" pairs')
             return
-        
+
         symbols = {pairs[i]:pairs[i+1] for i in range(0, len(pairs), 2)}
-        matched_libcs = find_libc({'symbols': symbols})
+        matched_libcs = libcdb.search_by_symbol_offsets(symbols, raw=True, unstrip=not args.no_unstrip, offline=args.offline)
         for libc in matched_libcs:
-            handle_remote_libc(args, libc)
+            print_libc(libc)
+            fetch_libc(args, libc)
 
     elif args.libc_command == 'hash':
         for hash_value in args.hash_value:
-            matched_libcs = find_libc({args.hash_type: hash_value})
-            for libc in matched_libcs:
-                handle_remote_libc(args, libc)
+            libs_id = None
+            hash_type = args.hash_type
+
+            # Search and download libc
+            exe_path = libcdb.search_by_hash(hash_value, hash_type, not args.no_unstrip, args.offline)
+
+            if exe_path:
+                libs_id = read(exe_path + ".id").decode()
+
+            if not libs_id:
+                log.warn_once("Empty libs id in %s %s", hash_type, hash_value)
+                continue
+
+            # Dump common symbols
+            exe = ELF(exe_path, checksec=False)
+            symbols = {}
+            for symbol in dump_basic_symbols(exe, []):
+                if symbol in exe.symbols:
+                    symbols[symbol] = exe.symbols[symbol]
+                else:
+                    symbols[symbol] = 0
+                    log.warn('%s is not found', symbol)
+
+            libc = libcdb._pack_libs_info(exe_path, libs_id, "", symbols)
+            print_libc(libc)
+            fetch_libc(args, libc, hash_type)
 
     elif args.libc_command == 'file':
         from hashlib import md5, sha1, sha256
         for file in args.files:
             if not os.path.exists(file) or not os.path.isfile(file):
                 log.failure('File does not exist %s', args.file)
                 continue
-            
-            if args.unstrip:
+
+            if not args.no_unstrip:
                 libcdb.unstrip_libc(file)
 
             exe = ELF(file, checksec=False)
@@ -233,15 +275,29 @@ def main(args):
 
             # Always dump the basic list of common symbols
             log.indented('%s', text.green('Symbols:'))
-            synthetic_symbols = collect_synthetic_symbols(exe)
+            symbols = dump_basic_symbols(exe, args.symbols)
 
-            symbols = common_symbols + (args.symbols or []) + synthetic_symbols
-            symbols.sort()
             for symbol in symbols:
                 if symbol not in exe.symbols:
                     log.indented('%25s = %s', symbol, text.red('not found'))
                 else:
                     log.indented('%25s = %#x', symbol, translate_offset(exe.symbols[symbol], args, exe))
 
+    elif args.libc_command == 'download':
+        if args.save_path:
+            save_path = os.path.abspath(args.save_path)
+        else:
+            save_path = context.local_libcdb
+
+        if not os.path.exists(save_path):
+            log.info("Download libc-database to %s", save_path)
+            process = subprocess.Popen(["git", "clone", "https://github.com/niklasb/libc-database", save_path])
+            process.wait()
+
+        log.info("Fetch libc categories and symbol offsets")
+        process = subprocess.Popen(["./get"] + args.categories, cwd=save_path)
+        process.wait()
+
+
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/context/__init__.py

@@ -360,6 +360,7 @@ class ContextType(object):
         'endian': 'little',
         'gdbinit': "",
         'kernel': None,
+        'local_libcdb': "/var/lib/libc-database",
         'log_level': logging.INFO,
         'log_file': _devnull(),
         'log_console': sys.stdout,
@@ -1070,6 +1071,10 @@ def log_console(self, stream):
             stream = open(stream, 'wt')
         return stream
 
+    @_validator
+    def local_libcdb(self, path):
+        return path
+
     @property
     def mask(self):
         return (1 << self.bits) - 1

pwnlib/elf/elf.py

@@ -459,6 +459,7 @@ def _describe(self, *a, **kw):
     def get_machine_arch(self):
         return {
             ('EM_X86_64', 64): 'amd64',
+            ('EM_X86_64', 32): 'amd64', # x32 ABI
             ('EM_386', 32): 'i386',
             ('EM_486', 32): 'i386',
             ('EM_ARM', 32): 'arm',