From 0957bb914e3e3a314caf340d09b31d66083947e2 Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Fri, 23 Feb 2024 14:42:28 +0800 Subject: [PATCH 01/10] Add local libc database provider for libcdb --- pwnlib/args.py | 5 +++++ pwnlib/context/__init__.py | 5 +++++ pwnlib/libcdb.py | 20 +++++++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pwnlib/args.py b/pwnlib/args.py index 6af4a34cd..0801eb444 100644 --- a/pwnlib/args.py +++ b/pwnlib/args.py @@ -159,6 +159,10 @@ def STDERR(v): """Sends logging to ``stderr`` by default, instead of ``stdout``""" context.log_console = sys.stderr +def LOCAL_LIBCDB(v): + """Sets local libcdb path""" + context.defaults['local_libcdb'] = v + hooks = { 'LOG_LEVEL': LOG_LEVEL, 'LOG_FILE': LOG_FILE, @@ -170,6 +174,7 @@ def STDERR(v): 'NOASLR': NOASLR, 'NOPTRACE': NOPTRACE, 'STDERR': STDERR, + 'LOCAL_LIBCDB': LOCAL_LIBCDB, } def initialize(): diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index 6b3f636b7..549241b3a 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -360,6 +360,7 @@ class ContextType(object): 'endian': 'little', 'gdbinit': "", 'kernel': None, + 'local_libcdb': "/var/lib/libc-database", 'log_level': logging.INFO, 'log_file': _devnull(), 'log_console': sys.stdout, @@ -1071,6 +1072,10 @@ def log_console(self, stream): stream = open(stream, 'wt') return stream + @_validator + def local_libcdb(self, path): + return path + @property def mask(self): return (1 << self.bits) - 1 diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index 885a0eea2..50dcfd40f 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -11,6 +11,7 @@ from pwnlib.context import context from pwnlib.elf import ELF +from pwnlib.filesystem.path import Path from pwnlib.log import getLogger from pwnlib.tubes.process import process from pwnlib.util.fiddling import enhex @@ -126,7 +127,24 @@ def provider_local_system(hex_encoded_id, hash_type): return local_libc.data return None -PROVIDERS = [provider_local_system, provider_libcdb, provider_libc_rip] +# Offline search https://github.com/niklasb/libc-database for hash type +def provider_local_database(hex_encoded_id, hash_type): + assert context.local_libcdb + assert hash_type in HASHES + + localdb = Path(context.local_libcdb) + if not localdb.is_dir(): + log.warn_once("%s does not exist, please download libc-database first.", str(localdb)) + return None + + log.debug("Searching local libc database, %s: %s", hash_type, hex_encoded_id) + for libc_path in localdb.rglob("*.so"): + if hex_encoded_id == HASHES[hash_type](libc_path): + return read(libc_path) + + return None + +PROVIDERS = [provider_local_database, provider_local_system, provider_libcdb, provider_libc_rip] def search_by_hash(hex_encoded_id, hash_type='build_id', unstrip=True): assert hash_type in HASHES, hash_type From ad6d0c908bbe65491498c58fb535060848360582 Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Fri, 23 Feb 2024 16:51:21 +0800 Subject: [PATCH 02/10] Remove unnecessary assert --- pwnlib/libcdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index 50dcfd40f..e012a9e51 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -130,7 +130,6 @@ def provider_local_system(hex_encoded_id, hash_type): # Offline search https://github.com/niklasb/libc-database for hash type def provider_local_database(hex_encoded_id, hash_type): assert context.local_libcdb - assert hash_type in HASHES localdb = Path(context.local_libcdb) if not localdb.is_dir(): From f3325d99190b7e662c3bfaafe4d5a163b411a10b Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Sat, 24 Feb 2024 13:52:00 +0800 Subject: [PATCH 03/10] Add docstring for local_libcdb --- pwnlib/args.py | 3 ++- pwnlib/context/__init__.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pwnlib/args.py b/pwnlib/args.py index 0801eb444..411f7a158 100644 --- a/pwnlib/args.py +++ b/pwnlib/args.py @@ -160,7 +160,8 @@ def STDERR(v): context.log_console = sys.stderr def LOCAL_LIBCDB(v): - """Sets local libcdb path""" + """Sets path to local libc-databse via ``context.local_libcdb``, e.g. + ``LOCAL_LIBCDB='/path/to/libc-databse'``""" context.defaults['local_libcdb'] = v hooks = { diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index 549241b3a..86c162b58 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -1074,6 +1074,21 @@ def log_console(self, stream): @_validator def local_libcdb(self, path): + """ + Sets path to local libc-database, get more information for libc-database: + https://github.com/niklasb/libc-database + + Works in offline search mode of :attr:`pwnlib.libcdb`. + + The default value is ``/var/lib/libc-database``. + + When `context.local_libcdb` is default value, no warning will be issued for path check + during `pwnlib.libcdb` searching by local database provider. + + Examples: + + >>> context.local_libcdb = '/path/to/libc-database' + """ return path @property From c8b629da98715ffa71f3ec049c0df20be30cdfae Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Sat, 24 Feb 2024 13:53:54 +0800 Subject: [PATCH 04/10] Suppress warning output while `context.libdb` sets default --- pwnlib/libcdb.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index e012a9e51..5cee5009b 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -40,6 +40,10 @@ # Retry failed lookups after some time NEGATIVE_CACHE_EXPIRY = 60 * 60 * 24 * 7 # 1 week +# The default value of `context.local_libcdb`, when user doesn't modify `context.local_libcdb` +# suppress the warning output for path check in local database provider. +DEFAULT_LOCAL_LIBCDB = '/var/lib/libc-database' + # https://gitlab.com/libcdb/libcdb wasn't updated after 2019, # but still is a massive database of older libc binaries. def provider_libcdb(hex_encoded_id, hash_type): @@ -129,11 +133,13 @@ def provider_local_system(hex_encoded_id, hash_type): # Offline search https://github.com/niklasb/libc-database for hash type def provider_local_database(hex_encoded_id, hash_type): - assert context.local_libcdb + if not context.local_libcdb: + log.error("`context.local_libcdb` could not be null or empty string.") localdb = Path(context.local_libcdb) if not localdb.is_dir(): - log.warn_once("%s does not exist, please download libc-database first.", str(localdb)) + if context.local_libcdb != DEFAULT_LOCAL_LIBCDB: + log.warn_once("%s does not exist, please download libc-database first.", context.local_libcdb) return None log.debug("Searching local libc database, %s: %s", hash_type, hex_encoded_id) From de063a73fb13a5b0fd6010ddcfd560c2920f8bc6 Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Sat, 24 Feb 2024 14:04:08 +0800 Subject: [PATCH 05/10] Testing the local system's libc first --- pwnlib/libcdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index 5cee5009b..619783383 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -149,7 +149,7 @@ def provider_local_database(hex_encoded_id, hash_type): return None -PROVIDERS = [provider_local_database, provider_local_system, provider_libcdb, provider_libc_rip] +PROVIDERS = [provider_local_system, provider_local_database, provider_libcdb, provider_libc_rip] def search_by_hash(hex_encoded_id, hash_type='build_id', unstrip=True): assert hash_type in HASHES, hash_type From 2f8395fc3a9c102106dbd5d655d593c071578c33 Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Sun, 25 Feb 2024 13:41:20 +0800 Subject: [PATCH 06/10] Set falsely `context.lcoal_libcdb` to turn off local libc-database integration --- pwnlib/libcdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index 619783383..5ccad223f 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -134,7 +134,7 @@ def provider_local_system(hex_encoded_id, hash_type): # Offline search https://github.com/niklasb/libc-database for hash type def provider_local_database(hex_encoded_id, hash_type): if not context.local_libcdb: - log.error("`context.local_libcdb` could not be null or empty string.") + return None localdb = Path(context.local_libcdb) if not localdb.is_dir(): From 33e728be75903a94ffcbf14b8aae2da581d91256 Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Sun, 25 Feb 2024 13:52:24 +0800 Subject: [PATCH 07/10] Fix docstring --- pwnlib/args.py | 2 +- pwnlib/context/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pwnlib/args.py b/pwnlib/args.py index 411f7a158..55c9beef1 100644 --- a/pwnlib/args.py +++ b/pwnlib/args.py @@ -160,7 +160,7 @@ def STDERR(v): context.log_console = sys.stderr def LOCAL_LIBCDB(v): - """Sets path to local libc-databse via ``context.local_libcdb``, e.g. + """Sets path to local libc-database via ``context.local_libcdb``, e.g. ``LOCAL_LIBCDB='/path/to/libc-databse'``""" context.defaults['local_libcdb'] = v diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index 86c162b58..1b781a777 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -1078,10 +1078,12 @@ def local_libcdb(self, path): Sets path to local libc-database, get more information for libc-database: https://github.com/niklasb/libc-database - Works in offline search mode of :attr:`pwnlib.libcdb`. + Works in :attr:`pwnlib.libcdb` when searching by local database provider. The default value is ``/var/lib/libc-database``. + Sets `context.local_libcdb` to falsely path will turn off local libc-database integration. + When `context.local_libcdb` is default value, no warning will be issued for path check during `pwnlib.libcdb` searching by local database provider. From 217632d140c7be88b67e9d3019378f560b84c51c Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Mon, 26 Feb 2024 14:00:40 +0800 Subject: [PATCH 08/10] Make path check in validator --- pwnlib/args.py | 2 +- pwnlib/context/__init__.py | 13 +++++++++---- pwnlib/libcdb.py | 6 ------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pwnlib/args.py b/pwnlib/args.py index 55c9beef1..f7985e7d6 100644 --- a/pwnlib/args.py +++ b/pwnlib/args.py @@ -162,7 +162,7 @@ def STDERR(v): def LOCAL_LIBCDB(v): """Sets path to local libc-database via ``context.local_libcdb``, e.g. ``LOCAL_LIBCDB='/path/to/libc-databse'``""" - context.defaults['local_libcdb'] = v + context.local_libcdb = v hooks = { 'LOG_LEVEL': LOG_LEVEL, diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index 1b781a777..7b24524b5 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -1082,15 +1082,20 @@ def local_libcdb(self, path): The default value is ``/var/lib/libc-database``. - Sets `context.local_libcdb` to falsely path will turn off local libc-database integration. - - When `context.local_libcdb` is default value, no warning will be issued for path check - during `pwnlib.libcdb` searching by local database provider. + Sets `context.local_libcdb` to empty string or `None` will turn off local libc-database integration. Examples: >>> context.local_libcdb = '/path/to/libc-database' + >>> context.local_libcdb = 'foobar' + Traceback (most recent call last): + ... + AttributeError: 'foobar' does not exist, please download libc-database first """ + + if not os.path.isdir(path): + raise AttributeError("'%s' does not exist, please download libc-database first" % path) + return path @property diff --git a/pwnlib/libcdb.py b/pwnlib/libcdb.py index 5ccad223f..329ed187f 100644 --- a/pwnlib/libcdb.py +++ b/pwnlib/libcdb.py @@ -40,10 +40,6 @@ # Retry failed lookups after some time NEGATIVE_CACHE_EXPIRY = 60 * 60 * 24 * 7 # 1 week -# The default value of `context.local_libcdb`, when user doesn't modify `context.local_libcdb` -# suppress the warning output for path check in local database provider. -DEFAULT_LOCAL_LIBCDB = '/var/lib/libc-database' - # https://gitlab.com/libcdb/libcdb wasn't updated after 2019, # but still is a massive database of older libc binaries. def provider_libcdb(hex_encoded_id, hash_type): @@ -138,8 +134,6 @@ def provider_local_database(hex_encoded_id, hash_type): localdb = Path(context.local_libcdb) if not localdb.is_dir(): - if context.local_libcdb != DEFAULT_LOCAL_LIBCDB: - log.warn_once("%s does not exist, please download libc-database first.", context.local_libcdb) return None log.debug("Searching local libc database, %s: %s", hash_type, hex_encoded_id) From ab0467e37ba44bbc0e782653bced5733495e72c2 Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Thu, 29 Feb 2024 20:24:27 +0800 Subject: [PATCH 09/10] Fix doctests --- pwnlib/context/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pwnlib/context/__init__.py b/pwnlib/context/__init__.py index 7b24524b5..9d6ebe71d 100644 --- a/pwnlib/context/__init__.py +++ b/pwnlib/context/__init__.py @@ -1086,7 +1086,7 @@ def local_libcdb(self, path): Examples: - >>> context.local_libcdb = '/path/to/libc-database' + >>> context.local_libcdb = pwnlib.data.elf.path >>> context.local_libcdb = 'foobar' Traceback (most recent call last): ... From f6c14b7b7094fd96dccb0a112a981cc9c7030a24 Mon Sep 17 00:00:00 2001 From: Th3S <46804083+the-soloist@users.noreply.github.com> Date: Thu, 29 Feb 2024 20:24:31 +0800 Subject: [PATCH 10/10] Add CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5187d79da..e253c3cf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,7 +71,9 @@ The table below shows which release corresponds to each branch, and what date th ## 4.14.0 (`dev`) +- [#2356][2356] Add local libc database provider for libcdb +[2356]: https://github.com/Gallopsled/pwntools/pull/2356 ## 4.13.0 (`beta`)