Skip to content

Commit

Permalink
Unbreak various things
Browse files Browse the repository at this point in the history 
 * A merge to reduce error spam during loading broke .so
   loading in at least some (maybe all?) cases, where find_library
   doesn't return an absolute path.
 * Prematurely pushed some in-progress test changes that were super broken, all fixed now.
  • Loading branch information
@ahupp
ahupp committed Feb 19, 2025
1 parent 36ecbf9 commit a1dd358
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 114 deletions.
16 changes: 10 additions & 6 deletions magic/loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

logger = logging.getLogger(__name__)


def _lib_candidates_linux():
"""Yield possible libmagic library names on Linux.
Expand Down Expand Up @@ -51,7 +52,7 @@ def _lib_candidates():
"darwin": _lib_candidates_macos,
"linux": _lib_candidates_linux,
"win32": _lib_candidates_windows,
"sunos5": _lib_candidates_linux,
"sunos5": _lib_candidates_linux,
}.get(sys.platform)
if func is None:
raise ImportError("python-magic: Unsupported platform: " + sys.platform)
Expand All @@ -61,17 +62,20 @@ def _lib_candidates():


def load_lib():
exc = []
for lib in _lib_candidates():
# find_library returns None when lib not found
if lib is None:
continue
if not os.path.exists(lib):
continue

try:
return ctypes.CDLL(lib)
except OSError:
logger.warning("Failed to load: " + lib, exc_info=True)
except OSError as e:
exc.append(e)

msg = "\n".join([str(e) for e in exc])

# It is better to raise an ImportError since we are importing magic module
raise ImportError("python-magic: failed to find libmagic. Check your installation")
raise ImportError(
"python-magic: failed to find libmagic. Check your installation: \n" + msg
)
238 changes: 130 additions & 108 deletions test/python_magic_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import shutil
import sys
import tempfile
from typing import List
import unittest

import pytest
Expand All @@ -19,140 +20,162 @@

import magic


@dataclass
class TestFile:
file_name: str
mime_results: list[str]
text_results: list[str]
no_check_elf_results: list[str] | None
mime_results: List[str]
text_results: List[str]
no_check_elf_results: List[str] | None
buf_equals_file: bool = True


# magic_descriptor is broken (?) in centos 7, so don't run those tests
SKIP_FROM_DESCRIPTOR = bool(os.environ.get("SKIP_FROM_DESCRIPTOR"))


COMMON_PLAIN = [
{},
{"check_soft": True},
{"check_soft": False},
{"check_json": True},
{"check_json": False},
]

NO_SOFT = {"check_soft": False}

COMMON_MIME = [{"mime": True, **k} for k in COMMON_PLAIN]
COMMON_PLAIN = [{}]
NO_SOFT = [{"check_soft": False}]
COMMON_MIME = [{"mime": True}]

CASES = {
"magic._pyc_": [
(COMMON_MIME, [
"application/octet-stream",
"text/x-bytecode.python",
"application/x-bytecode.python",
]),
b"magic._pyc_": [
(
COMMON_MIME,
[
"application/octet-stream",
"text/x-bytecode.python",
"application/x-bytecode.python",
],
),
(COMMON_PLAIN, ["python 2.4 byte-compiled"]),
(NO_SOFT, ["data"]),
],
"test.pdf": [
b"test.pdf": [
(COMMON_MIME, ["application/pdf"]),
(COMMON_PLAIN, [
"PDF document, version 1.2",
"PDF document, version 1.2, 2 pages",
"PDF document, version 1.2, 2 page(s)",
]),
(
COMMON_PLAIN,
[
"PDF document, version 1.2",
"PDF document, version 1.2, 2 pages",
"PDF document, version 1.2, 2 page(s)",
],
),
(NO_SOFT, ["ASCII text"]),
],
"test.gz": [
b"test.gz": [
(COMMON_MIME, ["application/gzip", "application/x-gzip"]),
(COMMON_PLAIN, [
'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated',
]),
({"extension": True}, [
# some versions return '' for the extensions of a gz file,
# including w/ the command line. Who knows...
"gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz",
"gz/tgz/tpz/zabw/svgz",
"",
"???",
]),
(
COMMON_PLAIN,
[
'gzip compressed data, was "test", from Unix, last modified: Sun Jun 29 01:32:52 2008',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size 15',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, original size modulo 2^32 15',
'gzip compressed data, was "test", last modified: Sun Jun 29 01:32:52 2008, from Unix, truncated',
],
),
(
[{"extension": True}],
[
# some versions return '' for the extensions of a gz file,
# including w/ the command line. Who knows...
"gz/tgz/tpz/zabw/svgz/adz/kmy/xcfgz",
"gz/tgz/tpz/zabw/svgz",
"",
"???",
],
),
(NO_SOFT, ["data"]),
],
"test.snappy.parquet": [
b"test.snappy.parquet": [
(COMMON_MIME, ["application/octet-stream"]),
(COMMON_PLAIN, ["Apache Parquet", "Par archive data"]),
(NO_SOFT, ["data"]),
],
"test.json": [
# TODO: soft, no_json
b"test.json": [
(COMMON_MIME, ["application/json"]),
(COMMON_PLAIN, ["JSON text data"]),
({"mime": True, "check_json": False}, [
"data",
]),
(NO_SOFT, ["JSON text data"])
(
[{"mime": True, "check_json": False}],
[
"text/plain",
],
),
(NO_SOFT, ["JSON text data"]),
],
"elf-NetBSD-x86_64-echo": [
b"elf-NetBSD-x86_64-echo": [
# TODO: soft, no elf
(COMMON_PLAIN, [
"ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)",
"ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped",
]),
(COMMON_MIME, [
"application/x-pie-executable",
"application/x-sharedlib",
]),
({"check_elf": False}, [
"ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)",
]),
(
COMMON_PLAIN,
[
"ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)",
"ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped",
],
),
(
COMMON_MIME,
[
"application/x-pie-executable",
"application/x-sharedlib",
],
),
(
[{"check_elf": False}],
[
"ELF 64-bit LSB shared object, x86-64, version 1 (SYSV)",
],
),
# TODO: sometimes
# "ELF 64-bit LSB pie executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /libexec/ld.elf_so, for NetBSD 8.0, not stripped",

(NO_SOFT, ["data"]),
],
"test.txt": [
b"text.txt": [
(COMMON_MIME, ["text/plain"]),
(COMMON_PLAIN, ["ASCII text"]),
({"mime_encoding": True}, [
"us-ascii",
]),
(
[{"mime_encoding": True}],
[
"us-ascii",
],
),
(NO_SOFT, ["ASCII text"]),
],
"text-iso8859-1.txt": [
({"mime_encoding": True}, [
"iso-8859-1",
]),
b"text-iso8859-1.txt": [
(
[{"mime_encoding": True}],
[
"iso-8859-1",
],
),
],
b"\xce\xbb": [
(COMMON_MIME, ["text/plain"]),
],
"b\xce\xbb".decode("utf-8"): [
(COMMON_MIME, ["text/plain"]),
b"name_use.jpg": [
([{"extension": True}], ["jpeg/jpg/jpe/jfif"]),
],
"name_use.jpg": [
({"extension": True}, [
"jpeg/jpg/jpe/jfif"
]),
b"keep-going.jpg": [
(COMMON_MIME, ["image/jpeg"]),
(
[{"mime": True, "keep_going": True}],
[
"image/jpeg\\012- application/octet-stream",
],
),
],
"keep-going.jpg": [
(COMMON_MIME, [
"image/jpeg"
]),
({"mime": True, "keep_going": True}, [
"image/jpeg\\012- application/octet-stream",
])
b"../../magic/loader.py": [
(
COMMON_MIME,
[
"text/x-python",
"text/x-script.python",
],
)
],
"test.py": [
(COMMON_MIME, [
"text/x-python",
"text/x-script.python",
])
]
}


class MagicTest(unittest.TestCase):
TESTDATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "testdata"))

Expand All @@ -165,7 +188,6 @@ def test_version(self):
def test_fs_encoding(self):
self.assertEqual("utf-8", sys.getfilesystemencoding().lower())


def test_from_file_str_and_bytes(self):
filename = os.path.join(self.TESTDATA_DIR, "test.pdf")

Expand All @@ -174,7 +196,6 @@ def test_from_file_str_and_bytes(self):
"application/pdf", magic.from_file(filename.encode("utf-8"), mime=True)
)


def test_all_cases(self):
# TODO:
# * MAGIC_EXTENSION not supported
Expand All @@ -184,21 +205,24 @@ def test_all_cases(self):
shutil.copyfile(os.path.join(MagicTest.TESTDATA_DIR, "lambda"), dest)
os.environ["TZ"] = "UTC"
try:
for file_name, cases in CASES:
filename = os.path.join(self.TESTDATA_DIR, file_name)
for flags, outputs in cases:
m = magic.Magic(**flags)
with open(filename) as f:
self.assertIn(m.from_descriptor(f.fileno()), outputs)

self.assertIn(m.from_file(filename), outputs)

fname_bytes = filename.encode("utf-8")
self.assertIn(m.from_file(fname_bytes), outputs)

with open(file_name, "rb") as f:
buf_result = m.from_buffer(f.read(1024))
self.assertIn(buf_result, outputs)
for filename, cases in CASES.items():
filename = os.path.join(self.TESTDATA_DIR.encode("utf-8"), filename)
print("test case ", filename, file=sys.stderr)
for flag_variants, outputs in cases:
for flags in flag_variants:
print("flags", flags, file=sys.stderr)
m = magic.Magic(**flags)
with open(filename) as f:
self.assertIn(m.from_descriptor(f.fileno()), outputs)

self.assertIn(m.from_file(filename), outputs)

fname_str = filename.decode("utf-8")
self.assertIn(m.from_file(fname_str), outputs)

with open(filename, "rb") as f:
buf_result = m.from_buffer(f.read(1024))
self.assertIn(buf_result, outputs)
finally:
del os.environ["TZ"]
os.unlink(dest)
Expand All @@ -222,7 +246,6 @@ def test_unicode_result_raw(self):
else:
raise unittest.SkipTest("Magic file doesn't return expected type.")


def test_errors(self):
m = magic.Magic()
self.assertRaises(IOError, m.from_file, "nonexistent")
Expand All @@ -233,7 +256,6 @@ def test_errors(self):
finally:
del os.environ["MAGIC"]


def test_rethrow(self):
old = magic.magic_buffer
try:
Expand Down

0 comments on commit a1dd358

Please sign in to comment.