Skip to content

NTFS alternate stream + sparse file; Minor tidy-up. #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
axelriet wants to merge 5 commits into
base: mdb.master
Choose a base branch
from
+49 −19
Open

NTFS alternate stream + sparse file; Minor tidy-up. #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ce912dc
Minor _WIN32 tidy-up.
axelriet Jan 1, 2026
62f84fb
MDB_NTFSSTREAM: supports embedding the lock file as an NTFS alternate…
axelriet Jan 1, 2026
2317128
Fix bogous trace format string.
axelriet Jan 1, 2026
fc73f59
abort() => __fastfail() when MDB_BUGCHECK is defined to 1
axelriet Jan 1, 2026
90d0ab0
MDB_NTFSSPARSE: enable NTFS sparse data file.
axelriet Jan 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 16 additions & 12 deletions libraries/liblmdb/lmdb.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,29 +311,33 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
* @{
*/
/** mmap at a fixed address (experimental) */
#define MDB_FIXEDMAP 0x01
#define MDB_FIXEDMAP 0x00000001
/** no environment directory */
#define MDB_NOSUBDIR 0x4000
#define MDB_NOSUBDIR 0x00004000
/** don't fsync after commit */
#define MDB_NOSYNC 0x10000
#define MDB_NOSYNC 0x00010000
/** read only */
#define MDB_RDONLY 0x20000
#define MDB_RDONLY 0x00020000
/** don't fsync metapage after commit */
#define MDB_NOMETASYNC 0x40000
#define MDB_NOMETASYNC 0x00040000
/** use writable mmap */
#define MDB_WRITEMAP 0x80000
#define MDB_WRITEMAP 0x00080000
/** use asynchronous msync when #MDB_WRITEMAP is used */
#define MDB_MAPASYNC 0x100000
#define MDB_MAPASYNC 0x00100000
/** tie reader locktable slots to #MDB_txn objects instead of to threads */
#define MDB_NOTLS 0x200000
#define MDB_NOTLS 0x00200000
/** don't do any locking, caller must manage their own locks */
#define MDB_NOLOCK 0x400000
#define MDB_NOLOCK 0x00400000
/** don't do readahead (no effect on Windows) */
#define MDB_NORDAHEAD 0x800000
#define MDB_NORDAHEAD 0x00800000
/** don't initialize malloc'd memory before writing to datafile */
#define MDB_NOMEMINIT 0x1000000
#define MDB_NOMEMINIT 0x01000000
/** use the previous snapshot rather than the latest one */
#define MDB_PREVSNAPSHOT 0x2000000
#define MDB_PREVSNAPSHOT 0x02000000
/** store the lock file as an NTFS stream within the main data file */
#define MDB_NTFSSTREAM 0x04000000
/** Use a sparse file for the main data file */
#define MDB_NTFSSPARSE 0x08000000
/** @} */

/** @defgroup mdb_dbi_open Database Flags
Expand Down
40 changes: 33 additions & 7 deletions libraries/liblmdb/mdb.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,9 @@ union semun {
#ifdef _WIN32
#define MDB_USE_HASH 1
#define MDB_PIDLOCK 0
#ifndef MDB_BUGCHECK
#define MDB_BUGCHECK 0
#endif
#define THREAD_RET DWORD
#define pthread_t HANDLE
#define pthread_mutex_t HANDLE
Expand Down Expand Up @@ -1824,7 +1827,11 @@ mdb_assert_fail(MDB_env *env, const char *expr_txt,
if (env->me_assert_func)
env->me_assert_func(env, buf);
fprintf(stderr, "%s\n", buf);
#if (MDB_BUGCHECK) > 0
__fastfail(FAST_FAIL_FATAL_APP_EXIT);
#else
abort();
#endif
}
#else
# define mdb_assert0(env, expr, expr_txt) ((void) 0)
Expand Down Expand Up @@ -3831,7 +3838,9 @@ mdb_page_flush(MDB_txn *txn, int keep)
#endif
) {
if (n) {
#ifndef _WIN32
retry_write:
#endif
/* Write previous page(s) */
DPRINTF(("committing page %"Z"u", pgno));
#ifdef _WIN32
Expand Down Expand Up @@ -4455,13 +4464,17 @@ mdb_env_write_meta(MDB_txn *txn)
r2 = pwrite(env->me_fd, ptr, len, off);
(void)r2; /* Silence warnings. We don't care about pwrite's return value */
#endif
#ifndef _WIN32
fail:
#endif
env->me_flags |= MDB_FATAL_ERROR;
return rc;
}
/* MIPS has cache coherency issues, this is a no-op everywhere else */
CACHEFLUSH(env->me_map + off, len, DCACHE);
#ifndef _WIN32
done:
#endif
/* Memory ordering issues are irrelevant; since the entire writer
* is wrapped by wmutex, all of these changes will become visible
* after the wmutex is unlocked. Since the DB is multi-version,
Expand Down Expand Up @@ -4677,7 +4690,7 @@ mdb_env_set_mapsize(MDB_env *env, mdb_size_t size)
env->me_mapsize = size;
if (env->me_psize)
env->me_maxpg = env->me_mapsize / env->me_psize;
MDB_TRACE(("%p, %"Yu"", env, size));
MDB_TRACE(("%p, %"Z"u", env, size));
return MDB_SUCCESS;
}

Expand Down Expand Up @@ -4751,12 +4764,13 @@ typedef struct MDB_name {
} MDB_name;

/** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */
static const mdb_nchar_t *const mdb_suffixes[2][2] = {
{ MDB_NAME("/data.mdb"), MDB_NAME("") },
{ MDB_NAME("/lock.mdb"), MDB_NAME("-lock") }
static const mdb_nchar_t *const mdb_suffixes[3][2] = {
{ MDB_NAME("/data.mdb"), MDB_NAME("") },
{ MDB_NAME("/lock.mdb"), MDB_NAME("-lock") },
{ MDB_NAME("/data.mdb:lock"), MDB_NAME(":lock") }
};

#define MDB_SUFFLEN 9 /**< Max string length in #mdb_suffixes[] */
#define MDB_SUFFLEN 14 /**< Max string length in #mdb_suffixes[] */

/** Set up filename + scratch area for filename suffix, for opening files.
* It should be freed with #mdb_fname_destroy().
Expand Down Expand Up @@ -4839,7 +4853,7 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,

if (fname->mn_alloced) /* modifiable copy */
mdb_name_cpy(fname->mn_val + fname->mn_len,
mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]);
mdb_suffixes[(which==MDB_O_LOCKS)?(1+F_ISSET(env->me_flags, MDB_NTFSSTREAM)):0][F_ISSET(env->me_flags, MDB_NOSUBDIR)]);

/* The directory must already exist. Usually the file need not.
* MDB_O_META requires the file because we already created it using
Expand Down Expand Up @@ -4912,6 +4926,18 @@ mdb_fopen(const MDB_env *env, MDB_name *fname,
# endif
}
}
#else
else if ((which == MDB_O_RDWR) && (F_ISSET(env->me_flags, MDB_NTFSSPARSE)))
{
DWORD BytesReturned;

if (!DeviceIoControl(fd, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &BytesReturned, NULL))
{
rc = ErrCode();
CloseHandle(fd);
fd = INVALID_HANDLE_VALUE;
}
}
#endif /* !_WIN32 */

*res = fd;
Expand Down Expand Up @@ -5608,7 +5634,7 @@ mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl)
*/
#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \
MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT)
MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT|MDB_NTFSSTREAM|MDB_NTFSSPARSE)

#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS)
# error "Persistent DB flags & env flags overlap, but both go in mm_flags"
Expand Down