diff --git a/configure.ac b/configure.ac index c2a902ba6..bd077c043 100644 --- a/configure.ac +++ b/configure.ac @@ -1,1297 +1,1305 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) define(_CLIENT_VERSION_MINOR, 15) define(_CLIENT_VERSION_REVISION, 1) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_IS_RELEASE, true) define(_COPYRIGHT_YEAR, 2017) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin]]) AC_INIT([Bitcoin ABC],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_REVISION],[https://github.com/Bitcoin-ABC/bitcoin-abc/issues],[bitcoin-abc],[https://bitcoinabc.org/]) AC_CONFIG_SRCDIR([src/validation.cpp]) AC_CONFIG_HEADERS([src/config/bitcoin-config.h]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) BITCOIN_DAEMON_NAME=bitcoind BITCOIN_GUI_NAME=bitcoin-qt BITCOIN_CLI_NAME=bitcoin-cli BITCOIN_TX_NAME=bitcoin-tx BITCOIN_SEEDER_NAME=bitcoin-seeder AC_CANONICAL_HOST AH_TOP([#ifndef BITCOIN_BITCOIN_CONFIG_H]) AH_TOP([#define BITCOIN_BITCOIN_CONFIG_H]) AH_BOTTOM([#endif // BITCOIN_BITCOIN_CONFIG_H]) dnl faketime breaks configure and is only needed for make. Disable it here. unset FAKETIME dnl Automake init set-up and checks AM_INIT_AUTOMAKE([no-define subdir-objects foreign]) dnl faketime messes with timestamps and causes configure to be re-run. dnl --disable-maintainer-mode can be used to bypass this. AM_MAINTAINER_MODE([enable]) dnl make the compilation flags quiet unless V=1 is used m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl Compiler checks (here before libtool). if test "x${CXXFLAGS+set}" = "xset"; then CXXFLAGS_overridden=yes else CXXFLAGS_overridden=no fi AC_PROG_CXX dnl By default, libtool for mingw refuses to link static libs into a dll for dnl fear of mixing pic/non-pic objects, and import/export complications. Since dnl we have those under control, re-enable that functionality. case $host in *mingw*) lt_cv_deplibs_check_method="pass_all" ;; esac dnl Require C++11 compiler (no GNU extensions) AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) dnl Check if -latomic is required for CHECK_ATOMIC dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. m4_ifdef([AC_PROG_OBJCXX],[ if test "x${OBJCXX+set}" = "x"; then OBJCXX="${CXX}" fi AC_PROG_OBJCXX ]) dnl Libtool init checks. LT_INIT([pic-only]) dnl Check/return PATH for base programs. AC_PATH_TOOL(AR, ar) AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) dnl Python 3.x is supported from 3.4 on (see https://github.com/bitcoin/bitcoin/issues/7893) AC_PATH_PROGS([PYTHON], [python3.6 python3.5 python3.4 python3 python2.7 python2 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(HEXDUMP,hexdump) AC_PATH_TOOL(READELF, readelf) AC_PATH_TOOL(CPPFILT, c++filt) AC_PATH_TOOL(OBJCOPY, objcopy) AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) # Enable wallet AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], [disable wallet (enabled by default)])], [enable_wallet=$enableval], [enable_wallet=yes]) AC_ARG_WITH([miniupnpc], [AS_HELP_STRING([--with-miniupnpc], [enable UPNP (default is yes if libminiupnpc is found)])], [use_upnp=$withval], [use_upnp=auto]) AC_ARG_ENABLE([upnp-default], [AS_HELP_STRING([--enable-upnp-default], [if UPNP is enabled, turn it on at startup (default is no)])], [use_upnp_default=$enableval], [use_upnp_default=no]) AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[do not compile tests (default is to compile)]), [use_tests=$enableval], [use_tests=yes]) AC_ARG_ENABLE(gui-tests, AS_HELP_STRING([--disable-gui-tests],[do not compile GUI tests (default is to compile if GUI and tests enabled)]), [use_gui_tests=$enableval], [use_gui_tests=$use_tests]) AC_ARG_ENABLE(bench, AS_HELP_STRING([--disable-bench],[do not compile benchmarks (default is to compile)]), [use_bench=$enableval], [use_bench=yes]) AC_ARG_ENABLE([extended-rpc-tests], AS_HELP_STRING([--enable-extended-rpc-tests],[enable expensive RPC tests when using lcov (default no)]), [use_extended_rpc_tests=$enableval], [use_extended_rpc_tests=no]) AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], [enable QR code support (default is yes if qt is enabled and libqrencode is found)])], [use_qr=$withval], [use_qr=auto]) AC_ARG_ENABLE([hardening], [AS_HELP_STRING([--disable-hardening], [do not attempt to harden the resulting executables (default is to harden)])], [use_hardening=$enableval], [use_hardening=yes]) AC_ARG_ENABLE([reduce-exports], [AS_HELP_STRING([--enable-reduce-exports], [attempt to reduce exported symbols in the resulting executables (default is no)])], [use_reduce_exports=$enableval], [use_reduce_exports=no]) AC_ARG_ENABLE([ccache], [AS_HELP_STRING([--disable-ccache], [do not use ccache for building (default is to use if found)])], [use_ccache=$enableval], [use_ccache=auto]) AC_ARG_ENABLE([lcov], [AS_HELP_STRING([--enable-lcov], [enable lcov testing (default is no)])], [use_lcov=yes], [use_lcov=no]) AC_ARG_ENABLE([glibc-back-compat], [AS_HELP_STRING([--enable-glibc-back-compat], [enable backwards compatibility with glibc])], [use_glibc_compat=$enableval], [use_glibc_compat=no]) AC_ARG_WITH([system-univalue], [AS_HELP_STRING([--with-system-univalue], [Build with system UniValue (default is no)])], [system_univalue=$withval], [system_univalue=no] ) AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], [use_zmq=$enableval], [use_zmq=yes]) AC_ARG_WITH([protoc-bindir],[AS_HELP_STRING([--with-protoc-bindir=BIN_DIR],[specify protoc bin path])], [protoc_bin_path=$withval], []) AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, enable_man=yes) AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) # Enable debug AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], [use debug compiler flags and macros (default is no)])], [enable_debug=$enableval], [enable_debug=no]) # Enable ASAN AC_ARG_ENABLE([asan], [AS_HELP_STRING([--enable-asan], [enable address sanitizer compiler flags (implies --enable-debug, default is no)])], [enable_asan=$enableval], [enable_asan=no]) # Enable TSAN AC_ARG_ENABLE([tsan], [AS_HELP_STRING([--enable-tsan], [enable thread sanitizer compiler flags; requires 64-bit target architecture (implies --enable-debug, default is no)])], [enable_tsan=$enableval], [enable_tsan=no]) # Enable UBSAN AC_ARG_ENABLE([ubsan], [AS_HELP_STRING([--enable-ubsan], [enable undefined behavior sanitizer compiler flags (implies --enable-debug, default is no)])], [enable_ubsan=$enableval], [enable_ubsan=no]) # Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], [Treat certain compiler warnings as errors (default is no)])], [enable_werror=$enableval], [enable_werror=no]) AC_LANG_PUSH([C++]) AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) if test "x$enable_asan" = xyes; then case $host in *mingw*) AC_MSG_ERROR("Windows builds cannot use ASAN") ;; esac enable_debug=yes CFLAGS="$CFLAGS -fsanitize=address -fno-omit-frame-pointer" CXXFLAGS="$CXXFLAGS -fsanitize=address -fno-omit-frame-pointer" LDFLAGS="$LDFLAGS -fsanitize=address" fi if test "x$enable_tsan" = xyes; then case $host in *mingw*) AC_MSG_ERROR("Windows builds cannot use TSAN") ;; esac enable_debug=yes CFLAGS="$CFLAGS -fsanitize=thread -fno-omit-frame-pointer" CXXFLAGS="$CXXFLAGS -fsanitize=thread -fno-omit-frame-pointer" LDFLAGS="$LDFLAGS -fsanitize=thread" fi if test "x$enable_ubsan" = xyes; then case $host in *mingw*) AC_MSG_ERROR("Windows builds cannot use UBSAN") ;; esac enable_debug=yes CFLAGS="$CFLAGS -fsanitize=undefined -fno-omit-frame-pointer" CXXFLAGS="$CXXFLAGS -fsanitize=undefined -fno-omit-frame-pointer" LDFLAGS="$LDFLAGS -fsanitize=undefined" fi if test "x$enable_debug" = xyes; then CPPFLAGS="$CPPFLAGS -DDEBUG -DDEBUG_LOCKORDER" if test "x$GCC" = xyes; then CFLAGS="$CFLAGS -g3 -O0" fi if test "x$GXX" = xyes; then CXXFLAGS="$CXXFLAGS -g3 -O0" fi fi ERROR_CXXFLAGS= if test "x$enable_werror" = "xyes"; then if test "x$CXXFLAG_WERROR" = "x"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wall],[CXXFLAGS="$CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wextra],[CXXFLAGS="$CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat],[CXXFLAGS="$CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[CXXFLAGS="$CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat-security],[CXXFLAGS="$CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all ## unknown options if any other warning is produced. Test the -Wfoo case, and ## set the -Wno-foo case if it works. AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[CXXFLAGS="$CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wself-assign],[CXXFLAGS="$CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[CXXFLAGS="$CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[CXXFLAGS="$CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) + + # Check for optional instruction set support. Enabling these does _not_ imply that all code will + # be compiled with them, rather that specific objects/libs may use them after checking for runtime + # compatibility. + AX_CHECK_COMPILE_FLAG([-msse4.2],[[enable_sse42=yes; SSE42_CXXFLAGS="-msse4.2"]],,[[$CXXFLAG_WERROR]]) + fi CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], [build bitcoin-cli bitcoin-tx (default=yes)])], [build_bitcoin_utils=$withval], [build_bitcoin_utils=yes]) AC_ARG_WITH([libs], [AS_HELP_STRING([--with-libs], [build libraries (default=yes)])], [build_bitcoin_libs=$withval], [build_bitcoin_libs=yes]) AC_ARG_WITH([daemon], [AS_HELP_STRING([--with-daemon], [build bitcoind daemon (default=yes)])], [build_bitcoind=$withval], [build_bitcoind=yes]) AC_ARG_WITH([seeder], [AS_HELP_STRING([--with-seeder], [build seeder (default=yes)])], [build_bitcoin_seeder=$withval], [build_bitcoin_seeder=yes]) use_pkgconfig=yes case $host in *mingw*) #pkgconfig does more harm than good with MinGW use_pkgconfig=no TARGET_OS=windows AC_CHECK_LIB([mingwthrd], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([kernel32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([winspool], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([shell32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([ole32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([rpcrt4], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([advapi32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([ws2_32], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([mswsock], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([shlwapi], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([iphlpapi], [main],, AC_MSG_ERROR(lib missing)) AC_CHECK_LIB([crypt32], [main],, AC_MSG_ERROR(lib missing)) # -static is interpreted by libtool, where it has a different meaning. # In libtool-speak, it's -all-static. AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) AC_PATH_PROG([MAKENSIS], [makensis], none) if test x$MAKENSIS = xnone; then AC_MSG_WARN("makensis not found. Cannot create installer.") fi AC_PATH_TOOL(WINDRES, windres, none) if test x$WINDRES = xnone; then AC_MSG_ERROR("windres not found") fi CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -DBOOST_THREAD_USE_LIB" LEVELDB_TARGET_FLAGS="-DOS_WINDOWS" if test "x$CXXFLAGS_overridden" = "xno"; then CXXFLAGS="$CXXFLAGS -w" fi case $host in i?86-*) WINDOWS_BITS=32 ;; x86_64-*) WINDOWS_BITS=64 ;; *) AC_MSG_ERROR("Could not determine win32/win64 for installer") ;; esac AC_SUBST(WINDOWS_BITS) dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override dnl its command here, with the predeps/postdeps removed, and -static inserted. Postdeps are dnl also overridden to prevent their insertion later. dnl This should only affect dll's. archive_cmds_CXX="\$CC -shared \$libobjs \$deplibs \$compiler_flags -static -o \$output_objdir/\$soname \${wl}--enable-auto-image-base -Xlinker --out-implib -Xlinker \$lib" postdeps_CXX= ;; *darwin*) TARGET_OS=darwin LEVELDB_TARGET_FLAGS="-DOS_MACOSX" if test x$cross_compiling != xyes; then BUILD_OS=darwin AC_CHECK_PROG([PORT],port, port) if test x$PORT = xport; then dnl add default macports paths CPPFLAGS="$CPPFLAGS -isystem /opt/local/include" LIBS="$LIBS -L/opt/local/lib" if test -d /opt/local/include/db48; then CPPFLAGS="$CPPFLAGS -I/opt/local/include/db48" LIBS="$LIBS -L/opt/local/lib/db48" fi fi AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_CHECK_PROG([BREW],brew, brew) if test x$BREW = xbrew; then dnl These Homebrew packages may be keg-only, meaning that they won't be found dnl in expected paths because they may conflict with system files. Ask dnl Homebrew where each one is located, then adjust paths accordingly. dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). openssl_prefix=`$BREW --prefix openssl 2>/dev/null` bdb_prefix=`$BREW --prefix berkeley-db4 2>/dev/null` qt5_prefix=`$BREW --prefix qt5 2>/dev/null` if test x$openssl_prefix != x; then PKG_CONFIG_PATH="$openssl_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH fi if test x$bdb_prefix != x; then CPPFLAGS="$CPPFLAGS -I$bdb_prefix/include" LIBS="$LIBS -L$bdb_prefix/lib" fi if test x$qt5_prefix != x; then PKG_CONFIG_PATH="$qt5_prefix/lib/pkgconfig:$PKG_CONFIG_PATH" export PKG_CONFIG_PATH fi fi else case $build_os in *darwin*) BUILD_OS=darwin ;; *) AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, dnl which may not exist in the path. Stripping the .a is not dnl necessary, so just disable it. old_striplib= ;; esac fi AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"]) CPPFLAGS="$CPPFLAGS -DMAC_OSX" OBJCXXFLAGS="$CXXFLAGS" ;; *linux*) TARGET_OS=linux LEVELDB_TARGET_FLAGS="-DOS_LINUX" ;; *freebsd*) LEVELDB_TARGET_FLAGS="-DOS_FREEBSD" ;; *openbsd*) LEVELDB_TARGET_FLAGS="-DOS_OPENBSD" ;; *) OTHER_OS=`echo ${host_os} | awk '{print toupper($0)}'` AC_MSG_WARN([Guessing LevelDB OS as OS_${OTHER_OS}, please check whether this is correct, if not add an entry to configure.ac.]) LEVELDB_TARGET_FLAGS="-DOS_${OTHER_OS}" ;; esac if test x$use_pkgconfig = xyes; then m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR(PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh.)]) m4_ifdef([PKG_PROG_PKG_CONFIG], [ PKG_PROG_PKG_CONFIG if test x"$PKG_CONFIG" = "x"; then AC_MSG_ERROR(pkg-config not found.) fi ]) fi if test x$use_extended_rpc_tests != xno; then AC_SUBST(EXTENDED_RPC_TESTS, -extended) fi if test x$use_lcov = xyes; then if test x$LCOV = x; then AC_MSG_ERROR("lcov testing requested but lcov not found") fi if test x$GCOV = x; then AC_MSG_ERROR("lcov testing requested but gcov not found") fi if test x$PYTHON = x; then AC_MSG_ERROR("lcov testing requested but python not found") fi if test x$GENHTML = x; then AC_MSG_ERROR("lcov testing requested but genhtml not found") fi LCOV="$LCOV --gcov-tool=$GCOV" AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"], [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")]) AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")]) fi dnl Check for endianness AC_C_BIGENDIAN dnl Check for pthread compile/link requirements AX_PTHREAD # The following macro will add the necessary defines to bitcoin-config.h, but # they also need to be passed down to any subprojects. Pull the results out of # the cache and add them to CPPFLAGS. AC_SYS_LARGEFILE # detect POSIX or GNU variant of strerror_r AC_FUNC_STRERROR_R if test x$ac_cv_sys_file_offset_bits != x && test x$ac_cv_sys_file_offset_bits != xno && test x$ac_cv_sys_file_offset_bits != xunknown; then CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi if test x$ac_cv_sys_large_files != x && test x$ac_cv_sys_large_files != xno && test x$ac_cv_sys_large_files != xunknown; then CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi AX_CHECK_LINK_FLAG([[-Wl,--large-address-aware]], [LDFLAGS="$LDFLAGS -Wl,--large-address-aware"]) AX_GCC_FUNC_ATTRIBUTE([visibility]) AX_GCC_FUNC_ATTRIBUTE([dllexport]) AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then #glibc absorbed clock_gettime in 2.17. librt (its previous location) is safe to link #in anyway for back-compat. AC_CHECK_LIB([rt],[clock_gettime],, AC_MSG_ERROR(lib missing)) #__fdelt_chk's params and return type have changed from long unsigned int to long int. # See which one is present here. AC_MSG_CHECKING(__fdelt_chk type) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#ifdef _FORTIFY_SOURCE #undef _FORTIFY_SOURCE #endif #define _FORTIFY_SOURCE 2 #include extern "C" long unsigned int __fdelt_warn(long unsigned int);]],[[]])], [ fdelt_type="long unsigned int"], [ fdelt_type="long int"]) AC_MSG_RESULT($fdelt_type) AC_DEFINE_UNQUOTED(FDELT_TYPE, $fdelt_type,[parameter and return value type for __fdelt_chk]) else AC_SEARCH_LIBS([clock_gettime],[rt]) fi if test x$TARGET_OS != xwindows; then # All windows code is PIC, forcing it on just adds useless compile warnings AX_CHECK_COMPILE_FLAG([-fPIC],[PIC_FLAGS="-fPIC"]) fi if test x$use_hardening != xno; then AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -U_FORTIFY_SOURCE" ]) HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -D_FORTIFY_SOURCE=2" ]) AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"]) AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"]) AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"]) AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"]) AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"]) if test x$TARGET_OS != xwindows; then AX_CHECK_COMPILE_FLAG([-fPIE],[PIE_FLAGS="-fPIE"]) AX_CHECK_LINK_FLAG([[-pie]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"]) fi case $host in *mingw*) AC_CHECK_LIB([ssp], [main],, AC_MSG_ERROR(lib missing)) ;; esac fi dnl this flag screws up non-darwin gcc even when the check fails. special-case it. if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"]) fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h]) AC_CHECK_DECLS([strnlen]) # Check for daemon(3), unrelated to --with-daemon (although used by it) AC_CHECK_DECLS([daemon]) AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, [#if HAVE_ENDIAN_H #include #elif HAVE_SYS_ENDIAN_H #include #endif]) AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, [#if HAVE_BYTESWAP_H #include #endif]) AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll]) dnl Check for MSG_NOSIGNAL AC_MSG_CHECKING(for MSG_NOSIGNAL) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ int f = MSG_NOSIGNAL; ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MSG_NOSIGNAL, 1,[Define this symbol if you have MSG_NOSIGNAL]) ], [ AC_MSG_RESULT(no)] ) dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas) AC_MSG_CHECKING(for mallopt M_ARENA_MAX) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ mallopt(M_ARENA_MAX, 1); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOPT_ARENA_MAX, 1,[Define this symbol if you have mallopt with M_ARENA_MAX]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING([for visibility attribute]) AC_LINK_IFELSE([AC_LANG_SOURCE([ int foo_def( void ) __attribute__((visibility("default"))); int main(){} ])], [ AC_DEFINE(HAVE_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) AC_MSG_RESULT(yes) ], [ AC_MSG_RESULT(no) if test x$use_reduce_exports = xyes; then AC_MSG_ERROR([Cannot find a working visibility attribute. Use --disable-reduce-exports.]) fi ] ) # Check for different ways of gathering OS randomness AC_MSG_CHECKING(for Linux getrandom syscall) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include #include ]], [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYS_GETRANDOM, 1,[Define this symbol if the Linux getrandom system call is available]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING(for getentropy) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ getentropy(nullptr, 32) ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY, 1,[Define this symbol if the BSD getentropy system call is available]) ], [ AC_MSG_RESULT(no)] ) AC_MSG_CHECKING(for sysctl KERN_ARND) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include #include ]], [[ static const int name[2] = {CTL_KERN, KERN_ARND}; sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL_ARND, 1,[Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], [ AC_MSG_RESULT(no)] ) # Check for reduced exports if test x$use_reduce_exports = xyes; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[RE_CXXFLAGS="-fvisibility=hidden"], [AC_MSG_ERROR([Cannot set default symbol visibility. Use --disable-reduce-exports.])]) fi LEVELDB_CPPFLAGS= LIBLEVELDB= LIBMEMENV= AM_CONDITIONAL([EMBEDDED_LEVELDB],[true]) AC_SUBST(LEVELDB_CPPFLAGS) AC_SUBST(LIBLEVELDB) AC_SUBST(LIBMEMENV) if test x$enable_wallet != xno; then dnl Check for libdb_cxx only if wallet enabled BITCOIN_FIND_BDB48 fi dnl Check for libminiupnpc (optional) if test x$use_upnp != xno; then AC_CHECK_HEADERS( [miniupnpc/miniwget.h miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], [AC_CHECK_LIB([miniupnpc], [main],[MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], [have_miniupnpc=no] ) fi BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus BITCOIN_QT_CONFIGURE([$use_pkgconfig], [qt5]) if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$build_bitcoin_seeder$use_tests$use_bench = xnononononono; then use_boost=no else use_boost=yes fi if test x$use_boost = xyes; then dnl Minimum required Boost version define(MINIMUM_REQUIRED_BOOST, 1.47.0) dnl Check for boost libs AX_BOOST_BASE([MINIMUM_REQUIRED_BOOST]) AX_BOOST_SYSTEM AX_BOOST_FILESYSTEM AX_BOOST_PROGRAM_OPTIONS AX_BOOST_THREAD AX_BOOST_CHRONO if test x$use_reduce_exports = xyes; then AC_MSG_CHECKING([for working boost reduced exports]) TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$BOOST_CPPFLAGS $CPPFLAGS" AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ @%:@include ]], [[ #if BOOST_VERSION >= 104900 // Everything is okay #else # error Boost version is too old #endif ]])],[ AC_MSG_RESULT(yes) ],[ AC_MSG_ERROR([boost versions < 1.49 are known to be broken with reduced exports. Use --disable-reduce-exports.]) ]) CPPFLAGS="$TEMP_CPPFLAGS" fi fi if test x$use_reduce_exports = xyes; then CXXFLAGS="$CXXFLAGS $RE_CXXFLAGS" AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]], [RELDFLAGS="-Wl,--exclude-libs,ALL"]) fi if test x$use_tests = xyes; then if test x$HEXDUMP = x; then AC_MSG_ERROR(hexdump is required for tests) fi if test x$use_boost = xyes; then AX_BOOST_UNIT_TEST_FRAMEWORK dnl Determine if -DBOOST_TEST_DYN_LINK is needed AC_MSG_CHECKING([for dynamic linked boost test]) TEMP_LIBS="$LIBS" LIBS="$LIBS $BOOST_LDFLAGS $BOOST_UNIT_TEST_FRAMEWORK_LIB" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_LINK_IFELSE([AC_LANG_SOURCE([ #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MAIN #include ])], [AC_MSG_RESULT(yes)] [TESTDEFS="$TESTDEFS -DBOOST_TEST_DYN_LINK"], [AC_MSG_RESULT(no)]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" fi fi if test x$use_boost = xyes; then BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB $BOOST_PROGRAM_OPTIONS_LIB $BOOST_THREAD_LIB $BOOST_CHRONO_LIB" dnl If boost (prior to 1.57) was built without c++11, it emulated scoped enums dnl using c++98 constructs. Unfortunately, this implementation detail leaked into dnl the abi. This was fixed in 1.57. dnl When building against that installed version using c++11, the headers pick up dnl on the native c++11 scoped enum support and enable it, however it will fail to dnl link. This can be worked around by disabling c++11 scoped enums if linking will dnl fail. dnl BOOST_NO_SCOPED_ENUMS was changed to BOOST_NO_CXX11_SCOPED_ENUMS in 1.51. TEMP_LIBS="$LIBS" LIBS="$BOOST_LIBS $LIBS" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_MSG_CHECKING([for mismatched boost c++11 scoped enums]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include "boost/config.hpp" #include "boost/version.hpp" #if !defined(BOOST_NO_SCOPED_ENUMS) && !defined(BOOST_NO_CXX11_SCOPED_ENUMS) && BOOST_VERSION < 105700 #define BOOST_NO_SCOPED_ENUMS #define BOOST_NO_CXX11_SCOPED_ENUMS #define CHECK #endif #include "boost/filesystem.hpp" ]],[[ #if defined(CHECK) boost::filesystem::copy_file("foo", "bar"); #else choke; #endif ]])], [AC_MSG_RESULT(mismatched); BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_SCOPED_ENUMS -DBOOST_NO_CXX11_SCOPED_ENUMS"], [AC_MSG_RESULT(ok)]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" dnl Boost >= 1.50 uses sleep_for rather than the now-deprecated sleep, however dnl it was broken from 1.50 to 1.52 when backed by nanosleep. Use sleep_for if dnl a working version is available, else fall back to sleep. sleep was removed dnl after 1.56. dnl If neither is available, abort. TEMP_LIBS="$LIBS" LIBS="$BOOST_LIBS $LIBS" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include #include ]],[[ #if BOOST_VERSION >= 105000 && (!defined(BOOST_HAS_NANOSLEEP) || BOOST_VERSION >= 105200) boost::this_thread::sleep_for(boost::chrono::milliseconds(0)); #else choke me #endif ]])], [boost_sleep=yes; AC_DEFINE(HAVE_WORKING_BOOST_SLEEP_FOR, 1, [Define this symbol if boost sleep_for works])], [boost_sleep=no]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" if test x$boost_sleep != xyes; then TEMP_LIBS="$LIBS" LIBS="$BOOST_LIBS $LIBS" TEMP_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include #include #include ]],[[ #if BOOST_VERSION <= 105600 boost::this_thread::sleep(boost::posix_time::milliseconds(0)); #else choke me #endif ]])], [boost_sleep=yes; AC_DEFINE(HAVE_WORKING_BOOST_SLEEP, 1, [Define this symbol if boost sleep works])], [boost_sleep=no]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" fi if test x$boost_sleep != xyes; then AC_MSG_ERROR(No working boost sleep implementation found.) fi fi if test x$use_pkgconfig = xyes; then : dnl m4_ifdef( [PKG_CHECK_MODULES], [ PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) BITCOIN_QT_CHECK([PKG_CHECK_MODULES([PROTOBUF], [protobuf], [have_protobuf=yes], [BITCOIN_QT_FAIL(libprotobuf not found)])]) if test x$use_qr != xno; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) if test x$TARGET_OS != xwindows; then PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) fi fi if test "x$use_zmq" = "xyes"; then PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) AC_MSG_WARN([libzmq version 4.x or greater not found, disabling]) use_zmq=no]) else AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) fi ] ) else AC_CHECK_HEADER([openssl/crypto.h],,AC_MSG_ERROR(libcrypto headers missing)) AC_CHECK_LIB([crypto], [main],CRYPTO_LIBS=-lcrypto, AC_MSG_ERROR(libcrypto missing)) AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),) AC_CHECK_LIB([ssl], [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing)) if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) if test x$TARGET_OS != xwindows; then AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) fi fi if test "x$use_zmq" = "xyes"; then AC_CHECK_HEADER([zmq.h], [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], [AC_MSG_WARN([zmq.h not found, disabling zmq support]) use_zmq=no AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) AC_CHECK_LIB([zmq],[zmq_ctx_shutdown],ZMQ_LIBS=-lzmq, [AC_MSG_WARN([libzmq >= 4.0 not found, disabling zmq support]) use_zmq=no AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions])]) else AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) fi if test "x$use_zmq" = "xyes"; then dnl Assume libzmq was built for static linking case $host in *mingw*) ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" ;; esac fi BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) if test x$use_qr != xno; then BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) BITCOIN_QT_CHECK([AC_CHECK_HEADER([qrencode.h],, have_qrencode=no)]) fi fi save_CXXFLAGS="${CXXFLAGS}" CXXFLAGS="${CXXFLAGS} ${CRYPTO_CFLAGS} ${SSL_CFLAGS}" AC_CHECK_DECLS([EVP_MD_CTX_new],,,[AC_INCLUDES_DEFAULT #include ]) CXXFLAGS="${save_CXXFLAGS}" dnl univalue check need_bundled_univalue=yes if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononono; then need_bundled_univalue=no else if test x$system_univalue != xno ; then found_univalue=no if test x$use_pkgconfig = xyes; then : #NOP m4_ifdef( [PKG_CHECK_MODULES], [ PKG_CHECK_MODULES([UNIVALUE],[libunivalue],[found_univalue=yes],[true]) ] ) else AC_CHECK_HEADER([univalue.h],[ AC_CHECK_LIB([univalue], [main],[ UNIVALUE_LIBS=-lunivalue found_univalue=yes ],[true]) ],[true]) fi if test x$found_univalue = xyes ; then system_univalue=yes need_bundled_univalue=no elif test x$system_univalue = xyes ; then AC_MSG_ERROR([univalue not found]) else system_univalue=no fi fi if test x$need_bundled_univalue = xyes ; then UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' UNIVALUE_LIBS='univalue/libunivalue.la' fi fi AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) AC_SUBST(UNIVALUE_CFLAGS) AC_SUBST(UNIVALUE_LIBS) BITCOIN_QT_PATH_PROGS([PROTOC], [protoc],$protoc_bin_path) AC_MSG_CHECKING([whether to build bitcoind]) AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) AC_MSG_RESULT($build_bitcoind) AC_MSG_CHECKING([whether to build bitcoin-seeder]) AM_CONDITIONAL([BUILD_BITCOIN_SEEDER], [test x$build_bitcoin_seeder = xyes]) AC_MSG_RESULT($build_bitcoin_seeder) AC_MSG_CHECKING([whether to build utils (bitcoin-cli bitcoin-tx)]) AM_CONDITIONAL([BUILD_BITCOIN_UTILS], [test x$build_bitcoin_utils = xyes]) AC_MSG_RESULT($build_bitcoin_utils) AC_MSG_CHECKING([whether to build libraries]) AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test x$build_bitcoin_libs = xyes]) if test x$build_bitcoin_libs = xyes; then AC_DEFINE(HAVE_CONSENSUS_LIB, 1, [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP if test "x$use_ccache" != "xno"; then AC_MSG_CHECKING(if ccache should be used) if test x$CCACHE = x; then if test "x$use_ccache" = "xyes"; then AC_MSG_ERROR([ccache not found.]); else use_ccache=no fi else use_ccache=yes CC="$ac_cv_path_CCACHE $CC" CXX="$ac_cv_path_CCACHE $CXX" fi AC_MSG_RESULT($use_ccache) fi if test "x$use_ccache" = "xyes"; then AX_CHECK_PREPROC_FLAG([-Qunused-arguments],[CPPFLAGS="-Qunused-arguments $CPPFLAGS"]) fi dnl enable wallet AC_MSG_CHECKING([if wallet should be enabled]) if test x$enable_wallet != xno; then AC_MSG_RESULT(yes) AC_DEFINE_UNQUOTED([ENABLE_WALLET],[1],[Define to 1 to enable wallet functions]) else AC_MSG_RESULT(no) fi dnl enable upnp support AC_MSG_CHECKING([whether to build with support for UPnP]) if test x$have_miniupnpc = xno; then if test x$use_upnp = xyes; then AC_MSG_ERROR("UPnP requested but cannot be built. use --without-miniupnpc") fi AC_MSG_RESULT(no) else if test x$use_upnp != xno; then AC_MSG_RESULT(yes) AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes upnp_setting=0 if test x$use_upnp_default != xno; then use_upnp_default=yes upnp_setting=1 fi AC_MSG_RESULT($use_upnp_default) AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) if test x$TARGET_OS = xwindows; then MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB" fi else AC_MSG_RESULT(no) fi fi dnl these are only used when qt is enabled BUILD_TEST_QT="" if test x$bitcoin_enable_qt != xno; then dnl enable dbus support AC_MSG_CHECKING([whether to build GUI with support for D-Bus]) if test x$bitcoin_enable_qt_dbus != xno; then AC_DEFINE([USE_DBUS],[1],[Define if dbus support should be compiled in]) fi AC_MSG_RESULT($bitcoin_enable_qt_dbus) dnl enable qr support AC_MSG_CHECKING([whether to build GUI with support for QR codes]) if test x$have_qrencode = xno; then if test x$use_qr = xyes; then AC_MSG_ERROR("QR support requested but cannot be built. use --without-qrencode") fi AC_MSG_RESULT(no) else if test x$use_qr != xno; then AC_MSG_RESULT(yes) AC_DEFINE([USE_QRCODE],[1],[Define if QR support should be compiled in]) use_qr=yes else AC_MSG_RESULT(no) fi fi if test x$XGETTEXT = x; then AC_MSG_WARN("xgettext is required to update qt translations") fi AC_MSG_CHECKING([whether to build test_bitcoin-qt]) if test x$use_gui_tests$bitcoin_enable_qt_test = xyesyes; then AC_MSG_RESULT([yes]) BUILD_TEST_QT="yes" else AC_MSG_RESULT([no]) fi fi AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) AC_MSG_CHECKING([whether to build test_bitcoin]) if test x$use_tests = xyes; then AC_MSG_RESULT([yes]) BUILD_TEST="yes" else AC_MSG_RESULT([no]) BUILD_TEST="" fi AC_MSG_CHECKING([whether to reduce exports]) if test x$use_reduce_exports = xyes; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) fi if test x$build_bitcoin_utils$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests = xnononononono; then AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-seeder --with-gui --enable-bench or --enable-tests]) fi AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) +AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) AC_DEFINE(CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION, [Build revision]) AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build]) AC_DEFINE(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE, [Version is release]) AC_DEFINE(COPYRIGHT_YEAR, _COPYRIGHT_YEAR, [Copyright year]) AC_DEFINE(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS", [Copyright holder(s) before %s replacement]) AC_DEFINE(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION", [Replacement for %s in copyright holders string]) define(_COPYRIGHT_HOLDERS_FINAL, [patsubst(_COPYRIGHT_HOLDERS, [%s], [_COPYRIGHT_HOLDERS_SUBSTITUTION])]) AC_DEFINE(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL", [Copyright holder(s)]) AC_SUBST(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR) AC_SUBST(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR) AC_SUBST(CLIENT_VERSION_REVISION, _CLIENT_VERSION_REVISION) AC_SUBST(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD) AC_SUBST(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE) AC_SUBST(COPYRIGHT_YEAR, _COPYRIGHT_YEAR) AC_SUBST(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS") AC_SUBST(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION") AC_SUBST(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL") AC_SUBST(BITCOIN_DAEMON_NAME) AC_SUBST(BITCOIN_GUI_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) AC_SUBST(BITCOIN_SEEDER_NAME) AC_SUBST(RELDFLAGS) AC_SUBST(ERROR_CXXFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) +AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(USE_UPNP) AC_SUBST(USE_QRCODE) AC_SUBST(BOOST_LIBS) AC_SUBST(TESTDEFS) AC_SUBST(LEVELDB_TARGET_FLAGS) AC_SUBST(MINIUPNPC_CPPFLAGS) AC_SUBST(MINIUPNPC_LIBS) AC_SUBST(CRYPTO_LIBS) AC_SUBST(SSL_LIBS) AC_SUBST(EVENT_LIBS) AC_SUBST(EVENT_PTHREADS_LIBS) AC_SUBST(ZMQ_LIBS) AC_SUBST(PROTOBUF_LIBS) AC_SUBST(QR_LIBS) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py]) AC_CONFIG_FILES([qa/pull-tester/tests_config.py],[chmod +x qa/pull-tester/tests_config.py]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AC_CONFIG_LINKS([qa/pull-tester/rpc-tests.py:qa/pull-tester/rpc-tests.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. dnl Until this is fixed upstream and we've synced, we'll just un-export them. CPPFLAGS_TEMP="$CPPFLAGS" unset CPPFLAGS CPPFLAGS="$CPPFLAGS_TEMP" LDFLAGS_TEMP="$LDFLAGS" unset LDFLAGS LDFLAGS="$LDFLAGS_TEMP" LIBS_TEMP="$LIBS" unset LIBS LIBS="$LIBS_TEMP" PKGCONFIG_PATH_TEMP="$PKG_CONFIG_PATH" unset PKG_CONFIG_PATH PKG_CONFIG_PATH="$PKGCONFIG_PATH_TEMP" PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" unset PKG_CONFIG_LIBDIR PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" if test x$need_bundled_univalue = xyes; then AC_CONFIG_SUBDIRS([src/univalue]) fi ac_configure_args="${ac_configure_args} --disable-shared --with-pic --with-bignum=no --enable-module-recovery" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT dnl Taken from https://wiki.debian.org/RpathIssue case $host in *-*-linux-gnu) AC_MSG_RESULT([Fixing libtool for -rpath problems.]) sed < libtool > libtool-2 \ 's/^hardcode_libdir_flag_spec.*$'/'hardcode_libdir_flag_spec=" -D__LIBTOOL_IS_A_FOOL__ "/' mv libtool-2 libtool chmod 755 libtool ;; esac dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows case ${OS} in *Windows*) sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' qa/pull-tester/tests_config.py > qa/pull-tester/tests_config-2.py mv qa/pull-tester/tests_config-2.py qa/pull-tester/tests_config.py ;; esac -echo +echo echo "Options used to compile and link:" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" if test x$bitcoin_enable_qt != xno; then echo " qt version = $bitcoin_qt_got_major_vers" echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" echo " with test = $use_tests" echo " with bench = $use_bench" echo " with upnp = $use_upnp" echo " debug enabled = $enable_debug" echo " werror = $enable_werror" echo echo " sanitizers " echo " asan = $enable_asan" echo " tsan = $enable_tsan" echo " ubsan = $enable_ubsan" echo echo " target os = $TARGET_OS" echo " build os = $BUILD_OS" echo echo " CC = $CC" echo " CFLAGS = $CFLAGS" echo " CPPFLAGS = $CPPFLAGS" echo " CXX = $CXX" echo " CXXFLAGS = $CXXFLAGS" echo " LDFLAGS = $LDFLAGS" -echo +echo diff --git a/src/Makefile.am b/src/Makefile.am index 4f8f44cdc..4e7d4243d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,565 +1,566 @@ # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. DIST_SUBDIRS = secp256k1 univalue AM_LDFLAGS = $(PTHREAD_CFLAGS) $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS) AM_CPPFLAGS = $(HARDENED_CPPFLAGS) EXTRA_LIBRARIES = if EMBEDDED_UNIVALUE LIBUNIVALUE = univalue/libunivalue.la $(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) else LIBUNIVALUE = $(UNIVALUE_LIBS) endif BITCOIN_CONFIG_INCLUDES=-I$(builddir)/config BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) BITCOIN_SEEDER_INCLUDES = -I$(srcdir)/seeder BITCOIN_SEEDER_INCLUDES += $(BITCOIN_INCLUDES) LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a LIBBITCOIN_UTIL=libbitcoin_util.a LIBBITCOIN_CRYPTO=crypto/libbitcoin_crypto.a LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la if ENABLE_ZMQ LIBBITCOIN_ZMQ=libbitcoin_zmq.a endif if BUILD_BITCOIN_LIBS LIBBITCOINCONSENSUS=libbitcoinconsensus.la endif if BUILD_BITCOIN_SEEDER LIBBITCOIN_SEEDER=libbitcoin_seeder.a endif if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a endif $(LIBSECP256K1): $(wildcard secp256k1/src/*) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ $(LIBBITCOIN_CRYPTO) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_CLI) \ $(LIBBITCOIN_SEEDER) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_ZMQ) lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) bin_PROGRAMS = noinst_PROGRAMS = TESTS = BENCHMARKS = if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif if BUILD_BITCOIN_SEEDER bin_PROGRAMS += bitcoin-seeder endif if BUILD_BITCOIN_UTILS bin_PROGRAMS += bitcoin-cli bitcoin-tx endif .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ addrdb.h \ addrman.h \ base58.h \ bloom.h \ blockencodings.h \ chain.h \ chainparams.h \ chainparamsbase.h \ chainparamsseeds.h \ checkpoints.h \ checkqueue.h \ clientversion.h \ coins.h \ compat.h \ compat/byteswap.h \ compat/endian.h \ compat/sanity.h \ compressor.h \ config.h \ consensus/consensus.h \ core_io.h \ core_memusage.h \ cuckoocache.h \ globals.h \ httprpc.h \ httpserver.h \ indirectmap.h \ init.h \ key.h \ keystore.h \ dbwrapper.h \ limitedmap.h \ memusage.h \ merkleblock.h \ miner.h \ net.h \ net_processing.h \ netaddress.h \ netbase.h \ netmessagemaker.h \ noui.h \ policy/fees.h \ policy/policy.h \ pow.h \ protocol.h \ random.h \ reverselock.h \ rpc/blockchain.h \ rpc/client.h \ rpc/misc.h \ rpc/protocol.h \ rpc/server.h \ rpc/register.h \ scheduler.h \ script/scriptcache.h \ script/sigcache.h \ script/sign.h \ script/standard.h \ script/ismine.h \ streams.h \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ support/cleanse.h \ support/events.h \ support/lockedpool.h \ sync.h \ threadsafety.h \ threadinterrupt.h \ timedata.h \ torcontrol.h \ txdb.h \ txmempool.h \ ui_interface.h \ undo.h \ util.h \ utilmoneystr.h \ utiltime.h \ validation.h \ validationinterface.h \ versionbits.h \ wallet/coincontrol.h \ wallet/crypter.h \ wallet/db.h \ wallet/finaltx.h \ wallet/rpcdump.h \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ zmq/zmqnotificationinterface.h \ zmq/zmqpublishnotifier.h obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj @$(top_srcdir)/share/genbuild.sh $(abs_top_builddir)/src/obj/build.h \ $(abs_top_srcdir) libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # server: shared between bitcoind and bitcoin-qt libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ addrman.cpp \ addrdb.cpp \ bloom.cpp \ blockencodings.cpp \ chain.cpp \ checkpoints.cpp \ config.cpp \ globals.cpp \ httprpc.cpp \ httpserver.cpp \ init.cpp \ dbwrapper.cpp \ merkleblock.cpp \ miner.cpp \ net.cpp \ net_processing.cpp \ noui.cpp \ policy/fees.cpp \ policy/policy.cpp \ pow.cpp \ rest.cpp \ rpc/abc.cpp \ rpc/blockchain.cpp \ rpc/mining.cpp \ rpc/misc.cpp \ rpc/net.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ script/scriptcache.cpp \ script/sigcache.cpp \ script/ismine.cpp \ timedata.cpp \ torcontrol.cpp \ txdb.cpp \ txmempool.cpp \ ui_interface.cpp \ validation.cpp \ validationinterface.cpp \ versionbits.cpp \ $(BITCOIN_CORE_H) if ENABLE_ZMQ libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_zmq_a_SOURCES = \ zmq/zmqabstractnotifier.cpp \ zmq/zmqnotificationinterface.cpp \ zmq/zmqpublishnotifier.cpp endif # wallet: shared between bitcoind and bitcoin-qt, but only linked # when wallet enabled libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ wallet/crypter.cpp \ wallet/db.cpp \ wallet/finaltx.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ $(BITCOIN_CORE_H) # crypto primitives library crypto_libbitcoin_crypto_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_CONFIG_INCLUDES) crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ crypto/chacha20.h \ crypto/chacha20.cpp \ crypto/common.h \ crypto/hmac_sha256.cpp \ crypto/hmac_sha256.h \ crypto/hmac_sha512.cpp \ crypto/hmac_sha512.h \ crypto/ripemd160.cpp \ crypto/ripemd160.h \ crypto/sha1.cpp \ crypto/sha1.h \ crypto/sha256.cpp \ crypto/sha256.h \ crypto/sha512.cpp \ crypto/sha512.h # consensus: shared between all executables that validate any consensus rules. libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ amount.h \ arith_uint256.cpp \ arith_uint256.h \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ consensus/validation.h \ hash.cpp \ hash.h \ prevector.h \ primitives/block.cpp \ primitives/block.h \ primitives/transaction.cpp \ primitives/transaction.h \ pubkey.cpp \ pubkey.h \ script/bitcoinconsensus.cpp \ script/interpreter.cpp \ script/interpreter.h \ script/script.cpp \ script/script.h \ script/script_error.cpp \ script/script_error.h \ serialize.h \ tinyformat.h \ uint256.cpp \ uint256.h \ utilstrencodings.cpp \ utilstrencodings.h \ version.h # common: shared between bitcoind, and bitcoin-qt and non-server tools libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ amount.cpp \ base58.cpp \ chainparams.cpp \ coins.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ key.cpp \ keystore.cpp \ netaddress.cpp \ netbase.cpp \ protocol.cpp \ scheduler.cpp \ script/sign.cpp \ script/standard.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) # util: shared between all executables. # This library *must* be included to make sure that the glibc # backward-compatibility objects and their sanity checks are linked. libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ compat/glibc_sanity.cpp \ compat/glibcxx_sanity.cpp \ compat/strnlen.cpp \ random.cpp \ rpc/protocol.cpp \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ util.cpp \ utilmoneystr.cpp \ utilstrencodings.cpp \ utiltime.cpp \ $(BITCOIN_CORE_H) if GLIBC_BACK_COMPAT libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp endif # cli: shared between bitcoin-cli and bitcoin-qt libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_cli_a_SOURCES = \ rpc/client.cpp \ $(BITCOIN_CORE_H) # seeder library libbitcoin_seeder_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_SEEDER_INCLUDES) libbitcoin_seeder_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_seeder_a_SOURCES = \ seeder/bitcoin.cpp \ seeder/bitcoin.h \ seeder/compat.h \ seeder/db.cpp \ seeder/db.h \ seeder/dns.c \ seeder/dns.h \ seeder/netbase.cpp \ seeder/netbase.h \ seeder/protocol.cpp \ seeder/protocol.h \ seeder/strlcpy.h \ seeder/util.cpp \ seeder/util.h nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h # # bitcoind binary # bitcoind_SOURCES = bitcoind.cpp bitcoind_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) bitcoind_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoind_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS bitcoind_SOURCES += bitcoind-res.rc endif bitcoind_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ + $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp bitcoin_cli_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) bitcoin_cli_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoin_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS bitcoin_cli_SOURCES += bitcoin-cli-res.rc endif bitcoin_cli_LDADD = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) bitcoin_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) # # bitcoin-seeder binary # bitcoin_seeder_SOURCES = seeder/main.cpp bitcoin_seeder_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_SEEDER_INCLUDES) bitcoin_seeder_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoin_seeder_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) bitcoin_seeder_LDADD = \ $(LIBBITCOIN_SEEDER) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) bitcoin_seeder_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) # # bitcoin-tx binary # bitcoin_tx_SOURCES = bitcoin-tx.cpp bitcoin_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) bitcoin_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bitcoin_tx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS bitcoin_tx_SOURCES += bitcoin-tx-res.rc endif bitcoin_tx_LDADD = \ $(LIBUNIVALUE) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) bitcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) # # bitcoinconsensus library # if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h libbitcoinconsensus_la_SOURCES = $(crypto_libbitcoin_crypto_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) if GLIBC_BACK_COMPAT libbitcoinconsensus_la_SOURCES += compat/glibc_compat.cpp endif libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) endif # CTAES_DIST = crypto/ctaes/bench.c CTAES_DIST += crypto/ctaes/ctaes.c CTAES_DIST += crypto/ctaes/ctaes.h CTAES_DIST += crypto/ctaes/README.md CTAES_DIST += crypto/ctaes/test.c CLEANFILES = $(EXTRA_LIBRARIES) CLEANFILES += *.gcda *.gcno CLEANFILES += compat/*.gcda compat/*.gcno CLEANFILES += consensus/*.gcda consensus/*.gcno CLEANFILES += crypto/*.gcda crypto/*.gcno CLEANFILES += policy/*.gcda policy/*.gcno CLEANFILES += primitives/*.gcda primitives/*.gcno CLEANFILES += script/*.gcda script/*.gcno CLEANFILES += support/*.gcda support/*.gcno CLEANFILES += univalue/*.gcda univalue/*.gcno CLEANFILES += wallet/*.gcda wallet/*.gcno CLEANFILES += wallet/test/*.gcda wallet/test/*.gcno CLEANFILES += zmq/*.gcda zmq/*.gcno DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) clean-local: -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -rf test/__pycache__ .rc.o: @test -f $(WINDRES) ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ .mm.o: $(AM_V_CXX) $(OBJCXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ $(CPPFLAGS) $(AM_CXXFLAGS) $(QT_INCLUDES) $(AM_CXXFLAGS) $(PIE_FLAGS) $(CXXFLAGS) -c -o $@ $< check-symbols: $(bin_PROGRAMS) if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." $(AM_V_at) READELF=$(READELF) CPPFILT=$(CPPFILT) $(top_srcdir)/contrib/devtools/symbol-check.py < $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." $(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS) endif %.pb.cc %.pb.h: %.proto @test -f $(PROTOC) $(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$( "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 358f39cbe..ac38141f4 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -1,137 +1,149 @@ # Copyright (c) 2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. LIBLEVELDB_INT = leveldb/libleveldb.a LIBMEMENV_INT = leveldb/libmemenv.a +LIBLEVELDB_SSE42_INT = leveldb/libleveldb_sse42.a EXTRA_LIBRARIES += $(LIBLEVELDB_INT) EXTRA_LIBRARIES += $(LIBMEMENV_INT) +EXTRA_LIBRARIES += $(LIBLEVELDB_SSE42_INT) LIBLEVELDB += $(LIBLEVELDB_INT) LIBMEMENV += $(LIBMEMENV_INT) +LIBLEVELDB_SSE42 = $(LIBLEVELDB_SSE42_INT) LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv LEVELDB_CPPFLAGS_INT = LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb LEVELDB_CPPFLAGS_INT += $(LEVELDB_TARGET_FLAGS) LEVELDB_CPPFLAGS_INT += -DLEVELDB_ATOMIC_PRESENT LEVELDB_CPPFLAGS_INT += -D__STDC_LIMIT_MACROS if TARGET_WINDOWS LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1 else LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX endif leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) leveldb_libleveldb_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) leveldb_libleveldb_a_SOURCES= leveldb_libleveldb_a_SOURCES += leveldb/port/atomic_pointer.h leveldb_libleveldb_a_SOURCES += leveldb/port/port_example.h leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.h leveldb_libleveldb_a_SOURCES += leveldb/port/win/stdint.h leveldb_libleveldb_a_SOURCES += leveldb/port/port.h leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.h leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h leveldb_libleveldb_a_SOURCES += leveldb/table/block.h leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h leveldb_libleveldb_a_SOURCES += leveldb/table/format.h leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h +leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h leveldb_libleveldb_a_SOURCES += leveldb/util/random.h leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc if TARGET_WINDOWS leveldb_libleveldb_a_SOURCES += leveldb/util/env_win.cc leveldb_libleveldb_a_SOURCES += leveldb/port/port_win.cc else leveldb_libleveldb_a_SOURCES += leveldb/port/port_posix.cc endif leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS) leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS) leveldb_libmemenv_a_SOURCES = leveldb/helpers/memenv/memenv.cc leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h + +leveldb_libleveldb_sse42_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS) +leveldb_libleveldb_sse42_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS) +if ENABLE_SSE42 +leveldb_libleveldb_sse42_a_CPPFLAGS += -DLEVELDB_PLATFORM_POSIX_SSE +leveldb_libleveldb_sse42_a_CXXFLAGS += $(SSE42_CXXFLAGS) +endif +leveldb_libleveldb_sse42_a_SOURCES = leveldb/port/port_posix_sse.cc diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index edc3c4b29..960ae4bc8 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -1,463 +1,463 @@ # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. bin_PROGRAMS += qt/bitcoin-qt EXTRA_LIBRARIES += qt/libbitcoinqt.a # bitcoin qt core # QT_TS = \ qt/locale/bitcoin_af.ts \ qt/locale/bitcoin_af_ZA.ts \ qt/locale/bitcoin_ar.ts \ qt/locale/bitcoin_be_BY.ts \ qt/locale/bitcoin_bg_BG.ts \ qt/locale/bitcoin_bg.ts \ qt/locale/bitcoin_ca_ES.ts \ qt/locale/bitcoin_ca.ts \ qt/locale/bitcoin_ca@valencia.ts \ qt/locale/bitcoin_cs.ts \ qt/locale/bitcoin_cy.ts \ qt/locale/bitcoin_da.ts \ qt/locale/bitcoin_de.ts \ qt/locale/bitcoin_el_GR.ts \ qt/locale/bitcoin_el.ts \ qt/locale/bitcoin_en_GB.ts \ qt/locale/bitcoin_en.ts \ qt/locale/bitcoin_eo.ts \ qt/locale/bitcoin_es_AR.ts \ qt/locale/bitcoin_es_CL.ts \ qt/locale/bitcoin_es_CO.ts \ qt/locale/bitcoin_es_DO.ts \ qt/locale/bitcoin_es_ES.ts \ qt/locale/bitcoin_es_MX.ts \ qt/locale/bitcoin_es.ts \ qt/locale/bitcoin_es_UY.ts \ qt/locale/bitcoin_es_VE.ts \ qt/locale/bitcoin_et_EE.ts \ qt/locale/bitcoin_et.ts \ qt/locale/bitcoin_eu_ES.ts \ qt/locale/bitcoin_fa_IR.ts \ qt/locale/bitcoin_fa.ts \ qt/locale/bitcoin_fi.ts \ qt/locale/bitcoin_fr_CA.ts \ qt/locale/bitcoin_fr_FR.ts \ qt/locale/bitcoin_fr.ts \ qt/locale/bitcoin_gl.ts \ qt/locale/bitcoin_he.ts \ qt/locale/bitcoin_hi_IN.ts \ qt/locale/bitcoin_hr.ts \ qt/locale/bitcoin_hu.ts \ qt/locale/bitcoin_id_ID.ts \ qt/locale/bitcoin_it_IT.ts \ qt/locale/bitcoin_it.ts \ qt/locale/bitcoin_ja.ts \ qt/locale/bitcoin_ka.ts \ qt/locale/bitcoin_kk_KZ.ts \ qt/locale/bitcoin_ko_KR.ts \ qt/locale/bitcoin_ku_IQ.ts \ qt/locale/bitcoin_ky.ts \ qt/locale/bitcoin_la.ts \ qt/locale/bitcoin_lt.ts \ qt/locale/bitcoin_lv_LV.ts \ qt/locale/bitcoin_mk_MK.ts \ qt/locale/bitcoin_mn.ts \ qt/locale/bitcoin_ms_MY.ts \ qt/locale/bitcoin_nb.ts \ qt/locale/bitcoin_ne.ts \ qt/locale/bitcoin_nl.ts \ qt/locale/bitcoin_pam.ts \ qt/locale/bitcoin_pl.ts \ qt/locale/bitcoin_pt_BR.ts \ qt/locale/bitcoin_pt_PT.ts \ qt/locale/bitcoin_ro_RO.ts \ qt/locale/bitcoin_ro.ts \ qt/locale/bitcoin_ru_RU.ts \ qt/locale/bitcoin_ru.ts \ qt/locale/bitcoin_sk.ts \ qt/locale/bitcoin_sl_SI.ts \ qt/locale/bitcoin_sq.ts \ qt/locale/bitcoin_sr@latin.ts \ qt/locale/bitcoin_sr.ts \ qt/locale/bitcoin_sv.ts \ qt/locale/bitcoin_ta.ts \ qt/locale/bitcoin_th_TH.ts \ qt/locale/bitcoin_tr_TR.ts \ qt/locale/bitcoin_tr.ts \ qt/locale/bitcoin_uk.ts \ qt/locale/bitcoin_ur_PK.ts \ qt/locale/bitcoin_uz@Cyrl.ts \ qt/locale/bitcoin_vi.ts \ qt/locale/bitcoin_vi_VN.ts \ qt/locale/bitcoin_zh_CN.ts \ qt/locale/bitcoin_zh_HK.ts \ qt/locale/bitcoin_zh.ts \ qt/locale/bitcoin_zh_TW.ts QT_FORMS_UI = \ qt/forms/addressbookpage.ui \ qt/forms/askpassphrasedialog.ui \ qt/forms/coincontroldialog.ui \ qt/forms/editaddressdialog.ui \ qt/forms/helpmessagedialog.ui \ qt/forms/intro.ui \ qt/forms/modaloverlay.ui \ qt/forms/openuridialog.ui \ qt/forms/optionsdialog.ui \ qt/forms/overviewpage.ui \ qt/forms/receivecoinsdialog.ui \ qt/forms/receiverequestdialog.ui \ qt/forms/debugwindow.ui \ qt/forms/sendcoinsdialog.ui \ qt/forms/sendcoinsentry.ui \ qt/forms/signverifymessagedialog.ui \ qt/forms/transactiondescdialog.ui QT_MOC_CPP = \ qt/moc_addressbookpage.cpp \ qt/moc_addresstablemodel.cpp \ qt/moc_askpassphrasedialog.cpp \ qt/moc_bantablemodel.cpp \ qt/moc_bitcoinaddressvalidator.cpp \ qt/moc_bitcoinamountfield.cpp \ qt/moc_bitcoingui.cpp \ qt/moc_bitcoinunits.cpp \ qt/moc_clientmodel.cpp \ qt/moc_coincontroldialog.cpp \ qt/moc_coincontroltreewidget.cpp \ qt/moc_csvmodelwriter.cpp \ qt/moc_editaddressdialog.cpp \ qt/moc_guiutil.cpp \ qt/moc_intro.cpp \ qt/moc_macdockiconhandler.cpp \ qt/moc_macnotificationhandler.cpp \ qt/moc_modaloverlay.cpp \ qt/moc_notificator.cpp \ qt/moc_openuridialog.cpp \ qt/moc_optionsdialog.cpp \ qt/moc_optionsmodel.cpp \ qt/moc_overviewpage.cpp \ qt/moc_peertablemodel.cpp \ qt/moc_paymentserver.cpp \ qt/moc_qvalidatedlineedit.cpp \ qt/moc_qvaluecombobox.cpp \ qt/moc_receivecoinsdialog.cpp \ qt/moc_receiverequestdialog.cpp \ qt/moc_recentrequeststablemodel.cpp \ qt/moc_rpcconsole.cpp \ qt/moc_sendcoinsdialog.cpp \ qt/moc_sendcoinsentry.cpp \ qt/moc_signverifymessagedialog.cpp \ qt/moc_splashscreen.cpp \ qt/moc_trafficgraphwidget.cpp \ qt/moc_transactiondesc.cpp \ qt/moc_transactiondescdialog.cpp \ qt/moc_transactionfilterproxy.cpp \ qt/moc_transactiontablemodel.cpp \ qt/moc_transactionview.cpp \ qt/moc_utilitydialog.cpp \ qt/moc_walletframe.cpp \ qt/moc_walletmodel.cpp \ qt/moc_walletview.cpp BITCOIN_MM = \ qt/macdockiconhandler.mm \ qt/macnotificationhandler.mm QT_MOC = \ qt/bitcoin.moc \ qt/bitcoinamountfield.moc \ qt/intro.moc \ qt/overviewpage.moc \ qt/rpcconsole.moc QT_QRC_CPP = qt/qrc_bitcoin.cpp QT_QRC = qt/bitcoin.qrc QT_QRC_LOCALE_CPP = qt/qrc_bitcoin_locale.cpp QT_QRC_LOCALE = qt/bitcoin_locale.qrc PROTOBUF_CC = qt/paymentrequest.pb.cc PROTOBUF_H = qt/paymentrequest.pb.h PROTOBUF_PROTO = qt/paymentrequest.proto BITCOIN_QT_H = \ qt/addressbookpage.h \ qt/addresstablemodel.h \ qt/askpassphrasedialog.h \ qt/bantablemodel.h \ qt/bitcoinaddressvalidator.h \ qt/bitcoinamountfield.h \ qt/bitcoingui.h \ qt/bitcoinunits.h \ qt/clientmodel.h \ qt/coincontroldialog.h \ qt/coincontroltreewidget.h \ qt/csvmodelwriter.h \ qt/editaddressdialog.h \ qt/guiconstants.h \ qt/guiutil.h \ qt/intro.h \ qt/macdockiconhandler.h \ qt/macnotificationhandler.h \ qt/modaloverlay.h \ qt/networkstyle.h \ qt/notificator.h \ qt/openuridialog.h \ qt/optionsdialog.h \ qt/optionsmodel.h \ qt/overviewpage.h \ qt/paymentrequestplus.h \ qt/paymentserver.h \ qt/peertablemodel.h \ qt/platformstyle.h \ qt/qvalidatedlineedit.h \ qt/qvaluecombobox.h \ qt/receivecoinsdialog.h \ qt/receiverequestdialog.h \ qt/recentrequeststablemodel.h \ qt/rpcconsole.h \ qt/sendcoinsdialog.h \ qt/sendcoinsentry.h \ qt/signverifymessagedialog.h \ qt/splashscreen.h \ qt/trafficgraphwidget.h \ qt/transactiondesc.h \ qt/transactiondescdialog.h \ qt/transactionfilterproxy.h \ qt/transactionrecord.h \ qt/transactiontablemodel.h \ qt/transactionview.h \ qt/utilitydialog.h \ qt/walletframe.h \ qt/walletmodel.h \ qt/walletmodeltransaction.h \ qt/walletview.h \ qt/winshutdownmonitor.h RES_ICONS = \ qt/res/icons/add.png \ qt/res/icons/address-book.png \ qt/res/icons/about.png \ qt/res/icons/about_qt.png \ qt/res/icons/bitcoin.ico \ qt/res/icons/bitcoin_testnet.ico \ qt/res/icons/bitcoin.png \ qt/res/icons/chevron.png \ qt/res/icons/clock1.png \ qt/res/icons/clock2.png \ qt/res/icons/clock3.png \ qt/res/icons/clock4.png \ qt/res/icons/clock5.png \ qt/res/icons/configure.png \ qt/res/icons/connect0.png \ qt/res/icons/connect1.png \ qt/res/icons/connect2.png \ qt/res/icons/connect3.png \ qt/res/icons/connect4.png \ qt/res/icons/debugwindow.png \ qt/res/icons/edit.png \ qt/res/icons/editcopy.png \ qt/res/icons/editpaste.png \ qt/res/icons/export.png \ qt/res/icons/eye.png \ qt/res/icons/eye_minus.png \ qt/res/icons/eye_plus.png \ qt/res/icons/filesave.png \ qt/res/icons/fontbigger.png \ qt/res/icons/fontsmaller.png \ qt/res/icons/hd_disabled.png \ qt/res/icons/hd_enabled.png \ qt/res/icons/history.png \ qt/res/icons/info.png \ qt/res/icons/key.png \ qt/res/icons/lock_closed.png \ qt/res/icons/lock_open.png \ qt/res/icons/network_disabled.png \ qt/res/icons/open.png \ qt/res/icons/overview.png \ qt/res/icons/quit.png \ qt/res/icons/receive.png \ qt/res/icons/remove.png \ qt/res/icons/send.png \ qt/res/icons/synced.png \ qt/res/icons/transaction0.png \ qt/res/icons/transaction2.png \ qt/res/icons/transaction_abandoned.png \ qt/res/icons/transaction_conflicted.png \ qt/res/icons/tx_inout.png \ qt/res/icons/tx_input.png \ qt/res/icons/tx_output.png \ qt/res/icons/tx_mined.png \ qt/res/icons/warning.png \ qt/res/icons/verify.png BITCOIN_QT_BASE_CPP = \ qt/bantablemodel.cpp \ qt/bitcoinaddressvalidator.cpp \ qt/bitcoinamountfield.cpp \ qt/bitcoingui.cpp \ qt/bitcoinunits.cpp \ qt/clientmodel.cpp \ qt/csvmodelwriter.cpp \ qt/guiutil.cpp \ qt/intro.cpp \ qt/modaloverlay.cpp \ qt/networkstyle.cpp \ qt/notificator.cpp \ qt/optionsdialog.cpp \ qt/optionsmodel.cpp \ qt/peertablemodel.cpp \ qt/platformstyle.cpp \ qt/qvalidatedlineedit.cpp \ qt/qvaluecombobox.cpp \ qt/rpcconsole.cpp \ qt/splashscreen.cpp \ qt/trafficgraphwidget.cpp \ qt/utilitydialog.cpp BITCOIN_QT_WINDOWS_CPP = qt/winshutdownmonitor.cpp BITCOIN_QT_WALLET_CPP = \ qt/addressbookpage.cpp \ qt/addresstablemodel.cpp \ qt/askpassphrasedialog.cpp \ qt/coincontroldialog.cpp \ qt/coincontroltreewidget.cpp \ qt/editaddressdialog.cpp \ qt/openuridialog.cpp \ qt/overviewpage.cpp \ qt/paymentrequestplus.cpp \ qt/paymentserver.cpp \ qt/receivecoinsdialog.cpp \ qt/receiverequestdialog.cpp \ qt/recentrequeststablemodel.cpp \ qt/sendcoinsdialog.cpp \ qt/sendcoinsentry.cpp \ qt/signverifymessagedialog.cpp \ qt/transactiondesc.cpp \ qt/transactiondescdialog.cpp \ qt/transactionfilterproxy.cpp \ qt/transactionrecord.cpp \ qt/transactiontablemodel.cpp \ qt/transactionview.cpp \ qt/walletframe.cpp \ qt/walletmodel.cpp \ qt/walletmodeltransaction.cpp \ qt/walletview.cpp BITCOIN_QT_CPP = $(BITCOIN_QT_BASE_CPP) if TARGET_WINDOWS BITCOIN_QT_CPP += $(BITCOIN_QT_WINDOWS_CPP) endif if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) endif RES_IMAGES = RES_MOVIES = $(wildcard $(srcdir)/qt/res/movies/spinner-*.png) BITCOIN_RC = qt/res/bitcoin-qt-res.rc BITCOIN_QT_INCLUDES = -I$(builddir)/qt -I$(srcdir)/qt -I$(srcdir)/qt/forms \ -I$(builddir)/qt/forms -DQT_NO_KEYWORDS qt_libbitcoinqt_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(PROTOBUF_CFLAGS) $(QR_CFLAGS) qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(PROTOBUF_PROTO) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) nodist_qt_libbitcoinqt_a_SOURCES = $(QT_MOC_CPP) $(QT_MOC) $(PROTOBUF_CC) \ $(PROTOBUF_H) $(QT_QRC_CPP) $(QT_QRC_LOCALE_CPP) # forms/foo.h -> forms/ui_foo.h QT_FORMS_H=$(join $(dir $(QT_FORMS_UI)),$(addprefix ui_, $(notdir $(QT_FORMS_UI:.ui=.h)))) # Most files will depend on the forms and moc files as includes. Generate them # before anything else. $(QT_MOC): $(QT_FORMS_H) $(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) : | $(QT_MOC) #Generating these with a half-written protobuf header leads to wacky results. #This makes sure it's done. $(QT_MOC): $(PROTOBUF_H) $(QT_MOC_CPP): $(PROTOBUF_H) # bitcoin-qt binary # qt_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(PROTOBUF_CFLAGS) $(QR_CFLAGS) qt_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_bitcoin_qt_SOURCES = qt/bitcoin.cpp if TARGET_DARWIN qt_bitcoin_qt_SOURCES += $(BITCOIN_MM) endif if TARGET_WINDOWS qt_bitcoin_qt_SOURCES += $(BITCOIN_RC) endif qt_bitcoin_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) if ENABLE_WALLET qt_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET) endif if ENABLE_ZMQ qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ +qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_bitcoin_qt_LIBTOOLFLAGS = --tag CXX #locale/foo.ts -> locale/foo.qm QT_QM=$(QT_TS:.ts=.qm) SECONDARY: $(QT_QM) $(srcdir)/qt/bitcoinstrings.cpp: $(libbitcoin_server_a_SOURCES) $(libbitcoin_wallet_a_SOURCES) $(libbitcoin_common_a_SOURCES) $(libbitcoin_zmq_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) $(libbitcoin_util_a_SOURCES) @test -n $(XGETTEXT) || echo "xgettext is required for updating translations" $(AM_V_GEN) cd $(srcdir); XGETTEXT=$(XGETTEXT) PACKAGE_NAME="$(PACKAGE_NAME)" COPYRIGHT_HOLDERS="$(COPYRIGHT_HOLDERS)" COPYRIGHT_HOLDERS_SUBSTITUTION="$(COPYRIGHT_HOLDERS_SUBSTITUTION)" $(PYTHON) ../share/qt/extract_strings_qt.py $^ translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCOIN_QT_BASE_CPP) qt/bitcoin.cpp $(BITCOIN_QT_WINDOWS_CPP) $(BITCOIN_QT_WALLET_CPP) $(BITCOIN_QT_H) $(BITCOIN_MM) @test -n $(LUPDATE) || echo "lupdate is required for updating translations" $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LUPDATE) $^ -locations relative -no-obsolete -ts $(srcdir)/qt/locale/bitcoin_en.ts $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) @test -f $(RCC) @cp -f $< $(@D)/temp_$( $@ @rm $(@D)/temp_$( $@ CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno CLEANFILES += $(CLEAN_QT) bitcoin_qt_clean: FORCE rm -f $(CLEAN_QT) $(qt_libbitcoinqt_a_OBJECTS) $(qt_bitcoin_qt_OBJECTS) qt/bitcoin-qt$(EXEEXT) $(LIBBITCOINQT) bitcoin_qt : qt/bitcoin-qt$(EXEEXT) ui_%.h: %.ui @test -f $(UIC) @$(MKDIR_P) $(@D) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(UIC) -o $@ $< || (echo "Error creating $@"; false) %.moc: %.cpp $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES) $(MOC_DEFS) $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ moc_%.cpp: %.h $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES) $(MOC_DEFS) $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ %.qm: %.ts @test -f $(LRELEASE) @$(MKDIR_P) $(@D) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LRELEASE) -silent $< -qm $@ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 039f8ac54..82e5d4bf5 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -1,64 +1,64 @@ # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. bin_PROGRAMS += qt/test/test_bitcoin-qt TESTS += qt/test/test_bitcoin-qt TEST_QT_MOC_CPP = \ qt/test/moc_compattests.cpp \ qt/test/moc_rpcnestedtests.cpp \ qt/test/moc_uritests.cpp if ENABLE_WALLET TEST_QT_MOC_CPP += qt/test/moc_paymentservertests.cpp endif TEST_QT_H = \ qt/test/compattests.h \ qt/test/rpcnestedtests.h \ qt/test/uritests.h \ qt/test/paymentrequestdata.h \ qt/test/paymentservertests.h qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(PROTOBUF_CFLAGS) qt_test_test_bitcoin_qt_SOURCES = \ qt/test/compattests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ qt/test/uritests.cpp \ $(TEST_QT_H) if ENABLE_WALLET qt_test_test_bitcoin_qt_SOURCES += \ qt/test/paymentservertests.cpp endif nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER) if ENABLE_WALLET qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET) endif if ENABLE_ZMQ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ - $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ + $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_test_test_bitcoin_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) CLEAN_BITCOIN_QT_TEST = $(TEST_QT_MOC_CPP) qt/test/*.gcda qt/test/*.gcno CLEANFILES += $(CLEAN_BITCOIN_QT_TEST) test_bitcoin_qt : qt/test/test_bitcoin-qt$(EXEEXT) test_bitcoin_qt_check : qt/test/test_bitcoin-qt$(EXEEXT) FORCE $(MAKE) check-TESTS TESTS=$^ test_bitcoin_qt_clean: FORCE rm -f $(CLEAN_BITCOIN_QT_TEST) $(qt_test_test_bitcoin_qt_OBJECTS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b494d5d73..96beb7186 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -1,220 +1,221 @@ # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. TESTS += test/test_bitcoin LOG_DRIVER = $(srcdir)/test/test-bitcoin-driver EXTRA_DIST += test/test-bitcoin-driver bin_PROGRAMS += test/test_bitcoin noinst_PROGRAMS += test/test_bitcoin_fuzzy TEST_SRCDIR = test TEST_BINARY=test/test_bitcoin$(EXEEXT) EXTRA_DIST += \ test/bctest.py \ test/bitcoin-util-test.py \ test/data/bitcoin-util-test.json \ test/data/blanktxv1.hex \ test/data/blanktxv1.json \ test/data/blanktxv2.hex \ test/data/blanktxv2.json \ test/data/tt-delin1-out.hex \ test/data/tt-delin1-out.json \ test/data/tt-delout1-out.hex \ test/data/tt-delout1-out.json \ test/data/tt-locktime317000-out.hex \ test/data/tt-locktime317000-out.json \ test/data/tx394b54bb.hex \ test/data/txcreate1.hex \ test/data/txcreate1.json \ test/data/txcreate2.hex \ test/data/txcreate2.json \ test/data/txcreatedata1.hex \ test/data/txcreatedata1.json \ test/data/txcreatedata2.hex \ test/data/txcreatedata2.json \ test/data/txcreatedata_seq0.hex \ test/data/txcreatedata_seq0.json \ test/data/txcreatedata_seq1.hex \ test/data/txcreatedata_seq1.json \ test/data/txcreatemultisig1.hex \ test/data/txcreatemultisig1.json \ test/data/txcreatemultisig2.hex \ test/data/txcreatemultisig2.json \ test/data/txcreateoutpubkey1.hex \ test/data/txcreateoutpubkey1.json \ test/data/txcreatescript1.hex \ test/data/txcreatescript1.json \ test/data/txcreatescript2.hex \ test/data/txcreatescript2.json \ test/data/txcreatesignv1.hex \ test/data/txcreatesignv1.json \ test/data/txcreatesignv2.hex JSON_TEST_FILES = \ test/data/script_tests.json \ test/data/base58_keys_valid.json \ test/data/base58_encode_decode.json \ test/data/base58_keys_invalid.json \ test/data/tx_invalid.json \ test/data/tx_valid.json \ test/data/sighash.json RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) # test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ test/scriptnum10.h \ test/addrman_tests.cpp \ test/amount_tests.cpp \ test/allocator_tests.cpp \ test/base32_tests.cpp \ test/base58_tests.cpp \ test/base64_tests.cpp \ test/bip32_tests.cpp \ test/blockcheck_tests.cpp \ test/blockencodings_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/coins_tests.cpp \ test/compress_tests.cpp \ test/config_tests.cpp \ test/crypto_tests.cpp \ test/cuckoocache_tests.cpp \ test/DoS_tests.cpp \ test/excessiveblock_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ test/inv_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ test/dbwrapper_tests.cpp \ test/main_tests.cpp \ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/miner_tests.cpp \ test/multisig_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ test/pmt_tests.cpp \ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_antireplay_tests.cpp \ test/script_P2SH_tests.cpp \ test/script_tests.cpp \ test/scriptflags.cpp \ test/scriptflags.h \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ test/sighash_tests.cpp \ test/sigopcount_tests.cpp \ test/sigutil.cpp \ test/sigutil.h \ test/skiplist_tests.cpp \ test/streams_tests.cpp \ test/test_bitcoin.cpp \ test/test_bitcoin.h \ test/test_random.h \ test/testutil.cpp \ test/testutil.h \ test/timedata_tests.cpp \ test/transaction_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ test/uint256_tests.cpp \ test/undo_tests.cpp \ test/univalue_tests.cpp \ test/util_tests.cpp \ test/validation_tests.cpp if ENABLE_WALLET BITCOIN_TESTS += \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ wallet/test/accounting_tests.cpp \ wallet/test/wallet_tests.cpp \ wallet/test/crypto_tests.cpp endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) $(EVENT_CFLAGS) -test_test_bitcoin_LDADD = $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ - $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) +test_test_bitcoin_LDADD = $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ + $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) + test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif test_test_bitcoin_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ test_test_bitcoin_LDADD += $(ZMQ_LIBS) endif # # test_bitcoin_fuzzy binary # test_test_bitcoin_fuzzy_SOURCES = test/test_bitcoin_fuzzy.cpp test_test_bitcoin_fuzzy_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_test_bitcoin_fuzzy_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_fuzzy_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_test_bitcoin_fuzzy_LDADD = \ $(LIBUNIVALUE) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBSECP256K1) test_test_bitcoin_fuzzy_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) # nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) $(BITCOIN_TESTS): $(GENERATED_TEST_FILES) CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno $(GENERATED_TEST_FILES) CLEANFILES += $(CLEAN_BITCOIN_TEST) # This file is problematic for out-of-tree builds if it exists. DISTCLEANFILES += test/buildenv.pyc bitcoin_test: $(TEST_BINARY) bitcoin_test_check: $(TEST_BINARY) FORCE $(MAKE) check-TESTS TESTS=$^ bitcoin_test_clean : FORCE rm -f $(CLEAN_BITCOIN_TEST) $(test_test_bitcoin_OBJECTS) $(TEST_BINARY) check-local: @echo "Running test/bitcoin-util-test.py..." $(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(PYTHON) $(srcdir)/test/bitcoin-util-test.py $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check if EMBEDDED_UNIVALUE $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check endif %.json.h: %.json @$(MKDIR_P) $(@D) @{ \ echo "namespace json_tests{" && \ echo "static unsigned const char $(*F)[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" diff --git a/src/leveldb/Makefile b/src/leveldb/Makefile index 07a5a1ead..f7cc7d736 100644 --- a/src/leveldb/Makefile +++ b/src/leveldb/Makefile @@ -1,414 +1,424 @@ # Copyright (c) 2011 The LevelDB Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. See the AUTHORS file for names of contributors. #----------------------------------------------- # Uncomment exactly one of the lines labelled (A), (B), and (C) below # to switch between compilation modes. # (A) Production use (optimized mode) OPT ?= -O2 -DNDEBUG # (B) Debug mode, w/ full line-level debugging symbols # OPT ?= -g2 # (C) Profiling mode: opt, but w/debugging symbols # OPT ?= -O2 -g2 -DNDEBUG #----------------------------------------------- # detect what platform we're building on $(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \ ./build_detect_platform build_config.mk ./) # this file is generated by the previous line to set build flags and sources include build_config.mk TESTS = \ db/autocompact_test \ db/c_test \ db/corruption_test \ db/db_test \ db/dbformat_test \ db/fault_injection_test \ db/filename_test \ db/log_test \ db/recovery_test \ db/skiplist_test \ db/version_edit_test \ db/version_set_test \ db/write_batch_test \ helpers/memenv/memenv_test \ issues/issue178_test \ issues/issue200_test \ table/filter_block_test \ table/table_test \ util/arena_test \ util/bloom_test \ util/cache_test \ util/coding_test \ util/crc32c_test \ + util/env_posix_test \ util/env_test \ util/hash_test UTILS = \ db/db_bench \ db/leveldbutil # Put the object files in a subdirectory, but the application at the top of the object dir. PROGNAMES := $(notdir $(TESTS) $(UTILS)) # On Linux may need libkyotocabinet-dev for dependency. BENCHMARKS = \ doc/bench/db_bench_sqlite3 \ doc/bench/db_bench_tree_db CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) LDFLAGS += $(PLATFORM_LDFLAGS) LIBS += $(PLATFORM_LIBS) SIMULATOR_OUTDIR=out-ios-x86 DEVICE_OUTDIR=out-ios-arm ifeq ($(PLATFORM), IOS) # Note: iOS should probably be using libtool, not ar. AR=xcrun ar SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path) DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path) DEVICE_CFLAGS = -isysroot "$(DEVICESDK)" -arch armv6 -arch armv7 -arch armv7s -arch arm64 SIMULATOR_CFLAGS = -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64 STATIC_OUTDIR=out-ios-universal else STATIC_OUTDIR=out-static SHARED_OUTDIR=out-shared STATIC_PROGRAMS := $(addprefix $(STATIC_OUTDIR)/, $(PROGNAMES)) SHARED_PROGRAMS := $(addprefix $(SHARED_OUTDIR)/, db_bench) endif STATIC_LIBOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(SOURCES:.cc=.o)) STATIC_MEMENVOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) DEVICE_LIBOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(SOURCES:.cc=.o)) DEVICE_MEMENVOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) SIMULATOR_LIBOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(SOURCES:.cc=.o)) SIMULATOR_MEMENVOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) SHARED_LIBOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(SOURCES:.cc=.o)) SHARED_MEMENVOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o)) TESTUTIL := $(STATIC_OUTDIR)/util/testutil.o TESTHARNESS := $(STATIC_OUTDIR)/util/testharness.o $(TESTUTIL) STATIC_TESTOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(TESTS))) STATIC_UTILOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(UTILS))) STATIC_ALLOBJS := $(STATIC_LIBOBJECTS) $(STATIC_MEMENVOBJECTS) $(STATIC_TESTOBJS) $(STATIC_UTILOBJS) $(TESTHARNESS) DEVICE_ALLOBJS := $(DEVICE_LIBOBJECTS) $(DEVICE_MEMENVOBJECTS) SIMULATOR_ALLOBJS := $(SIMULATOR_LIBOBJECTS) $(SIMULATOR_MEMENVOBJECTS) default: all # Should we build shared libraries? ifneq ($(PLATFORM_SHARED_EXT),) # Many leveldb test apps use non-exported API's. Only build a subset for testing. SHARED_ALLOBJS := $(SHARED_LIBOBJECTS) $(SHARED_MEMENVOBJECTS) $(TESTHARNESS) ifneq ($(PLATFORM_SHARED_VERSIONED),true) SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED_LIB2 = $(SHARED_LIB1) SHARED_LIB3 = $(SHARED_LIB1) SHARED_LIBS = $(SHARED_LIB1) SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a else # Update db.h if you change these. SHARED_VERSION_MAJOR = 1 -SHARED_VERSION_MINOR = 19 +SHARED_VERSION_MINOR = 20 SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED_LIB2 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR) SHARED_LIB3 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR).$(SHARED_VERSION_MINOR) SHARED_LIBS = $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2) $(SHARED_OUTDIR)/$(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1): $(SHARED_OUTDIR)/$(SHARED_LIB3) ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2): $(SHARED_OUTDIR)/$(SHARED_LIB3) ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB2) SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a endif $(SHARED_OUTDIR)/$(SHARED_LIB3): $(SHARED_LIBOBJECTS) $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED_LIB2) $(SHARED_LIBOBJECTS) -o $(SHARED_OUTDIR)/$(SHARED_LIB3) $(LIBS) endif # PLATFORM_SHARED_EXT all: $(SHARED_LIBS) $(SHARED_PROGRAMS) $(STATIC_OUTDIR)/libleveldb.a $(STATIC_OUTDIR)/libmemenv.a $(STATIC_PROGRAMS) check: $(STATIC_PROGRAMS) for t in $(notdir $(TESTS)); do echo "***** Running $$t"; $(STATIC_OUTDIR)/$$t || exit 1; done clean: -rm -rf out-static out-shared out-ios-x86 out-ios-arm out-ios-universal -rm -f build_config.mk -rm -rf ios-x86 ios-arm $(STATIC_OUTDIR): mkdir $@ $(STATIC_OUTDIR)/db: | $(STATIC_OUTDIR) mkdir $@ $(STATIC_OUTDIR)/helpers/memenv: | $(STATIC_OUTDIR) mkdir -p $@ $(STATIC_OUTDIR)/port: | $(STATIC_OUTDIR) mkdir $@ $(STATIC_OUTDIR)/table: | $(STATIC_OUTDIR) mkdir $@ $(STATIC_OUTDIR)/util: | $(STATIC_OUTDIR) mkdir $@ .PHONY: STATIC_OBJDIRS STATIC_OBJDIRS: \ $(STATIC_OUTDIR)/db \ $(STATIC_OUTDIR)/port \ $(STATIC_OUTDIR)/table \ $(STATIC_OUTDIR)/util \ $(STATIC_OUTDIR)/helpers/memenv $(SHARED_OUTDIR): mkdir $@ $(SHARED_OUTDIR)/db: | $(SHARED_OUTDIR) mkdir $@ $(SHARED_OUTDIR)/helpers/memenv: | $(SHARED_OUTDIR) mkdir -p $@ $(SHARED_OUTDIR)/port: | $(SHARED_OUTDIR) mkdir $@ $(SHARED_OUTDIR)/table: | $(SHARED_OUTDIR) mkdir $@ $(SHARED_OUTDIR)/util: | $(SHARED_OUTDIR) mkdir $@ .PHONY: SHARED_OBJDIRS SHARED_OBJDIRS: \ $(SHARED_OUTDIR)/db \ $(SHARED_OUTDIR)/port \ $(SHARED_OUTDIR)/table \ $(SHARED_OUTDIR)/util \ $(SHARED_OUTDIR)/helpers/memenv $(DEVICE_OUTDIR): mkdir $@ $(DEVICE_OUTDIR)/db: | $(DEVICE_OUTDIR) mkdir $@ $(DEVICE_OUTDIR)/helpers/memenv: | $(DEVICE_OUTDIR) mkdir -p $@ $(DEVICE_OUTDIR)/port: | $(DEVICE_OUTDIR) mkdir $@ $(DEVICE_OUTDIR)/table: | $(DEVICE_OUTDIR) mkdir $@ $(DEVICE_OUTDIR)/util: | $(DEVICE_OUTDIR) mkdir $@ .PHONY: DEVICE_OBJDIRS DEVICE_OBJDIRS: \ $(DEVICE_OUTDIR)/db \ $(DEVICE_OUTDIR)/port \ $(DEVICE_OUTDIR)/table \ $(DEVICE_OUTDIR)/util \ $(DEVICE_OUTDIR)/helpers/memenv $(SIMULATOR_OUTDIR): mkdir $@ $(SIMULATOR_OUTDIR)/db: | $(SIMULATOR_OUTDIR) mkdir $@ $(SIMULATOR_OUTDIR)/helpers/memenv: | $(SIMULATOR_OUTDIR) mkdir -p $@ $(SIMULATOR_OUTDIR)/port: | $(SIMULATOR_OUTDIR) mkdir $@ $(SIMULATOR_OUTDIR)/table: | $(SIMULATOR_OUTDIR) mkdir $@ $(SIMULATOR_OUTDIR)/util: | $(SIMULATOR_OUTDIR) mkdir $@ .PHONY: SIMULATOR_OBJDIRS SIMULATOR_OBJDIRS: \ $(SIMULATOR_OUTDIR)/db \ $(SIMULATOR_OUTDIR)/port \ $(SIMULATOR_OUTDIR)/table \ $(SIMULATOR_OUTDIR)/util \ $(SIMULATOR_OUTDIR)/helpers/memenv $(STATIC_ALLOBJS): | STATIC_OBJDIRS $(DEVICE_ALLOBJS): | DEVICE_OBJDIRS $(SIMULATOR_ALLOBJS): | SIMULATOR_OBJDIRS $(SHARED_ALLOBJS): | SHARED_OBJDIRS ifeq ($(PLATFORM), IOS) $(DEVICE_OUTDIR)/libleveldb.a: $(DEVICE_LIBOBJECTS) rm -f $@ $(AR) -rs $@ $(DEVICE_LIBOBJECTS) $(SIMULATOR_OUTDIR)/libleveldb.a: $(SIMULATOR_LIBOBJECTS) rm -f $@ $(AR) -rs $@ $(SIMULATOR_LIBOBJECTS) $(DEVICE_OUTDIR)/libmemenv.a: $(DEVICE_MEMENVOBJECTS) rm -f $@ $(AR) -rs $@ $(DEVICE_MEMENVOBJECTS) $(SIMULATOR_OUTDIR)/libmemenv.a: $(SIMULATOR_MEMENVOBJECTS) rm -f $@ $(AR) -rs $@ $(SIMULATOR_MEMENVOBJECTS) # For iOS, create universal object libraries to be used on both the simulator and # a device. $(STATIC_OUTDIR)/libleveldb.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a lipo -create $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a -output $@ $(STATIC_OUTDIR)/libmemenv.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a lipo -create $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a -output $@ else $(STATIC_OUTDIR)/libleveldb.a:$(STATIC_LIBOBJECTS) rm -f $@ $(AR) -rs $@ $(STATIC_LIBOBJECTS) $(STATIC_OUTDIR)/libmemenv.a:$(STATIC_MEMENVOBJECTS) rm -f $@ $(AR) -rs $@ $(STATIC_MEMENVOBJECTS) endif $(SHARED_MEMENVLIB):$(SHARED_MEMENVOBJECTS) rm -f $@ $(AR) -rs $@ $(SHARED_MEMENVOBJECTS) $(STATIC_OUTDIR)/db_bench:db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS) $(STATIC_OUTDIR)/db_bench_sqlite3:doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) $(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS) $(STATIC_OUTDIR)/db_bench_tree_db:doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) $(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS) $(STATIC_OUTDIR)/leveldbutil:db/leveldbutil.cc $(STATIC_LIBOBJECTS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/leveldbutil.cc $(STATIC_LIBOBJECTS) -o $@ $(LIBS) $(STATIC_OUTDIR)/arena_test:util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/autocompact_test:db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/bloom_test:util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/c_test:$(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/cache_test:util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/coding_test:util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/corruption_test:db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/crc32c_test:util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/db_test:db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/dbformat_test:db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) +$(STATIC_OUTDIR)/env_posix_test:util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) + $(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_posix_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) + $(STATIC_OUTDIR)/env_test:util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/fault_injection_test:db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/filename_test:db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/filter_block_test:table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/hash_test:util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/issue178_test:issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/issue200_test:issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/log_test:db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/recovery_test:db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/table_test:table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/skiplist_test:db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/version_edit_test:db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/version_set_test:db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/write_batch_test:db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) $(CXX) $(LDFLAGS) $(CXXFLAGS) db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(STATIC_OUTDIR)/memenv_test:$(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) $(XCRUN) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) -o $@ $(LIBS) $(SHARED_OUTDIR)/db_bench:$(SHARED_OUTDIR)/db/db_bench.o $(SHARED_LIBS) $(TESTUTIL) $(XCRUN) $(CXX) $(LDFLAGS) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SHARED_OUTDIR)/db/db_bench.o $(TESTUTIL) $(SHARED_OUTDIR)/$(SHARED_LIB3) -o $@ $(LIBS) .PHONY: run-shared run-shared: $(SHARED_OUTDIR)/db_bench LD_LIBRARY_PATH=$(SHARED_OUTDIR) $(SHARED_OUTDIR)/db_bench $(SIMULATOR_OUTDIR)/%.o: %.cc xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@ $(DEVICE_OUTDIR)/%.o: %.cc xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) $(DEVICE_CFLAGS) -c $< -o $@ $(SIMULATOR_OUTDIR)/%.o: %.c xcrun -sdk iphonesimulator $(CC) $(CFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@ $(DEVICE_OUTDIR)/%.o: %.c xcrun -sdk iphoneos $(CC) $(CFLAGS) $(DEVICE_CFLAGS) -c $< -o $@ $(STATIC_OUTDIR)/%.o: %.cc $(CXX) $(CXXFLAGS) -c $< -o $@ $(STATIC_OUTDIR)/%.o: %.c $(CC) $(CFLAGS) -c $< -o $@ $(SHARED_OUTDIR)/%.o: %.cc $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@ $(SHARED_OUTDIR)/%.o: %.c $(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@ + +$(STATIC_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc + $(CXX) $(CXXFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@ + +$(SHARED_OUTDIR)/port/port_posix_sse.o: port/port_posix_sse.cc + $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(PLATFORM_SSEFLAGS) -c $< -o $@ diff --git a/src/leveldb/README.md b/src/leveldb/README.md index c75b185e0..a010c5085 100644 --- a/src/leveldb/README.md +++ b/src/leveldb/README.md @@ -1,173 +1,174 @@ **LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.** [![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb) Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) # Features * Keys and values are arbitrary byte arrays. * Data is stored sorted by key. * Callers can provide a custom comparison function to override the sort order. * The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`. * Multiple changes can be made in one atomic batch. * Users can create a transient snapshot to get a consistent view of data. * Forward and backward iteration is supported over the data. * Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/). * External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions. # Documentation - [LevelDB library documentation](https://rawgit.com/google/leveldb/master/doc/index.html) is online and bundled with the source code. + [LevelDB library documentation](https://github.com/google/leveldb/blob/master/doc/index.md) is online and bundled with the source code. # Limitations * This is not a SQL database. It does not have a relational data model, it does not support SQL queries, and it has no support for indexes. * Only a single process (possibly multi-threaded) can access a particular database at a time. * There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library. # Contributing to the leveldb Project The leveldb project welcomes contributions. leveldb's primary goal is to be a reliable and fast key/value store. Changes that are in line with the features/limitations outlined above, and meet the requirements below, will be considered. Contribution requirements: 1. **POSIX only**. We _generally_ will only accept changes that are both compiled, and tested on a POSIX platform - usually Linux. Very small changes will sometimes be accepted, but consider that more of an exception than the rule. 2. **Stable API**. We strive very hard to maintain a stable API. Changes that require changes for projects using leveldb _might_ be rejected without sufficient benefit to the project. 3. **Tests**: All changes must be accompanied by a new (or changed) test, or a sufficient explanation as to why a new (or changed) test is not required. ## Submitting a Pull Request Before any pull request will be accepted the author must first sign a Contributor License Agreement (CLA) at https://cla.developers.google.com/. In order to keep the commit timeline linear [squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits) your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase) on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed with the internal repository at Google. More information at GitHub's [About Git rebase](https://help.github.com/articles/about-git-rebase/) page. # Performance Here is a performance report (with explanations) from the run of the included db_bench program. The results are somewhat noisy, but should be enough to get a ballpark performance estimate. ## Setup We use a database with a million entries. Each entry has a 16 byte key, and a 100 byte value. Values used by the benchmark compress to about half their original size. LevelDB: version 1.1 Date: Sun May 1 12:11:26 2011 CPU: 4 x Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz CPUCache: 4096 KB Keys: 16 bytes each Values: 100 bytes each (50 bytes after compression) Entries: 1000000 Raw Size: 110.6 MB (estimated) File Size: 62.9 MB (estimated) ## Write performance The "fill" benchmarks create a brand new database, in either sequential, or random order. The "fillsync" benchmark flushes data from the operating system to the disk after every operation; the other write operations leave the data sitting in the operating system buffer cache for a while. The "overwrite" benchmark does random writes that update existing keys in the database. fillseq : 1.765 micros/op; 62.7 MB/s fillsync : 268.409 micros/op; 0.4 MB/s (10000 ops) fillrandom : 2.460 micros/op; 45.0 MB/s overwrite : 2.380 micros/op; 46.5 MB/s Each "op" above corresponds to a write of a single key/value pair. I.e., a random write benchmark goes at approximately 400,000 writes per second. Each "fillsync" operation costs much less (0.3 millisecond) than a disk seek (typically 10 milliseconds). We suspect that this is because the hard disk itself is buffering the update in its memory and responding before the data has been written to the platter. This may or may not be safe based on whether or not the hard disk has enough power to save its memory in the event of a power failure. ## Read performance We list the performance of reading sequentially in both the forward and reverse direction, and also the performance of a random lookup. Note that the database created by the benchmark is quite small. Therefore the report characterizes the performance of leveldb when the working set fits in memory. The cost of reading a piece of data that is not present in the operating system buffer cache will be dominated by the one or two disk seeks needed to fetch the data from disk. Write performance will be mostly unaffected by whether or not the working set fits in memory. - readrandom : 16.677 micros/op; (approximately 60,000 reads per second) - readseq : 0.476 micros/op; 232.3 MB/s - readreverse : 0.724 micros/op; 152.9 MB/s + readrandom : 16.677 micros/op; (approximately 60,000 reads per second) + readseq : 0.476 micros/op; 232.3 MB/s + readreverse : 0.724 micros/op; 152.9 MB/s LevelDB compacts its underlying storage data in the background to improve read performance. The results listed above were done immediately after a lot of random writes. The results after compactions (which are usually triggered automatically) are better. - readrandom : 11.602 micros/op; (approximately 85,000 reads per second) - readseq : 0.423 micros/op; 261.8 MB/s - readreverse : 0.663 micros/op; 166.9 MB/s + readrandom : 11.602 micros/op; (approximately 85,000 reads per second) + readseq : 0.423 micros/op; 261.8 MB/s + readreverse : 0.663 micros/op; 166.9 MB/s Some of the high cost of reads comes from repeated decompression of blocks read from disk. If we supply enough cache to the leveldb so it can hold the uncompressed blocks in memory, the read performance improves again: - readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction) - readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction) + readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction) + readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction) ## Repository contents -See doc/index.html for more explanation. See doc/impl.html for a brief overview of the implementation. +See [doc/index.md](doc/index.md) for more explanation. See +[doc/impl.md](doc/impl.md) for a brief overview of the implementation. The public interface is in include/*.h. Callers should not include or rely on the details of any other header files in this package. Those internal APIs may be changed without warning. Guide to header files: * **include/db.h**: Main interface to the DB: Start here * **include/options.h**: Control over the behavior of an entire database, and also control over the behavior of individual reads and writes. -* **include/comparator.h**: Abstraction for user-specified comparison function. +* **include/comparator.h**: Abstraction for user-specified comparison function. If you want just bytewise comparison of keys, you can use the default comparator, but clients can write their own comparator implementations if they want custom ordering (e.g. to handle different character encodings, etc.) * **include/iterator.h**: Interface for iterating over data. You can get an iterator from a DB object. * **include/write_batch.h**: Interface for atomically applying multiple updates to a database. * **include/slice.h**: A simple module for maintaining a pointer and a length into some other byte array. * **include/status.h**: Status is returned from many of the public interfaces and is used to report success and various kinds of errors. -* **include/env.h**: +* **include/env.h**: Abstraction of the OS environment. A posix implementation of this interface is in util/env_posix.cc * **include/table.h, include/table_builder.h**: Lower-level modules that most clients probably won't use directly diff --git a/src/leveldb/build_detect_platform b/src/leveldb/build_detect_platform index d7edab1d8..4a9471590 100755 --- a/src/leveldb/build_detect_platform +++ b/src/leveldb/build_detect_platform @@ -1,231 +1,259 @@ #!/bin/sh # # Detects OS we're compiling on and outputs a file specified by the first # argument, which in turn gets read while processing Makefile. # # The output will set the following variables: # CC C Compiler path # CXX C++ Compiler path # PLATFORM_LDFLAGS Linker flags # PLATFORM_LIBS Libraries flags # PLATFORM_SHARED_EXT Extension for shared libraries # PLATFORM_SHARED_LDFLAGS Flags for building shared library # This flag is embedded just before the name # of the shared library without intervening spaces # PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library # PLATFORM_CCFLAGS C compiler flags # PLATFORM_CXXFLAGS C++ compiler flags. Will contain: # PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned # shared libraries, empty otherwise. # # The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following: # # -DLEVELDB_ATOMIC_PRESENT if is present # -DLEVELDB_PLATFORM_POSIX for Posix-based platforms # -DSNAPPY if the Snappy library is present # OUTPUT=$1 PREFIX=$2 if test -z "$OUTPUT" || test -z "$PREFIX"; then echo "usage: $0 " >&2 exit 1 fi # Delete existing output, if it exists rm -f $OUTPUT touch $OUTPUT if test -z "$CC"; then CC=cc fi if test -z "$CXX"; then CXX=g++ fi if test -z "$TMPDIR"; then TMPDIR=/tmp fi # Detect OS if test -z "$TARGET_OS"; then TARGET_OS=`uname -s` fi COMMON_FLAGS= CROSS_COMPILE= PLATFORM_CCFLAGS= PLATFORM_CXXFLAGS= PLATFORM_LDFLAGS= PLATFORM_LIBS= PLATFORM_SHARED_EXT="so" PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl," PLATFORM_SHARED_CFLAGS="-fPIC" PLATFORM_SHARED_VERSIONED=true +PLATFORM_SSEFLAGS= MEMCMP_FLAG= if [ "$CXX" = "g++" ]; then # Use libc's memcmp instead of GCC's memcmp. This results in ~40% # performance improvement on readrandom under gcc 4.4.3 on Linux/x86. MEMCMP_FLAG="-fno-builtin-memcmp" fi case "$TARGET_OS" in CYGWIN_*) PLATFORM=OS_LINUX COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN" PLATFORM_LDFLAGS="-lpthread" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; Darwin) PLATFORM=OS_MACOSX COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX" PLATFORM_SHARED_EXT=dylib [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd` PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; Linux) PLATFORM=OS_LINUX COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX" PLATFORM_LDFLAGS="-pthread" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; SunOS) PLATFORM=OS_SOLARIS COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS" PLATFORM_LIBS="-lpthread -lrt" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; FreeBSD) PLATFORM=OS_FREEBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD" PLATFORM_LIBS="-lpthread" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; GNU/kFreeBSD) PLATFORM=OS_KFREEBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_KFREEBSD" PLATFORM_LIBS="-lpthread" PORT_FILE=port/port_posix.cc ;; NetBSD) PLATFORM=OS_NETBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD" PLATFORM_LIBS="-lpthread -lgcc_s" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; OpenBSD) PLATFORM=OS_OPENBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD" PLATFORM_LDFLAGS="-pthread" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; DragonFly) PLATFORM=OS_DRAGONFLYBSD COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD" PLATFORM_LIBS="-lpthread" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc ;; OS_ANDROID_CROSSCOMPILE) PLATFORM=OS_ANDROID COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX" PLATFORM_LDFLAGS="" # All pthread features are in the Android C library PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc CROSS_COMPILE=true ;; HP-UX) PLATFORM=OS_HPUX COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX" PLATFORM_LDFLAGS="-pthread" PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc # man ld: +h internal_name PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl," ;; IOS) PLATFORM=IOS COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX" [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd` PORT_FILE=port/port_posix.cc + PORT_SSE_FILE=port/port_posix_sse.cc PLATFORM_SHARED_EXT= PLATFORM_SHARED_LDFLAGS= PLATFORM_SHARED_CFLAGS= PLATFORM_SHARED_VERSIONED= ;; OS_WINDOWS_CROSSCOMPILE | NATIVE_WINDOWS) PLATFORM=OS_WINDOWS COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_WINDOWS -DLEVELDB_PLATFORM_WINDOWS -DWINVER=0x0500 -D__USE_MINGW_ANSI_STDIO=1" PLATFORM_SOURCES="util/env_win.cc" PLATFORM_LIBS="-lshlwapi" PORT_FILE=port/port_win.cc CROSS_COMPILE=true ;; *) echo "Unknown platform!" >&2 exit 1 esac # We want to make a list of all cc files within util, db, table, and helpers # except for the test and benchmark files. By default, find will output a list # of all files matching either rule, so we need to append -print to make the # prune take effect. DIRS="$PREFIX/db $PREFIX/util $PREFIX/table" set -f # temporarily disable globbing so that our patterns aren't expanded PRUNE_TEST="-name *test*.cc -prune" PRUNE_BENCH="-name *_bench.cc -prune" PRUNE_TOOL="-name leveldbutil.cc -prune" PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "` set +f # re-enable globbing # The sources consist of the portable files, plus the platform-specific port # file. -echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT +echo "SOURCES=$PORTABLE_FILES $PORT_FILE $PORT_SSE_FILE" >> $OUTPUT echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT if [ "$CROSS_COMPILE" = "true" ]; then # Cross-compiling; do not try any compilation tests. true else CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$" # If -std=c++0x works, use as fallback for when memory barriers # are not available. $CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null < int main() {} EOF if [ "$?" = 0 ]; then COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT" PLATFORM_CXXFLAGS="-std=c++0x" else COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX" fi # Test whether tcmalloc is available $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null </dev/null + + # Test if gcc SSE 4.2 is supported + $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -msse4.2 2>/dev/null </dev/null +fi + +# Use the SSE 4.2 CRC32C intrinsics iff runtime checks indicate compiler supports them. +if [ -n "$PLATFORM_SSEFLAGS" ]; then + PLATFORM_SSEFLAGS="$PLATFORM_SSEFLAGS -DLEVELDB_PLATFORM_POSIX_SSE" fi PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" echo "CC=$CC" >> $OUTPUT echo "CXX=$CXX" >> $OUTPUT echo "PLATFORM=$PLATFORM" >> $OUTPUT echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT +echo "PLATFORM_SSEFLAGS=$PLATFORM_SSEFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT diff --git a/src/leveldb/db/db_bench.cc b/src/leveldb/db/db_bench.cc index 7a0f5e08c..3ad19a512 100644 --- a/src/leveldb/db/db_bench.cc +++ b/src/leveldb/db/db_bench.cc @@ -1,1000 +1,1020 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include #include #include #include "db/db_impl.h" #include "db/version_set.h" #include "leveldb/cache.h" #include "leveldb/db.h" #include "leveldb/env.h" #include "leveldb/write_batch.h" #include "port/port.h" #include "util/crc32c.h" #include "util/histogram.h" #include "util/mutexlock.h" #include "util/random.h" #include "util/testutil.h" // Comma-separated list of operations to run in the specified order // Actual benchmarks: // fillseq -- write N values in sequential key order in async mode // fillrandom -- write N values in random key order in async mode // overwrite -- overwrite N values in random key order in async mode // fillsync -- write N/100 values in random key order in sync mode // fill100K -- write N/1000 100K values in random order in async mode // deleteseq -- delete N keys in sequential order // deleterandom -- delete N keys in random order // readseq -- read N times sequentially // readreverse -- read N times in reverse order // readrandom -- read N times in random order // readmissing -- read N missing keys in random order // readhot -- read N times in random order from 1% section of DB // seekrandom -- N random seeks // open -- cost of opening a DB // crc32c -- repeated crc32c of 4K of data // acquireload -- load N*1000 times // Meta operations: // compact -- Compact the entire DB // stats -- Print DB stats // sstables -- Print sstable info // heapprofile -- Dump a heap profile (if supported by this port) static const char* FLAGS_benchmarks = "fillseq," "fillsync," "fillrandom," "overwrite," "readrandom," "readrandom," // Extra run to allow previous compactions to quiesce "readseq," "readreverse," "compact," "readrandom," "readseq," "readreverse," "fill100K," "crc32c," "snappycomp," "snappyuncomp," "acquireload," ; // Number of key/values to place in database static int FLAGS_num = 1000000; // Number of read operations to do. If negative, do FLAGS_num reads. static int FLAGS_reads = -1; // Number of concurrent threads to run. static int FLAGS_threads = 1; // Size of each value static int FLAGS_value_size = 100; // Arrange to generate values that shrink to this fraction of // their original size after compression static double FLAGS_compression_ratio = 0.5; // Print histogram of operation timings static bool FLAGS_histogram = false; // Number of bytes to buffer in memtable before compacting // (initialized to default value by "main") static int FLAGS_write_buffer_size = 0; +// Number of bytes written to each file. +// (initialized to default value by "main") +static int FLAGS_max_file_size = 0; + +// Approximate size of user data packed per block (before compression. +// (initialized to default value by "main") +static int FLAGS_block_size = 0; + // Number of bytes to use as a cache of uncompressed data. // Negative means use default settings. static int FLAGS_cache_size = -1; // Maximum number of files to keep open at the same time (use default if == 0) static int FLAGS_open_files = 0; // Bloom filter bits per key. // Negative means use default settings. static int FLAGS_bloom_bits = -1; // If true, do not destroy the existing database. If you set this // flag and also specify a benchmark that wants a fresh database, that // benchmark will fail. static bool FLAGS_use_existing_db = false; // If true, reuse existing log/MANIFEST files when re-opening a database. static bool FLAGS_reuse_logs = false; // Use the db with the following name. static const char* FLAGS_db = NULL; namespace leveldb { namespace { +leveldb::Env* g_env = NULL; // Helper for quickly generating random data. class RandomGenerator { private: std::string data_; int pos_; public: RandomGenerator() { // We use a limited amount of data over and over again and ensure // that it is larger than the compression window (32KB), and also // large enough to serve all typical value sizes we want to write. Random rnd(301); std::string piece; while (data_.size() < 1048576) { // Add a short fragment that is as compressible as specified // by FLAGS_compression_ratio. test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); data_.append(piece); } pos_ = 0; } Slice Generate(size_t len) { if (pos_ + len > data_.size()) { pos_ = 0; assert(len < data_.size()); } pos_ += len; return Slice(data_.data() + pos_ - len, len); } }; #if defined(__linux) static Slice TrimSpace(Slice s) { size_t start = 0; while (start < s.size() && isspace(s[start])) { start++; } size_t limit = s.size(); while (limit > start && isspace(s[limit-1])) { limit--; } return Slice(s.data() + start, limit - start); } #endif static void AppendWithSpace(std::string* str, Slice msg) { if (msg.empty()) return; if (!str->empty()) { str->push_back(' '); } str->append(msg.data(), msg.size()); } class Stats { private: double start_; double finish_; double seconds_; int done_; int next_report_; int64_t bytes_; double last_op_finish_; Histogram hist_; std::string message_; public: Stats() { Start(); } void Start() { next_report_ = 100; last_op_finish_ = start_; hist_.Clear(); done_ = 0; bytes_ = 0; seconds_ = 0; - start_ = Env::Default()->NowMicros(); + start_ = g_env->NowMicros(); finish_ = start_; message_.clear(); } void Merge(const Stats& other) { hist_.Merge(other.hist_); done_ += other.done_; bytes_ += other.bytes_; seconds_ += other.seconds_; if (other.start_ < start_) start_ = other.start_; if (other.finish_ > finish_) finish_ = other.finish_; // Just keep the messages from one thread if (message_.empty()) message_ = other.message_; } void Stop() { - finish_ = Env::Default()->NowMicros(); + finish_ = g_env->NowMicros(); seconds_ = (finish_ - start_) * 1e-6; } void AddMessage(Slice msg) { AppendWithSpace(&message_, msg); } void FinishedSingleOp() { if (FLAGS_histogram) { - double now = Env::Default()->NowMicros(); + double now = g_env->NowMicros(); double micros = now - last_op_finish_; hist_.Add(micros); if (micros > 20000) { fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); fflush(stderr); } last_op_finish_ = now; } done_++; if (done_ >= next_report_) { if (next_report_ < 1000) next_report_ += 100; else if (next_report_ < 5000) next_report_ += 500; else if (next_report_ < 10000) next_report_ += 1000; else if (next_report_ < 50000) next_report_ += 5000; else if (next_report_ < 100000) next_report_ += 10000; else if (next_report_ < 500000) next_report_ += 50000; else next_report_ += 100000; fprintf(stderr, "... finished %d ops%30s\r", done_, ""); fflush(stderr); } } void AddBytes(int64_t n) { bytes_ += n; } void Report(const Slice& name) { // Pretend at least one op was done in case we are running a benchmark // that does not call FinishedSingleOp(). if (done_ < 1) done_ = 1; std::string extra; if (bytes_ > 0) { // Rate is computed on actual elapsed time, not the sum of per-thread // elapsed times. double elapsed = (finish_ - start_) * 1e-6; char rate[100]; snprintf(rate, sizeof(rate), "%6.1f MB/s", (bytes_ / 1048576.0) / elapsed); extra = rate; } AppendWithSpace(&extra, message_); fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", name.ToString().c_str(), seconds_ * 1e6 / done_, (extra.empty() ? "" : " "), extra.c_str()); if (FLAGS_histogram) { fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); } fflush(stdout); } }; // State shared by all concurrent executions of the same benchmark. struct SharedState { port::Mutex mu; port::CondVar cv; int total; // Each thread goes through the following states: // (1) initializing // (2) waiting for others to be initialized // (3) running // (4) done int num_initialized; int num_done; bool start; SharedState() : cv(&mu) { } }; // Per-thread state for concurrent executions of the same benchmark. struct ThreadState { int tid; // 0..n-1 when running in n threads Random rand; // Has different seeds for different threads Stats stats; SharedState* shared; ThreadState(int index) : tid(index), rand(1000 + index) { } }; } // namespace class Benchmark { private: Cache* cache_; const FilterPolicy* filter_policy_; DB* db_; int num_; int value_size_; int entries_per_batch_; WriteOptions write_options_; int reads_; int heap_counter_; void PrintHeader() { const int kKeySize = 16; PrintEnvironment(); fprintf(stdout, "Keys: %d bytes each\n", kKeySize); fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n", FLAGS_value_size, static_cast(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); fprintf(stdout, "Entries: %d\n", num_); fprintf(stdout, "RawSize: %.1f MB (estimated)\n", ((static_cast(kKeySize + FLAGS_value_size) * num_) / 1048576.0)); fprintf(stdout, "FileSize: %.1f MB (estimated)\n", (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) / 1048576.0)); PrintWarnings(); fprintf(stdout, "------------------------------------------------\n"); } void PrintWarnings() { #if defined(__GNUC__) && !defined(__OPTIMIZE__) fprintf(stdout, "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" ); #endif #ifndef NDEBUG fprintf(stdout, "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); #endif // See if snappy is working by attempting to compress a compressible string const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"; std::string compressed; if (!port::Snappy_Compress(text, sizeof(text), &compressed)) { fprintf(stdout, "WARNING: Snappy compression is not enabled\n"); } else if (compressed.size() >= sizeof(text)) { fprintf(stdout, "WARNING: Snappy compression is not effective\n"); } } void PrintEnvironment() { fprintf(stderr, "LevelDB: version %d.%d\n", kMajorVersion, kMinorVersion); #if defined(__linux) time_t now = time(NULL); fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); if (cpuinfo != NULL) { char line[1000]; int num_cpus = 0; std::string cpu_type; std::string cache_size; while (fgets(line, sizeof(line), cpuinfo) != NULL) { const char* sep = strchr(line, ':'); if (sep == NULL) { continue; } Slice key = TrimSpace(Slice(line, sep - 1 - line)); Slice val = TrimSpace(Slice(sep + 1)); if (key == "model name") { ++num_cpus; cpu_type = val.ToString(); } else if (key == "cache size") { cache_size = val.ToString(); } } fclose(cpuinfo); fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); } #endif } public: Benchmark() : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL), filter_policy_(FLAGS_bloom_bits >= 0 ? NewBloomFilterPolicy(FLAGS_bloom_bits) : NULL), db_(NULL), num_(FLAGS_num), value_size_(FLAGS_value_size), entries_per_batch_(1), reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), heap_counter_(0) { std::vector files; - Env::Default()->GetChildren(FLAGS_db, &files); + g_env->GetChildren(FLAGS_db, &files); for (size_t i = 0; i < files.size(); i++) { if (Slice(files[i]).starts_with("heap-")) { - Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]); + g_env->DeleteFile(std::string(FLAGS_db) + "/" + files[i]); } } if (!FLAGS_use_existing_db) { DestroyDB(FLAGS_db, Options()); } } ~Benchmark() { delete db_; delete cache_; delete filter_policy_; } void Run() { PrintHeader(); Open(); const char* benchmarks = FLAGS_benchmarks; while (benchmarks != NULL) { const char* sep = strchr(benchmarks, ','); Slice name; if (sep == NULL) { name = benchmarks; benchmarks = NULL; } else { name = Slice(benchmarks, sep - benchmarks); benchmarks = sep + 1; } // Reset parameters that may be overridden below num_ = FLAGS_num; reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); value_size_ = FLAGS_value_size; entries_per_batch_ = 1; write_options_ = WriteOptions(); void (Benchmark::*method)(ThreadState*) = NULL; bool fresh_db = false; int num_threads = FLAGS_threads; if (name == Slice("open")) { method = &Benchmark::OpenBench; num_ /= 10000; if (num_ < 1) num_ = 1; } else if (name == Slice("fillseq")) { fresh_db = true; method = &Benchmark::WriteSeq; } else if (name == Slice("fillbatch")) { fresh_db = true; entries_per_batch_ = 1000; method = &Benchmark::WriteSeq; } else if (name == Slice("fillrandom")) { fresh_db = true; method = &Benchmark::WriteRandom; } else if (name == Slice("overwrite")) { fresh_db = false; method = &Benchmark::WriteRandom; } else if (name == Slice("fillsync")) { fresh_db = true; num_ /= 1000; write_options_.sync = true; method = &Benchmark::WriteRandom; } else if (name == Slice("fill100K")) { fresh_db = true; num_ /= 1000; value_size_ = 100 * 1000; method = &Benchmark::WriteRandom; } else if (name == Slice("readseq")) { method = &Benchmark::ReadSequential; } else if (name == Slice("readreverse")) { method = &Benchmark::ReadReverse; } else if (name == Slice("readrandom")) { method = &Benchmark::ReadRandom; } else if (name == Slice("readmissing")) { method = &Benchmark::ReadMissing; } else if (name == Slice("seekrandom")) { method = &Benchmark::SeekRandom; } else if (name == Slice("readhot")) { method = &Benchmark::ReadHot; } else if (name == Slice("readrandomsmall")) { reads_ /= 1000; method = &Benchmark::ReadRandom; } else if (name == Slice("deleteseq")) { method = &Benchmark::DeleteSeq; } else if (name == Slice("deleterandom")) { method = &Benchmark::DeleteRandom; } else if (name == Slice("readwhilewriting")) { num_threads++; // Add extra thread for writing method = &Benchmark::ReadWhileWriting; } else if (name == Slice("compact")) { method = &Benchmark::Compact; } else if (name == Slice("crc32c")) { method = &Benchmark::Crc32c; } else if (name == Slice("acquireload")) { method = &Benchmark::AcquireLoad; } else if (name == Slice("snappycomp")) { method = &Benchmark::SnappyCompress; } else if (name == Slice("snappyuncomp")) { method = &Benchmark::SnappyUncompress; } else if (name == Slice("heapprofile")) { HeapProfile(); } else if (name == Slice("stats")) { PrintStats("leveldb.stats"); } else if (name == Slice("sstables")) { PrintStats("leveldb.sstables"); } else { if (name != Slice()) { // No error message for empty name fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str()); } } if (fresh_db) { if (FLAGS_use_existing_db) { fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n", name.ToString().c_str()); method = NULL; } else { delete db_; db_ = NULL; DestroyDB(FLAGS_db, Options()); Open(); } } if (method != NULL) { RunBenchmark(num_threads, name, method); } } } private: struct ThreadArg { Benchmark* bm; SharedState* shared; ThreadState* thread; void (Benchmark::*method)(ThreadState*); }; static void ThreadBody(void* v) { ThreadArg* arg = reinterpret_cast(v); SharedState* shared = arg->shared; ThreadState* thread = arg->thread; { MutexLock l(&shared->mu); shared->num_initialized++; if (shared->num_initialized >= shared->total) { shared->cv.SignalAll(); } while (!shared->start) { shared->cv.Wait(); } } thread->stats.Start(); (arg->bm->*(arg->method))(thread); thread->stats.Stop(); { MutexLock l(&shared->mu); shared->num_done++; if (shared->num_done >= shared->total) { shared->cv.SignalAll(); } } } void RunBenchmark(int n, Slice name, void (Benchmark::*method)(ThreadState*)) { SharedState shared; shared.total = n; shared.num_initialized = 0; shared.num_done = 0; shared.start = false; ThreadArg* arg = new ThreadArg[n]; for (int i = 0; i < n; i++) { arg[i].bm = this; arg[i].method = method; arg[i].shared = &shared; arg[i].thread = new ThreadState(i); arg[i].thread->shared = &shared; - Env::Default()->StartThread(ThreadBody, &arg[i]); + g_env->StartThread(ThreadBody, &arg[i]); } shared.mu.Lock(); while (shared.num_initialized < n) { shared.cv.Wait(); } shared.start = true; shared.cv.SignalAll(); while (shared.num_done < n) { shared.cv.Wait(); } shared.mu.Unlock(); for (int i = 1; i < n; i++) { arg[0].thread->stats.Merge(arg[i].thread->stats); } arg[0].thread->stats.Report(name); for (int i = 0; i < n; i++) { delete arg[i].thread; } delete[] arg; } void Crc32c(ThreadState* thread) { // Checksum about 500MB of data total const int size = 4096; const char* label = "(4K per op)"; std::string data(size, 'x'); int64_t bytes = 0; uint32_t crc = 0; while (bytes < 500 * 1048576) { crc = crc32c::Value(data.data(), size); thread->stats.FinishedSingleOp(); bytes += size; } // Print so result is not dead fprintf(stderr, "... crc=0x%x\r", static_cast(crc)); thread->stats.AddBytes(bytes); thread->stats.AddMessage(label); } void AcquireLoad(ThreadState* thread) { int dummy; port::AtomicPointer ap(&dummy); int count = 0; void *ptr = NULL; thread->stats.AddMessage("(each op is 1000 loads)"); while (count < 100000) { for (int i = 0; i < 1000; i++) { ptr = ap.Acquire_Load(); } count++; thread->stats.FinishedSingleOp(); } if (ptr == NULL) exit(1); // Disable unused variable warning. } void SnappyCompress(ThreadState* thread) { RandomGenerator gen; Slice input = gen.Generate(Options().block_size); int64_t bytes = 0; int64_t produced = 0; bool ok = true; std::string compressed; while (ok && bytes < 1024 * 1048576) { // Compress 1G ok = port::Snappy_Compress(input.data(), input.size(), &compressed); produced += compressed.size(); bytes += input.size(); thread->stats.FinishedSingleOp(); } if (!ok) { thread->stats.AddMessage("(snappy failure)"); } else { char buf[100]; snprintf(buf, sizeof(buf), "(output: %.1f%%)", (produced * 100.0) / bytes); thread->stats.AddMessage(buf); thread->stats.AddBytes(bytes); } } void SnappyUncompress(ThreadState* thread) { RandomGenerator gen; Slice input = gen.Generate(Options().block_size); std::string compressed; bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed); int64_t bytes = 0; char* uncompressed = new char[input.size()]; while (ok && bytes < 1024 * 1048576) { // Compress 1G ok = port::Snappy_Uncompress(compressed.data(), compressed.size(), uncompressed); bytes += input.size(); thread->stats.FinishedSingleOp(); } delete[] uncompressed; if (!ok) { thread->stats.AddMessage("(snappy failure)"); } else { thread->stats.AddBytes(bytes); } } void Open() { assert(db_ == NULL); Options options; + options.env = g_env; options.create_if_missing = !FLAGS_use_existing_db; options.block_cache = cache_; options.write_buffer_size = FLAGS_write_buffer_size; + options.max_file_size = FLAGS_max_file_size; + options.block_size = FLAGS_block_size; options.max_open_files = FLAGS_open_files; options.filter_policy = filter_policy_; options.reuse_logs = FLAGS_reuse_logs; Status s = DB::Open(options, FLAGS_db, &db_); if (!s.ok()) { fprintf(stderr, "open error: %s\n", s.ToString().c_str()); exit(1); } } void OpenBench(ThreadState* thread) { for (int i = 0; i < num_; i++) { delete db_; Open(); thread->stats.FinishedSingleOp(); } } void WriteSeq(ThreadState* thread) { DoWrite(thread, true); } void WriteRandom(ThreadState* thread) { DoWrite(thread, false); } void DoWrite(ThreadState* thread, bool seq) { if (num_ != FLAGS_num) { char msg[100]; snprintf(msg, sizeof(msg), "(%d ops)", num_); thread->stats.AddMessage(msg); } RandomGenerator gen; WriteBatch batch; Status s; int64_t bytes = 0; for (int i = 0; i < num_; i += entries_per_batch_) { batch.Clear(); for (int j = 0; j < entries_per_batch_; j++) { const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); char key[100]; snprintf(key, sizeof(key), "%016d", k); batch.Put(key, gen.Generate(value_size_)); bytes += value_size_ + strlen(key); thread->stats.FinishedSingleOp(); } s = db_->Write(write_options_, &batch); if (!s.ok()) { fprintf(stderr, "put error: %s\n", s.ToString().c_str()); exit(1); } } thread->stats.AddBytes(bytes); } void ReadSequential(ThreadState* thread) { Iterator* iter = db_->NewIterator(ReadOptions()); int i = 0; int64_t bytes = 0; for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) { bytes += iter->key().size() + iter->value().size(); thread->stats.FinishedSingleOp(); ++i; } delete iter; thread->stats.AddBytes(bytes); } void ReadReverse(ThreadState* thread) { Iterator* iter = db_->NewIterator(ReadOptions()); int i = 0; int64_t bytes = 0; for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) { bytes += iter->key().size() + iter->value().size(); thread->stats.FinishedSingleOp(); ++i; } delete iter; thread->stats.AddBytes(bytes); } void ReadRandom(ThreadState* thread) { ReadOptions options; std::string value; int found = 0; for (int i = 0; i < reads_; i++) { char key[100]; const int k = thread->rand.Next() % FLAGS_num; snprintf(key, sizeof(key), "%016d", k); if (db_->Get(options, key, &value).ok()) { found++; } thread->stats.FinishedSingleOp(); } char msg[100]; snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); thread->stats.AddMessage(msg); } void ReadMissing(ThreadState* thread) { ReadOptions options; std::string value; for (int i = 0; i < reads_; i++) { char key[100]; const int k = thread->rand.Next() % FLAGS_num; snprintf(key, sizeof(key), "%016d.", k); db_->Get(options, key, &value); thread->stats.FinishedSingleOp(); } } void ReadHot(ThreadState* thread) { ReadOptions options; std::string value; const int range = (FLAGS_num + 99) / 100; for (int i = 0; i < reads_; i++) { char key[100]; const int k = thread->rand.Next() % range; snprintf(key, sizeof(key), "%016d", k); db_->Get(options, key, &value); thread->stats.FinishedSingleOp(); } } void SeekRandom(ThreadState* thread) { ReadOptions options; int found = 0; for (int i = 0; i < reads_; i++) { Iterator* iter = db_->NewIterator(options); char key[100]; const int k = thread->rand.Next() % FLAGS_num; snprintf(key, sizeof(key), "%016d", k); iter->Seek(key); if (iter->Valid() && iter->key() == key) found++; delete iter; thread->stats.FinishedSingleOp(); } char msg[100]; snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); thread->stats.AddMessage(msg); } void DoDelete(ThreadState* thread, bool seq) { RandomGenerator gen; WriteBatch batch; Status s; for (int i = 0; i < num_; i += entries_per_batch_) { batch.Clear(); for (int j = 0; j < entries_per_batch_; j++) { const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); char key[100]; snprintf(key, sizeof(key), "%016d", k); batch.Delete(key); thread->stats.FinishedSingleOp(); } s = db_->Write(write_options_, &batch); if (!s.ok()) { fprintf(stderr, "del error: %s\n", s.ToString().c_str()); exit(1); } } } void DeleteSeq(ThreadState* thread) { DoDelete(thread, true); } void DeleteRandom(ThreadState* thread) { DoDelete(thread, false); } void ReadWhileWriting(ThreadState* thread) { if (thread->tid > 0) { ReadRandom(thread); } else { // Special thread that keeps writing until other threads are done. RandomGenerator gen; while (true) { { MutexLock l(&thread->shared->mu); if (thread->shared->num_done + 1 >= thread->shared->num_initialized) { // Other threads have finished break; } } const int k = thread->rand.Next() % FLAGS_num; char key[100]; snprintf(key, sizeof(key), "%016d", k); Status s = db_->Put(write_options_, key, gen.Generate(value_size_)); if (!s.ok()) { fprintf(stderr, "put error: %s\n", s.ToString().c_str()); exit(1); } } // Do not count any of the preceding work/delay in stats. thread->stats.Start(); } } void Compact(ThreadState* thread) { db_->CompactRange(NULL, NULL); } void PrintStats(const char* key) { std::string stats; if (!db_->GetProperty(key, &stats)) { stats = "(failed)"; } fprintf(stdout, "\n%s\n", stats.c_str()); } static void WriteToFile(void* arg, const char* buf, int n) { reinterpret_cast(arg)->Append(Slice(buf, n)); } void HeapProfile() { char fname[100]; snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_); WritableFile* file; - Status s = Env::Default()->NewWritableFile(fname, &file); + Status s = g_env->NewWritableFile(fname, &file); if (!s.ok()) { fprintf(stderr, "%s\n", s.ToString().c_str()); return; } bool ok = port::GetHeapProfile(WriteToFile, file); delete file; if (!ok) { fprintf(stderr, "heap profiling not supported\n"); - Env::Default()->DeleteFile(fname); + g_env->DeleteFile(fname); } } }; } // namespace leveldb int main(int argc, char** argv) { FLAGS_write_buffer_size = leveldb::Options().write_buffer_size; + FLAGS_max_file_size = leveldb::Options().max_file_size; + FLAGS_block_size = leveldb::Options().block_size; FLAGS_open_files = leveldb::Options().max_open_files; std::string default_db_path; for (int i = 1; i < argc; i++) { double d; int n; char junk; if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { FLAGS_compression_ratio = d; } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && (n == 0 || n == 1)) { FLAGS_histogram = n; } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && (n == 0 || n == 1)) { FLAGS_use_existing_db = n; } else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 && (n == 0 || n == 1)) { FLAGS_reuse_logs = n; } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { FLAGS_num = n; } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { FLAGS_reads = n; } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) { FLAGS_threads = n; } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { FLAGS_value_size = n; } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) { FLAGS_write_buffer_size = n; + } else if (sscanf(argv[i], "--max_file_size=%d%c", &n, &junk) == 1) { + FLAGS_max_file_size = n; + } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) { + FLAGS_block_size = n; } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) { FLAGS_cache_size = n; } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) { FLAGS_bloom_bits = n; } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) { FLAGS_open_files = n; } else if (strncmp(argv[i], "--db=", 5) == 0) { FLAGS_db = argv[i] + 5; } else { fprintf(stderr, "Invalid flag '%s'\n", argv[i]); exit(1); } } + leveldb::g_env = leveldb::Env::Default(); + // Choose a location for the test database if none given with --db= if (FLAGS_db == NULL) { - leveldb::Env::Default()->GetTestDirectory(&default_db_path); + leveldb::g_env->GetTestDirectory(&default_db_path); default_db_path += "/dbbench"; FLAGS_db = default_db_path.c_str(); } leveldb::Benchmark benchmark; benchmark.Run(); return 0; } diff --git a/src/leveldb/db/db_impl.cc b/src/leveldb/db/db_impl.cc index 60f4e66e5..f43ad7679 100644 --- a/src/leveldb/db/db_impl.cc +++ b/src/leveldb/db/db_impl.cc @@ -1,1567 +1,1568 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/db_impl.h" #include #include #include #include #include #include #include "db/builder.h" #include "db/db_iter.h" #include "db/dbformat.h" #include "db/filename.h" #include "db/log_reader.h" #include "db/log_writer.h" #include "db/memtable.h" #include "db/table_cache.h" #include "db/version_set.h" #include "db/write_batch_internal.h" #include "leveldb/db.h" #include "leveldb/env.h" #include "leveldb/status.h" #include "leveldb/table.h" #include "leveldb/table_builder.h" #include "port/port.h" #include "table/block.h" #include "table/merger.h" #include "table/two_level_iterator.h" #include "util/coding.h" #include "util/logging.h" #include "util/mutexlock.h" namespace leveldb { const int kNumNonTableCacheFiles = 10; // Information kept for every waiting writer struct DBImpl::Writer { Status status; WriteBatch* batch; bool sync; bool done; port::CondVar cv; explicit Writer(port::Mutex* mu) : cv(mu) { } }; struct DBImpl::CompactionState { Compaction* const compaction; // Sequence numbers < smallest_snapshot are not significant since we // will never have to service a snapshot below smallest_snapshot. // Therefore if we have seen a sequence number S <= smallest_snapshot, // we can drop all entries for the same key with sequence numbers < S. SequenceNumber smallest_snapshot; // Files produced by compaction struct Output { uint64_t number; uint64_t file_size; InternalKey smallest, largest; }; std::vector outputs; // State kept for output being generated WritableFile* outfile; TableBuilder* builder; uint64_t total_bytes; Output* current_output() { return &outputs[outputs.size()-1]; } explicit CompactionState(Compaction* c) : compaction(c), outfile(NULL), builder(NULL), total_bytes(0) { } }; // Fix user-supplied options to be reasonable template static void ClipToRange(T* ptr, V minvalue, V maxvalue) { if (static_cast(*ptr) > maxvalue) *ptr = maxvalue; if (static_cast(*ptr) < minvalue) *ptr = minvalue; } Options SanitizeOptions(const std::string& dbname, const InternalKeyComparator* icmp, const InternalFilterPolicy* ipolicy, const Options& src) { Options result = src; result.comparator = icmp; result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL; ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); ClipToRange(&result.write_buffer_size, 64<<10, 1<<30); + ClipToRange(&result.max_file_size, 1<<20, 1<<30); ClipToRange(&result.block_size, 1<<10, 4<<20); if (result.info_log == NULL) { // Open a log file in the same directory as the db src.env->CreateDir(dbname); // In case it does not exist src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname)); Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log); if (!s.ok()) { // No place suitable for logging result.info_log = NULL; } } if (result.block_cache == NULL) { result.block_cache = NewLRUCache(8 << 20); } return result; } DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) : env_(raw_options.env), internal_comparator_(raw_options.comparator), internal_filter_policy_(raw_options.filter_policy), options_(SanitizeOptions(dbname, &internal_comparator_, &internal_filter_policy_, raw_options)), owns_info_log_(options_.info_log != raw_options.info_log), owns_cache_(options_.block_cache != raw_options.block_cache), dbname_(dbname), db_lock_(NULL), shutting_down_(NULL), bg_cv_(&mutex_), mem_(NULL), imm_(NULL), logfile_(NULL), logfile_number_(0), log_(NULL), seed_(0), tmp_batch_(new WriteBatch), bg_compaction_scheduled_(false), manual_compaction_(NULL) { has_imm_.Release_Store(NULL); // Reserve ten files or so for other uses and give the rest to TableCache. const int table_cache_size = options_.max_open_files - kNumNonTableCacheFiles; table_cache_ = new TableCache(dbname_, &options_, table_cache_size); versions_ = new VersionSet(dbname_, &options_, table_cache_, &internal_comparator_); } DBImpl::~DBImpl() { // Wait for background work to finish mutex_.Lock(); shutting_down_.Release_Store(this); // Any non-NULL value is ok while (bg_compaction_scheduled_) { bg_cv_.Wait(); } mutex_.Unlock(); if (db_lock_ != NULL) { env_->UnlockFile(db_lock_); } delete versions_; if (mem_ != NULL) mem_->Unref(); if (imm_ != NULL) imm_->Unref(); delete tmp_batch_; delete log_; delete logfile_; delete table_cache_; if (owns_info_log_) { delete options_.info_log; } if (owns_cache_) { delete options_.block_cache; } } Status DBImpl::NewDB() { VersionEdit new_db; new_db.SetComparatorName(user_comparator()->Name()); new_db.SetLogNumber(0); new_db.SetNextFile(2); new_db.SetLastSequence(0); const std::string manifest = DescriptorFileName(dbname_, 1); WritableFile* file; Status s = env_->NewWritableFile(manifest, &file); if (!s.ok()) { return s; } { log::Writer log(file); std::string record; new_db.EncodeTo(&record); s = log.AddRecord(record); if (s.ok()) { s = file->Close(); } } delete file; if (s.ok()) { // Make "CURRENT" file that points to the new manifest file. s = SetCurrentFile(env_, dbname_, 1); } else { env_->DeleteFile(manifest); } return s; } void DBImpl::MaybeIgnoreError(Status* s) const { if (s->ok() || options_.paranoid_checks) { // No change needed } else { Log(options_.info_log, "Ignoring error %s", s->ToString().c_str()); *s = Status::OK(); } } void DBImpl::DeleteObsoleteFiles() { if (!bg_error_.ok()) { // After a background error, we don't know whether a new version may // or may not have been committed, so we cannot safely garbage collect. return; } // Make a set of all of the live files std::set live = pending_outputs_; versions_->AddLiveFiles(&live); std::vector filenames; env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose uint64_t number; FileType type; for (size_t i = 0; i < filenames.size(); i++) { if (ParseFileName(filenames[i], &number, &type)) { bool keep = true; switch (type) { case kLogFile: keep = ((number >= versions_->LogNumber()) || (number == versions_->PrevLogNumber())); break; case kDescriptorFile: // Keep my manifest file, and any newer incarnations' // (in case there is a race that allows other incarnations) keep = (number >= versions_->ManifestFileNumber()); break; case kTableFile: keep = (live.find(number) != live.end()); break; case kTempFile: // Any temp files that are currently being written to must // be recorded in pending_outputs_, which is inserted into "live" keep = (live.find(number) != live.end()); break; case kCurrentFile: case kDBLockFile: case kInfoLogFile: keep = true; break; } if (!keep) { if (type == kTableFile) { table_cache_->Evict(number); } Log(options_.info_log, "Delete type=%d #%lld\n", int(type), static_cast(number)); env_->DeleteFile(dbname_ + "/" + filenames[i]); } } } } Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { mutex_.AssertHeld(); // Ignore error from CreateDir since the creation of the DB is // committed only when the descriptor is created, and this directory // may already exist from a previous failed creation attempt. env_->CreateDir(dbname_); assert(db_lock_ == NULL); Status s = env_->LockFile(LockFileName(dbname_), &db_lock_); if (!s.ok()) { return s; } if (!env_->FileExists(CurrentFileName(dbname_))) { if (options_.create_if_missing) { s = NewDB(); if (!s.ok()) { return s; } } else { return Status::InvalidArgument( dbname_, "does not exist (create_if_missing is false)"); } } else { if (options_.error_if_exists) { return Status::InvalidArgument( dbname_, "exists (error_if_exists is true)"); } } s = versions_->Recover(save_manifest); if (!s.ok()) { return s; } SequenceNumber max_sequence(0); // Recover from all newer log files than the ones named in the // descriptor (new log files may have been added by the previous // incarnation without registering them in the descriptor). // // Note that PrevLogNumber() is no longer used, but we pay // attention to it in case we are recovering a database // produced by an older version of leveldb. const uint64_t min_log = versions_->LogNumber(); const uint64_t prev_log = versions_->PrevLogNumber(); std::vector filenames; s = env_->GetChildren(dbname_, &filenames); if (!s.ok()) { return s; } std::set expected; versions_->AddLiveFiles(&expected); uint64_t number; FileType type; std::vector logs; for (size_t i = 0; i < filenames.size(); i++) { if (ParseFileName(filenames[i], &number, &type)) { expected.erase(number); if (type == kLogFile && ((number >= min_log) || (number == prev_log))) logs.push_back(number); } } if (!expected.empty()) { char buf[50]; snprintf(buf, sizeof(buf), "%d missing files; e.g.", static_cast(expected.size())); return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); } // Recover in the order in which the logs were generated std::sort(logs.begin(), logs.end()); for (size_t i = 0; i < logs.size(); i++) { s = RecoverLogFile(logs[i], (i == logs.size() - 1), save_manifest, edit, &max_sequence); if (!s.ok()) { return s; } // The previous incarnation may not have written any MANIFEST // records after allocating this log number. So we manually // update the file number allocation counter in VersionSet. versions_->MarkFileNumberUsed(logs[i]); } if (versions_->LastSequence() < max_sequence) { versions_->SetLastSequence(max_sequence); } return Status::OK(); } Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest, VersionEdit* edit, SequenceNumber* max_sequence) { struct LogReporter : public log::Reader::Reporter { Env* env; Logger* info_log; const char* fname; Status* status; // NULL if options_.paranoid_checks==false virtual void Corruption(size_t bytes, const Status& s) { Log(info_log, "%s%s: dropping %d bytes; %s", (this->status == NULL ? "(ignoring error) " : ""), fname, static_cast(bytes), s.ToString().c_str()); if (this->status != NULL && this->status->ok()) *this->status = s; } }; mutex_.AssertHeld(); // Open the log file std::string fname = LogFileName(dbname_, log_number); SequentialFile* file; Status status = env_->NewSequentialFile(fname, &file); if (!status.ok()) { MaybeIgnoreError(&status); return status; } // Create the log reader. LogReporter reporter; reporter.env = env_; reporter.info_log = options_.info_log; reporter.fname = fname.c_str(); reporter.status = (options_.paranoid_checks ? &status : NULL); // We intentionally make log::Reader do checksumming even if // paranoid_checks==false so that corruptions cause entire commits // to be skipped instead of propagating bad information (like overly // large sequence numbers). log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); Log(options_.info_log, "Recovering log #%llu", (unsigned long long) log_number); // Read all the records and add to a memtable std::string scratch; Slice record; WriteBatch batch; int compactions = 0; MemTable* mem = NULL; while (reader.ReadRecord(&record, &scratch) && status.ok()) { if (record.size() < 12) { reporter.Corruption( record.size(), Status::Corruption("log record too small")); continue; } WriteBatchInternal::SetContents(&batch, record); if (mem == NULL) { mem = new MemTable(internal_comparator_); mem->Ref(); } status = WriteBatchInternal::InsertInto(&batch, mem); MaybeIgnoreError(&status); if (!status.ok()) { break; } const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) + WriteBatchInternal::Count(&batch) - 1; if (last_seq > *max_sequence) { *max_sequence = last_seq; } if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) { compactions++; *save_manifest = true; status = WriteLevel0Table(mem, edit, NULL); mem->Unref(); mem = NULL; if (!status.ok()) { // Reflect errors immediately so that conditions like full // file-systems cause the DB::Open() to fail. break; } } } delete file; // See if we should keep reusing the last log file. if (status.ok() && options_.reuse_logs && last_log && compactions == 0) { assert(logfile_ == NULL); assert(log_ == NULL); assert(mem_ == NULL); uint64_t lfile_size; if (env_->GetFileSize(fname, &lfile_size).ok() && env_->NewAppendableFile(fname, &logfile_).ok()) { Log(options_.info_log, "Reusing old log %s \n", fname.c_str()); log_ = new log::Writer(logfile_, lfile_size); logfile_number_ = log_number; if (mem != NULL) { mem_ = mem; mem = NULL; } else { // mem can be NULL if lognum exists but was empty. mem_ = new MemTable(internal_comparator_); mem_->Ref(); } } } if (mem != NULL) { // mem did not get reused; compact it. if (status.ok()) { *save_manifest = true; status = WriteLevel0Table(mem, edit, NULL); } mem->Unref(); } return status; } Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base) { mutex_.AssertHeld(); const uint64_t start_micros = env_->NowMicros(); FileMetaData meta; meta.number = versions_->NewFileNumber(); pending_outputs_.insert(meta.number); Iterator* iter = mem->NewIterator(); Log(options_.info_log, "Level-0 table #%llu: started", (unsigned long long) meta.number); Status s; { mutex_.Unlock(); s = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta); mutex_.Lock(); } Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s", (unsigned long long) meta.number, (unsigned long long) meta.file_size, s.ToString().c_str()); delete iter; pending_outputs_.erase(meta.number); // Note that if file_size is zero, the file has been deleted and // should not be added to the manifest. int level = 0; if (s.ok() && meta.file_size > 0) { const Slice min_user_key = meta.smallest.user_key(); const Slice max_user_key = meta.largest.user_key(); if (base != NULL) { level = base->PickLevelForMemTableOutput(min_user_key, max_user_key); } edit->AddFile(level, meta.number, meta.file_size, meta.smallest, meta.largest); } CompactionStats stats; stats.micros = env_->NowMicros() - start_micros; stats.bytes_written = meta.file_size; stats_[level].Add(stats); return s; } void DBImpl::CompactMemTable() { mutex_.AssertHeld(); assert(imm_ != NULL); // Save the contents of the memtable as a new Table VersionEdit edit; Version* base = versions_->current(); base->Ref(); Status s = WriteLevel0Table(imm_, &edit, base); base->Unref(); if (s.ok() && shutting_down_.Acquire_Load()) { s = Status::IOError("Deleting DB during memtable compaction"); } // Replace immutable memtable with the generated Table if (s.ok()) { edit.SetPrevLogNumber(0); edit.SetLogNumber(logfile_number_); // Earlier logs no longer needed s = versions_->LogAndApply(&edit, &mutex_); } if (s.ok()) { // Commit to the new state imm_->Unref(); imm_ = NULL; has_imm_.Release_Store(NULL); DeleteObsoleteFiles(); } else { RecordBackgroundError(s); } } void DBImpl::CompactRange(const Slice* begin, const Slice* end) { int max_level_with_files = 1; { MutexLock l(&mutex_); Version* base = versions_->current(); for (int level = 1; level < config::kNumLevels; level++) { if (base->OverlapInLevel(level, begin, end)) { max_level_with_files = level; } } } TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap for (int level = 0; level < max_level_with_files; level++) { TEST_CompactRange(level, begin, end); } } void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) { assert(level >= 0); assert(level + 1 < config::kNumLevels); InternalKey begin_storage, end_storage; ManualCompaction manual; manual.level = level; manual.done = false; if (begin == NULL) { manual.begin = NULL; } else { begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek); manual.begin = &begin_storage; } if (end == NULL) { manual.end = NULL; } else { end_storage = InternalKey(*end, 0, static_cast(0)); manual.end = &end_storage; } MutexLock l(&mutex_); while (!manual.done && !shutting_down_.Acquire_Load() && bg_error_.ok()) { if (manual_compaction_ == NULL) { // Idle manual_compaction_ = &manual; MaybeScheduleCompaction(); } else { // Running either my compaction or another compaction. bg_cv_.Wait(); } } if (manual_compaction_ == &manual) { // Cancel my manual compaction since we aborted early for some reason. manual_compaction_ = NULL; } } Status DBImpl::TEST_CompactMemTable() { // NULL batch means just wait for earlier writes to be done Status s = Write(WriteOptions(), NULL); if (s.ok()) { // Wait until the compaction completes MutexLock l(&mutex_); while (imm_ != NULL && bg_error_.ok()) { bg_cv_.Wait(); } if (imm_ != NULL) { s = bg_error_; } } return s; } void DBImpl::RecordBackgroundError(const Status& s) { mutex_.AssertHeld(); if (bg_error_.ok()) { bg_error_ = s; bg_cv_.SignalAll(); } } void DBImpl::MaybeScheduleCompaction() { mutex_.AssertHeld(); if (bg_compaction_scheduled_) { // Already scheduled } else if (shutting_down_.Acquire_Load()) { // DB is being deleted; no more background compactions } else if (!bg_error_.ok()) { // Already got an error; no more changes } else if (imm_ == NULL && manual_compaction_ == NULL && !versions_->NeedsCompaction()) { // No work to be done } else { bg_compaction_scheduled_ = true; env_->Schedule(&DBImpl::BGWork, this); } } void DBImpl::BGWork(void* db) { reinterpret_cast(db)->BackgroundCall(); } void DBImpl::BackgroundCall() { MutexLock l(&mutex_); assert(bg_compaction_scheduled_); if (shutting_down_.Acquire_Load()) { // No more background work when shutting down. } else if (!bg_error_.ok()) { // No more background work after a background error. } else { BackgroundCompaction(); } bg_compaction_scheduled_ = false; // Previous compaction may have produced too many files in a level, // so reschedule another compaction if needed. MaybeScheduleCompaction(); bg_cv_.SignalAll(); } void DBImpl::BackgroundCompaction() { mutex_.AssertHeld(); if (imm_ != NULL) { CompactMemTable(); return; } Compaction* c; bool is_manual = (manual_compaction_ != NULL); InternalKey manual_end; if (is_manual) { ManualCompaction* m = manual_compaction_; c = versions_->CompactRange(m->level, m->begin, m->end); m->done = (c == NULL); if (c != NULL) { manual_end = c->input(0, c->num_input_files(0) - 1)->largest; } Log(options_.info_log, "Manual compaction at level-%d from %s .. %s; will stop at %s\n", m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"), (m->end ? m->end->DebugString().c_str() : "(end)"), (m->done ? "(end)" : manual_end.DebugString().c_str())); } else { c = versions_->PickCompaction(); } Status status; if (c == NULL) { // Nothing to do } else if (!is_manual && c->IsTrivialMove()) { // Move file to next level assert(c->num_input_files(0) == 1); FileMetaData* f = c->input(0, 0); c->edit()->DeleteFile(c->level(), f->number); c->edit()->AddFile(c->level() + 1, f->number, f->file_size, f->smallest, f->largest); status = versions_->LogAndApply(c->edit(), &mutex_); if (!status.ok()) { RecordBackgroundError(status); } VersionSet::LevelSummaryStorage tmp; Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n", static_cast(f->number), c->level() + 1, static_cast(f->file_size), status.ToString().c_str(), versions_->LevelSummary(&tmp)); } else { CompactionState* compact = new CompactionState(c); status = DoCompactionWork(compact); if (!status.ok()) { RecordBackgroundError(status); } CleanupCompaction(compact); c->ReleaseInputs(); DeleteObsoleteFiles(); } delete c; if (status.ok()) { // Done } else if (shutting_down_.Acquire_Load()) { // Ignore compaction errors found during shutting down } else { Log(options_.info_log, "Compaction error: %s", status.ToString().c_str()); } if (is_manual) { ManualCompaction* m = manual_compaction_; if (!status.ok()) { m->done = true; } if (!m->done) { // We only compacted part of the requested range. Update *m // to the range that is left to be compacted. m->tmp_storage = manual_end; m->begin = &m->tmp_storage; } manual_compaction_ = NULL; } } void DBImpl::CleanupCompaction(CompactionState* compact) { mutex_.AssertHeld(); if (compact->builder != NULL) { // May happen if we get a shutdown call in the middle of compaction compact->builder->Abandon(); delete compact->builder; } else { assert(compact->outfile == NULL); } delete compact->outfile; for (size_t i = 0; i < compact->outputs.size(); i++) { const CompactionState::Output& out = compact->outputs[i]; pending_outputs_.erase(out.number); } delete compact; } Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) { assert(compact != NULL); assert(compact->builder == NULL); uint64_t file_number; { mutex_.Lock(); file_number = versions_->NewFileNumber(); pending_outputs_.insert(file_number); CompactionState::Output out; out.number = file_number; out.smallest.Clear(); out.largest.Clear(); compact->outputs.push_back(out); mutex_.Unlock(); } // Make the output file std::string fname = TableFileName(dbname_, file_number); Status s = env_->NewWritableFile(fname, &compact->outfile); if (s.ok()) { compact->builder = new TableBuilder(options_, compact->outfile); } return s; } Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, Iterator* input) { assert(compact != NULL); assert(compact->outfile != NULL); assert(compact->builder != NULL); const uint64_t output_number = compact->current_output()->number; assert(output_number != 0); // Check for iterator errors Status s = input->status(); const uint64_t current_entries = compact->builder->NumEntries(); if (s.ok()) { s = compact->builder->Finish(); } else { compact->builder->Abandon(); } const uint64_t current_bytes = compact->builder->FileSize(); compact->current_output()->file_size = current_bytes; compact->total_bytes += current_bytes; delete compact->builder; compact->builder = NULL; // Finish and check for file errors if (s.ok()) { s = compact->outfile->Sync(); } if (s.ok()) { s = compact->outfile->Close(); } delete compact->outfile; compact->outfile = NULL; if (s.ok() && current_entries > 0) { // Verify that the table is usable Iterator* iter = table_cache_->NewIterator(ReadOptions(), output_number, current_bytes); s = iter->status(); delete iter; if (s.ok()) { Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes", (unsigned long long) output_number, compact->compaction->level(), (unsigned long long) current_entries, (unsigned long long) current_bytes); } } return s; } Status DBImpl::InstallCompactionResults(CompactionState* compact) { mutex_.AssertHeld(); Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", compact->compaction->num_input_files(0), compact->compaction->level(), compact->compaction->num_input_files(1), compact->compaction->level() + 1, static_cast(compact->total_bytes)); // Add compaction outputs compact->compaction->AddInputDeletions(compact->compaction->edit()); const int level = compact->compaction->level(); for (size_t i = 0; i < compact->outputs.size(); i++) { const CompactionState::Output& out = compact->outputs[i]; compact->compaction->edit()->AddFile( level + 1, out.number, out.file_size, out.smallest, out.largest); } return versions_->LogAndApply(compact->compaction->edit(), &mutex_); } Status DBImpl::DoCompactionWork(CompactionState* compact) { const uint64_t start_micros = env_->NowMicros(); int64_t imm_micros = 0; // Micros spent doing imm_ compactions Log(options_.info_log, "Compacting %d@%d + %d@%d files", compact->compaction->num_input_files(0), compact->compaction->level(), compact->compaction->num_input_files(1), compact->compaction->level() + 1); assert(versions_->NumLevelFiles(compact->compaction->level()) > 0); assert(compact->builder == NULL); assert(compact->outfile == NULL); if (snapshots_.empty()) { compact->smallest_snapshot = versions_->LastSequence(); } else { compact->smallest_snapshot = snapshots_.oldest()->number_; } // Release mutex while we're actually doing the compaction work mutex_.Unlock(); Iterator* input = versions_->MakeInputIterator(compact->compaction); input->SeekToFirst(); Status status; ParsedInternalKey ikey; std::string current_user_key; bool has_current_user_key = false; SequenceNumber last_sequence_for_key = kMaxSequenceNumber; for (; input->Valid() && !shutting_down_.Acquire_Load(); ) { // Prioritize immutable compaction work if (has_imm_.NoBarrier_Load() != NULL) { const uint64_t imm_start = env_->NowMicros(); mutex_.Lock(); if (imm_ != NULL) { CompactMemTable(); bg_cv_.SignalAll(); // Wakeup MakeRoomForWrite() if necessary } mutex_.Unlock(); imm_micros += (env_->NowMicros() - imm_start); } Slice key = input->key(); if (compact->compaction->ShouldStopBefore(key) && compact->builder != NULL) { status = FinishCompactionOutputFile(compact, input); if (!status.ok()) { break; } } // Handle key/value, add to state, etc. bool drop = false; if (!ParseInternalKey(key, &ikey)) { // Do not hide error keys current_user_key.clear(); has_current_user_key = false; last_sequence_for_key = kMaxSequenceNumber; } else { if (!has_current_user_key || user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) != 0) { // First occurrence of this user key current_user_key.assign(ikey.user_key.data(), ikey.user_key.size()); has_current_user_key = true; last_sequence_for_key = kMaxSequenceNumber; } if (last_sequence_for_key <= compact->smallest_snapshot) { // Hidden by an newer entry for same user key drop = true; // (A) } else if (ikey.type == kTypeDeletion && ikey.sequence <= compact->smallest_snapshot && compact->compaction->IsBaseLevelForKey(ikey.user_key)) { // For this user key: // (1) there is no data in higher levels // (2) data in lower levels will have larger sequence numbers // (3) data in layers that are being compacted here and have // smaller sequence numbers will be dropped in the next // few iterations of this loop (by rule (A) above). // Therefore this deletion marker is obsolete and can be dropped. drop = true; } last_sequence_for_key = ikey.sequence; } #if 0 Log(options_.info_log, " Compact: %s, seq %d, type: %d %d, drop: %d, is_base: %d, " "%d smallest_snapshot: %d", ikey.user_key.ToString().c_str(), (int)ikey.sequence, ikey.type, kTypeValue, drop, compact->compaction->IsBaseLevelForKey(ikey.user_key), (int)last_sequence_for_key, (int)compact->smallest_snapshot); #endif if (!drop) { // Open output file if necessary if (compact->builder == NULL) { status = OpenCompactionOutputFile(compact); if (!status.ok()) { break; } } if (compact->builder->NumEntries() == 0) { compact->current_output()->smallest.DecodeFrom(key); } compact->current_output()->largest.DecodeFrom(key); compact->builder->Add(key, input->value()); // Close output file if it is big enough if (compact->builder->FileSize() >= compact->compaction->MaxOutputFileSize()) { status = FinishCompactionOutputFile(compact, input); if (!status.ok()) { break; } } } input->Next(); } if (status.ok() && shutting_down_.Acquire_Load()) { status = Status::IOError("Deleting DB during compaction"); } if (status.ok() && compact->builder != NULL) { status = FinishCompactionOutputFile(compact, input); } if (status.ok()) { status = input->status(); } delete input; input = NULL; CompactionStats stats; stats.micros = env_->NowMicros() - start_micros - imm_micros; for (int which = 0; which < 2; which++) { for (int i = 0; i < compact->compaction->num_input_files(which); i++) { stats.bytes_read += compact->compaction->input(which, i)->file_size; } } for (size_t i = 0; i < compact->outputs.size(); i++) { stats.bytes_written += compact->outputs[i].file_size; } mutex_.Lock(); stats_[compact->compaction->level() + 1].Add(stats); if (status.ok()) { status = InstallCompactionResults(compact); } if (!status.ok()) { RecordBackgroundError(status); } VersionSet::LevelSummaryStorage tmp; Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp)); return status; } namespace { struct IterState { port::Mutex* mu; Version* version; MemTable* mem; MemTable* imm; }; static void CleanupIteratorState(void* arg1, void* arg2) { IterState* state = reinterpret_cast(arg1); state->mu->Lock(); state->mem->Unref(); if (state->imm != NULL) state->imm->Unref(); state->version->Unref(); state->mu->Unlock(); delete state; } } // namespace Iterator* DBImpl::NewInternalIterator(const ReadOptions& options, SequenceNumber* latest_snapshot, uint32_t* seed) { IterState* cleanup = new IterState; mutex_.Lock(); *latest_snapshot = versions_->LastSequence(); // Collect together all needed child iterators std::vector list; list.push_back(mem_->NewIterator()); mem_->Ref(); if (imm_ != NULL) { list.push_back(imm_->NewIterator()); imm_->Ref(); } versions_->current()->AddIterators(options, &list); Iterator* internal_iter = NewMergingIterator(&internal_comparator_, &list[0], list.size()); versions_->current()->Ref(); cleanup->mu = &mutex_; cleanup->mem = mem_; cleanup->imm = imm_; cleanup->version = versions_->current(); internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, NULL); *seed = ++seed_; mutex_.Unlock(); return internal_iter; } Iterator* DBImpl::TEST_NewInternalIterator() { SequenceNumber ignored; uint32_t ignored_seed; return NewInternalIterator(ReadOptions(), &ignored, &ignored_seed); } int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() { MutexLock l(&mutex_); return versions_->MaxNextLevelOverlappingBytes(); } Status DBImpl::Get(const ReadOptions& options, const Slice& key, std::string* value) { Status s; MutexLock l(&mutex_); SequenceNumber snapshot; if (options.snapshot != NULL) { snapshot = reinterpret_cast(options.snapshot)->number_; } else { snapshot = versions_->LastSequence(); } MemTable* mem = mem_; MemTable* imm = imm_; Version* current = versions_->current(); mem->Ref(); if (imm != NULL) imm->Ref(); current->Ref(); bool have_stat_update = false; Version::GetStats stats; // Unlock while reading from files and memtables { mutex_.Unlock(); // First look in the memtable, then in the immutable memtable (if any). LookupKey lkey(key, snapshot); if (mem->Get(lkey, value, &s)) { // Done } else if (imm != NULL && imm->Get(lkey, value, &s)) { // Done } else { s = current->Get(options, lkey, value, &stats); have_stat_update = true; } mutex_.Lock(); } if (have_stat_update && current->UpdateStats(stats)) { MaybeScheduleCompaction(); } mem->Unref(); if (imm != NULL) imm->Unref(); current->Unref(); return s; } Iterator* DBImpl::NewIterator(const ReadOptions& options) { SequenceNumber latest_snapshot; uint32_t seed; Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed); return NewDBIterator( this, user_comparator(), iter, (options.snapshot != NULL ? reinterpret_cast(options.snapshot)->number_ : latest_snapshot), seed); } void DBImpl::RecordReadSample(Slice key) { MutexLock l(&mutex_); if (versions_->current()->RecordReadSample(key)) { MaybeScheduleCompaction(); } } const Snapshot* DBImpl::GetSnapshot() { MutexLock l(&mutex_); return snapshots_.New(versions_->LastSequence()); } void DBImpl::ReleaseSnapshot(const Snapshot* s) { MutexLock l(&mutex_); snapshots_.Delete(reinterpret_cast(s)); } // Convenience methods Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) { return DB::Put(o, key, val); } Status DBImpl::Delete(const WriteOptions& options, const Slice& key) { return DB::Delete(options, key); } Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { Writer w(&mutex_); w.batch = my_batch; w.sync = options.sync; w.done = false; MutexLock l(&mutex_); writers_.push_back(&w); while (!w.done && &w != writers_.front()) { w.cv.Wait(); } if (w.done) { return w.status; } // May temporarily unlock and wait. Status status = MakeRoomForWrite(my_batch == NULL); uint64_t last_sequence = versions_->LastSequence(); Writer* last_writer = &w; if (status.ok() && my_batch != NULL) { // NULL batch is for compactions WriteBatch* updates = BuildBatchGroup(&last_writer); WriteBatchInternal::SetSequence(updates, last_sequence + 1); last_sequence += WriteBatchInternal::Count(updates); // Add to log and apply to memtable. We can release the lock // during this phase since &w is currently responsible for logging // and protects against concurrent loggers and concurrent writes // into mem_. { mutex_.Unlock(); status = log_->AddRecord(WriteBatchInternal::Contents(updates)); bool sync_error = false; if (status.ok() && options.sync) { status = logfile_->Sync(); if (!status.ok()) { sync_error = true; } } if (status.ok()) { status = WriteBatchInternal::InsertInto(updates, mem_); } mutex_.Lock(); if (sync_error) { // The state of the log file is indeterminate: the log record we // just added may or may not show up when the DB is re-opened. // So we force the DB into a mode where all future writes fail. RecordBackgroundError(status); } } if (updates == tmp_batch_) tmp_batch_->Clear(); versions_->SetLastSequence(last_sequence); } while (true) { Writer* ready = writers_.front(); writers_.pop_front(); if (ready != &w) { ready->status = status; ready->done = true; ready->cv.Signal(); } if (ready == last_writer) break; } // Notify new head of write queue if (!writers_.empty()) { writers_.front()->cv.Signal(); } return status; } // REQUIRES: Writer list must be non-empty // REQUIRES: First writer must have a non-NULL batch WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { assert(!writers_.empty()); Writer* first = writers_.front(); WriteBatch* result = first->batch; assert(result != NULL); size_t size = WriteBatchInternal::ByteSize(first->batch); // Allow the group to grow up to a maximum size, but if the // original write is small, limit the growth so we do not slow // down the small write too much. size_t max_size = 1 << 20; if (size <= (128<<10)) { max_size = size + (128<<10); } *last_writer = first; std::deque::iterator iter = writers_.begin(); ++iter; // Advance past "first" for (; iter != writers_.end(); ++iter) { Writer* w = *iter; if (w->sync && !first->sync) { // Do not include a sync write into a batch handled by a non-sync write. break; } if (w->batch != NULL) { size += WriteBatchInternal::ByteSize(w->batch); if (size > max_size) { // Do not make batch too big break; } // Append to *result if (result == first->batch) { // Switch to temporary batch instead of disturbing caller's batch result = tmp_batch_; assert(WriteBatchInternal::Count(result) == 0); WriteBatchInternal::Append(result, first->batch); } WriteBatchInternal::Append(result, w->batch); } *last_writer = w; } return result; } // REQUIRES: mutex_ is held // REQUIRES: this thread is currently at the front of the writer queue Status DBImpl::MakeRoomForWrite(bool force) { mutex_.AssertHeld(); assert(!writers_.empty()); bool allow_delay = !force; Status s; while (true) { if (!bg_error_.ok()) { // Yield previous error s = bg_error_; break; } else if ( allow_delay && versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger) { // We are getting close to hitting a hard limit on the number of // L0 files. Rather than delaying a single write by several // seconds when we hit the hard limit, start delaying each // individual write by 1ms to reduce latency variance. Also, // this delay hands over some CPU to the compaction thread in // case it is sharing the same core as the writer. mutex_.Unlock(); env_->SleepForMicroseconds(1000); allow_delay = false; // Do not delay a single write more than once mutex_.Lock(); } else if (!force && (mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) { // There is room in current memtable break; } else if (imm_ != NULL) { // We have filled up the current memtable, but the previous // one is still being compacted, so we wait. Log(options_.info_log, "Current memtable full; waiting...\n"); bg_cv_.Wait(); } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) { // There are too many level-0 files. Log(options_.info_log, "Too many L0 files; waiting...\n"); bg_cv_.Wait(); } else { // Attempt to switch to a new memtable and trigger compaction of old assert(versions_->PrevLogNumber() == 0); uint64_t new_log_number = versions_->NewFileNumber(); WritableFile* lfile = NULL; s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile); if (!s.ok()) { // Avoid chewing through file number space in a tight loop. versions_->ReuseFileNumber(new_log_number); break; } delete log_; delete logfile_; logfile_ = lfile; logfile_number_ = new_log_number; log_ = new log::Writer(lfile); imm_ = mem_; has_imm_.Release_Store(imm_); mem_ = new MemTable(internal_comparator_); mem_->Ref(); force = false; // Do not force another compaction if have room MaybeScheduleCompaction(); } } return s; } bool DBImpl::GetProperty(const Slice& property, std::string* value) { value->clear(); MutexLock l(&mutex_); Slice in = property; Slice prefix("leveldb."); if (!in.starts_with(prefix)) return false; in.remove_prefix(prefix.size()); if (in.starts_with("num-files-at-level")) { in.remove_prefix(strlen("num-files-at-level")); uint64_t level; bool ok = ConsumeDecimalNumber(&in, &level) && in.empty(); if (!ok || level >= config::kNumLevels) { return false; } else { char buf[100]; snprintf(buf, sizeof(buf), "%d", versions_->NumLevelFiles(static_cast(level))); *value = buf; return true; } } else if (in == "stats") { char buf[200]; snprintf(buf, sizeof(buf), " Compactions\n" "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n" "--------------------------------------------------\n" ); value->append(buf); for (int level = 0; level < config::kNumLevels; level++) { int files = versions_->NumLevelFiles(level); if (stats_[level].micros > 0 || files > 0) { snprintf( buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", level, files, versions_->NumLevelBytes(level) / 1048576.0, stats_[level].micros / 1e6, stats_[level].bytes_read / 1048576.0, stats_[level].bytes_written / 1048576.0); value->append(buf); } } return true; } else if (in == "sstables") { *value = versions_->current()->DebugString(); return true; } else if (in == "approximate-memory-usage") { size_t total_usage = options_.block_cache->TotalCharge(); if (mem_) { total_usage += mem_->ApproximateMemoryUsage(); } if (imm_) { total_usage += imm_->ApproximateMemoryUsage(); } char buf[50]; snprintf(buf, sizeof(buf), "%llu", static_cast(total_usage)); value->append(buf); return true; } return false; } void DBImpl::GetApproximateSizes( const Range* range, int n, uint64_t* sizes) { // TODO(opt): better implementation Version* v; { MutexLock l(&mutex_); versions_->current()->Ref(); v = versions_->current(); } for (int i = 0; i < n; i++) { // Convert user_key into a corresponding internal key. InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek); InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek); uint64_t start = versions_->ApproximateOffsetOf(v, k1); uint64_t limit = versions_->ApproximateOffsetOf(v, k2); sizes[i] = (limit >= start ? limit - start : 0); } { MutexLock l(&mutex_); v->Unref(); } } // Default implementations of convenience methods that subclasses of DB // can call if they wish Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) { WriteBatch batch; batch.Put(key, value); return Write(opt, &batch); } Status DB::Delete(const WriteOptions& opt, const Slice& key) { WriteBatch batch; batch.Delete(key); return Write(opt, &batch); } DB::~DB() { } Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { *dbptr = NULL; DBImpl* impl = new DBImpl(options, dbname); impl->mutex_.Lock(); VersionEdit edit; // Recover handles create_if_missing, error_if_exists bool save_manifest = false; Status s = impl->Recover(&edit, &save_manifest); if (s.ok() && impl->mem_ == NULL) { // Create new log and a corresponding memtable. uint64_t new_log_number = impl->versions_->NewFileNumber(); WritableFile* lfile; s = options.env->NewWritableFile(LogFileName(dbname, new_log_number), &lfile); if (s.ok()) { edit.SetLogNumber(new_log_number); impl->logfile_ = lfile; impl->logfile_number_ = new_log_number; impl->log_ = new log::Writer(lfile); impl->mem_ = new MemTable(impl->internal_comparator_); impl->mem_->Ref(); } } if (s.ok() && save_manifest) { edit.SetPrevLogNumber(0); // No older logs needed after recovery. edit.SetLogNumber(impl->logfile_number_); s = impl->versions_->LogAndApply(&edit, &impl->mutex_); } if (s.ok()) { impl->DeleteObsoleteFiles(); impl->MaybeScheduleCompaction(); } impl->mutex_.Unlock(); if (s.ok()) { assert(impl->mem_ != NULL); *dbptr = impl; } else { delete impl; } return s; } Snapshot::~Snapshot() { } Status DestroyDB(const std::string& dbname, const Options& options) { Env* env = options.env; std::vector filenames; // Ignore error in case directory does not exist env->GetChildren(dbname, &filenames); if (filenames.empty()) { return Status::OK(); } FileLock* lock; const std::string lockname = LockFileName(dbname); Status result = env->LockFile(lockname, &lock); if (result.ok()) { uint64_t number; FileType type; for (size_t i = 0; i < filenames.size(); i++) { if (ParseFileName(filenames[i], &number, &type) && type != kDBLockFile) { // Lock file will be deleted at end Status del = env->DeleteFile(dbname + "/" + filenames[i]); if (result.ok() && !del.ok()) { result = del; } } } env->UnlockFile(lock); // Ignore error since state is already gone env->DeleteFile(lockname); env->DeleteDir(dbname); // Ignore error in case dir contains other files } return result; } } // namespace leveldb diff --git a/src/leveldb/db/log_format.h b/src/leveldb/db/log_format.h index a8c06efe1..356e69fca 100644 --- a/src/leveldb/db/log_format.h +++ b/src/leveldb/db/log_format.h @@ -1,35 +1,35 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // Log format information shared by reader and writer. -// See ../doc/log_format.txt for more detail. +// See ../doc/log_format.md for more detail. #ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_ #define STORAGE_LEVELDB_DB_LOG_FORMAT_H_ namespace leveldb { namespace log { enum RecordType { // Zero is reserved for preallocated files kZeroType = 0, kFullType = 1, // For fragments kFirstType = 2, kMiddleType = 3, kLastType = 4 }; static const int kMaxRecordType = kLastType; static const int kBlockSize = 32768; // Header is checksum (4 bytes), length (2 bytes), type (1 byte). static const int kHeaderSize = 4 + 2 + 1; } // namespace log } // namespace leveldb #endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_ diff --git a/src/leveldb/db/version_set.cc b/src/leveldb/db/version_set.cc index a5e0f77a6..b1256f90e 100644 --- a/src/leveldb/db/version_set.cc +++ b/src/leveldb/db/version_set.cc @@ -1,1522 +1,1535 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/version_set.h" #include #include #include "db/filename.h" #include "db/log_reader.h" #include "db/log_writer.h" #include "db/memtable.h" #include "db/table_cache.h" #include "leveldb/env.h" #include "leveldb/table_builder.h" #include "table/merger.h" #include "table/two_level_iterator.h" #include "util/coding.h" #include "util/logging.h" namespace leveldb { -static const int kTargetFileSize = 2 * 1048576; +static int TargetFileSize(const Options* options) { + return options->max_file_size; +} // Maximum bytes of overlaps in grandparent (i.e., level+2) before we // stop building a single file in a level->level+1 compaction. -static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize; +static int64_t MaxGrandParentOverlapBytes(const Options* options) { + return 10 * TargetFileSize(options); +} // Maximum number of bytes in all compacted files. We avoid expanding // the lower level file set of a compaction if it would make the // total compaction cover more than this many bytes. -static const int64_t kExpandedCompactionByteSizeLimit = 25 * kTargetFileSize; +static int64_t ExpandedCompactionByteSizeLimit(const Options* options) { + return 25 * TargetFileSize(options); +} -static double MaxBytesForLevel(int level) { +static double MaxBytesForLevel(const Options* options, int level) { // Note: the result for level zero is not really used since we set // the level-0 compaction threshold based on number of files. - double result = 10 * 1048576.0; // Result for both level-0 and level-1 + + // Result for both level-0 and level-1 + double result = 10. * 1048576.0; while (level > 1) { result *= 10; level--; } return result; } -static uint64_t MaxFileSizeForLevel(int level) { - return kTargetFileSize; // We could vary per level to reduce number of files? +static uint64_t MaxFileSizeForLevel(const Options* options, int level) { + // We could vary per level to reduce number of files? + return TargetFileSize(options); } static int64_t TotalFileSize(const std::vector& files) { int64_t sum = 0; for (size_t i = 0; i < files.size(); i++) { sum += files[i]->file_size; } return sum; } Version::~Version() { assert(refs_ == 0); // Remove from linked list prev_->next_ = next_; next_->prev_ = prev_; // Drop references to files for (int level = 0; level < config::kNumLevels; level++) { for (size_t i = 0; i < files_[level].size(); i++) { FileMetaData* f = files_[level][i]; assert(f->refs > 0); f->refs--; if (f->refs <= 0) { delete f; } } } } int FindFile(const InternalKeyComparator& icmp, const std::vector& files, const Slice& key) { uint32_t left = 0; uint32_t right = files.size(); while (left < right) { uint32_t mid = (left + right) / 2; const FileMetaData* f = files[mid]; if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) { // Key at "mid.largest" is < "target". Therefore all // files at or before "mid" are uninteresting. left = mid + 1; } else { // Key at "mid.largest" is >= "target". Therefore all files // after "mid" are uninteresting. right = mid; } } return right; } static bool AfterFile(const Comparator* ucmp, const Slice* user_key, const FileMetaData* f) { // NULL user_key occurs before all keys and is therefore never after *f return (user_key != NULL && ucmp->Compare(*user_key, f->largest.user_key()) > 0); } static bool BeforeFile(const Comparator* ucmp, const Slice* user_key, const FileMetaData* f) { // NULL user_key occurs after all keys and is therefore never before *f return (user_key != NULL && ucmp->Compare(*user_key, f->smallest.user_key()) < 0); } bool SomeFileOverlapsRange( const InternalKeyComparator& icmp, bool disjoint_sorted_files, const std::vector& files, const Slice* smallest_user_key, const Slice* largest_user_key) { const Comparator* ucmp = icmp.user_comparator(); if (!disjoint_sorted_files) { // Need to check against all files for (size_t i = 0; i < files.size(); i++) { const FileMetaData* f = files[i]; if (AfterFile(ucmp, smallest_user_key, f) || BeforeFile(ucmp, largest_user_key, f)) { // No overlap } else { return true; // Overlap } } return false; } // Binary search over file list uint32_t index = 0; if (smallest_user_key != NULL) { // Find the earliest possible internal key for smallest_user_key InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); index = FindFile(icmp, files, small.Encode()); } if (index >= files.size()) { // beginning of range is after all files, so no overlap. return false; } return !BeforeFile(ucmp, largest_user_key, files[index]); } // An internal iterator. For a given version/level pair, yields // information about the files in the level. For a given entry, key() // is the largest key that occurs in the file, and value() is an // 16-byte value containing the file number and file size, both // encoded using EncodeFixed64. class Version::LevelFileNumIterator : public Iterator { public: LevelFileNumIterator(const InternalKeyComparator& icmp, const std::vector* flist) : icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid } virtual bool Valid() const { return index_ < flist_->size(); } virtual void Seek(const Slice& target) { index_ = FindFile(icmp_, *flist_, target); } virtual void SeekToFirst() { index_ = 0; } virtual void SeekToLast() { index_ = flist_->empty() ? 0 : flist_->size() - 1; } virtual void Next() { assert(Valid()); index_++; } virtual void Prev() { assert(Valid()); if (index_ == 0) { index_ = flist_->size(); // Marks as invalid } else { index_--; } } Slice key() const { assert(Valid()); return (*flist_)[index_]->largest.Encode(); } Slice value() const { assert(Valid()); EncodeFixed64(value_buf_, (*flist_)[index_]->number); EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size); return Slice(value_buf_, sizeof(value_buf_)); } virtual Status status() const { return Status::OK(); } private: const InternalKeyComparator icmp_; const std::vector* const flist_; uint32_t index_; // Backing store for value(). Holds the file number and size. mutable char value_buf_[16]; }; static Iterator* GetFileIterator(void* arg, const ReadOptions& options, const Slice& file_value) { TableCache* cache = reinterpret_cast(arg); if (file_value.size() != 16) { return NewErrorIterator( Status::Corruption("FileReader invoked with unexpected value")); } else { return cache->NewIterator(options, DecodeFixed64(file_value.data()), DecodeFixed64(file_value.data() + 8)); } } Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, int level) const { return NewTwoLevelIterator( new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator, vset_->table_cache_, options); } void Version::AddIterators(const ReadOptions& options, std::vector* iters) { // Merge all level zero files together since they may overlap for (size_t i = 0; i < files_[0].size(); i++) { iters->push_back( vset_->table_cache_->NewIterator( options, files_[0][i]->number, files_[0][i]->file_size)); } // For levels > 0, we can use a concatenating iterator that sequentially // walks through the non-overlapping files in the level, opening them // lazily. for (int level = 1; level < config::kNumLevels; level++) { if (!files_[level].empty()) { iters->push_back(NewConcatenatingIterator(options, level)); } } } // Callback from TableCache::Get() namespace { enum SaverState { kNotFound, kFound, kDeleted, kCorrupt, }; struct Saver { SaverState state; const Comparator* ucmp; Slice user_key; std::string* value; }; } static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { Saver* s = reinterpret_cast(arg); ParsedInternalKey parsed_key; if (!ParseInternalKey(ikey, &parsed_key)) { s->state = kCorrupt; } else { if (s->ucmp->Compare(parsed_key.user_key, s->user_key) == 0) { s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted; if (s->state == kFound) { s->value->assign(v.data(), v.size()); } } } } static bool NewestFirst(FileMetaData* a, FileMetaData* b) { return a->number > b->number; } void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, bool (*func)(void*, int, FileMetaData*)) { // TODO(sanjay): Change Version::Get() to use this function. const Comparator* ucmp = vset_->icmp_.user_comparator(); // Search level-0 in order from newest to oldest. std::vector tmp; tmp.reserve(files_[0].size()); for (uint32_t i = 0; i < files_[0].size(); i++) { FileMetaData* f = files_[0][i]; if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && ucmp->Compare(user_key, f->largest.user_key()) <= 0) { tmp.push_back(f); } } if (!tmp.empty()) { std::sort(tmp.begin(), tmp.end(), NewestFirst); for (uint32_t i = 0; i < tmp.size(); i++) { if (!(*func)(arg, 0, tmp[i])) { return; } } } // Search other levels. for (int level = 1; level < config::kNumLevels; level++) { size_t num_files = files_[level].size(); if (num_files == 0) continue; // Binary search to find earliest index whose largest key >= internal_key. uint32_t index = FindFile(vset_->icmp_, files_[level], internal_key); if (index < num_files) { FileMetaData* f = files_[level][index]; if (ucmp->Compare(user_key, f->smallest.user_key()) < 0) { // All of "f" is past any data for user_key } else { if (!(*func)(arg, level, f)) { return; } } } } } Status Version::Get(const ReadOptions& options, const LookupKey& k, std::string* value, GetStats* stats) { Slice ikey = k.internal_key(); Slice user_key = k.user_key(); const Comparator* ucmp = vset_->icmp_.user_comparator(); Status s; stats->seek_file = NULL; stats->seek_file_level = -1; FileMetaData* last_file_read = NULL; int last_file_read_level = -1; // We can search level-by-level since entries never hop across // levels. Therefore we are guaranteed that if we find data // in an smaller level, later levels are irrelevant. std::vector tmp; FileMetaData* tmp2; for (int level = 0; level < config::kNumLevels; level++) { size_t num_files = files_[level].size(); if (num_files == 0) continue; // Get the list of files to search in this level FileMetaData* const* files = &files_[level][0]; if (level == 0) { // Level-0 files may overlap each other. Find all files that // overlap user_key and process them in order from newest to oldest. tmp.reserve(num_files); for (uint32_t i = 0; i < num_files; i++) { FileMetaData* f = files[i]; if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && ucmp->Compare(user_key, f->largest.user_key()) <= 0) { tmp.push_back(f); } } if (tmp.empty()) continue; std::sort(tmp.begin(), tmp.end(), NewestFirst); files = &tmp[0]; num_files = tmp.size(); } else { // Binary search to find earliest index whose largest key >= ikey. uint32_t index = FindFile(vset_->icmp_, files_[level], ikey); if (index >= num_files) { files = NULL; num_files = 0; } else { tmp2 = files[index]; if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) { // All of "tmp2" is past any data for user_key files = NULL; num_files = 0; } else { files = &tmp2; num_files = 1; } } } for (uint32_t i = 0; i < num_files; ++i) { if (last_file_read != NULL && stats->seek_file == NULL) { // We have had more than one seek for this read. Charge the 1st file. stats->seek_file = last_file_read; stats->seek_file_level = last_file_read_level; } FileMetaData* f = files[i]; last_file_read = f; last_file_read_level = level; Saver saver; saver.state = kNotFound; saver.ucmp = ucmp; saver.user_key = user_key; saver.value = value; s = vset_->table_cache_->Get(options, f->number, f->file_size, ikey, &saver, SaveValue); if (!s.ok()) { return s; } switch (saver.state) { case kNotFound: break; // Keep searching in other files case kFound: return s; case kDeleted: s = Status::NotFound(Slice()); // Use empty error message for speed return s; case kCorrupt: s = Status::Corruption("corrupted key for ", user_key); return s; } } } return Status::NotFound(Slice()); // Use an empty error message for speed } bool Version::UpdateStats(const GetStats& stats) { FileMetaData* f = stats.seek_file; if (f != NULL) { f->allowed_seeks--; if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) { file_to_compact_ = f; file_to_compact_level_ = stats.seek_file_level; return true; } } return false; } bool Version::RecordReadSample(Slice internal_key) { ParsedInternalKey ikey; if (!ParseInternalKey(internal_key, &ikey)) { return false; } struct State { GetStats stats; // Holds first matching file int matches; static bool Match(void* arg, int level, FileMetaData* f) { State* state = reinterpret_cast(arg); state->matches++; if (state->matches == 1) { // Remember first match. state->stats.seek_file = f; state->stats.seek_file_level = level; } // We can stop iterating once we have a second match. return state->matches < 2; } }; State state; state.matches = 0; ForEachOverlapping(ikey.user_key, internal_key, &state, &State::Match); // Must have at least two matches since we want to merge across // files. But what if we have a single file that contains many // overwrites and deletions? Should we have another mechanism for // finding such files? if (state.matches >= 2) { // 1MB cost is about 1 seek (see comment in Builder::Apply). return UpdateStats(state.stats); } return false; } void Version::Ref() { ++refs_; } void Version::Unref() { assert(this != &vset_->dummy_versions_); assert(refs_ >= 1); --refs_; if (refs_ == 0) { delete this; } } bool Version::OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key) { return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level], smallest_user_key, largest_user_key); } int Version::PickLevelForMemTableOutput( const Slice& smallest_user_key, const Slice& largest_user_key) { int level = 0; if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) { // Push to next level if there is no overlap in next level, // and the #bytes overlapping in the level after that are limited. InternalKey start(smallest_user_key, kMaxSequenceNumber, kValueTypeForSeek); InternalKey limit(largest_user_key, 0, static_cast(0)); std::vector overlaps; while (level < config::kMaxMemCompactLevel) { if (OverlapInLevel(level + 1, &smallest_user_key, &largest_user_key)) { break; } if (level + 2 < config::kNumLevels) { // Check that file does not overlap too many grandparent bytes. GetOverlappingInputs(level + 2, &start, &limit, &overlaps); const int64_t sum = TotalFileSize(overlaps); - if (sum > kMaxGrandParentOverlapBytes) { + if (sum > MaxGrandParentOverlapBytes(vset_->options_)) { break; } } level++; } } return level; } // Store in "*inputs" all files in "level" that overlap [begin,end] void Version::GetOverlappingInputs( int level, const InternalKey* begin, const InternalKey* end, std::vector* inputs) { assert(level >= 0); assert(level < config::kNumLevels); inputs->clear(); Slice user_begin, user_end; if (begin != NULL) { user_begin = begin->user_key(); } if (end != NULL) { user_end = end->user_key(); } const Comparator* user_cmp = vset_->icmp_.user_comparator(); for (size_t i = 0; i < files_[level].size(); ) { FileMetaData* f = files_[level][i++]; const Slice file_start = f->smallest.user_key(); const Slice file_limit = f->largest.user_key(); if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) { // "f" is completely before specified range; skip it } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) { // "f" is completely after specified range; skip it } else { inputs->push_back(f); if (level == 0) { // Level-0 files may overlap each other. So check if the newly // added file has expanded the range. If so, restart search. if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) { user_begin = file_start; inputs->clear(); i = 0; } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) { user_end = file_limit; inputs->clear(); i = 0; } } } } } std::string Version::DebugString() const { std::string r; for (int level = 0; level < config::kNumLevels; level++) { // E.g., // --- level 1 --- // 17:123['a' .. 'd'] // 20:43['e' .. 'g'] r.append("--- level "); AppendNumberTo(&r, level); r.append(" ---\n"); const std::vector& files = files_[level]; for (size_t i = 0; i < files.size(); i++) { r.push_back(' '); AppendNumberTo(&r, files[i]->number); r.push_back(':'); AppendNumberTo(&r, files[i]->file_size); r.append("["); r.append(files[i]->smallest.DebugString()); r.append(" .. "); r.append(files[i]->largest.DebugString()); r.append("]\n"); } } return r; } // A helper class so we can efficiently apply a whole sequence // of edits to a particular state without creating intermediate // Versions that contain full copies of the intermediate state. class VersionSet::Builder { private: // Helper to sort by v->files_[file_number].smallest struct BySmallestKey { const InternalKeyComparator* internal_comparator; bool operator()(FileMetaData* f1, FileMetaData* f2) const { int r = internal_comparator->Compare(f1->smallest, f2->smallest); if (r != 0) { return (r < 0); } else { // Break ties by file number return (f1->number < f2->number); } } }; typedef std::set FileSet; struct LevelState { std::set deleted_files; FileSet* added_files; }; VersionSet* vset_; Version* base_; LevelState levels_[config::kNumLevels]; public: // Initialize a builder with the files from *base and other info from *vset Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) { base_->Ref(); BySmallestKey cmp; cmp.internal_comparator = &vset_->icmp_; for (int level = 0; level < config::kNumLevels; level++) { levels_[level].added_files = new FileSet(cmp); } } ~Builder() { for (int level = 0; level < config::kNumLevels; level++) { const FileSet* added = levels_[level].added_files; std::vector to_unref; to_unref.reserve(added->size()); for (FileSet::const_iterator it = added->begin(); it != added->end(); ++it) { to_unref.push_back(*it); } delete added; for (uint32_t i = 0; i < to_unref.size(); i++) { FileMetaData* f = to_unref[i]; f->refs--; if (f->refs <= 0) { delete f; } } } base_->Unref(); } // Apply all of the edits in *edit to the current state. void Apply(VersionEdit* edit) { // Update compaction pointers for (size_t i = 0; i < edit->compact_pointers_.size(); i++) { const int level = edit->compact_pointers_[i].first; vset_->compact_pointer_[level] = edit->compact_pointers_[i].second.Encode().ToString(); } // Delete files const VersionEdit::DeletedFileSet& del = edit->deleted_files_; for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin(); iter != del.end(); ++iter) { const int level = iter->first; const uint64_t number = iter->second; levels_[level].deleted_files.insert(number); } // Add new files for (size_t i = 0; i < edit->new_files_.size(); i++) { const int level = edit->new_files_[i].first; FileMetaData* f = new FileMetaData(edit->new_files_[i].second); f->refs = 1; // We arrange to automatically compact this file after // a certain number of seeks. Let's assume: // (1) One seek costs 10ms // (2) Writing or reading 1MB costs 10ms (100MB/s) // (3) A compaction of 1MB does 25MB of IO: // 1MB read from this level // 10-12MB read from next level (boundaries may be misaligned) // 10-12MB written to next level // This implies that 25 seeks cost the same as the compaction // of 1MB of data. I.e., one seek costs approximately the // same as the compaction of 40KB of data. We are a little // conservative and allow approximately one seek for every 16KB // of data before triggering a compaction. f->allowed_seeks = (f->file_size / 16384); if (f->allowed_seeks < 100) f->allowed_seeks = 100; levels_[level].deleted_files.erase(f->number); levels_[level].added_files->insert(f); } } // Save the current state in *v. void SaveTo(Version* v) { BySmallestKey cmp; cmp.internal_comparator = &vset_->icmp_; for (int level = 0; level < config::kNumLevels; level++) { // Merge the set of added files with the set of pre-existing files. // Drop any deleted files. Store the result in *v. const std::vector& base_files = base_->files_[level]; std::vector::const_iterator base_iter = base_files.begin(); std::vector::const_iterator base_end = base_files.end(); const FileSet* added = levels_[level].added_files; v->files_[level].reserve(base_files.size() + added->size()); for (FileSet::const_iterator added_iter = added->begin(); added_iter != added->end(); ++added_iter) { // Add all smaller files listed in base_ for (std::vector::const_iterator bpos = std::upper_bound(base_iter, base_end, *added_iter, cmp); base_iter != bpos; ++base_iter) { MaybeAddFile(v, level, *base_iter); } MaybeAddFile(v, level, *added_iter); } // Add remaining base files for (; base_iter != base_end; ++base_iter) { MaybeAddFile(v, level, *base_iter); } #ifndef NDEBUG // Make sure there is no overlap in levels > 0 if (level > 0) { for (uint32_t i = 1; i < v->files_[level].size(); i++) { const InternalKey& prev_end = v->files_[level][i-1]->largest; const InternalKey& this_begin = v->files_[level][i]->smallest; if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) { fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", prev_end.DebugString().c_str(), this_begin.DebugString().c_str()); abort(); } } } #endif } } void MaybeAddFile(Version* v, int level, FileMetaData* f) { if (levels_[level].deleted_files.count(f->number) > 0) { // File is deleted: do nothing } else { std::vector* files = &v->files_[level]; if (level > 0 && !files->empty()) { // Must not overlap assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest, f->smallest) < 0); } f->refs++; files->push_back(f); } } }; VersionSet::VersionSet(const std::string& dbname, const Options* options, TableCache* table_cache, const InternalKeyComparator* cmp) : env_(options->env), dbname_(dbname), options_(options), table_cache_(table_cache), icmp_(*cmp), next_file_number_(2), manifest_file_number_(0), // Filled by Recover() last_sequence_(0), log_number_(0), prev_log_number_(0), descriptor_file_(NULL), descriptor_log_(NULL), dummy_versions_(this), current_(NULL) { AppendVersion(new Version(this)); } VersionSet::~VersionSet() { current_->Unref(); assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty delete descriptor_log_; delete descriptor_file_; } void VersionSet::AppendVersion(Version* v) { // Make "v" current assert(v->refs_ == 0); assert(v != current_); if (current_ != NULL) { current_->Unref(); } current_ = v; v->Ref(); // Append to linked list v->prev_ = dummy_versions_.prev_; v->next_ = &dummy_versions_; v->prev_->next_ = v; v->next_->prev_ = v; } Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { if (edit->has_log_number_) { assert(edit->log_number_ >= log_number_); assert(edit->log_number_ < next_file_number_); } else { edit->SetLogNumber(log_number_); } if (!edit->has_prev_log_number_) { edit->SetPrevLogNumber(prev_log_number_); } edit->SetNextFile(next_file_number_); edit->SetLastSequence(last_sequence_); Version* v = new Version(this); { Builder builder(this, current_); builder.Apply(edit); builder.SaveTo(v); } Finalize(v); // Initialize new descriptor log file if necessary by creating // a temporary file that contains a snapshot of the current version. std::string new_manifest_file; Status s; if (descriptor_log_ == NULL) { // No reason to unlock *mu here since we only hit this path in the // first call to LogAndApply (when opening the database). assert(descriptor_file_ == NULL); new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_); edit->SetNextFile(next_file_number_); s = env_->NewWritableFile(new_manifest_file, &descriptor_file_); if (s.ok()) { descriptor_log_ = new log::Writer(descriptor_file_); s = WriteSnapshot(descriptor_log_); } } // Unlock during expensive MANIFEST log write { mu->Unlock(); // Write new record to MANIFEST log if (s.ok()) { std::string record; edit->EncodeTo(&record); s = descriptor_log_->AddRecord(record); if (s.ok()) { s = descriptor_file_->Sync(); } if (!s.ok()) { Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str()); } } // If we just created a new descriptor file, install it by writing a // new CURRENT file that points to it. if (s.ok() && !new_manifest_file.empty()) { s = SetCurrentFile(env_, dbname_, manifest_file_number_); } mu->Lock(); } // Install the new version if (s.ok()) { AppendVersion(v); log_number_ = edit->log_number_; prev_log_number_ = edit->prev_log_number_; } else { delete v; if (!new_manifest_file.empty()) { delete descriptor_log_; delete descriptor_file_; descriptor_log_ = NULL; descriptor_file_ = NULL; env_->DeleteFile(new_manifest_file); } } return s; } Status VersionSet::Recover(bool *save_manifest) { struct LogReporter : public log::Reader::Reporter { Status* status; virtual void Corruption(size_t bytes, const Status& s) { if (this->status->ok()) *this->status = s; } }; // Read "CURRENT" file, which contains a pointer to the current manifest file std::string current; Status s = ReadFileToString(env_, CurrentFileName(dbname_), ¤t); if (!s.ok()) { return s; } if (current.empty() || current[current.size()-1] != '\n') { return Status::Corruption("CURRENT file does not end with newline"); } current.resize(current.size() - 1); std::string dscname = dbname_ + "/" + current; SequentialFile* file; s = env_->NewSequentialFile(dscname, &file); if (!s.ok()) { return s; } bool have_log_number = false; bool have_prev_log_number = false; bool have_next_file = false; bool have_last_sequence = false; uint64_t next_file = 0; uint64_t last_sequence = 0; uint64_t log_number = 0; uint64_t prev_log_number = 0; Builder builder(this, current_); { LogReporter reporter; reporter.status = &s; log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { VersionEdit edit; s = edit.DecodeFrom(record); if (s.ok()) { if (edit.has_comparator_ && edit.comparator_ != icmp_.user_comparator()->Name()) { s = Status::InvalidArgument( edit.comparator_ + " does not match existing comparator ", icmp_.user_comparator()->Name()); } } if (s.ok()) { builder.Apply(&edit); } if (edit.has_log_number_) { log_number = edit.log_number_; have_log_number = true; } if (edit.has_prev_log_number_) { prev_log_number = edit.prev_log_number_; have_prev_log_number = true; } if (edit.has_next_file_number_) { next_file = edit.next_file_number_; have_next_file = true; } if (edit.has_last_sequence_) { last_sequence = edit.last_sequence_; have_last_sequence = true; } } } delete file; file = NULL; if (s.ok()) { if (!have_next_file) { s = Status::Corruption("no meta-nextfile entry in descriptor"); } else if (!have_log_number) { s = Status::Corruption("no meta-lognumber entry in descriptor"); } else if (!have_last_sequence) { s = Status::Corruption("no last-sequence-number entry in descriptor"); } if (!have_prev_log_number) { prev_log_number = 0; } MarkFileNumberUsed(prev_log_number); MarkFileNumberUsed(log_number); } if (s.ok()) { Version* v = new Version(this); builder.SaveTo(v); // Install recovered version Finalize(v); AppendVersion(v); manifest_file_number_ = next_file; next_file_number_ = next_file + 1; last_sequence_ = last_sequence; log_number_ = log_number; prev_log_number_ = prev_log_number; // See if we can reuse the existing MANIFEST file. if (ReuseManifest(dscname, current)) { // No need to save new manifest } else { *save_manifest = true; } } return s; } bool VersionSet::ReuseManifest(const std::string& dscname, const std::string& dscbase) { if (!options_->reuse_logs) { return false; } FileType manifest_type; uint64_t manifest_number; uint64_t manifest_size; if (!ParseFileName(dscbase, &manifest_number, &manifest_type) || manifest_type != kDescriptorFile || !env_->GetFileSize(dscname, &manifest_size).ok() || // Make new compacted MANIFEST if old one is too big - manifest_size >= kTargetFileSize) { + manifest_size >= TargetFileSize(options_)) { return false; } assert(descriptor_file_ == NULL); assert(descriptor_log_ == NULL); Status r = env_->NewAppendableFile(dscname, &descriptor_file_); if (!r.ok()) { Log(options_->info_log, "Reuse MANIFEST: %s\n", r.ToString().c_str()); assert(descriptor_file_ == NULL); return false; } Log(options_->info_log, "Reusing MANIFEST %s\n", dscname.c_str()); descriptor_log_ = new log::Writer(descriptor_file_, manifest_size); manifest_file_number_ = manifest_number; return true; } void VersionSet::MarkFileNumberUsed(uint64_t number) { if (next_file_number_ <= number) { next_file_number_ = number + 1; } } void VersionSet::Finalize(Version* v) { // Precomputed best level for next compaction int best_level = -1; double best_score = -1; for (int level = 0; level < config::kNumLevels-1; level++) { double score; if (level == 0) { // We treat level-0 specially by bounding the number of files // instead of number of bytes for two reasons: // // (1) With larger write-buffer sizes, it is nice not to do too // many level-0 compactions. // // (2) The files in level-0 are merged on every read and // therefore we wish to avoid too many files when the individual // file size is small (perhaps because of a small write-buffer // setting, or very high compression ratios, or lots of // overwrites/deletions). score = v->files_[level].size() / static_cast(config::kL0_CompactionTrigger); } else { // Compute the ratio of current size to size limit. const uint64_t level_bytes = TotalFileSize(v->files_[level]); - score = static_cast(level_bytes) / MaxBytesForLevel(level); + score = + static_cast(level_bytes) / MaxBytesForLevel(options_, level); } if (score > best_score) { best_level = level; best_score = score; } } v->compaction_level_ = best_level; v->compaction_score_ = best_score; } Status VersionSet::WriteSnapshot(log::Writer* log) { // TODO: Break up into multiple records to reduce memory usage on recovery? // Save metadata VersionEdit edit; edit.SetComparatorName(icmp_.user_comparator()->Name()); // Save compaction pointers for (int level = 0; level < config::kNumLevels; level++) { if (!compact_pointer_[level].empty()) { InternalKey key; key.DecodeFrom(compact_pointer_[level]); edit.SetCompactPointer(level, key); } } // Save files for (int level = 0; level < config::kNumLevels; level++) { const std::vector& files = current_->files_[level]; for (size_t i = 0; i < files.size(); i++) { const FileMetaData* f = files[i]; edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest); } } std::string record; edit.EncodeTo(&record); return log->AddRecord(record); } int VersionSet::NumLevelFiles(int level) const { assert(level >= 0); assert(level < config::kNumLevels); return current_->files_[level].size(); } const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { // Update code if kNumLevels changes assert(config::kNumLevels == 7); snprintf(scratch->buffer, sizeof(scratch->buffer), "files[ %d %d %d %d %d %d %d ]", int(current_->files_[0].size()), int(current_->files_[1].size()), int(current_->files_[2].size()), int(current_->files_[3].size()), int(current_->files_[4].size()), int(current_->files_[5].size()), int(current_->files_[6].size())); return scratch->buffer; } uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { uint64_t result = 0; for (int level = 0; level < config::kNumLevels; level++) { const std::vector& files = v->files_[level]; for (size_t i = 0; i < files.size(); i++) { if (icmp_.Compare(files[i]->largest, ikey) <= 0) { // Entire file is before "ikey", so just add the file size result += files[i]->file_size; } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) { // Entire file is after "ikey", so ignore if (level > 0) { // Files other than level 0 are sorted by meta->smallest, so // no further files in this level will contain data for // "ikey". break; } } else { // "ikey" falls in the range for this table. Add the // approximate offset of "ikey" within the table. Table* tableptr; Iterator* iter = table_cache_->NewIterator( ReadOptions(), files[i]->number, files[i]->file_size, &tableptr); if (tableptr != NULL) { result += tableptr->ApproximateOffsetOf(ikey.Encode()); } delete iter; } } } return result; } void VersionSet::AddLiveFiles(std::set* live) { for (Version* v = dummy_versions_.next_; v != &dummy_versions_; v = v->next_) { for (int level = 0; level < config::kNumLevels; level++) { const std::vector& files = v->files_[level]; for (size_t i = 0; i < files.size(); i++) { live->insert(files[i]->number); } } } } int64_t VersionSet::NumLevelBytes(int level) const { assert(level >= 0); assert(level < config::kNumLevels); return TotalFileSize(current_->files_[level]); } int64_t VersionSet::MaxNextLevelOverlappingBytes() { int64_t result = 0; std::vector overlaps; for (int level = 1; level < config::kNumLevels - 1; level++) { for (size_t i = 0; i < current_->files_[level].size(); i++) { const FileMetaData* f = current_->files_[level][i]; current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest, &overlaps); const int64_t sum = TotalFileSize(overlaps); if (sum > result) { result = sum; } } } return result; } // Stores the minimal range that covers all entries in inputs in // *smallest, *largest. // REQUIRES: inputs is not empty void VersionSet::GetRange(const std::vector& inputs, InternalKey* smallest, InternalKey* largest) { assert(!inputs.empty()); smallest->Clear(); largest->Clear(); for (size_t i = 0; i < inputs.size(); i++) { FileMetaData* f = inputs[i]; if (i == 0) { *smallest = f->smallest; *largest = f->largest; } else { if (icmp_.Compare(f->smallest, *smallest) < 0) { *smallest = f->smallest; } if (icmp_.Compare(f->largest, *largest) > 0) { *largest = f->largest; } } } } // Stores the minimal range that covers all entries in inputs1 and inputs2 // in *smallest, *largest. // REQUIRES: inputs is not empty void VersionSet::GetRange2(const std::vector& inputs1, const std::vector& inputs2, InternalKey* smallest, InternalKey* largest) { std::vector all = inputs1; all.insert(all.end(), inputs2.begin(), inputs2.end()); GetRange(all, smallest, largest); } Iterator* VersionSet::MakeInputIterator(Compaction* c) { ReadOptions options; options.verify_checksums = options_->paranoid_checks; options.fill_cache = false; // Level-0 files have to be merged together. For other levels, // we will make a concatenating iterator per level. // TODO(opt): use concatenating iterator for level-0 if there is no overlap const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2); Iterator** list = new Iterator*[space]; int num = 0; for (int which = 0; which < 2; which++) { if (!c->inputs_[which].empty()) { if (c->level() + which == 0) { const std::vector& files = c->inputs_[which]; for (size_t i = 0; i < files.size(); i++) { list[num++] = table_cache_->NewIterator( options, files[i]->number, files[i]->file_size); } } else { // Create concatenating iterator for the files from this level list[num++] = NewTwoLevelIterator( new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]), &GetFileIterator, table_cache_, options); } } } assert(num <= space); Iterator* result = NewMergingIterator(&icmp_, list, num); delete[] list; return result; } Compaction* VersionSet::PickCompaction() { Compaction* c; int level; // We prefer compactions triggered by too much data in a level over // the compactions triggered by seeks. const bool size_compaction = (current_->compaction_score_ >= 1); const bool seek_compaction = (current_->file_to_compact_ != NULL); if (size_compaction) { level = current_->compaction_level_; assert(level >= 0); assert(level+1 < config::kNumLevels); - c = new Compaction(level); + c = new Compaction(options_, level); // Pick the first file that comes after compact_pointer_[level] for (size_t i = 0; i < current_->files_[level].size(); i++) { FileMetaData* f = current_->files_[level][i]; if (compact_pointer_[level].empty() || icmp_.Compare(f->largest.Encode(), compact_pointer_[level]) > 0) { c->inputs_[0].push_back(f); break; } } if (c->inputs_[0].empty()) { // Wrap-around to the beginning of the key space c->inputs_[0].push_back(current_->files_[level][0]); } } else if (seek_compaction) { level = current_->file_to_compact_level_; - c = new Compaction(level); + c = new Compaction(options_, level); c->inputs_[0].push_back(current_->file_to_compact_); } else { return NULL; } c->input_version_ = current_; c->input_version_->Ref(); // Files in level 0 may overlap each other, so pick up all overlapping ones if (level == 0) { InternalKey smallest, largest; GetRange(c->inputs_[0], &smallest, &largest); // Note that the next call will discard the file we placed in // c->inputs_[0] earlier and replace it with an overlapping set // which will include the picked file. current_->GetOverlappingInputs(0, &smallest, &largest, &c->inputs_[0]); assert(!c->inputs_[0].empty()); } SetupOtherInputs(c); return c; } void VersionSet::SetupOtherInputs(Compaction* c) { const int level = c->level(); InternalKey smallest, largest; GetRange(c->inputs_[0], &smallest, &largest); current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]); // Get entire range covered by compaction InternalKey all_start, all_limit; GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit); // See if we can grow the number of inputs in "level" without // changing the number of "level+1" files we pick up. if (!c->inputs_[1].empty()) { std::vector expanded0; current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0); const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); const int64_t expanded0_size = TotalFileSize(expanded0); if (expanded0.size() > c->inputs_[0].size() && - inputs1_size + expanded0_size < kExpandedCompactionByteSizeLimit) { + inputs1_size + expanded0_size < + ExpandedCompactionByteSizeLimit(options_)) { InternalKey new_start, new_limit; GetRange(expanded0, &new_start, &new_limit); std::vector expanded1; current_->GetOverlappingInputs(level+1, &new_start, &new_limit, &expanded1); if (expanded1.size() == c->inputs_[1].size()) { Log(options_->info_log, "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n", level, int(c->inputs_[0].size()), int(c->inputs_[1].size()), long(inputs0_size), long(inputs1_size), int(expanded0.size()), int(expanded1.size()), long(expanded0_size), long(inputs1_size)); smallest = new_start; largest = new_limit; c->inputs_[0] = expanded0; c->inputs_[1] = expanded1; GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit); } } } // Compute the set of grandparent files that overlap this compaction // (parent == level+1; grandparent == level+2) if (level + 2 < config::kNumLevels) { current_->GetOverlappingInputs(level + 2, &all_start, &all_limit, &c->grandparents_); } if (false) { Log(options_->info_log, "Compacting %d '%s' .. '%s'", level, smallest.DebugString().c_str(), largest.DebugString().c_str()); } // Update the place where we will do the next compaction for this level. // We update this immediately instead of waiting for the VersionEdit // to be applied so that if the compaction fails, we will try a different // key range next time. compact_pointer_[level] = largest.Encode().ToString(); c->edit_.SetCompactPointer(level, largest); } Compaction* VersionSet::CompactRange( int level, const InternalKey* begin, const InternalKey* end) { std::vector inputs; current_->GetOverlappingInputs(level, begin, end, &inputs); if (inputs.empty()) { return NULL; } // Avoid compacting too much in one shot in case the range is large. // But we cannot do this for level-0 since level-0 files can overlap // and we must not pick one file and drop another older file if the // two files overlap. if (level > 0) { - const uint64_t limit = MaxFileSizeForLevel(level); + const uint64_t limit = MaxFileSizeForLevel(options_, level); uint64_t total = 0; for (size_t i = 0; i < inputs.size(); i++) { uint64_t s = inputs[i]->file_size; total += s; if (total >= limit) { inputs.resize(i + 1); break; } } } - Compaction* c = new Compaction(level); + Compaction* c = new Compaction(options_, level); c->input_version_ = current_; c->input_version_->Ref(); c->inputs_[0] = inputs; SetupOtherInputs(c); return c; } -Compaction::Compaction(int level) +Compaction::Compaction(const Options* options, int level) : level_(level), - max_output_file_size_(MaxFileSizeForLevel(level)), + max_output_file_size_(MaxFileSizeForLevel(options, level)), input_version_(NULL), grandparent_index_(0), seen_key_(false), overlapped_bytes_(0) { for (int i = 0; i < config::kNumLevels; i++) { level_ptrs_[i] = 0; } } Compaction::~Compaction() { if (input_version_ != NULL) { input_version_->Unref(); } } bool Compaction::IsTrivialMove() const { + const VersionSet* vset = input_version_->vset_; // Avoid a move if there is lots of overlapping grandparent data. // Otherwise, the move could create a parent file that will require // a very expensive merge later on. - return (num_input_files(0) == 1 && - num_input_files(1) == 0 && - TotalFileSize(grandparents_) <= kMaxGrandParentOverlapBytes); + return (num_input_files(0) == 1 && num_input_files(1) == 0 && + TotalFileSize(grandparents_) <= + MaxGrandParentOverlapBytes(vset->options_)); } void Compaction::AddInputDeletions(VersionEdit* edit) { for (int which = 0; which < 2; which++) { for (size_t i = 0; i < inputs_[which].size(); i++) { edit->DeleteFile(level_ + which, inputs_[which][i]->number); } } } bool Compaction::IsBaseLevelForKey(const Slice& user_key) { // Maybe use binary search to find right entry instead of linear search? const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) { const std::vector& files = input_version_->files_[lvl]; for (; level_ptrs_[lvl] < files.size(); ) { FileMetaData* f = files[level_ptrs_[lvl]]; if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { // We've advanced far enough if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) { // Key falls in this file's range, so definitely not base level return false; } break; } level_ptrs_[lvl]++; } } return true; } bool Compaction::ShouldStopBefore(const Slice& internal_key) { + const VersionSet* vset = input_version_->vset_; // Scan to find earliest grandparent file that contains key. - const InternalKeyComparator* icmp = &input_version_->vset_->icmp_; + const InternalKeyComparator* icmp = &vset->icmp_; while (grandparent_index_ < grandparents_.size() && icmp->Compare(internal_key, grandparents_[grandparent_index_]->largest.Encode()) > 0) { if (seen_key_) { overlapped_bytes_ += grandparents_[grandparent_index_]->file_size; } grandparent_index_++; } seen_key_ = true; - if (overlapped_bytes_ > kMaxGrandParentOverlapBytes) { + if (overlapped_bytes_ > MaxGrandParentOverlapBytes(vset->options_)) { // Too much overlap for current output; start new output overlapped_bytes_ = 0; return true; } else { return false; } } void Compaction::ReleaseInputs() { if (input_version_ != NULL) { input_version_->Unref(); input_version_ = NULL; } } } // namespace leveldb diff --git a/src/leveldb/db/version_set.h b/src/leveldb/db/version_set.h index 1dec74567..c4e7ac360 100644 --- a/src/leveldb/db/version_set.h +++ b/src/leveldb/db/version_set.h @@ -1,398 +1,398 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // The representation of a DBImpl consists of a set of Versions. The // newest version is called "current". Older versions may be kept // around to provide a consistent view to live iterators. // // Each Version keeps track of a set of Table files per level. The // entire set of versions is maintained in a VersionSet. // // Version,VersionSet are thread-compatible, but require external // synchronization on all accesses. #ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_ #define STORAGE_LEVELDB_DB_VERSION_SET_H_ #include #include #include #include "db/dbformat.h" #include "db/version_edit.h" #include "port/port.h" #include "port/thread_annotations.h" namespace leveldb { namespace log { class Writer; } class Compaction; class Iterator; class MemTable; class TableBuilder; class TableCache; class Version; class VersionSet; class WritableFile; // Return the smallest index i such that files[i]->largest >= key. // Return files.size() if there is no such file. // REQUIRES: "files" contains a sorted list of non-overlapping files. extern int FindFile(const InternalKeyComparator& icmp, const std::vector& files, const Slice& key); // Returns true iff some file in "files" overlaps the user key range // [*smallest,*largest]. // smallest==NULL represents a key smaller than all keys in the DB. // largest==NULL represents a key largest than all keys in the DB. // REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges // in sorted order. extern bool SomeFileOverlapsRange( const InternalKeyComparator& icmp, bool disjoint_sorted_files, const std::vector& files, const Slice* smallest_user_key, const Slice* largest_user_key); class Version { public: // Append to *iters a sequence of iterators that will // yield the contents of this Version when merged together. // REQUIRES: This version has been saved (see VersionSet::SaveTo) void AddIterators(const ReadOptions&, std::vector* iters); // Lookup the value for key. If found, store it in *val and // return OK. Else return a non-OK status. Fills *stats. // REQUIRES: lock is not held struct GetStats { FileMetaData* seek_file; int seek_file_level; }; Status Get(const ReadOptions&, const LookupKey& key, std::string* val, GetStats* stats); // Adds "stats" into the current state. Returns true if a new // compaction may need to be triggered, false otherwise. // REQUIRES: lock is held bool UpdateStats(const GetStats& stats); // Record a sample of bytes read at the specified internal key. // Samples are taken approximately once every config::kReadBytesPeriod // bytes. Returns true if a new compaction may need to be triggered. // REQUIRES: lock is held bool RecordReadSample(Slice key); // Reference count management (so Versions do not disappear out from // under live iterators) void Ref(); void Unref(); void GetOverlappingInputs( int level, const InternalKey* begin, // NULL means before all keys const InternalKey* end, // NULL means after all keys std::vector* inputs); // Returns true iff some file in the specified level overlaps // some part of [*smallest_user_key,*largest_user_key]. // smallest_user_key==NULL represents a key smaller than all keys in the DB. // largest_user_key==NULL represents a key largest than all keys in the DB. bool OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key); // Return the level at which we should place a new memtable compaction // result that covers the range [smallest_user_key,largest_user_key]. int PickLevelForMemTableOutput(const Slice& smallest_user_key, const Slice& largest_user_key); int NumFiles(int level) const { return files_[level].size(); } // Return a human readable string that describes this version's contents. std::string DebugString() const; private: friend class Compaction; friend class VersionSet; class LevelFileNumIterator; Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const; // Call func(arg, level, f) for every file that overlaps user_key in // order from newest to oldest. If an invocation of func returns // false, makes no more calls. // // REQUIRES: user portion of internal_key == user_key. void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, bool (*func)(void*, int, FileMetaData*)); VersionSet* vset_; // VersionSet to which this Version belongs Version* next_; // Next version in linked list Version* prev_; // Previous version in linked list int refs_; // Number of live refs to this version // List of files per level std::vector files_[config::kNumLevels]; // Next file to compact based on seek stats. FileMetaData* file_to_compact_; int file_to_compact_level_; // Level that should be compacted next and its compaction score. // Score < 1 means compaction is not strictly needed. These fields // are initialized by Finalize(). double compaction_score_; int compaction_level_; explicit Version(VersionSet* vset) : vset_(vset), next_(this), prev_(this), refs_(0), file_to_compact_(NULL), file_to_compact_level_(-1), compaction_score_(-1), compaction_level_(-1) { } ~Version(); // No copying allowed Version(const Version&); void operator=(const Version&); }; class VersionSet { public: VersionSet(const std::string& dbname, const Options* options, TableCache* table_cache, const InternalKeyComparator*); ~VersionSet(); // Apply *edit to the current version to form a new descriptor that // is both saved to persistent state and installed as the new // current version. Will release *mu while actually writing to the file. // REQUIRES: *mu is held on entry. // REQUIRES: no other thread concurrently calls LogAndApply() Status LogAndApply(VersionEdit* edit, port::Mutex* mu) EXCLUSIVE_LOCKS_REQUIRED(mu); // Recover the last saved descriptor from persistent storage. Status Recover(bool *save_manifest); // Return the current version. Version* current() const { return current_; } // Return the current manifest file number uint64_t ManifestFileNumber() const { return manifest_file_number_; } // Allocate and return a new file number uint64_t NewFileNumber() { return next_file_number_++; } // Arrange to reuse "file_number" unless a newer file number has // already been allocated. // REQUIRES: "file_number" was returned by a call to NewFileNumber(). void ReuseFileNumber(uint64_t file_number) { if (next_file_number_ == file_number + 1) { next_file_number_ = file_number; } } // Return the number of Table files at the specified level. int NumLevelFiles(int level) const; // Return the combined file size of all files at the specified level. int64_t NumLevelBytes(int level) const; // Return the last sequence number. uint64_t LastSequence() const { return last_sequence_; } // Set the last sequence number to s. void SetLastSequence(uint64_t s) { assert(s >= last_sequence_); last_sequence_ = s; } // Mark the specified file number as used. void MarkFileNumberUsed(uint64_t number); // Return the current log file number. uint64_t LogNumber() const { return log_number_; } // Return the log file number for the log file that is currently // being compacted, or zero if there is no such log file. uint64_t PrevLogNumber() const { return prev_log_number_; } // Pick level and inputs for a new compaction. // Returns NULL if there is no compaction to be done. // Otherwise returns a pointer to a heap-allocated object that // describes the compaction. Caller should delete the result. Compaction* PickCompaction(); // Return a compaction object for compacting the range [begin,end] in // the specified level. Returns NULL if there is nothing in that // level that overlaps the specified range. Caller should delete // the result. Compaction* CompactRange( int level, const InternalKey* begin, const InternalKey* end); // Return the maximum overlapping data (in bytes) at next level for any // file at a level >= 1. int64_t MaxNextLevelOverlappingBytes(); // Create an iterator that reads over the compaction inputs for "*c". // The caller should delete the iterator when no longer needed. Iterator* MakeInputIterator(Compaction* c); // Returns true iff some level needs a compaction. bool NeedsCompaction() const { Version* v = current_; return (v->compaction_score_ >= 1) || (v->file_to_compact_ != NULL); } // Add all files listed in any live version to *live. // May also mutate some internal state. void AddLiveFiles(std::set* live); // Return the approximate offset in the database of the data for // "key" as of version "v". uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key); // Return a human-readable short (single-line) summary of the number // of files per level. Uses *scratch as backing store. struct LevelSummaryStorage { char buffer[100]; }; const char* LevelSummary(LevelSummaryStorage* scratch) const; private: class Builder; friend class Compaction; friend class Version; bool ReuseManifest(const std::string& dscname, const std::string& dscbase); void Finalize(Version* v); void GetRange(const std::vector& inputs, InternalKey* smallest, InternalKey* largest); void GetRange2(const std::vector& inputs1, const std::vector& inputs2, InternalKey* smallest, InternalKey* largest); void SetupOtherInputs(Compaction* c); // Save current contents to *log Status WriteSnapshot(log::Writer* log); void AppendVersion(Version* v); Env* const env_; const std::string dbname_; const Options* const options_; TableCache* const table_cache_; const InternalKeyComparator icmp_; uint64_t next_file_number_; uint64_t manifest_file_number_; uint64_t last_sequence_; uint64_t log_number_; uint64_t prev_log_number_; // 0 or backing store for memtable being compacted // Opened lazily WritableFile* descriptor_file_; log::Writer* descriptor_log_; Version dummy_versions_; // Head of circular doubly-linked list of versions. Version* current_; // == dummy_versions_.prev_ // Per-level key at which the next compaction at that level should start. // Either an empty string, or a valid InternalKey. std::string compact_pointer_[config::kNumLevels]; // No copying allowed VersionSet(const VersionSet&); void operator=(const VersionSet&); }; // A Compaction encapsulates information about a compaction. class Compaction { public: ~Compaction(); // Return the level that is being compacted. Inputs from "level" // and "level+1" will be merged to produce a set of "level+1" files. int level() const { return level_; } // Return the object that holds the edits to the descriptor done // by this compaction. VersionEdit* edit() { return &edit_; } // "which" must be either 0 or 1 int num_input_files(int which) const { return inputs_[which].size(); } // Return the ith input file at "level()+which" ("which" must be 0 or 1). FileMetaData* input(int which, int i) const { return inputs_[which][i]; } // Maximum size of files to build during this compaction. uint64_t MaxOutputFileSize() const { return max_output_file_size_; } // Is this a trivial compaction that can be implemented by just // moving a single input file to the next level (no merging or splitting) bool IsTrivialMove() const; // Add all inputs to this compaction as delete operations to *edit. void AddInputDeletions(VersionEdit* edit); // Returns true if the information we have available guarantees that // the compaction is producing data in "level+1" for which no data exists // in levels greater than "level+1". bool IsBaseLevelForKey(const Slice& user_key); // Returns true iff we should stop building the current output // before processing "internal_key". bool ShouldStopBefore(const Slice& internal_key); // Release the input version for the compaction, once the compaction // is successful. void ReleaseInputs(); private: friend class Version; friend class VersionSet; - explicit Compaction(int level); + Compaction(const Options* options, int level); int level_; uint64_t max_output_file_size_; Version* input_version_; VersionEdit edit_; // Each compaction reads inputs from "level_" and "level_+1" std::vector inputs_[2]; // The two sets of inputs // State used to check for number of of overlapping grandparent files // (parent == level_ + 1, grandparent == level_ + 2) std::vector grandparents_; size_t grandparent_index_; // Index in grandparent_starts_ bool seen_key_; // Some output key has been seen int64_t overlapped_bytes_; // Bytes of overlap between current output // and grandparent files // State for implementing IsBaseLevelForKey // level_ptrs_ holds indices into input_version_->levels_: our state // is that we are positioned at one of the file ranges for each // higher level than the ones involved in this compaction (i.e. for // all L >= level_ + 2). size_t level_ptrs_[config::kNumLevels]; }; } // namespace leveldb #endif // STORAGE_LEVELDB_DB_VERSION_SET_H_ diff --git a/src/leveldb/doc/doc.css b/src/leveldb/doc/doc.css deleted file mode 100644 index 700c564e4..000000000 --- a/src/leveldb/doc/doc.css +++ /dev/null @@ -1,89 +0,0 @@ -body { - margin-left: 0.5in; - margin-right: 0.5in; - background: white; - color: black; -} - -h1 { - margin-left: -0.2in; - font-size: 14pt; -} -h2 { - margin-left: -0in; - font-size: 12pt; -} -h3 { - margin-left: -0in; -} -h4 { - margin-left: -0in; -} -hr { - margin-left: -0in; -} - -/* Definition lists: definition term bold */ -dt { - font-weight: bold; -} - -address { - text-align: center; -} -code,samp,var { - color: blue; -} -kbd { - color: #600000; -} -div.note p { - float: right; - width: 3in; - margin-right: 0%; - padding: 1px; - border: 2px solid #6060a0; - background-color: #fffff0; -} - -ul { - margin-top: -0em; - margin-bottom: -0em; -} - -ol { - margin-top: -0em; - margin-bottom: -0em; -} - -UL.nobullets { - list-style-type: none; - list-style-image: none; - margin-left: -1em; -} - -p { - margin: 1em 0 1em 0; - padding: 0 0 0 0; -} - -pre { - line-height: 1.3em; - padding: 0.4em 0 0.8em 0; - margin: 0 0 0 0; - border: 0 0 0 0; - color: blue; -} - -.datatable { - margin-left: auto; - margin-right: auto; - margin-top: 2em; - margin-bottom: 2em; - border: 1px solid; -} - -.datatable td,th { - padding: 0 0.5em 0 0.5em; - text-align: right; -} diff --git a/src/leveldb/doc/impl.html b/src/leveldb/doc/impl.html deleted file mode 100644 index 6a468be09..000000000 --- a/src/leveldb/doc/impl.html +++ /dev/null @@ -1,213 +0,0 @@ - - - - -Leveldb file layout and compactions - - - - -

Files

- -The implementation of leveldb is similar in spirit to the -representation of a single - -Bigtable tablet (section 5.3). -However the organization of the files that make up the representation -is somewhat different and is explained below. - -

-Each database is represented by a set of files stored in a directory. -There are several different types of files as documented below: -

-

Log files

-

-A log file (*.log) stores a sequence of recent updates. Each update -is appended to the current log file. When the log file reaches a -pre-determined size (approximately 4MB by default), it is converted -to a sorted table (see below) and a new log file is created for future -updates. -

-A copy of the current log file is kept in an in-memory structure (the -memtable). This copy is consulted on every read so that read -operations reflect all logged updates. -

-

Sorted tables

-

-A sorted table (*.sst) stores a sequence of entries sorted by key. -Each entry is either a value for the key, or a deletion marker for the -key. (Deletion markers are kept around to hide obsolete values -present in older sorted tables). -

-The set of sorted tables are organized into a sequence of levels. The -sorted table generated from a log file is placed in a special young -level (also called level-0). When the number of young files exceeds a -certain threshold (currently four), all of the young files are merged -together with all of the overlapping level-1 files to produce a -sequence of new level-1 files (we create a new level-1 file for every -2MB of data.) -

-Files in the young level may contain overlapping keys. However files -in other levels have distinct non-overlapping key ranges. Consider -level number L where L >= 1. When the combined size of files in -level-L exceeds (10^L) MB (i.e., 10MB for level-1, 100MB for level-2, -...), one file in level-L, and all of the overlapping files in -level-(L+1) are merged to form a set of new files for level-(L+1). -These merges have the effect of gradually migrating new updates from -the young level to the largest level using only bulk reads and writes -(i.e., minimizing expensive seeks). - -

Manifest

-

-A MANIFEST file lists the set of sorted tables that make up each -level, the corresponding key ranges, and other important metadata. -A new MANIFEST file (with a new number embedded in the file name) -is created whenever the database is reopened. The MANIFEST file is -formatted as a log, and changes made to the serving state (as files -are added or removed) are appended to this log. -

-

Current

-

-CURRENT is a simple text file that contains the name of the latest -MANIFEST file. -

-

Info logs

-

-Informational messages are printed to files named LOG and LOG.old. -

-

Others

-

-Other files used for miscellaneous purposes may also be present -(LOCK, *.dbtmp). - -

Level 0

-When the log file grows above a certain size (1MB by default): -
    -
  • Create a brand new memtable and log file and direct future updates here -
  • In the background: -
      -
    • Write the contents of the previous memtable to an sstable -
    • Discard the memtable -
    • Delete the old log file and the old memtable -
    • Add the new sstable to the young (level-0) level. -
    -
- -

Compactions

- -

-When the size of level L exceeds its limit, we compact it in a -background thread. The compaction picks a file from level L and all -overlapping files from the next level L+1. Note that if a level-L -file overlaps only part of a level-(L+1) file, the entire file at -level-(L+1) is used as an input to the compaction and will be -discarded after the compaction. Aside: because level-0 is special -(files in it may overlap each other), we treat compactions from -level-0 to level-1 specially: a level-0 compaction may pick more than -one level-0 file in case some of these files overlap each other. - -

-A compaction merges the contents of the picked files to produce a -sequence of level-(L+1) files. We switch to producing a new -level-(L+1) file after the current output file has reached the target -file size (2MB). We also switch to a new output file when the key -range of the current output file has grown enough to overlap more than -ten level-(L+2) files. This last rule ensures that a later compaction -of a level-(L+1) file will not pick up too much data from level-(L+2). - -

-The old files are discarded and the new files are added to the serving -state. - -

-Compactions for a particular level rotate through the key space. In -more detail, for each level L, we remember the ending key of the last -compaction at level L. The next compaction for level L will pick the -first file that starts after this key (wrapping around to the -beginning of the key space if there is no such file). - -

-Compactions drop overwritten values. They also drop deletion markers -if there are no higher numbered levels that contain a file whose range -overlaps the current key. - -

Timing

- -Level-0 compactions will read up to four 1MB files from level-0, and -at worst all the level-1 files (10MB). I.e., we will read 14MB and -write 14MB. - -

-Other than the special level-0 compactions, we will pick one 2MB file -from level L. In the worst case, this will overlap ~ 12 files from -level L+1 (10 because level-(L+1) is ten times the size of level-L, -and another two at the boundaries since the file ranges at level-L -will usually not be aligned with the file ranges at level-L+1). The -compaction will therefore read 26MB and write 26MB. Assuming a disk -IO rate of 100MB/s (ballpark range for modern drives), the worst -compaction cost will be approximately 0.5 second. - -

-If we throttle the background writing to something small, say 10% of -the full 100MB/s speed, a compaction may take up to 5 seconds. If the -user is writing at 10MB/s, we might build up lots of level-0 files -(~50 to hold the 5*10MB). This may significantly increase the cost of -reads due to the overhead of merging more files together on every -read. - -

-Solution 1: To reduce this problem, we might want to increase the log -switching threshold when the number of level-0 files is large. Though -the downside is that the larger this threshold, the more memory we will -need to hold the corresponding memtable. - -

-Solution 2: We might want to decrease write rate artificially when the -number of level-0 files goes up. - -

-Solution 3: We work on reducing the cost of very wide merges. -Perhaps most of the level-0 files will have their blocks sitting -uncompressed in the cache and we will only need to worry about the -O(N) complexity in the merging iterator. - -

Number of files

- -Instead of always making 2MB files, we could make larger files for -larger levels to reduce the total file count, though at the expense of -more bursty compactions. Alternatively, we could shard the set of -files into multiple directories. - -

-An experiment on an ext3 filesystem on Feb 04, 2011 shows -the following timings to do 100K file opens in directories with -varying number of files: - - - - - -
Files in directoryMicroseconds to open a file
10009
1000010
10000016
-So maybe even the sharding is not necessary on modern filesystems? - -

Recovery

- -
    -
  • Read CURRENT to find name of the latest committed MANIFEST -
  • Read the named MANIFEST file -
  • Clean up stale files -
  • We could open all sstables here, but it is probably better to be lazy... -
  • Convert log chunk to a new level-0 sstable -
  • Start directing new writes to a new log file with recovered sequence# -
- -

Garbage collection of files

- -DeleteObsoleteFiles() is called at the end of every -compaction and at the end of recovery. It finds the names of all -files in the database. It deletes all log files that are not the -current log file. It deletes all table files that are not referenced -from some level and are not the output of an active compaction. - - - diff --git a/src/leveldb/doc/impl.md b/src/leveldb/doc/impl.md new file mode 100644 index 000000000..4b13f2a6b --- /dev/null +++ b/src/leveldb/doc/impl.md @@ -0,0 +1,170 @@ +## Files + +The implementation of leveldb is similar in spirit to the representation of a +single [Bigtable tablet (section 5.3)](http://research.google.com/archive/bigtable.html). +However the organization of the files that make up the representation is +somewhat different and is explained below. + +Each database is represented by a set of files stored in a directory. There are +several different types of files as documented below: + +### Log files + +A log file (*.log) stores a sequence of recent updates. Each update is appended +to the current log file. When the log file reaches a pre-determined size +(approximately 4MB by default), it is converted to a sorted table (see below) +and a new log file is created for future updates. + +A copy of the current log file is kept in an in-memory structure (the +`memtable`). This copy is consulted on every read so that read operations +reflect all logged updates. + +## Sorted tables + +A sorted table (*.ldb) stores a sequence of entries sorted by key. Each entry is +either a value for the key, or a deletion marker for the key. (Deletion markers +are kept around to hide obsolete values present in older sorted tables). + +The set of sorted tables are organized into a sequence of levels. The sorted +table generated from a log file is placed in a special **young** level (also +called level-0). When the number of young files exceeds a certain threshold +(currently four), all of the young files are merged together with all of the +overlapping level-1 files to produce a sequence of new level-1 files (we create +a new level-1 file for every 2MB of data.) + +Files in the young level may contain overlapping keys. However files in other +levels have distinct non-overlapping key ranges. Consider level number L where +L >= 1. When the combined size of files in level-L exceeds (10^L) MB (i.e., 10MB +for level-1, 100MB for level-2, ...), one file in level-L, and all of the +overlapping files in level-(L+1) are merged to form a set of new files for +level-(L+1). These merges have the effect of gradually migrating new updates +from the young level to the largest level using only bulk reads and writes +(i.e., minimizing expensive seeks). + +### Manifest + +A MANIFEST file lists the set of sorted tables that make up each level, the +corresponding key ranges, and other important metadata. A new MANIFEST file +(with a new number embedded in the file name) is created whenever the database +is reopened. The MANIFEST file is formatted as a log, and changes made to the +serving state (as files are added or removed) are appended to this log. + +### Current + +CURRENT is a simple text file that contains the name of the latest MANIFEST +file. + +### Info logs + +Informational messages are printed to files named LOG and LOG.old. + +### Others + +Other files used for miscellaneous purposes may also be present (LOCK, *.dbtmp). + +## Level 0 + +When the log file grows above a certain size (1MB by default): +Create a brand new memtable and log file and direct future updates here +In the background: +Write the contents of the previous memtable to an sstable +Discard the memtable +Delete the old log file and the old memtable +Add the new sstable to the young (level-0) level. + +## Compactions + +When the size of level L exceeds its limit, we compact it in a background +thread. The compaction picks a file from level L and all overlapping files from +the next level L+1. Note that if a level-L file overlaps only part of a +level-(L+1) file, the entire file at level-(L+1) is used as an input to the +compaction and will be discarded after the compaction. Aside: because level-0 +is special (files in it may overlap each other), we treat compactions from +level-0 to level-1 specially: a level-0 compaction may pick more than one +level-0 file in case some of these files overlap each other. + +A compaction merges the contents of the picked files to produce a sequence of +level-(L+1) files. We switch to producing a new level-(L+1) file after the +current output file has reached the target file size (2MB). We also switch to a +new output file when the key range of the current output file has grown enough +to overlap more than ten level-(L+2) files. This last rule ensures that a later +compaction of a level-(L+1) file will not pick up too much data from +level-(L+2). + +The old files are discarded and the new files are added to the serving state. + +Compactions for a particular level rotate through the key space. In more detail, +for each level L, we remember the ending key of the last compaction at level L. +The next compaction for level L will pick the first file that starts after this +key (wrapping around to the beginning of the key space if there is no such +file). + +Compactions drop overwritten values. They also drop deletion markers if there +are no higher numbered levels that contain a file whose range overlaps the +current key. + +### Timing + +Level-0 compactions will read up to four 1MB files from level-0, and at worst +all the level-1 files (10MB). I.e., we will read 14MB and write 14MB. + +Other than the special level-0 compactions, we will pick one 2MB file from level +L. In the worst case, this will overlap ~ 12 files from level L+1 (10 because +level-(L+1) is ten times the size of level-L, and another two at the boundaries +since the file ranges at level-L will usually not be aligned with the file +ranges at level-L+1). The compaction will therefore read 26MB and write 26MB. +Assuming a disk IO rate of 100MB/s (ballpark range for modern drives), the worst +compaction cost will be approximately 0.5 second. + +If we throttle the background writing to something small, say 10% of the full +100MB/s speed, a compaction may take up to 5 seconds. If the user is writing at +10MB/s, we might build up lots of level-0 files (~50 to hold the 5*10MB). This +may significantly increase the cost of reads due to the overhead of merging more +files together on every read. + +Solution 1: To reduce this problem, we might want to increase the log switching +threshold when the number of level-0 files is large. Though the downside is that +the larger this threshold, the more memory we will need to hold the +corresponding memtable. + +Solution 2: We might want to decrease write rate artificially when the number of +level-0 files goes up. + +Solution 3: We work on reducing the cost of very wide merges. Perhaps most of +the level-0 files will have their blocks sitting uncompressed in the cache and +we will only need to worry about the O(N) complexity in the merging iterator. + +### Number of files + +Instead of always making 2MB files, we could make larger files for larger levels +to reduce the total file count, though at the expense of more bursty +compactions. Alternatively, we could shard the set of files into multiple +directories. + +An experiment on an ext3 filesystem on Feb 04, 2011 shows the following timings +to do 100K file opens in directories with varying number of files: + + +| Files in directory | Microseconds to open a file | +|-------------------:|----------------------------:| +| 1000 | 9 | +| 10000 | 10 | +| 100000 | 16 | + +So maybe even the sharding is not necessary on modern filesystems? + +## Recovery + +* Read CURRENT to find name of the latest committed MANIFEST +* Read the named MANIFEST file +* Clean up stale files +* We could open all sstables here, but it is probably better to be lazy... +* Convert log chunk to a new level-0 sstable +* Start directing new writes to a new log file with recovered sequence# + +## Garbage collection of files + +`DeleteObsoleteFiles()` is called at the end of every compaction and at the end +of recovery. It finds the names of all files in the database. It deletes all log +files that are not the current log file. It deletes all table files that are not +referenced from some level and are not the output of an active compaction. diff --git a/src/leveldb/doc/index.html b/src/leveldb/doc/index.html deleted file mode 100644 index 215519258..000000000 --- a/src/leveldb/doc/index.html +++ /dev/null @@ -1,549 +0,0 @@ - - - - -Leveldb - - - -

Leveldb

-
Jeff Dean, Sanjay Ghemawat
-

-The leveldb library provides a persistent key value store. Keys and -values are arbitrary byte arrays. The keys are ordered within the key -value store according to a user-specified comparator function. - -

-

Opening A Database

-

-A leveldb database has a name which corresponds to a file system -directory. All of the contents of database are stored in this -directory. The following example shows how to open a database, -creating it if necessary: -

-

-  #include <cassert>
-  #include "leveldb/db.h"
-
-  leveldb::DB* db;
-  leveldb::Options options;
-  options.create_if_missing = true;
-  leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
-  assert(status.ok());
-  ...
-
-If you want to raise an error if the database already exists, add -the following line before the leveldb::DB::Open call: -
-  options.error_if_exists = true;
-
-

Status

-

-You may have noticed the leveldb::Status type above. Values of this -type are returned by most functions in leveldb that may encounter an -error. You can check if such a result is ok, and also print an -associated error message: -

-

-   leveldb::Status s = ...;
-   if (!s.ok()) cerr << s.ToString() << endl;
-
-

Closing A Database

-

-When you are done with a database, just delete the database object. -Example: -

-

-  ... open the db as described above ...
-  ... do something with db ...
-  delete db;
-
-

Reads And Writes

-

-The database provides Put, Delete, and Get methods to -modify/query the database. For example, the following code -moves the value stored under key1 to key2. -

-  std::string value;
-  leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
-  if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value);
-  if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1);
-
- -

Atomic Updates

-

-Note that if the process dies after the Put of key2 but before the -delete of key1, the same value may be left stored under multiple keys. -Such problems can be avoided by using the WriteBatch class to -atomically apply a set of updates: -

-

-  #include "leveldb/write_batch.h"
-  ...
-  std::string value;
-  leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
-  if (s.ok()) {
-    leveldb::WriteBatch batch;
-    batch.Delete(key1);
-    batch.Put(key2, value);
-    s = db->Write(leveldb::WriteOptions(), &batch);
-  }
-
-The WriteBatch holds a sequence of edits to be made to the database, -and these edits within the batch are applied in order. Note that we -called Delete before Put so that if key1 is identical to key2, -we do not end up erroneously dropping the value entirely. -

-Apart from its atomicity benefits, WriteBatch may also be used to -speed up bulk updates by placing lots of individual mutations into the -same batch. - -

Synchronous Writes

-By default, each write to leveldb is asynchronous: it -returns after pushing the write from the process into the operating -system. The transfer from operating system memory to the underlying -persistent storage happens asynchronously. The sync flag -can be turned on for a particular write to make the write operation -not return until the data being written has been pushed all the way to -persistent storage. (On Posix systems, this is implemented by calling -either fsync(...) or fdatasync(...) or -msync(..., MS_SYNC) before the write operation returns.) -
-  leveldb::WriteOptions write_options;
-  write_options.sync = true;
-  db->Put(write_options, ...);
-
-Asynchronous writes are often more than a thousand times as fast as -synchronous writes. The downside of asynchronous writes is that a -crash of the machine may cause the last few updates to be lost. Note -that a crash of just the writing process (i.e., not a reboot) will not -cause any loss since even when sync is false, an update -is pushed from the process memory into the operating system before it -is considered done. - -

-Asynchronous writes can often be used safely. For example, when -loading a large amount of data into the database you can handle lost -updates by restarting the bulk load after a crash. A hybrid scheme is -also possible where every Nth write is synchronous, and in the event -of a crash, the bulk load is restarted just after the last synchronous -write finished by the previous run. (The synchronous write can update -a marker that describes where to restart on a crash.) - -

-WriteBatch provides an alternative to asynchronous writes. -Multiple updates may be placed in the same WriteBatch and -applied together using a synchronous write (i.e., -write_options.sync is set to true). The extra cost of -the synchronous write will be amortized across all of the writes in -the batch. - -

-

Concurrency

-

-A database may only be opened by one process at a time. -The leveldb implementation acquires a lock from the -operating system to prevent misuse. Within a single process, the -same leveldb::DB object may be safely shared by multiple -concurrent threads. I.e., different threads may write into or fetch -iterators or call Get on the same database without any -external synchronization (the leveldb implementation will -automatically do the required synchronization). However other objects -(like Iterator and WriteBatch) may require external synchronization. -If two threads share such an object, they must protect access to it -using their own locking protocol. More details are available in -the public header files. -

-

Iteration

-

-The following example demonstrates how to print all key,value pairs -in a database. -

-

-  leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());
-  for (it->SeekToFirst(); it->Valid(); it->Next()) {
-    cout << it->key().ToString() << ": "  << it->value().ToString() << endl;
-  }
-  assert(it->status().ok());  // Check for any errors found during the scan
-  delete it;
-
-The following variation shows how to process just the keys in the -range [start,limit): -

-

-  for (it->Seek(start);
-       it->Valid() && it->key().ToString() < limit;
-       it->Next()) {
-    ...
-  }
-
-You can also process entries in reverse order. (Caveat: reverse -iteration may be somewhat slower than forward iteration.) -

-

-  for (it->SeekToLast(); it->Valid(); it->Prev()) {
-    ...
-  }
-
-

Snapshots

-

-Snapshots provide consistent read-only views over the entire state of -the key-value store. ReadOptions::snapshot may be non-NULL to indicate -that a read should operate on a particular version of the DB state. -If ReadOptions::snapshot is NULL, the read will operate on an -implicit snapshot of the current state. -

-Snapshots are created by the DB::GetSnapshot() method: -

-

-  leveldb::ReadOptions options;
-  options.snapshot = db->GetSnapshot();
-  ... apply some updates to db ...
-  leveldb::Iterator* iter = db->NewIterator(options);
-  ... read using iter to view the state when the snapshot was created ...
-  delete iter;
-  db->ReleaseSnapshot(options.snapshot);
-
-Note that when a snapshot is no longer needed, it should be released -using the DB::ReleaseSnapshot interface. This allows the -implementation to get rid of state that was being maintained just to -support reading as of that snapshot. -

Slice

-

-The return value of the it->key() and it->value() calls above -are instances of the leveldb::Slice type. Slice is a simple -structure that contains a length and a pointer to an external byte -array. Returning a Slice is a cheaper alternative to returning a -std::string since we do not need to copy potentially large keys and -values. In addition, leveldb methods do not return null-terminated -C-style strings since leveldb keys and values are allowed to -contain '\0' bytes. -

-C++ strings and null-terminated C-style strings can be easily converted -to a Slice: -

-

-   leveldb::Slice s1 = "hello";
-
-   std::string str("world");
-   leveldb::Slice s2 = str;
-
-A Slice can be easily converted back to a C++ string: -
-   std::string str = s1.ToString();
-   assert(str == std::string("hello"));
-
-Be careful when using Slices since it is up to the caller to ensure that -the external byte array into which the Slice points remains live while -the Slice is in use. For example, the following is buggy: -

-

-   leveldb::Slice slice;
-   if (...) {
-     std::string str = ...;
-     slice = str;
-   }
-   Use(slice);
-
-When the if statement goes out of scope, str will be destroyed and the -backing storage for slice will disappear. -

-

Comparators

-

-The preceding examples used the default ordering function for key, -which orders bytes lexicographically. You can however supply a custom -comparator when opening a database. For example, suppose each -database key consists of two numbers and we should sort by the first -number, breaking ties by the second number. First, define a proper -subclass of leveldb::Comparator that expresses these rules: -

-

-  class TwoPartComparator : public leveldb::Comparator {
-   public:
-    // Three-way comparison function:
-    //   if a < b: negative result
-    //   if a > b: positive result
-    //   else: zero result
-    int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const {
-      int a1, a2, b1, b2;
-      ParseKey(a, &a1, &a2);
-      ParseKey(b, &b1, &b2);
-      if (a1 < b1) return -1;
-      if (a1 > b1) return +1;
-      if (a2 < b2) return -1;
-      if (a2 > b2) return +1;
-      return 0;
-    }
-
-    // Ignore the following methods for now:
-    const char* Name() const { return "TwoPartComparator"; }
-    void FindShortestSeparator(std::string*, const leveldb::Slice&) const { }
-    void FindShortSuccessor(std::string*) const { }
-  };
-
-Now create a database using this custom comparator: -

-

-  TwoPartComparator cmp;
-  leveldb::DB* db;
-  leveldb::Options options;
-  options.create_if_missing = true;
-  options.comparator = &cmp;
-  leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
-  ...
-
-

Backwards compatibility

-

-The result of the comparator's Name method is attached to the -database when it is created, and is checked on every subsequent -database open. If the name changes, the leveldb::DB::Open call will -fail. Therefore, change the name if and only if the new key format -and comparison function are incompatible with existing databases, and -it is ok to discard the contents of all existing databases. -

-You can however still gradually evolve your key format over time with -a little bit of pre-planning. For example, you could store a version -number at the end of each key (one byte should suffice for most uses). -When you wish to switch to a new key format (e.g., adding an optional -third part to the keys processed by TwoPartComparator), -(a) keep the same comparator name (b) increment the version number -for new keys (c) change the comparator function so it uses the -version numbers found in the keys to decide how to interpret them. -

-

Performance

-

-Performance can be tuned by changing the default values of the -types defined in include/leveldb/options.h. - -

-

Block size

-

-leveldb groups adjacent keys together into the same block and such a -block is the unit of transfer to and from persistent storage. The -default block size is approximately 4096 uncompressed bytes. -Applications that mostly do bulk scans over the contents of the -database may wish to increase this size. Applications that do a lot -of point reads of small values may wish to switch to a smaller block -size if performance measurements indicate an improvement. There isn't -much benefit in using blocks smaller than one kilobyte, or larger than -a few megabytes. Also note that compression will be more effective -with larger block sizes. -

-

Compression

-

-Each block is individually compressed before being written to -persistent storage. Compression is on by default since the default -compression method is very fast, and is automatically disabled for -uncompressible data. In rare cases, applications may want to disable -compression entirely, but should only do so if benchmarks show a -performance improvement: -

-

-  leveldb::Options options;
-  options.compression = leveldb::kNoCompression;
-  ... leveldb::DB::Open(options, name, ...) ....
-
-

Cache

-

-The contents of the database are stored in a set of files in the -filesystem and each file stores a sequence of compressed blocks. If -options.cache is non-NULL, it is used to cache frequently used -uncompressed block contents. -

-

-  #include "leveldb/cache.h"
-
-  leveldb::Options options;
-  options.cache = leveldb::NewLRUCache(100 * 1048576);  // 100MB cache
-  leveldb::DB* db;
-  leveldb::DB::Open(options, name, &db);
-  ... use the db ...
-  delete db
-  delete options.cache;
-
-Note that the cache holds uncompressed data, and therefore it should -be sized according to application level data sizes, without any -reduction from compression. (Caching of compressed blocks is left to -the operating system buffer cache, or any custom Env -implementation provided by the client.) -

-When performing a bulk read, the application may wish to disable -caching so that the data processed by the bulk read does not end up -displacing most of the cached contents. A per-iterator option can be -used to achieve this: -

-

-  leveldb::ReadOptions options;
-  options.fill_cache = false;
-  leveldb::Iterator* it = db->NewIterator(options);
-  for (it->SeekToFirst(); it->Valid(); it->Next()) {
-    ...
-  }
-
-

Key Layout

-

-Note that the unit of disk transfer and caching is a block. Adjacent -keys (according to the database sort order) will usually be placed in -the same block. Therefore the application can improve its performance -by placing keys that are accessed together near each other and placing -infrequently used keys in a separate region of the key space. -

-For example, suppose we are implementing a simple file system on top -of leveldb. The types of entries we might wish to store are: -

-

-   filename -> permission-bits, length, list of file_block_ids
-   file_block_id -> data
-
-We might want to prefix filename keys with one letter (say '/') and the -file_block_id keys with a different letter (say '0') so that scans -over just the metadata do not force us to fetch and cache bulky file -contents. -

-

Filters

-

-Because of the way leveldb data is organized on disk, -a single Get() call may involve multiple reads from disk. -The optional FilterPolicy mechanism can be used to reduce -the number of disk reads substantially. -

-   leveldb::Options options;
-   options.filter_policy = NewBloomFilterPolicy(10);
-   leveldb::DB* db;
-   leveldb::DB::Open(options, "/tmp/testdb", &db);
-   ... use the database ...
-   delete db;
-   delete options.filter_policy;
-
-The preceding code associates a -Bloom filter -based filtering policy with the database. Bloom filter based -filtering relies on keeping some number of bits of data in memory per -key (in this case 10 bits per key since that is the argument we passed -to NewBloomFilterPolicy). This filter will reduce the number of unnecessary -disk reads needed for Get() calls by a factor of -approximately a 100. Increasing the bits per key will lead to a -larger reduction at the cost of more memory usage. We recommend that -applications whose working set does not fit in memory and that do a -lot of random reads set a filter policy. -

-If you are using a custom comparator, you should ensure that the filter -policy you are using is compatible with your comparator. For example, -consider a comparator that ignores trailing spaces when comparing keys. -NewBloomFilterPolicy must not be used with such a comparator. -Instead, the application should provide a custom filter policy that -also ignores trailing spaces. For example: -

-  class CustomFilterPolicy : public leveldb::FilterPolicy {
-   private:
-    FilterPolicy* builtin_policy_;
-   public:
-    CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) { }
-    ~CustomFilterPolicy() { delete builtin_policy_; }
-
-    const char* Name() const { return "IgnoreTrailingSpacesFilter"; }
-
-    void CreateFilter(const Slice* keys, int n, std::string* dst) const {
-      // Use builtin bloom filter code after removing trailing spaces
-      std::vector<Slice> trimmed(n);
-      for (int i = 0; i < n; i++) {
-        trimmed[i] = RemoveTrailingSpaces(keys[i]);
-      }
-      return builtin_policy_->CreateFilter(&trimmed[i], n, dst);
-    }
-
-    bool KeyMayMatch(const Slice& key, const Slice& filter) const {
-      // Use builtin bloom filter code after removing trailing spaces
-      return builtin_policy_->KeyMayMatch(RemoveTrailingSpaces(key), filter);
-    }
-  };
-
-

-Advanced applications may provide a filter policy that does not use -a bloom filter but uses some other mechanism for summarizing a set -of keys. See leveldb/filter_policy.h for detail. -

-

Checksums

-

-leveldb associates checksums with all data it stores in the file system. -There are two separate controls provided over how aggressively these -checksums are verified: -

-

    -
  • ReadOptions::verify_checksums may be set to true to force - checksum verification of all data that is read from the file system on - behalf of a particular read. By default, no such verification is - done. -

    -

  • Options::paranoid_checks may be set to true before opening a - database to make the database implementation raise an error as soon as - it detects an internal corruption. Depending on which portion of the - database has been corrupted, the error may be raised when the database - is opened, or later by another database operation. By default, - paranoid checking is off so that the database can be used even if - parts of its persistent storage have been corrupted. -

    - If a database is corrupted (perhaps it cannot be opened when - paranoid checking is turned on), the leveldb::RepairDB function - may be used to recover as much of the data as possible -

    -

-

Approximate Sizes

-

-The GetApproximateSizes method can used to get the approximate -number of bytes of file system space used by one or more key ranges. -

-

-   leveldb::Range ranges[2];
-   ranges[0] = leveldb::Range("a", "c");
-   ranges[1] = leveldb::Range("x", "z");
-   uint64_t sizes[2];
-   leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes);
-
-The preceding call will set sizes[0] to the approximate number of -bytes of file system space used by the key range [a..c) and -sizes[1] to the approximate number of bytes used by the key range -[x..z). -

-

Environment

-

-All file operations (and other operating system calls) issued by the -leveldb implementation are routed through a leveldb::Env object. -Sophisticated clients may wish to provide their own Env -implementation to get better control. For example, an application may -introduce artificial delays in the file IO paths to limit the impact -of leveldb on other activities in the system. -

-

-  class SlowEnv : public leveldb::Env {
-    .. implementation of the Env interface ...
-  };
-
-  SlowEnv env;
-  leveldb::Options options;
-  options.env = &env;
-  Status s = leveldb::DB::Open(options, ...);
-
-

Porting

-

-leveldb may be ported to a new platform by providing platform -specific implementations of the types/methods/functions exported by -leveldb/port/port.h. See leveldb/port/port_example.h for more -details. -

-In addition, the new platform may need a new default leveldb::Env -implementation. See leveldb/util/env_posix.h for an example. - -

Other Information

- -

-Details about the leveldb implementation may be found in -the following documents: -

- - - diff --git a/src/leveldb/doc/index.md b/src/leveldb/doc/index.md new file mode 100644 index 000000000..be8569692 --- /dev/null +++ b/src/leveldb/doc/index.md @@ -0,0 +1,523 @@ +leveldb +======= + +_Jeff Dean, Sanjay Ghemawat_ + +The leveldb library provides a persistent key value store. Keys and values are +arbitrary byte arrays. The keys are ordered within the key value store +according to a user-specified comparator function. + +## Opening A Database + +A leveldb database has a name which corresponds to a file system directory. All +of the contents of database are stored in this directory. The following example +shows how to open a database, creating it if necessary: + +```c++ +#include +#include "leveldb/db.h" + +leveldb::DB* db; +leveldb::Options options; +options.create_if_missing = true; +leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); +assert(status.ok()); +... +``` + +If you want to raise an error if the database already exists, add the following +line before the `leveldb::DB::Open` call: + +```c++ +options.error_if_exists = true; +``` + +## Status + +You may have noticed the `leveldb::Status` type above. Values of this type are +returned by most functions in leveldb that may encounter an error. You can check +if such a result is ok, and also print an associated error message: + +```c++ +leveldb::Status s = ...; +if (!s.ok()) cerr << s.ToString() << endl; +``` + +## Closing A Database + +When you are done with a database, just delete the database object. Example: + +```c++ +... open the db as described above ... +... do something with db ... +delete db; +``` + +## Reads And Writes + +The database provides Put, Delete, and Get methods to modify/query the database. +For example, the following code moves the value stored under key1 to key2. + +```c++ +std::string value; +leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); +if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value); +if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1); +``` + +## Atomic Updates + +Note that if the process dies after the Put of key2 but before the delete of +key1, the same value may be left stored under multiple keys. Such problems can +be avoided by using the `WriteBatch` class to atomically apply a set of updates: + +```c++ +#include "leveldb/write_batch.h" +... +std::string value; +leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); +if (s.ok()) { + leveldb::WriteBatch batch; + batch.Delete(key1); + batch.Put(key2, value); + s = db->Write(leveldb::WriteOptions(), &batch); +} +``` + +The `WriteBatch` holds a sequence of edits to be made to the database, and these +edits within the batch are applied in order. Note that we called Delete before +Put so that if key1 is identical to key2, we do not end up erroneously dropping +the value entirely. + +Apart from its atomicity benefits, `WriteBatch` may also be used to speed up +bulk updates by placing lots of individual mutations into the same batch. + +## Synchronous Writes + +By default, each write to leveldb is asynchronous: it returns after pushing the +write from the process into the operating system. The transfer from operating +system memory to the underlying persistent storage happens asynchronously. The +sync flag can be turned on for a particular write to make the write operation +not return until the data being written has been pushed all the way to +persistent storage. (On Posix systems, this is implemented by calling either +`fsync(...)` or `fdatasync(...)` or `msync(..., MS_SYNC)` before the write +operation returns.) + +```c++ +leveldb::WriteOptions write_options; +write_options.sync = true; +db->Put(write_options, ...); +``` + +Asynchronous writes are often more than a thousand times as fast as synchronous +writes. The downside of asynchronous writes is that a crash of the machine may +cause the last few updates to be lost. Note that a crash of just the writing +process (i.e., not a reboot) will not cause any loss since even when sync is +false, an update is pushed from the process memory into the operating system +before it is considered done. + +Asynchronous writes can often be used safely. For example, when loading a large +amount of data into the database you can handle lost updates by restarting the +bulk load after a crash. A hybrid scheme is also possible where every Nth write +is synchronous, and in the event of a crash, the bulk load is restarted just +after the last synchronous write finished by the previous run. (The synchronous +write can update a marker that describes where to restart on a crash.) + +`WriteBatch` provides an alternative to asynchronous writes. Multiple updates +may be placed in the same WriteBatch and applied together using a synchronous +write (i.e., `write_options.sync` is set to true). The extra cost of the +synchronous write will be amortized across all of the writes in the batch. + +## Concurrency + +A database may only be opened by one process at a time. The leveldb +implementation acquires a lock from the operating system to prevent misuse. +Within a single process, the same `leveldb::DB` object may be safely shared by +multiple concurrent threads. I.e., different threads may write into or fetch +iterators or call Get on the same database without any external synchronization +(the leveldb implementation will automatically do the required synchronization). +However other objects (like Iterator and `WriteBatch`) may require external +synchronization. If two threads share such an object, they must protect access +to it using their own locking protocol. More details are available in the public +header files. + +## Iteration + +The following example demonstrates how to print all key,value pairs in a +database. + +```c++ +leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions()); +for (it->SeekToFirst(); it->Valid(); it->Next()) { + cout << it->key().ToString() << ": " << it->value().ToString() << endl; +} +assert(it->status().ok()); // Check for any errors found during the scan +delete it; +``` + +The following variation shows how to process just the keys in the range +[start,limit): + +```c++ +for (it->Seek(start); + it->Valid() && it->key().ToString() < limit; + it->Next()) { + ... +} +``` + +You can also process entries in reverse order. (Caveat: reverse iteration may be +somewhat slower than forward iteration.) + +```c++ +for (it->SeekToLast(); it->Valid(); it->Prev()) { + ... +} +``` + +## Snapshots + +Snapshots provide consistent read-only views over the entire state of the +key-value store. `ReadOptions::snapshot` may be non-NULL to indicate that a +read should operate on a particular version of the DB state. If +`ReadOptions::snapshot` is NULL, the read will operate on an implicit snapshot +of the current state. + +Snapshots are created by the `DB::GetSnapshot()` method: + +```c++ +leveldb::ReadOptions options; +options.snapshot = db->GetSnapshot(); +... apply some updates to db ... +leveldb::Iterator* iter = db->NewIterator(options); +... read using iter to view the state when the snapshot was created ... +delete iter; +db->ReleaseSnapshot(options.snapshot); +``` + +Note that when a snapshot is no longer needed, it should be released using the +`DB::ReleaseSnapshot` interface. This allows the implementation to get rid of +state that was being maintained just to support reading as of that snapshot. + +## Slice + +The return value of the `it->key()` and `it->value()` calls above are instances +of the `leveldb::Slice` type. Slice is a simple structure that contains a length +and a pointer to an external byte array. Returning a Slice is a cheaper +alternative to returning a `std::string` since we do not need to copy +potentially large keys and values. In addition, leveldb methods do not return +null-terminated C-style strings since leveldb keys and values are allowed to +contain `'\0'` bytes. + +C++ strings and null-terminated C-style strings can be easily converted to a +Slice: + +```c++ +leveldb::Slice s1 = "hello"; + +std::string str("world"); +leveldb::Slice s2 = str; +``` + +A Slice can be easily converted back to a C++ string: + +```c++ +std::string str = s1.ToString(); +assert(str == std::string("hello")); +``` + +Be careful when using Slices since it is up to the caller to ensure that the +external byte array into which the Slice points remains live while the Slice is +in use. For example, the following is buggy: + +```c++ +leveldb::Slice slice; +if (...) { + std::string str = ...; + slice = str; +} +Use(slice); +``` + +When the if statement goes out of scope, str will be destroyed and the backing +storage for slice will disappear. + +## Comparators + +The preceding examples used the default ordering function for key, which orders +bytes lexicographically. You can however supply a custom comparator when opening +a database. For example, suppose each database key consists of two numbers and +we should sort by the first number, breaking ties by the second number. First, +define a proper subclass of `leveldb::Comparator` that expresses these rules: + +```c++ +class TwoPartComparator : public leveldb::Comparator { + public: + // Three-way comparison function: + // if a < b: negative result + // if a > b: positive result + // else: zero result + int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const { + int a1, a2, b1, b2; + ParseKey(a, &a1, &a2); + ParseKey(b, &b1, &b2); + if (a1 < b1) return -1; + if (a1 > b1) return +1; + if (a2 < b2) return -1; + if (a2 > b2) return +1; + return 0; + } + + // Ignore the following methods for now: + const char* Name() const { return "TwoPartComparator"; } + void FindShortestSeparator(std::string*, const leveldb::Slice&) const {} + void FindShortSuccessor(std::string*) const {} +}; +``` + +Now create a database using this custom comparator: + +```c++ +TwoPartComparator cmp; +leveldb::DB* db; +leveldb::Options options; +options.create_if_missing = true; +options.comparator = &cmp; +leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); +... +``` + +### Backwards compatibility + +The result of the comparator's Name method is attached to the database when it +is created, and is checked on every subsequent database open. If the name +changes, the `leveldb::DB::Open` call will fail. Therefore, change the name if +and only if the new key format and comparison function are incompatible with +existing databases, and it is ok to discard the contents of all existing +databases. + +You can however still gradually evolve your key format over time with a little +bit of pre-planning. For example, you could store a version number at the end of +each key (one byte should suffice for most uses). When you wish to switch to a +new key format (e.g., adding an optional third part to the keys processed by +`TwoPartComparator`), (a) keep the same comparator name (b) increment the +version number for new keys (c) change the comparator function so it uses the +version numbers found in the keys to decide how to interpret them. + +## Performance + +Performance can be tuned by changing the default values of the types defined in +`include/leveldb/options.h`. + +### Block size + +leveldb groups adjacent keys together into the same block and such a block is +the unit of transfer to and from persistent storage. The default block size is +approximately 4096 uncompressed bytes. Applications that mostly do bulk scans +over the contents of the database may wish to increase this size. Applications +that do a lot of point reads of small values may wish to switch to a smaller +block size if performance measurements indicate an improvement. There isn't much +benefit in using blocks smaller than one kilobyte, or larger than a few +megabytes. Also note that compression will be more effective with larger block +sizes. + +### Compression + +Each block is individually compressed before being written to persistent +storage. Compression is on by default since the default compression method is +very fast, and is automatically disabled for uncompressible data. In rare cases, +applications may want to disable compression entirely, but should only do so if +benchmarks show a performance improvement: + +```c++ +leveldb::Options options; +options.compression = leveldb::kNoCompression; +... leveldb::DB::Open(options, name, ...) .... +``` + +### Cache + +The contents of the database are stored in a set of files in the filesystem and +each file stores a sequence of compressed blocks. If options.cache is non-NULL, +it is used to cache frequently used uncompressed block contents. + +```c++ +#include "leveldb/cache.h" + +leveldb::Options options; +options.cache = leveldb::NewLRUCache(100 * 1048576); // 100MB cache +leveldb::DB* db; +leveldb::DB::Open(options, name, &db); +... use the db ... +delete db +delete options.cache; +``` + +Note that the cache holds uncompressed data, and therefore it should be sized +according to application level data sizes, without any reduction from +compression. (Caching of compressed blocks is left to the operating system +buffer cache, or any custom Env implementation provided by the client.) + +When performing a bulk read, the application may wish to disable caching so that +the data processed by the bulk read does not end up displacing most of the +cached contents. A per-iterator option can be used to achieve this: + +```c++ +leveldb::ReadOptions options; +options.fill_cache = false; +leveldb::Iterator* it = db->NewIterator(options); +for (it->SeekToFirst(); it->Valid(); it->Next()) { + ... +} +``` + +### Key Layout + +Note that the unit of disk transfer and caching is a block. Adjacent keys +(according to the database sort order) will usually be placed in the same block. +Therefore the application can improve its performance by placing keys that are +accessed together near each other and placing infrequently used keys in a +separate region of the key space. + +For example, suppose we are implementing a simple file system on top of leveldb. +The types of entries we might wish to store are: + + filename -> permission-bits, length, list of file_block_ids + file_block_id -> data + +We might want to prefix filename keys with one letter (say '/') and the +`file_block_id` keys with a different letter (say '0') so that scans over just +the metadata do not force us to fetch and cache bulky file contents. + +### Filters + +Because of the way leveldb data is organized on disk, a single `Get()` call may +involve multiple reads from disk. The optional FilterPolicy mechanism can be +used to reduce the number of disk reads substantially. + +```c++ +leveldb::Options options; +options.filter_policy = NewBloomFilterPolicy(10); +leveldb::DB* db; +leveldb::DB::Open(options, "/tmp/testdb", &db); +... use the database ... +delete db; +delete options.filter_policy; +``` + +The preceding code associates a Bloom filter based filtering policy with the +database. Bloom filter based filtering relies on keeping some number of bits of +data in memory per key (in this case 10 bits per key since that is the argument +we passed to `NewBloomFilterPolicy`). This filter will reduce the number of +unnecessary disk reads needed for Get() calls by a factor of approximately +a 100. Increasing the bits per key will lead to a larger reduction at the cost +of more memory usage. We recommend that applications whose working set does not +fit in memory and that do a lot of random reads set a filter policy. + +If you are using a custom comparator, you should ensure that the filter policy +you are using is compatible with your comparator. For example, consider a +comparator that ignores trailing spaces when comparing keys. +`NewBloomFilterPolicy` must not be used with such a comparator. Instead, the +application should provide a custom filter policy that also ignores trailing +spaces. For example: + +```c++ +class CustomFilterPolicy : public leveldb::FilterPolicy { + private: + FilterPolicy* builtin_policy_; + + public: + CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) {} + ~CustomFilterPolicy() { delete builtin_policy_; } + + const char* Name() const { return "IgnoreTrailingSpacesFilter"; } + + void CreateFilter(const Slice* keys, int n, std::string* dst) const { + // Use builtin bloom filter code after removing trailing spaces + std::vector trimmed(n); + for (int i = 0; i < n; i++) { + trimmed[i] = RemoveTrailingSpaces(keys[i]); + } + return builtin_policy_->CreateFilter(&trimmed[i], n, dst); + } +}; +``` + +Advanced applications may provide a filter policy that does not use a bloom +filter but uses some other mechanism for summarizing a set of keys. See +`leveldb/filter_policy.h` for detail. + +## Checksums + +leveldb associates checksums with all data it stores in the file system. There +are two separate controls provided over how aggressively these checksums are +verified: + +`ReadOptions::verify_checksums` may be set to true to force checksum +verification of all data that is read from the file system on behalf of a +particular read. By default, no such verification is done. + +`Options::paranoid_checks` may be set to true before opening a database to make +the database implementation raise an error as soon as it detects an internal +corruption. Depending on which portion of the database has been corrupted, the +error may be raised when the database is opened, or later by another database +operation. By default, paranoid checking is off so that the database can be used +even if parts of its persistent storage have been corrupted. + +If a database is corrupted (perhaps it cannot be opened when paranoid checking +is turned on), the `leveldb::RepairDB` function may be used to recover as much +of the data as possible + +## Approximate Sizes + +The `GetApproximateSizes` method can used to get the approximate number of bytes +of file system space used by one or more key ranges. + +```c++ +leveldb::Range ranges[2]; +ranges[0] = leveldb::Range("a", "c"); +ranges[1] = leveldb::Range("x", "z"); +uint64_t sizes[2]; +leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes); +``` + +The preceding call will set `sizes[0]` to the approximate number of bytes of +file system space used by the key range `[a..c)` and `sizes[1]` to the +approximate number of bytes used by the key range `[x..z)`. + +## Environment + +All file operations (and other operating system calls) issued by the leveldb +implementation are routed through a `leveldb::Env` object. Sophisticated clients +may wish to provide their own Env implementation to get better control. +For example, an application may introduce artificial delays in the file IO +paths to limit the impact of leveldb on other activities in the system. + +```c++ +class SlowEnv : public leveldb::Env { + ... implementation of the Env interface ... +}; + +SlowEnv env; +leveldb::Options options; +options.env = &env; +Status s = leveldb::DB::Open(options, ...); +``` + +## Porting + +leveldb may be ported to a new platform by providing platform specific +implementations of the types/methods/functions exported by +`leveldb/port/port.h`. See `leveldb/port/port_example.h` for more details. + +In addition, the new platform may need a new default `leveldb::Env` +implementation. See `leveldb/util/env_posix.h` for an example. + +## Other Information + +Details about the leveldb implementation may be found in the following +documents: + +1. [Implementation notes](impl.md) +2. [Format of an immutable Table file](table_format.md) +3. [Format of a log file](log_format.md) diff --git a/src/leveldb/doc/log_format.md b/src/leveldb/doc/log_format.md new file mode 100644 index 000000000..f32cb5d7d --- /dev/null +++ b/src/leveldb/doc/log_format.md @@ -0,0 +1,75 @@ +leveldb Log format +================== +The log file contents are a sequence of 32KB blocks. The only exception is that +the tail of the file may contain a partial block. + +Each block consists of a sequence of records: + + block := record* trailer? + record := + checksum: uint32 // crc32c of type and data[] ; little-endian + length: uint16 // little-endian + type: uint8 // One of FULL, FIRST, MIDDLE, LAST + data: uint8[length] + +A record never starts within the last six bytes of a block (since it won't fit). +Any leftover bytes here form the trailer, which must consist entirely of zero +bytes and must be skipped by readers. + +Aside: if exactly seven bytes are left in the current block, and a new non-zero +length record is added, the writer must emit a FIRST record (which contains zero +bytes of user data) to fill up the trailing seven bytes of the block and then +emit all of the user data in subsequent blocks. + +More types may be added in the future. Some Readers may skip record types they +do not understand, others may report that some data was skipped. + + FULL == 1 + FIRST == 2 + MIDDLE == 3 + LAST == 4 + +The FULL record contains the contents of an entire user record. + +FIRST, MIDDLE, LAST are types used for user records that have been split into +multiple fragments (typically because of block boundaries). FIRST is the type +of the first fragment of a user record, LAST is the type of the last fragment of +a user record, and MIDDLE is the type of all interior fragments of a user +record. + +Example: consider a sequence of user records: + + A: length 1000 + B: length 97270 + C: length 8000 + +**A** will be stored as a FULL record in the first block. + +**B** will be split into three fragments: first fragment occupies the rest of +the first block, second fragment occupies the entirety of the second block, and +the third fragment occupies a prefix of the third block. This will leave six +bytes free in the third block, which will be left empty as the trailer. + +**C** will be stored as a FULL record in the fourth block. + +---- + +## Some benefits over the recordio format: + +1. We do not need any heuristics for resyncing - just go to next block boundary + and scan. If there is a corruption, skip to the next block. As a + side-benefit, we do not get confused when part of the contents of one log + file are embedded as a record inside another log file. + +2. Splitting at approximate boundaries (e.g., for mapreduce) is simple: find the + next block boundary and skip records until we hit a FULL or FIRST record. + +3. We do not need extra buffering for large records. + +## Some downsides compared to recordio format: + +1. No packing of tiny records. This could be fixed by adding a new record type, + so it is a shortcoming of the current implementation, not necessarily the + format. + +2. No compression. Again, this could be fixed by adding new record types. diff --git a/src/leveldb/doc/log_format.txt b/src/leveldb/doc/log_format.txt deleted file mode 100644 index 4cca5ef6e..000000000 --- a/src/leveldb/doc/log_format.txt +++ /dev/null @@ -1,75 +0,0 @@ -The log file contents are a sequence of 32KB blocks. The only -exception is that the tail of the file may contain a partial block. - -Each block consists of a sequence of records: - block := record* trailer? - record := - checksum: uint32 // crc32c of type and data[] ; little-endian - length: uint16 // little-endian - type: uint8 // One of FULL, FIRST, MIDDLE, LAST - data: uint8[length] - -A record never starts within the last six bytes of a block (since it -won't fit). Any leftover bytes here form the trailer, which must -consist entirely of zero bytes and must be skipped by readers. - -Aside: if exactly seven bytes are left in the current block, and a new -non-zero length record is added, the writer must emit a FIRST record -(which contains zero bytes of user data) to fill up the trailing seven -bytes of the block and then emit all of the user data in subsequent -blocks. - -More types may be added in the future. Some Readers may skip record -types they do not understand, others may report that some data was -skipped. - -FULL == 1 -FIRST == 2 -MIDDLE == 3 -LAST == 4 - -The FULL record contains the contents of an entire user record. - -FIRST, MIDDLE, LAST are types used for user records that have been -split into multiple fragments (typically because of block boundaries). -FIRST is the type of the first fragment of a user record, LAST is the -type of the last fragment of a user record, and MIDDLE is the type of -all interior fragments of a user record. - -Example: consider a sequence of user records: - A: length 1000 - B: length 97270 - C: length 8000 -A will be stored as a FULL record in the first block. - -B will be split into three fragments: first fragment occupies the rest -of the first block, second fragment occupies the entirety of the -second block, and the third fragment occupies a prefix of the third -block. This will leave six bytes free in the third block, which will -be left empty as the trailer. - -C will be stored as a FULL record in the fourth block. - -=================== - -Some benefits over the recordio format: - -(1) We do not need any heuristics for resyncing - just go to next -block boundary and scan. If there is a corruption, skip to the next -block. As a side-benefit, we do not get confused when part of the -contents of one log file are embedded as a record inside another log -file. - -(2) Splitting at approximate boundaries (e.g., for mapreduce) is -simple: find the next block boundary and skip records until we -hit a FULL or FIRST record. - -(3) We do not need extra buffering for large records. - -Some downsides compared to recordio format: - -(1) No packing of tiny records. This could be fixed by adding a new -record type, so it is a shortcoming of the current implementation, -not necessarily the format. - -(2) No compression. Again, this could be fixed by adding new record types. diff --git a/src/leveldb/doc/table_format.md b/src/leveldb/doc/table_format.md new file mode 100644 index 000000000..5fe7e7241 --- /dev/null +++ b/src/leveldb/doc/table_format.md @@ -0,0 +1,107 @@ +leveldb File format +=================== + + + [data block 1] + [data block 2] + ... + [data block N] + [meta block 1] + ... + [meta block K] + [metaindex block] + [index block] + [Footer] (fixed size; starts at file_size - sizeof(Footer)) + + +The file contains internal pointers. Each such pointer is called +a BlockHandle and contains the following information: + + offset: varint64 + size: varint64 + +See [varints](https://developers.google.com/protocol-buffers/docs/encoding#varints) +for an explanation of varint64 format. + +1. The sequence of key/value pairs in the file are stored in sorted +order and partitioned into a sequence of data blocks. These blocks +come one after another at the beginning of the file. Each data block +is formatted according to the code in `block_builder.cc`, and then +optionally compressed. + +2. After the data blocks we store a bunch of meta blocks. The +supported meta block types are described below. More meta block types +may be added in the future. Each meta block is again formatted using +`block_builder.cc` and then optionally compressed. + +3. A "metaindex" block. It contains one entry for every other meta +block where the key is the name of the meta block and the value is a +BlockHandle pointing to that meta block. + +4. An "index" block. This block contains one entry per data block, +where the key is a string >= last key in that data block and before +the first key in the successive data block. The value is the +BlockHandle for the data block. + +5. At the very end of the file is a fixed length footer that contains +the BlockHandle of the metaindex and index blocks as well as a magic number. + + metaindex_handle: char[p]; // Block handle for metaindex + index_handle: char[q]; // Block handle for index + padding: char[40-p-q];// zeroed bytes to make fixed length + // (40==2*BlockHandle::kMaxEncodedLength) + magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) + +## "filter" Meta Block + +If a `FilterPolicy` was specified when the database was opened, a +filter block is stored in each table. The "metaindex" block contains +an entry that maps from `filter.` to the BlockHandle for the filter +block where `` is the string returned by the filter policy's +`Name()` method. + +The filter block stores a sequence of filters, where filter i contains +the output of `FilterPolicy::CreateFilter()` on all keys that are stored +in a block whose file offset falls within the range + + [ i*base ... (i+1)*base-1 ] + +Currently, "base" is 2KB. So for example, if blocks X and Y start in +the range `[ 0KB .. 2KB-1 ]`, all of the keys in X and Y will be +converted to a filter by calling `FilterPolicy::CreateFilter()`, and the +resulting filter will be stored as the first filter in the filter +block. + +The filter block is formatted as follows: + + [filter 0] + [filter 1] + [filter 2] + ... + [filter N-1] + + [offset of filter 0] : 4 bytes + [offset of filter 1] : 4 bytes + [offset of filter 2] : 4 bytes + ... + [offset of filter N-1] : 4 bytes + + [offset of beginning of offset array] : 4 bytes + lg(base) : 1 byte + +The offset array at the end of the filter block allows efficient +mapping from a data block offset to the corresponding filter. + +## "stats" Meta Block + +This meta block contains a bunch of stats. The key is the name +of the statistic. The value contains the statistic. + +TODO(postrelease): record following stats. + + data size + index size + key size (uncompressed) + value size (uncompressed) + number of entries + number of data blocks diff --git a/src/leveldb/doc/table_format.txt b/src/leveldb/doc/table_format.txt deleted file mode 100644 index ca8f9b446..000000000 --- a/src/leveldb/doc/table_format.txt +++ /dev/null @@ -1,104 +0,0 @@ -File format -=========== - - - [data block 1] - [data block 2] - ... - [data block N] - [meta block 1] - ... - [meta block K] - [metaindex block] - [index block] - [Footer] (fixed size; starts at file_size - sizeof(Footer)) - - -The file contains internal pointers. Each such pointer is called -a BlockHandle and contains the following information: - offset: varint64 - size: varint64 -See https://developers.google.com/protocol-buffers/docs/encoding#varints -for an explanation of varint64 format. - -(1) The sequence of key/value pairs in the file are stored in sorted -order and partitioned into a sequence of data blocks. These blocks -come one after another at the beginning of the file. Each data block -is formatted according to the code in block_builder.cc, and then -optionally compressed. - -(2) After the data blocks we store a bunch of meta blocks. The -supported meta block types are described below. More meta block types -may be added in the future. Each meta block is again formatted using -block_builder.cc and then optionally compressed. - -(3) A "metaindex" block. It contains one entry for every other meta -block where the key is the name of the meta block and the value is a -BlockHandle pointing to that meta block. - -(4) An "index" block. This block contains one entry per data block, -where the key is a string >= last key in that data block and before -the first key in the successive data block. The value is the -BlockHandle for the data block. - -(6) At the very end of the file is a fixed length footer that contains -the BlockHandle of the metaindex and index blocks as well as a magic number. - metaindex_handle: char[p]; // Block handle for metaindex - index_handle: char[q]; // Block handle for index - padding: char[40-p-q]; // zeroed bytes to make fixed length - // (40==2*BlockHandle::kMaxEncodedLength) - magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) - -"filter" Meta Block -------------------- - -If a "FilterPolicy" was specified when the database was opened, a -filter block is stored in each table. The "metaindex" block contains -an entry that maps from "filter." to the BlockHandle for the filter -block where "" is the string returned by the filter policy's -"Name()" method. - -The filter block stores a sequence of filters, where filter i contains -the output of FilterPolicy::CreateFilter() on all keys that are stored -in a block whose file offset falls within the range - - [ i*base ... (i+1)*base-1 ] - -Currently, "base" is 2KB. So for example, if blocks X and Y start in -the range [ 0KB .. 2KB-1 ], all of the keys in X and Y will be -converted to a filter by calling FilterPolicy::CreateFilter(), and the -resulting filter will be stored as the first filter in the filter -block. - -The filter block is formatted as follows: - - [filter 0] - [filter 1] - [filter 2] - ... - [filter N-1] - - [offset of filter 0] : 4 bytes - [offset of filter 1] : 4 bytes - [offset of filter 2] : 4 bytes - ... - [offset of filter N-1] : 4 bytes - - [offset of beginning of offset array] : 4 bytes - lg(base) : 1 byte - -The offset array at the end of the filter block allows efficient -mapping from a data block offset to the corresponding filter. - -"stats" Meta Block ------------------- - -This meta block contains a bunch of stats. The key is the name -of the statistic. The value contains the statistic. -TODO(postrelease): record following stats. - data size - index size - key size (uncompressed) - value size (uncompressed) - number of entries - number of data blocks diff --git a/src/leveldb/include/leveldb/db.h b/src/leveldb/include/leveldb/db.h index 9752cbad5..bfab10a0b 100644 --- a/src/leveldb/include/leveldb/db.h +++ b/src/leveldb/include/leveldb/db.h @@ -1,163 +1,163 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #ifndef STORAGE_LEVELDB_INCLUDE_DB_H_ #define STORAGE_LEVELDB_INCLUDE_DB_H_ #include #include #include "leveldb/iterator.h" #include "leveldb/options.h" namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 19; +static const int kMinorVersion = 20; struct Options; struct ReadOptions; struct WriteOptions; class WriteBatch; // Abstract handle to particular state of a DB. // A Snapshot is an immutable object and can therefore be safely // accessed from multiple threads without any external synchronization. class Snapshot { protected: virtual ~Snapshot(); }; // A range of keys struct Range { Slice start; // Included in the range Slice limit; // Not included in the range Range() { } Range(const Slice& s, const Slice& l) : start(s), limit(l) { } }; // A DB is a persistent ordered map from keys to values. // A DB is safe for concurrent access from multiple threads without // any external synchronization. class DB { public: // Open the database with the specified "name". // Stores a pointer to a heap-allocated database in *dbptr and returns // OK on success. // Stores NULL in *dbptr and returns a non-OK status on error. // Caller should delete *dbptr when it is no longer needed. static Status Open(const Options& options, const std::string& name, DB** dbptr); DB() { } virtual ~DB(); // Set the database entry for "key" to "value". Returns OK on success, // and a non-OK status on error. // Note: consider setting options.sync = true. virtual Status Put(const WriteOptions& options, const Slice& key, const Slice& value) = 0; // Remove the database entry (if any) for "key". Returns OK on // success, and a non-OK status on error. It is not an error if "key" // did not exist in the database. // Note: consider setting options.sync = true. virtual Status Delete(const WriteOptions& options, const Slice& key) = 0; // Apply the specified updates to the database. // Returns OK on success, non-OK on failure. // Note: consider setting options.sync = true. virtual Status Write(const WriteOptions& options, WriteBatch* updates) = 0; // If the database contains an entry for "key" store the // corresponding value in *value and return OK. // // If there is no entry for "key" leave *value unchanged and return // a status for which Status::IsNotFound() returns true. // // May return some other Status on an error. virtual Status Get(const ReadOptions& options, const Slice& key, std::string* value) = 0; // Return a heap-allocated iterator over the contents of the database. // The result of NewIterator() is initially invalid (caller must // call one of the Seek methods on the iterator before using it). // // Caller should delete the iterator when it is no longer needed. // The returned iterator should be deleted before this db is deleted. virtual Iterator* NewIterator(const ReadOptions& options) = 0; // Return a handle to the current DB state. Iterators created with // this handle will all observe a stable snapshot of the current DB // state. The caller must call ReleaseSnapshot(result) when the // snapshot is no longer needed. virtual const Snapshot* GetSnapshot() = 0; // Release a previously acquired snapshot. The caller must not // use "snapshot" after this call. virtual void ReleaseSnapshot(const Snapshot* snapshot) = 0; // DB implementations can export properties about their state // via this method. If "property" is a valid property understood by this // DB implementation, fills "*value" with its current value and returns // true. Otherwise returns false. // // // Valid property names include: // // "leveldb.num-files-at-level" - return the number of files at level , // where is an ASCII representation of a level number (e.g. "0"). // "leveldb.stats" - returns a multi-line string that describes statistics // about the internal operation of the DB. // "leveldb.sstables" - returns a multi-line string that describes all // of the sstables that make up the db contents. // "leveldb.approximate-memory-usage" - returns the approximate number of // bytes of memory in use by the DB. virtual bool GetProperty(const Slice& property, std::string* value) = 0; // For each i in [0,n-1], store in "sizes[i]", the approximate // file system space used by keys in "[range[i].start .. range[i].limit)". // // Note that the returned sizes measure file system space usage, so // if the user data compresses by a factor of ten, the returned // sizes will be one-tenth the size of the corresponding user data size. // // The results may not include the sizes of recently written data. virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) = 0; // Compact the underlying storage for the key range [*begin,*end]. // In particular, deleted and overwritten versions are discarded, // and the data is rearranged to reduce the cost of operations // needed to access the data. This operation should typically only // be invoked by users who understand the underlying implementation. // // begin==NULL is treated as a key before all keys in the database. // end==NULL is treated as a key after all keys in the database. // Therefore the following call will compact the entire database: // db->CompactRange(NULL, NULL); virtual void CompactRange(const Slice* begin, const Slice* end) = 0; private: // No copying allowed DB(const DB&); void operator=(const DB&); }; // Destroy the contents of the specified database. // Be very careful using this method. Status DestroyDB(const std::string& name, const Options& options); // If a DB cannot be opened, you may attempt to call this method to // resurrect as much of the contents of the database as possible. // Some data may be lost, so be careful when calling this function // on a database that contains important information. Status RepairDB(const std::string& dbname, const Options& options); } // namespace leveldb #endif // STORAGE_LEVELDB_INCLUDE_DB_H_ diff --git a/src/leveldb/include/leveldb/options.h b/src/leveldb/include/leveldb/options.h index 83a1ef39a..976e38122 100644 --- a/src/leveldb/include/leveldb/options.h +++ b/src/leveldb/include/leveldb/options.h @@ -1,201 +1,213 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ #define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ #include namespace leveldb { class Cache; class Comparator; class Env; class FilterPolicy; class Logger; class Snapshot; // DB contents are stored in a set of blocks, each of which holds a // sequence of key,value pairs. Each block may be compressed before // being stored in a file. The following enum describes which // compression method (if any) is used to compress a block. enum CompressionType { // NOTE: do not change the values of existing entries, as these are // part of the persistent format on disk. kNoCompression = 0x0, kSnappyCompression = 0x1 }; // Options to control the behavior of a database (passed to DB::Open) struct Options { // ------------------- // Parameters that affect behavior // Comparator used to define the order of keys in the table. // Default: a comparator that uses lexicographic byte-wise ordering // // REQUIRES: The client must ensure that the comparator supplied // here has the same name and orders keys *exactly* the same as the // comparator provided to previous open calls on the same DB. const Comparator* comparator; // If true, the database will be created if it is missing. // Default: false bool create_if_missing; // If true, an error is raised if the database already exists. // Default: false bool error_if_exists; // If true, the implementation will do aggressive checking of the // data it is processing and will stop early if it detects any // errors. This may have unforeseen ramifications: for example, a // corruption of one DB entry may cause a large number of entries to // become unreadable or for the entire DB to become unopenable. // Default: false bool paranoid_checks; // Use the specified object to interact with the environment, // e.g. to read/write files, schedule background work, etc. // Default: Env::Default() Env* env; // Any internal progress/error information generated by the db will // be written to info_log if it is non-NULL, or to a file stored // in the same directory as the DB contents if info_log is NULL. // Default: NULL Logger* info_log; // ------------------- // Parameters that affect performance // Amount of data to build up in memory (backed by an unsorted log // on disk) before converting to a sorted on-disk file. // // Larger values increase performance, especially during bulk loads. // Up to two write buffers may be held in memory at the same time, // so you may wish to adjust this parameter to control memory usage. // Also, a larger write buffer will result in a longer recovery time // the next time the database is opened. // // Default: 4MB size_t write_buffer_size; // Number of open files that can be used by the DB. You may need to // increase this if your database has a large working set (budget // one open file per 2MB of working set). // // Default: 1000 int max_open_files; // Control over blocks (user data is stored in a set of blocks, and // a block is the unit of reading from disk). // If non-NULL, use the specified cache for blocks. // If NULL, leveldb will automatically create and use an 8MB internal cache. // Default: NULL Cache* block_cache; // Approximate size of user data packed per block. Note that the // block size specified here corresponds to uncompressed data. The // actual size of the unit read from disk may be smaller if // compression is enabled. This parameter can be changed dynamically. // // Default: 4K size_t block_size; // Number of keys between restart points for delta encoding of keys. // This parameter can be changed dynamically. Most clients should // leave this parameter alone. // // Default: 16 int block_restart_interval; + // Leveldb will write up to this amount of bytes to a file before + // switching to a new one. + // Most clients should leave this parameter alone. However if your + // filesystem is more efficient with larger files, you could + // consider increasing the value. The downside will be longer + // compactions and hence longer latency/performance hiccups. + // Another reason to increase this parameter might be when you are + // initially populating a large database. + // + // Default: 2MB + size_t max_file_size; + // Compress blocks using the specified compression algorithm. This // parameter can be changed dynamically. // // Default: kSnappyCompression, which gives lightweight but fast // compression. // // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz: // ~200-500MB/s compression // ~400-800MB/s decompression // Note that these speeds are significantly faster than most // persistent storage speeds, and therefore it is typically never // worth switching to kNoCompression. Even if the input data is // incompressible, the kSnappyCompression implementation will // efficiently detect that and will switch to uncompressed mode. CompressionType compression; // EXPERIMENTAL: If true, append to existing MANIFEST and log files // when a database is opened. This can significantly speed up open. // // Default: currently false, but may become true later. bool reuse_logs; // If non-NULL, use the specified filter policy to reduce disk reads. // Many applications will benefit from passing the result of // NewBloomFilterPolicy() here. // // Default: NULL const FilterPolicy* filter_policy; // Create an Options object with default values for all fields. Options(); }; // Options that control read operations struct ReadOptions { // If true, all data read from underlying storage will be // verified against corresponding checksums. // Default: false bool verify_checksums; // Should the data read for this iteration be cached in memory? // Callers may wish to set this field to false for bulk scans. // Default: true bool fill_cache; // If "snapshot" is non-NULL, read as of the supplied snapshot // (which must belong to the DB that is being read and which must // not have been released). If "snapshot" is NULL, use an implicit // snapshot of the state at the beginning of this read operation. // Default: NULL const Snapshot* snapshot; ReadOptions() : verify_checksums(false), fill_cache(true), snapshot(NULL) { } }; // Options that control write operations struct WriteOptions { // If true, the write will be flushed from the operating system // buffer cache (by calling WritableFile::Sync()) before the write // is considered complete. If this flag is true, writes will be // slower. // // If this flag is false, and the machine crashes, some recent // writes may be lost. Note that if it is just the process that // crashes (i.e., the machine does not reboot), no writes will be // lost even if sync==false. // // In other words, a DB write with sync==false has similar // crash semantics as the "write()" system call. A DB write // with sync==true has similar crash semantics to a "write()" // system call followed by "fsync()". // // Default: false bool sync; WriteOptions() : sync(false) { } }; } // namespace leveldb #endif // STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ diff --git a/src/leveldb/port/port_example.h b/src/leveldb/port/port_example.h index ab9e489b3..97bd669a5 100644 --- a/src/leveldb/port/port_example.h +++ b/src/leveldb/port/port_example.h @@ -1,135 +1,141 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // This file contains the specification, but not the implementations, // of the types/operations/etc. that should be defined by a platform // specific port_.h file. Use this file as a reference for // how to port this package to a new platform. #ifndef STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ #define STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ namespace leveldb { namespace port { // TODO(jorlow): Many of these belong more in the environment class rather than // here. We should try moving them and see if it affects perf. // The following boolean constant must be true on a little-endian machine // and false otherwise. static const bool kLittleEndian = true /* or some other expression */; // ------------------ Threading ------------------- // A Mutex represents an exclusive lock. class Mutex { public: Mutex(); ~Mutex(); // Lock the mutex. Waits until other lockers have exited. // Will deadlock if the mutex is already locked by this thread. void Lock(); // Unlock the mutex. // REQUIRES: This mutex was locked by this thread. void Unlock(); // Optionally crash if this thread does not hold this mutex. // The implementation must be fast, especially if NDEBUG is // defined. The implementation is allowed to skip all checks. void AssertHeld(); }; class CondVar { public: explicit CondVar(Mutex* mu); ~CondVar(); // Atomically release *mu and block on this condition variable until // either a call to SignalAll(), or a call to Signal() that picks // this thread to wakeup. // REQUIRES: this thread holds *mu void Wait(); // If there are some threads waiting, wake up at least one of them. void Signal(); // Wake up all waiting threads. void SignallAll(); }; // Thread-safe initialization. // Used as follows: // static port::OnceType init_control = LEVELDB_ONCE_INIT; // static void Initializer() { ... do something ...; } // ... // port::InitOnce(&init_control, &Initializer); typedef intptr_t OnceType; #define LEVELDB_ONCE_INIT 0 extern void InitOnce(port::OnceType*, void (*initializer)()); // A type that holds a pointer that can be read or written atomically // (i.e., without word-tearing.) class AtomicPointer { private: intptr_t rep_; public: // Initialize to arbitrary value AtomicPointer(); // Initialize to hold v explicit AtomicPointer(void* v) : rep_(v) { } // Read and return the stored pointer with the guarantee that no // later memory access (read or write) by this thread can be // reordered ahead of this read. void* Acquire_Load() const; // Set v as the stored pointer with the guarantee that no earlier // memory access (read or write) by this thread can be reordered // after this store. void Release_Store(void* v); // Read the stored pointer with no ordering guarantees. void* NoBarrier_Load() const; // Set va as the stored pointer with no ordering guarantees. void NoBarrier_Store(void* v); }; // ------------------ Compression ------------------- // Store the snappy compression of "input[0,input_length-1]" in *output. // Returns false if snappy is not supported by this port. extern bool Snappy_Compress(const char* input, size_t input_length, std::string* output); // If input[0,input_length-1] looks like a valid snappy compressed // buffer, store the size of the uncompressed data in *result and // return true. Else return false. extern bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result); // Attempt to snappy uncompress input[0,input_length-1] into *output. // Returns true if successful, false if the input is invalid lightweight // compressed data. // // REQUIRES: at least the first "n" bytes of output[] must be writable // where "n" is the result of a successful call to // Snappy_GetUncompressedLength. extern bool Snappy_Uncompress(const char* input_data, size_t input_length, char* output); // ------------------ Miscellaneous ------------------- // If heap profiling is not supported, returns false. // Else repeatedly calls (*func)(arg, data, n) and then returns true. // The concatenation of all "data[0,n-1]" fragments is the heap profile. extern bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg); +// Extend the CRC to include the first n bytes of buf. +// +// Returns zero if the CRC cannot be extended using acceleration, else returns +// the newly extended CRC value (which may also be zero). +uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); + } // namespace port } // namespace leveldb #endif // STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ diff --git a/src/leveldb/port/port_posix.h b/src/leveldb/port/port_posix.h index ccca9939d..7e8213b22 100644 --- a/src/leveldb/port/port_posix.h +++ b/src/leveldb/port/port_posix.h @@ -1,158 +1,160 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // See port_example.h for documentation for the following types/functions. #ifndef STORAGE_LEVELDB_PORT_PORT_POSIX_H_ #define STORAGE_LEVELDB_PORT_PORT_POSIX_H_ #undef PLATFORM_IS_LITTLE_ENDIAN #if defined(OS_MACOSX) #include #if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER) #define PLATFORM_IS_LITTLE_ENDIAN \ (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN) #endif #elif defined(OS_SOLARIS) #include #ifdef _LITTLE_ENDIAN #define PLATFORM_IS_LITTLE_ENDIAN true #else #define PLATFORM_IS_LITTLE_ENDIAN false #endif #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) ||\ defined(OS_NETBSD) || defined(OS_DRAGONFLYBSD) #include #include #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #elif defined(OS_HPUX) #define PLATFORM_IS_LITTLE_ENDIAN false #elif defined(OS_ANDROID) // Due to a bug in the NDK x86 definition, // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android. // See http://code.google.com/p/android/issues/detail?id=39824 #include #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #else #include #endif #include #ifdef SNAPPY #include #endif #include #include #include "port/atomic_pointer.h" #ifndef PLATFORM_IS_LITTLE_ENDIAN #define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN) #endif #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ defined(OS_ANDROID) || defined(OS_HPUX) || defined(CYGWIN) // Use fread/fwrite/fflush on platforms without _unlocked variants #define fread_unlocked fread #define fwrite_unlocked fwrite #define fflush_unlocked fflush #endif #if defined(OS_FREEBSD) ||\ defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) // Use fsync() on platforms without fdatasync() #define fdatasync fsync #endif #if defined(OS_MACOSX) #define fdatasync(fd) fcntl(fd, F_FULLFSYNC, 0) #endif #if defined(OS_ANDROID) && __ANDROID_API__ < 9 // fdatasync() was only introduced in API level 9 on Android. Use fsync() // when targetting older platforms. #define fdatasync fsync #endif namespace leveldb { namespace port { static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN; #undef PLATFORM_IS_LITTLE_ENDIAN class CondVar; class Mutex { public: Mutex(); ~Mutex(); void Lock(); void Unlock(); void AssertHeld() { } private: friend class CondVar; pthread_mutex_t mu_; // No copying Mutex(const Mutex&); void operator=(const Mutex&); }; class CondVar { public: explicit CondVar(Mutex* mu); ~CondVar(); void Wait(); void Signal(); void SignalAll(); private: pthread_cond_t cv_; Mutex* mu_; }; typedef pthread_once_t OnceType; #define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT extern void InitOnce(OnceType* once, void (*initializer)()); inline bool Snappy_Compress(const char* input, size_t length, ::std::string* output) { #ifdef SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); output->resize(outlen); return true; #endif return false; } inline bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { #ifdef SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; #endif } inline bool Snappy_Uncompress(const char* input, size_t length, char* output) { #ifdef SNAPPY return snappy::RawUncompress(input, length, output); #else return false; #endif } inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { return false; } +uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); + } // namespace port } // namespace leveldb #endif // STORAGE_LEVELDB_PORT_PORT_POSIX_H_ diff --git a/src/leveldb/port/port_posix_sse.cc b/src/leveldb/port/port_posix_sse.cc new file mode 100644 index 000000000..1e519ba0b --- /dev/null +++ b/src/leveldb/port/port_posix_sse.cc @@ -0,0 +1,129 @@ +// Copyright 2016 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A portable implementation of crc32c, optimized to handle +// four bytes at a time. +// +// In a separate source file to allow this accelerated CRC32C function to be +// compiled with the appropriate compiler flags to enable x86 SSE 4.2 +// instructions. + +#include +#include +#include "port/port.h" + +#if defined(LEVELDB_PLATFORM_POSIX_SSE) + +#if defined(_MSC_VER) +#include +#elif defined(__GNUC__) && defined(__SSE4_2__) +#include +#include +#endif + +#endif // defined(LEVELDB_PLATFORM_POSIX_SSE) + +namespace leveldb { +namespace port { + +#if defined(LEVELDB_PLATFORM_POSIX_SSE) + +// Used to fetch a naturally-aligned 32-bit word in little endian byte-order +static inline uint32_t LE_LOAD32(const uint8_t *p) { + // SSE is x86 only, so ensured that |p| is always little-endian. + uint32_t word; + memcpy(&word, p, sizeof(word)); + return word; +} + +#if defined(_M_X64) || defined(__x86_64__) // LE_LOAD64 is only used on x64. + +// Used to fetch a naturally-aligned 64-bit word in little endian byte-order +static inline uint64_t LE_LOAD64(const uint8_t *p) { + uint64_t dword; + memcpy(&dword, p, sizeof(dword)); + return dword; +} + +#endif // defined(_M_X64) || defined(__x86_64__) + +static inline bool HaveSSE42() { +#if defined(_MSC_VER) + int cpu_info[4]; + __cpuid(cpu_info, 1); + return (cpu_info[2] & (1 << 20)) != 0; +#elif defined(__GNUC__) + unsigned int eax, ebx, ecx, edx; + __get_cpuid(1, &eax, &ebx, &ecx, &edx); + return (ecx & (1 << 20)) != 0; +#else + return false; +#endif +} + +#endif // defined(LEVELDB_PLATFORM_POSIX_SSE) + +// For further improvements see Intel publication at: +// http://download.intel.com/design/intarch/papers/323405.pdf +uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) { +#if !defined(LEVELDB_PLATFORM_POSIX_SSE) + return 0; +#else + static bool have = HaveSSE42(); + if (!have) { + return 0; + } + + const uint8_t *p = reinterpret_cast(buf); + const uint8_t *e = p + size; + uint32_t l = crc ^ 0xffffffffu; + +#define STEP1 do { \ + l = _mm_crc32_u8(l, *p++); \ +} while (0) +#define STEP4 do { \ + l = _mm_crc32_u32(l, LE_LOAD32(p)); \ + p += 4; \ +} while (0) +#define STEP8 do { \ + l = _mm_crc32_u64(l, LE_LOAD64(p)); \ + p += 8; \ +} while (0) + + if (size > 16) { + // Process unaligned bytes + for (unsigned int i = reinterpret_cast(p) % 8; i; --i) { + STEP1; + } + + // _mm_crc32_u64 is only available on x64. +#if defined(_M_X64) || defined(__x86_64__) + // Process 8 bytes at a time + while ((e-p) >= 8) { + STEP8; + } + // Process 4 bytes at a time + if ((e-p) >= 4) { + STEP4; + } +#else // !(defined(_M_X64) || defined(__x86_64__)) + // Process 4 bytes at a time + while ((e-p) >= 4) { + STEP4; + } +#endif // defined(_M_X64) || defined(__x86_64__) + } + // Process the last few bytes + while (p != e) { + STEP1; + } +#undef STEP8 +#undef STEP4 +#undef STEP1 + return l ^ 0xffffffffu; +#endif // defined(LEVELDB_PLATFORM_POSIX_SSE) +} + +} // namespace port +} // namespace leveldb diff --git a/src/leveldb/port/port_win.h b/src/leveldb/port/port_win.h index 45bf2f0ea..500900482 100644 --- a/src/leveldb/port/port_win.h +++ b/src/leveldb/port/port_win.h @@ -1,174 +1,176 @@ // LevelDB Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // See port_example.h for documentation for the following types/functions. // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of the University of California, Berkeley nor the // names of its contributors may be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // #ifndef STORAGE_LEVELDB_PORT_PORT_WIN_H_ #define STORAGE_LEVELDB_PORT_PORT_WIN_H_ #ifdef _MSC_VER #define snprintf _snprintf #define close _close #define fread_unlocked _fread_nolock #endif #include #include #ifdef SNAPPY #include #endif namespace leveldb { namespace port { // Windows is little endian (for now :p) static const bool kLittleEndian = true; class CondVar; class Mutex { public: Mutex(); ~Mutex(); void Lock(); void Unlock(); void AssertHeld(); private: friend class CondVar; // critical sections are more efficient than mutexes // but they are not recursive and can only be used to synchronize threads within the same process // we use opaque void * to avoid including windows.h in port_win.h void * cs_; // No copying Mutex(const Mutex&); void operator=(const Mutex&); }; // the Win32 API offers a dependable condition variable mechanism, but only starting with // Windows 2008 and Vista // no matter what we will implement our own condition variable with a semaphore // implementation as described in a paper written by Andrew D. Birrell in 2003 class CondVar { public: explicit CondVar(Mutex* mu); ~CondVar(); void Wait(); void Signal(); void SignalAll(); private: Mutex* mu_; Mutex wait_mtx_; long waiting_; void * sem1_; void * sem2_; }; class OnceType { public: // OnceType() : init_(false) {} OnceType(const OnceType &once) : init_(once.init_) {} OnceType(bool f) : init_(f) {} void InitOnce(void (*initializer)()) { mutex_.Lock(); if (!init_) { init_ = true; initializer(); } mutex_.Unlock(); } private: bool init_; Mutex mutex_; }; #define LEVELDB_ONCE_INIT false extern void InitOnce(port::OnceType*, void (*initializer)()); // Storage for a lock-free pointer class AtomicPointer { private: void * rep_; public: AtomicPointer() : rep_(NULL) { } explicit AtomicPointer(void* v); void* Acquire_Load() const; void Release_Store(void* v); void* NoBarrier_Load() const; void NoBarrier_Store(void* v); }; inline bool Snappy_Compress(const char* input, size_t length, ::std::string* output) { #ifdef SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); output->resize(outlen); return true; #endif return false; } inline bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { #ifdef SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; #endif } inline bool Snappy_Uncompress(const char* input, size_t length, char* output) { #ifdef SNAPPY return snappy::RawUncompress(input, length, output); #else return false; #endif } inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { return false; } +uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); + } } #endif // STORAGE_LEVELDB_PORT_PORT_WIN_H_ diff --git a/src/leveldb/table/filter_block.cc b/src/leveldb/table/filter_block.cc index 4e78b954f..1ed513417 100644 --- a/src/leveldb/table/filter_block.cc +++ b/src/leveldb/table/filter_block.cc @@ -1,111 +1,111 @@ // Copyright (c) 2012 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "table/filter_block.h" #include "leveldb/filter_policy.h" #include "util/coding.h" namespace leveldb { -// See doc/table_format.txt for an explanation of the filter block format. +// See doc/table_format.md for an explanation of the filter block format. // Generate new filter every 2KB of data static const size_t kFilterBaseLg = 11; static const size_t kFilterBase = 1 << kFilterBaseLg; FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy) : policy_(policy) { } void FilterBlockBuilder::StartBlock(uint64_t block_offset) { uint64_t filter_index = (block_offset / kFilterBase); assert(filter_index >= filter_offsets_.size()); while (filter_index > filter_offsets_.size()) { GenerateFilter(); } } void FilterBlockBuilder::AddKey(const Slice& key) { Slice k = key; start_.push_back(keys_.size()); keys_.append(k.data(), k.size()); } Slice FilterBlockBuilder::Finish() { if (!start_.empty()) { GenerateFilter(); } // Append array of per-filter offsets const uint32_t array_offset = result_.size(); for (size_t i = 0; i < filter_offsets_.size(); i++) { PutFixed32(&result_, filter_offsets_[i]); } PutFixed32(&result_, array_offset); result_.push_back(kFilterBaseLg); // Save encoding parameter in result return Slice(result_); } void FilterBlockBuilder::GenerateFilter() { const size_t num_keys = start_.size(); if (num_keys == 0) { // Fast path if there are no keys for this filter filter_offsets_.push_back(result_.size()); return; } // Make list of keys from flattened key structure start_.push_back(keys_.size()); // Simplify length computation tmp_keys_.resize(num_keys); for (size_t i = 0; i < num_keys; i++) { const char* base = keys_.data() + start_[i]; size_t length = start_[i+1] - start_[i]; tmp_keys_[i] = Slice(base, length); } // Generate filter for current set of keys and append to result_. filter_offsets_.push_back(result_.size()); policy_->CreateFilter(&tmp_keys_[0], static_cast(num_keys), &result_); tmp_keys_.clear(); keys_.clear(); start_.clear(); } FilterBlockReader::FilterBlockReader(const FilterPolicy* policy, const Slice& contents) : policy_(policy), data_(NULL), offset_(NULL), num_(0), base_lg_(0) { size_t n = contents.size(); if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array base_lg_ = contents[n-1]; uint32_t last_word = DecodeFixed32(contents.data() + n - 5); if (last_word > n - 5) return; data_ = contents.data(); offset_ = data_ + last_word; num_ = (n - 5 - last_word) / 4; } bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) { uint64_t index = block_offset >> base_lg_; if (index < num_) { uint32_t start = DecodeFixed32(offset_ + index*4); uint32_t limit = DecodeFixed32(offset_ + index*4 + 4); if (start <= limit && limit <= static_cast(offset_ - data_)) { Slice filter = Slice(data_ + start, limit - start); return policy_->KeyMayMatch(key, filter); } else if (start == limit) { // Empty filters do not match any keys return false; } } return true; // Errors are treated as potential matches } } diff --git a/src/leveldb/util/crc32c.cc b/src/leveldb/util/crc32c.cc index 6db9e7707..edd61cfd6 100644 --- a/src/leveldb/util/crc32c.cc +++ b/src/leveldb/util/crc32c.cc @@ -1,332 +1,350 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // // A portable implementation of crc32c, optimized to handle // four bytes at a time. #include "util/crc32c.h" #include + +#include "port/port.h" #include "util/coding.h" namespace leveldb { namespace crc32c { static const uint32_t table0_[256] = { 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 }; static const uint32_t table1_[256] = { 0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899, 0x4e8a61dc, 0x5d28f9ab, 0x69cf5132, 0x7a6dc945, 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21, 0xd39ea264, 0xc03c3a13, 0xf4db928a, 0xe7790afd, 0x3fc5f181, 0x2c6769f6, 0x1880c16f, 0x0b225918, 0x714f905d, 0x62ed082a, 0x560aa0b3, 0x45a838c4, 0xa2d13239, 0xb173aa4e, 0x859402d7, 0x96369aa0, 0xec5b53e5, 0xfff9cb92, 0xcb1e630b, 0xd8bcfb7c, 0x7f8be302, 0x6c297b75, 0x58ced3ec, 0x4b6c4b9b, 0x310182de, 0x22a31aa9, 0x1644b230, 0x05e62a47, 0xe29f20ba, 0xf13db8cd, 0xc5da1054, 0xd6788823, 0xac154166, 0xbfb7d911, 0x8b507188, 0x98f2e9ff, 0x404e1283, 0x53ec8af4, 0x670b226d, 0x74a9ba1a, 0x0ec4735f, 0x1d66eb28, 0x298143b1, 0x3a23dbc6, 0xdd5ad13b, 0xcef8494c, 0xfa1fe1d5, 0xe9bd79a2, 0x93d0b0e7, 0x80722890, 0xb4958009, 0xa737187e, 0xff17c604, 0xecb55e73, 0xd852f6ea, 0xcbf06e9d, 0xb19da7d8, 0xa23f3faf, 0x96d89736, 0x857a0f41, 0x620305bc, 0x71a19dcb, 0x45463552, 0x56e4ad25, 0x2c896460, 0x3f2bfc17, 0x0bcc548e, 0x186eccf9, 0xc0d23785, 0xd370aff2, 0xe797076b, 0xf4359f1c, 0x8e585659, 0x9dface2e, 0xa91d66b7, 0xbabffec0, 0x5dc6f43d, 0x4e646c4a, 0x7a83c4d3, 0x69215ca4, 0x134c95e1, 0x00ee0d96, 0x3409a50f, 0x27ab3d78, 0x809c2506, 0x933ebd71, 0xa7d915e8, 0xb47b8d9f, 0xce1644da, 0xddb4dcad, 0xe9537434, 0xfaf1ec43, 0x1d88e6be, 0x0e2a7ec9, 0x3acdd650, 0x296f4e27, 0x53028762, 0x40a01f15, 0x7447b78c, 0x67e52ffb, 0xbf59d487, 0xacfb4cf0, 0x981ce469, 0x8bbe7c1e, 0xf1d3b55b, 0xe2712d2c, 0xd69685b5, 0xc5341dc2, 0x224d173f, 0x31ef8f48, 0x050827d1, 0x16aabfa6, 0x6cc776e3, 0x7f65ee94, 0x4b82460d, 0x5820de7a, 0xfbc3faf9, 0xe861628e, 0xdc86ca17, 0xcf245260, 0xb5499b25, 0xa6eb0352, 0x920cabcb, 0x81ae33bc, 0x66d73941, 0x7575a136, 0x419209af, 0x523091d8, 0x285d589d, 0x3bffc0ea, 0x0f186873, 0x1cbaf004, 0xc4060b78, 0xd7a4930f, 0xe3433b96, 0xf0e1a3e1, 0x8a8c6aa4, 0x992ef2d3, 0xadc95a4a, 0xbe6bc23d, 0x5912c8c0, 0x4ab050b7, 0x7e57f82e, 0x6df56059, 0x1798a91c, 0x043a316b, 0x30dd99f2, 0x237f0185, 0x844819fb, 0x97ea818c, 0xa30d2915, 0xb0afb162, 0xcac27827, 0xd960e050, 0xed8748c9, 0xfe25d0be, 0x195cda43, 0x0afe4234, 0x3e19eaad, 0x2dbb72da, 0x57d6bb9f, 0x447423e8, 0x70938b71, 0x63311306, 0xbb8de87a, 0xa82f700d, 0x9cc8d894, 0x8f6a40e3, 0xf50789a6, 0xe6a511d1, 0xd242b948, 0xc1e0213f, 0x26992bc2, 0x353bb3b5, 0x01dc1b2c, 0x127e835b, 0x68134a1e, 0x7bb1d269, 0x4f567af0, 0x5cf4e287, 0x04d43cfd, 0x1776a48a, 0x23910c13, 0x30339464, 0x4a5e5d21, 0x59fcc556, 0x6d1b6dcf, 0x7eb9f5b8, 0x99c0ff45, 0x8a626732, 0xbe85cfab, 0xad2757dc, 0xd74a9e99, 0xc4e806ee, 0xf00fae77, 0xe3ad3600, 0x3b11cd7c, 0x28b3550b, 0x1c54fd92, 0x0ff665e5, 0x759baca0, 0x663934d7, 0x52de9c4e, 0x417c0439, 0xa6050ec4, 0xb5a796b3, 0x81403e2a, 0x92e2a65d, 0xe88f6f18, 0xfb2df76f, 0xcfca5ff6, 0xdc68c781, 0x7b5fdfff, 0x68fd4788, 0x5c1aef11, 0x4fb87766, 0x35d5be23, 0x26772654, 0x12908ecd, 0x013216ba, 0xe64b1c47, 0xf5e98430, 0xc10e2ca9, 0xd2acb4de, 0xa8c17d9b, 0xbb63e5ec, 0x8f844d75, 0x9c26d502, 0x449a2e7e, 0x5738b609, 0x63df1e90, 0x707d86e7, 0x0a104fa2, 0x19b2d7d5, 0x2d557f4c, 0x3ef7e73b, 0xd98eedc6, 0xca2c75b1, 0xfecbdd28, 0xed69455f, 0x97048c1a, 0x84a6146d, 0xb041bcf4, 0xa3e32483 }; static const uint32_t table2_[256] = { 0x00000000, 0xa541927e, 0x4f6f520d, 0xea2ec073, 0x9edea41a, 0x3b9f3664, 0xd1b1f617, 0x74f06469, 0x38513ec5, 0x9d10acbb, 0x773e6cc8, 0xd27ffeb6, 0xa68f9adf, 0x03ce08a1, 0xe9e0c8d2, 0x4ca15aac, 0x70a27d8a, 0xd5e3eff4, 0x3fcd2f87, 0x9a8cbdf9, 0xee7cd990, 0x4b3d4bee, 0xa1138b9d, 0x045219e3, 0x48f3434f, 0xedb2d131, 0x079c1142, 0xa2dd833c, 0xd62de755, 0x736c752b, 0x9942b558, 0x3c032726, 0xe144fb14, 0x4405696a, 0xae2ba919, 0x0b6a3b67, 0x7f9a5f0e, 0xdadbcd70, 0x30f50d03, 0x95b49f7d, 0xd915c5d1, 0x7c5457af, 0x967a97dc, 0x333b05a2, 0x47cb61cb, 0xe28af3b5, 0x08a433c6, 0xade5a1b8, 0x91e6869e, 0x34a714e0, 0xde89d493, 0x7bc846ed, 0x0f382284, 0xaa79b0fa, 0x40577089, 0xe516e2f7, 0xa9b7b85b, 0x0cf62a25, 0xe6d8ea56, 0x43997828, 0x37691c41, 0x92288e3f, 0x78064e4c, 0xdd47dc32, 0xc76580d9, 0x622412a7, 0x880ad2d4, 0x2d4b40aa, 0x59bb24c3, 0xfcfab6bd, 0x16d476ce, 0xb395e4b0, 0xff34be1c, 0x5a752c62, 0xb05bec11, 0x151a7e6f, 0x61ea1a06, 0xc4ab8878, 0x2e85480b, 0x8bc4da75, 0xb7c7fd53, 0x12866f2d, 0xf8a8af5e, 0x5de93d20, 0x29195949, 0x8c58cb37, 0x66760b44, 0xc337993a, 0x8f96c396, 0x2ad751e8, 0xc0f9919b, 0x65b803e5, 0x1148678c, 0xb409f5f2, 0x5e273581, 0xfb66a7ff, 0x26217bcd, 0x8360e9b3, 0x694e29c0, 0xcc0fbbbe, 0xb8ffdfd7, 0x1dbe4da9, 0xf7908dda, 0x52d11fa4, 0x1e704508, 0xbb31d776, 0x511f1705, 0xf45e857b, 0x80aee112, 0x25ef736c, 0xcfc1b31f, 0x6a802161, 0x56830647, 0xf3c29439, 0x19ec544a, 0xbcadc634, 0xc85da25d, 0x6d1c3023, 0x8732f050, 0x2273622e, 0x6ed23882, 0xcb93aafc, 0x21bd6a8f, 0x84fcf8f1, 0xf00c9c98, 0x554d0ee6, 0xbf63ce95, 0x1a225ceb, 0x8b277743, 0x2e66e53d, 0xc448254e, 0x6109b730, 0x15f9d359, 0xb0b84127, 0x5a968154, 0xffd7132a, 0xb3764986, 0x1637dbf8, 0xfc191b8b, 0x595889f5, 0x2da8ed9c, 0x88e97fe2, 0x62c7bf91, 0xc7862def, 0xfb850ac9, 0x5ec498b7, 0xb4ea58c4, 0x11abcaba, 0x655baed3, 0xc01a3cad, 0x2a34fcde, 0x8f756ea0, 0xc3d4340c, 0x6695a672, 0x8cbb6601, 0x29faf47f, 0x5d0a9016, 0xf84b0268, 0x1265c21b, 0xb7245065, 0x6a638c57, 0xcf221e29, 0x250cde5a, 0x804d4c24, 0xf4bd284d, 0x51fcba33, 0xbbd27a40, 0x1e93e83e, 0x5232b292, 0xf77320ec, 0x1d5de09f, 0xb81c72e1, 0xccec1688, 0x69ad84f6, 0x83834485, 0x26c2d6fb, 0x1ac1f1dd, 0xbf8063a3, 0x55aea3d0, 0xf0ef31ae, 0x841f55c7, 0x215ec7b9, 0xcb7007ca, 0x6e3195b4, 0x2290cf18, 0x87d15d66, 0x6dff9d15, 0xc8be0f6b, 0xbc4e6b02, 0x190ff97c, 0xf321390f, 0x5660ab71, 0x4c42f79a, 0xe90365e4, 0x032da597, 0xa66c37e9, 0xd29c5380, 0x77ddc1fe, 0x9df3018d, 0x38b293f3, 0x7413c95f, 0xd1525b21, 0x3b7c9b52, 0x9e3d092c, 0xeacd6d45, 0x4f8cff3b, 0xa5a23f48, 0x00e3ad36, 0x3ce08a10, 0x99a1186e, 0x738fd81d, 0xd6ce4a63, 0xa23e2e0a, 0x077fbc74, 0xed517c07, 0x4810ee79, 0x04b1b4d5, 0xa1f026ab, 0x4bdee6d8, 0xee9f74a6, 0x9a6f10cf, 0x3f2e82b1, 0xd50042c2, 0x7041d0bc, 0xad060c8e, 0x08479ef0, 0xe2695e83, 0x4728ccfd, 0x33d8a894, 0x96993aea, 0x7cb7fa99, 0xd9f668e7, 0x9557324b, 0x3016a035, 0xda386046, 0x7f79f238, 0x0b899651, 0xaec8042f, 0x44e6c45c, 0xe1a75622, 0xdda47104, 0x78e5e37a, 0x92cb2309, 0x378ab177, 0x437ad51e, 0xe63b4760, 0x0c158713, 0xa954156d, 0xe5f54fc1, 0x40b4ddbf, 0xaa9a1dcc, 0x0fdb8fb2, 0x7b2bebdb, 0xde6a79a5, 0x3444b9d6, 0x91052ba8 }; static const uint32_t table3_[256] = { 0x00000000, 0xdd45aab8, 0xbf672381, 0x62228939, 0x7b2231f3, 0xa6679b4b, 0xc4451272, 0x1900b8ca, 0xf64463e6, 0x2b01c95e, 0x49234067, 0x9466eadf, 0x8d665215, 0x5023f8ad, 0x32017194, 0xef44db2c, 0xe964b13d, 0x34211b85, 0x560392bc, 0x8b463804, 0x924680ce, 0x4f032a76, 0x2d21a34f, 0xf06409f7, 0x1f20d2db, 0xc2657863, 0xa047f15a, 0x7d025be2, 0x6402e328, 0xb9474990, 0xdb65c0a9, 0x06206a11, 0xd725148b, 0x0a60be33, 0x6842370a, 0xb5079db2, 0xac072578, 0x71428fc0, 0x136006f9, 0xce25ac41, 0x2161776d, 0xfc24ddd5, 0x9e0654ec, 0x4343fe54, 0x5a43469e, 0x8706ec26, 0xe524651f, 0x3861cfa7, 0x3e41a5b6, 0xe3040f0e, 0x81268637, 0x5c632c8f, 0x45639445, 0x98263efd, 0xfa04b7c4, 0x27411d7c, 0xc805c650, 0x15406ce8, 0x7762e5d1, 0xaa274f69, 0xb327f7a3, 0x6e625d1b, 0x0c40d422, 0xd1057e9a, 0xaba65fe7, 0x76e3f55f, 0x14c17c66, 0xc984d6de, 0xd0846e14, 0x0dc1c4ac, 0x6fe34d95, 0xb2a6e72d, 0x5de23c01, 0x80a796b9, 0xe2851f80, 0x3fc0b538, 0x26c00df2, 0xfb85a74a, 0x99a72e73, 0x44e284cb, 0x42c2eeda, 0x9f874462, 0xfda5cd5b, 0x20e067e3, 0x39e0df29, 0xe4a57591, 0x8687fca8, 0x5bc25610, 0xb4868d3c, 0x69c32784, 0x0be1aebd, 0xd6a40405, 0xcfa4bccf, 0x12e11677, 0x70c39f4e, 0xad8635f6, 0x7c834b6c, 0xa1c6e1d4, 0xc3e468ed, 0x1ea1c255, 0x07a17a9f, 0xdae4d027, 0xb8c6591e, 0x6583f3a6, 0x8ac7288a, 0x57828232, 0x35a00b0b, 0xe8e5a1b3, 0xf1e51979, 0x2ca0b3c1, 0x4e823af8, 0x93c79040, 0x95e7fa51, 0x48a250e9, 0x2a80d9d0, 0xf7c57368, 0xeec5cba2, 0x3380611a, 0x51a2e823, 0x8ce7429b, 0x63a399b7, 0xbee6330f, 0xdcc4ba36, 0x0181108e, 0x1881a844, 0xc5c402fc, 0xa7e68bc5, 0x7aa3217d, 0x52a0c93f, 0x8fe56387, 0xedc7eabe, 0x30824006, 0x2982f8cc, 0xf4c75274, 0x96e5db4d, 0x4ba071f5, 0xa4e4aad9, 0x79a10061, 0x1b838958, 0xc6c623e0, 0xdfc69b2a, 0x02833192, 0x60a1b8ab, 0xbde41213, 0xbbc47802, 0x6681d2ba, 0x04a35b83, 0xd9e6f13b, 0xc0e649f1, 0x1da3e349, 0x7f816a70, 0xa2c4c0c8, 0x4d801be4, 0x90c5b15c, 0xf2e73865, 0x2fa292dd, 0x36a22a17, 0xebe780af, 0x89c50996, 0x5480a32e, 0x8585ddb4, 0x58c0770c, 0x3ae2fe35, 0xe7a7548d, 0xfea7ec47, 0x23e246ff, 0x41c0cfc6, 0x9c85657e, 0x73c1be52, 0xae8414ea, 0xcca69dd3, 0x11e3376b, 0x08e38fa1, 0xd5a62519, 0xb784ac20, 0x6ac10698, 0x6ce16c89, 0xb1a4c631, 0xd3864f08, 0x0ec3e5b0, 0x17c35d7a, 0xca86f7c2, 0xa8a47efb, 0x75e1d443, 0x9aa50f6f, 0x47e0a5d7, 0x25c22cee, 0xf8878656, 0xe1873e9c, 0x3cc29424, 0x5ee01d1d, 0x83a5b7a5, 0xf90696d8, 0x24433c60, 0x4661b559, 0x9b241fe1, 0x8224a72b, 0x5f610d93, 0x3d4384aa, 0xe0062e12, 0x0f42f53e, 0xd2075f86, 0xb025d6bf, 0x6d607c07, 0x7460c4cd, 0xa9256e75, 0xcb07e74c, 0x16424df4, 0x106227e5, 0xcd278d5d, 0xaf050464, 0x7240aedc, 0x6b401616, 0xb605bcae, 0xd4273597, 0x09629f2f, 0xe6264403, 0x3b63eebb, 0x59416782, 0x8404cd3a, 0x9d0475f0, 0x4041df48, 0x22635671, 0xff26fcc9, 0x2e238253, 0xf36628eb, 0x9144a1d2, 0x4c010b6a, 0x5501b3a0, 0x88441918, 0xea669021, 0x37233a99, 0xd867e1b5, 0x05224b0d, 0x6700c234, 0xba45688c, 0xa345d046, 0x7e007afe, 0x1c22f3c7, 0xc167597f, 0xc747336e, 0x1a0299d6, 0x782010ef, 0xa565ba57, 0xbc65029d, 0x6120a825, 0x0302211c, 0xde478ba4, 0x31035088, 0xec46fa30, 0x8e647309, 0x5321d9b1, 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842 }; // Used to fetch a naturally-aligned 32-bit word in little endian byte-order static inline uint32_t LE_LOAD32(const uint8_t *p) { return DecodeFixed32(reinterpret_cast(p)); } +// Determine if the CPU running this program can accelerate the CRC32C +// calculation. +static bool CanAccelerateCRC32C() { + // port::AcceleretedCRC32C returns zero when unable to accelerate. + static const char kTestCRCBuffer[] = "TestCRCBuffer"; + static const char kBufSize = sizeof(kTestCRCBuffer) - 1; + static const uint32_t kTestCRCValue = 0xdcbc59fa; + + return port::AcceleratedCRC32C(0, kTestCRCBuffer, kBufSize) == kTestCRCValue; +} + uint32_t Extend(uint32_t crc, const char* buf, size_t size) { + static bool accelerate = CanAccelerateCRC32C(); + if (accelerate) { + return port::AcceleratedCRC32C(crc, buf, size); + } + const uint8_t *p = reinterpret_cast(buf); const uint8_t *e = p + size; uint32_t l = crc ^ 0xffffffffu; #define STEP1 do { \ int c = (l & 0xff) ^ *p++; \ l = table0_[c] ^ (l >> 8); \ } while (0) #define STEP4 do { \ uint32_t c = l ^ LE_LOAD32(p); \ p += 4; \ l = table3_[c & 0xff] ^ \ table2_[(c >> 8) & 0xff] ^ \ table1_[(c >> 16) & 0xff] ^ \ table0_[c >> 24]; \ } while (0) // Point x at first 4-byte aligned byte in string. This might be // just past the end of the string. const uintptr_t pval = reinterpret_cast(p); const uint8_t* x = reinterpret_cast(((pval + 3) >> 2) << 2); if (x <= e) { // Process bytes until finished or p is 4-byte aligned while (p != x) { STEP1; } } // Process bytes 16 at a time while ((e-p) >= 16) { STEP4; STEP4; STEP4; STEP4; } // Process bytes 4 at a time while ((e-p) >= 4) { STEP4; } // Process the last few bytes while (p != e) { STEP1; } #undef STEP4 #undef STEP1 return l ^ 0xffffffffu; } } // namespace crc32c } // namespace leveldb diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index e0fca52f4..dd852af35 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -1,621 +1,698 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #if !defined(LEVELDB_PLATFORM_WINDOWS) #include #include #include #include #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include "leveldb/env.h" #include "leveldb/slice.h" #include "port/port.h" #include "util/logging.h" #include "util/mutexlock.h" #include "util/posix_logger.h" +#include "util/env_posix_test_helper.h" namespace leveldb { namespace { +static int open_read_only_file_limit = -1; +static int mmap_limit = -1; + static Status IOError(const std::string& context, int err_number) { return Status::IOError(context, strerror(err_number)); } +// Helper class to limit resource usage to avoid exhaustion. +// Currently used to limit read-only file descriptors and mmap file usage +// so that we do not end up running out of file descriptors, virtual memory, +// or running into kernel performance problems for very large databases. +class Limiter { + public: + // Limit maximum number of resources to |n|. + Limiter(intptr_t n) { + SetAllowed(n); + } + + // If another resource is available, acquire it and return true. + // Else return false. + bool Acquire() { + if (GetAllowed() <= 0) { + return false; + } + MutexLock l(&mu_); + intptr_t x = GetAllowed(); + if (x <= 0) { + return false; + } else { + SetAllowed(x - 1); + return true; + } + } + + // Release a resource acquired by a previous call to Acquire() that returned + // true. + void Release() { + MutexLock l(&mu_); + SetAllowed(GetAllowed() + 1); + } + + private: + port::Mutex mu_; + port::AtomicPointer allowed_; + + intptr_t GetAllowed() const { + return reinterpret_cast(allowed_.Acquire_Load()); + } + + // REQUIRES: mu_ must be held + void SetAllowed(intptr_t v) { + allowed_.Release_Store(reinterpret_cast(v)); + } + + Limiter(const Limiter&); + void operator=(const Limiter&); +}; + class PosixSequentialFile: public SequentialFile { private: std::string filename_; FILE* file_; public: PosixSequentialFile(const std::string& fname, FILE* f) : filename_(fname), file_(f) { } virtual ~PosixSequentialFile() { fclose(file_); } virtual Status Read(size_t n, Slice* result, char* scratch) { Status s; size_t r = fread_unlocked(scratch, 1, n, file_); *result = Slice(scratch, r); if (r < n) { if (feof(file_)) { // We leave status as ok if we hit the end of the file } else { // A partial read with an error: return a non-ok status s = IOError(filename_, errno); } } return s; } virtual Status Skip(uint64_t n) { if (fseek(file_, n, SEEK_CUR)) { return IOError(filename_, errno); } return Status::OK(); } }; // pread() based random-access class PosixRandomAccessFile: public RandomAccessFile { private: std::string filename_; + bool temporary_fd_; // If true, fd_ is -1 and we open on every read. int fd_; + Limiter* limiter_; public: - PosixRandomAccessFile(const std::string& fname, int fd) - : filename_(fname), fd_(fd) { } - virtual ~PosixRandomAccessFile() { close(fd_); } + PosixRandomAccessFile(const std::string& fname, int fd, Limiter* limiter) + : filename_(fname), fd_(fd), limiter_(limiter) { + temporary_fd_ = !limiter->Acquire(); + if (temporary_fd_) { + // Open file on every access. + close(fd_); + fd_ = -1; + } + } + + virtual ~PosixRandomAccessFile() { + if (!temporary_fd_) { + close(fd_); + limiter_->Release(); + } + } virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { + int fd = fd_; + if (temporary_fd_) { + fd = open(filename_.c_str(), O_RDONLY); + if (fd < 0) { + return IOError(filename_, errno); + } + } + Status s; - ssize_t r = pread(fd_, scratch, n, static_cast(offset)); + ssize_t r = pread(fd, scratch, n, static_cast(offset)); *result = Slice(scratch, (r < 0) ? 0 : r); if (r < 0) { // An error: return a non-ok status s = IOError(filename_, errno); } - return s; - } -}; - -// Helper class to limit mmap file usage so that we do not end up -// running out virtual memory or running into kernel performance -// problems for very large databases. -class MmapLimiter { - public: - // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. - MmapLimiter() { - SetAllowed(sizeof(void*) >= 8 ? 1000 : 0); - } - - // If another mmap slot is available, acquire it and return true. - // Else return false. - bool Acquire() { - if (GetAllowed() <= 0) { - return false; - } - MutexLock l(&mu_); - intptr_t x = GetAllowed(); - if (x <= 0) { - return false; - } else { - SetAllowed(x - 1); - return true; + if (temporary_fd_) { + // Close the temporary file descriptor opened earlier. + close(fd); } + return s; } - - // Release a slot acquired by a previous call to Acquire() that returned true. - void Release() { - MutexLock l(&mu_); - SetAllowed(GetAllowed() + 1); - } - - private: - port::Mutex mu_; - port::AtomicPointer allowed_; - - intptr_t GetAllowed() const { - return reinterpret_cast(allowed_.Acquire_Load()); - } - - // REQUIRES: mu_ must be held - void SetAllowed(intptr_t v) { - allowed_.Release_Store(reinterpret_cast(v)); - } - - MmapLimiter(const MmapLimiter&); - void operator=(const MmapLimiter&); }; // mmap() based random-access class PosixMmapReadableFile: public RandomAccessFile { private: std::string filename_; void* mmapped_region_; size_t length_; - MmapLimiter* limiter_; + Limiter* limiter_; public: // base[0,length-1] contains the mmapped contents of the file. PosixMmapReadableFile(const std::string& fname, void* base, size_t length, - MmapLimiter* limiter) + Limiter* limiter) : filename_(fname), mmapped_region_(base), length_(length), limiter_(limiter) { } virtual ~PosixMmapReadableFile() { munmap(mmapped_region_, length_); limiter_->Release(); } virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { Status s; if (offset + n > length_) { *result = Slice(); s = IOError(filename_, EINVAL); } else { *result = Slice(reinterpret_cast(mmapped_region_) + offset, n); } return s; } }; class PosixWritableFile : public WritableFile { private: std::string filename_; FILE* file_; public: PosixWritableFile(const std::string& fname, FILE* f) : filename_(fname), file_(f) { } ~PosixWritableFile() { if (file_ != NULL) { // Ignoring any potential errors fclose(file_); } } virtual Status Append(const Slice& data) { size_t r = fwrite_unlocked(data.data(), 1, data.size(), file_); if (r != data.size()) { return IOError(filename_, errno); } return Status::OK(); } virtual Status Close() { Status result; if (fclose(file_) != 0) { result = IOError(filename_, errno); } file_ = NULL; return result; } virtual Status Flush() { if (fflush_unlocked(file_) != 0) { return IOError(filename_, errno); } return Status::OK(); } Status SyncDirIfManifest() { const char* f = filename_.c_str(); const char* sep = strrchr(f, '/'); Slice basename; std::string dir; if (sep == NULL) { dir = "."; basename = f; } else { dir = std::string(f, sep - f); basename = sep + 1; } Status s; if (basename.starts_with("MANIFEST")) { int fd = open(dir.c_str(), O_RDONLY); if (fd < 0) { s = IOError(dir, errno); } else { - if (fsync(fd) < 0) { + if (fsync(fd) < 0 && errno != EINVAL) { s = IOError(dir, errno); } close(fd); } } return s; } virtual Status Sync() { // Ensure new files referred to by the manifest are in the filesystem. Status s = SyncDirIfManifest(); if (!s.ok()) { return s; } if (fflush_unlocked(file_) != 0 || fdatasync(fileno(file_)) != 0) { s = Status::IOError(filename_, strerror(errno)); } return s; } }; static int LockOrUnlock(int fd, bool lock) { errno = 0; struct flock f; memset(&f, 0, sizeof(f)); f.l_type = (lock ? F_WRLCK : F_UNLCK); f.l_whence = SEEK_SET; f.l_start = 0; f.l_len = 0; // Lock/unlock entire file return fcntl(fd, F_SETLK, &f); } class PosixFileLock : public FileLock { public: int fd_; std::string name_; }; // Set of locked files. We keep a separate set instead of just // relying on fcntrl(F_SETLK) since fcntl(F_SETLK) does not provide // any protection against multiple uses from the same process. class PosixLockTable { private: port::Mutex mu_; std::set locked_files_; public: bool Insert(const std::string& fname) { MutexLock l(&mu_); return locked_files_.insert(fname).second; } void Remove(const std::string& fname) { MutexLock l(&mu_); locked_files_.erase(fname); } }; class PosixEnv : public Env { public: PosixEnv(); virtual ~PosixEnv() { char msg[] = "Destroying Env::Default()\n"; fwrite(msg, 1, sizeof(msg), stderr); abort(); } virtual Status NewSequentialFile(const std::string& fname, SequentialFile** result) { FILE* f = fopen(fname.c_str(), "r"); if (f == NULL) { *result = NULL; return IOError(fname, errno); } else { *result = new PosixSequentialFile(fname, f); return Status::OK(); } } virtual Status NewRandomAccessFile(const std::string& fname, RandomAccessFile** result) { *result = NULL; Status s; int fd = open(fname.c_str(), O_RDONLY); if (fd < 0) { s = IOError(fname, errno); } else if (mmap_limit_.Acquire()) { uint64_t size; s = GetFileSize(fname, &size); if (s.ok()) { void* base = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0); if (base != MAP_FAILED) { *result = new PosixMmapReadableFile(fname, base, size, &mmap_limit_); } else { s = IOError(fname, errno); } } close(fd); if (!s.ok()) { mmap_limit_.Release(); } } else { - *result = new PosixRandomAccessFile(fname, fd); + *result = new PosixRandomAccessFile(fname, fd, &fd_limit_); } return s; } virtual Status NewWritableFile(const std::string& fname, WritableFile** result) { Status s; FILE* f = fopen(fname.c_str(), "w"); if (f == NULL) { *result = NULL; s = IOError(fname, errno); } else { *result = new PosixWritableFile(fname, f); } return s; } virtual Status NewAppendableFile(const std::string& fname, WritableFile** result) { Status s; FILE* f = fopen(fname.c_str(), "a"); if (f == NULL) { *result = NULL; s = IOError(fname, errno); } else { *result = new PosixWritableFile(fname, f); } return s; } virtual bool FileExists(const std::string& fname) { return access(fname.c_str(), F_OK) == 0; } virtual Status GetChildren(const std::string& dir, std::vector* result) { result->clear(); DIR* d = opendir(dir.c_str()); if (d == NULL) { return IOError(dir, errno); } struct dirent* entry; while ((entry = readdir(d)) != NULL) { result->push_back(entry->d_name); } closedir(d); return Status::OK(); } virtual Status DeleteFile(const std::string& fname) { Status result; if (unlink(fname.c_str()) != 0) { result = IOError(fname, errno); } return result; } virtual Status CreateDir(const std::string& name) { Status result; if (mkdir(name.c_str(), 0755) != 0) { result = IOError(name, errno); } return result; } virtual Status DeleteDir(const std::string& name) { Status result; if (rmdir(name.c_str()) != 0) { result = IOError(name, errno); } return result; } virtual Status GetFileSize(const std::string& fname, uint64_t* size) { Status s; struct stat sbuf; if (stat(fname.c_str(), &sbuf) != 0) { *size = 0; s = IOError(fname, errno); } else { *size = sbuf.st_size; } return s; } virtual Status RenameFile(const std::string& src, const std::string& target) { Status result; if (rename(src.c_str(), target.c_str()) != 0) { result = IOError(src, errno); } return result; } virtual Status LockFile(const std::string& fname, FileLock** lock) { *lock = NULL; Status result; int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644); if (fd < 0) { result = IOError(fname, errno); } else if (!locks_.Insert(fname)) { close(fd); result = Status::IOError("lock " + fname, "already held by process"); } else if (LockOrUnlock(fd, true) == -1) { result = IOError("lock " + fname, errno); close(fd); locks_.Remove(fname); } else { PosixFileLock* my_lock = new PosixFileLock; my_lock->fd_ = fd; my_lock->name_ = fname; *lock = my_lock; } return result; } virtual Status UnlockFile(FileLock* lock) { PosixFileLock* my_lock = reinterpret_cast(lock); Status result; if (LockOrUnlock(my_lock->fd_, false) == -1) { result = IOError("unlock", errno); } locks_.Remove(my_lock->name_); close(my_lock->fd_); delete my_lock; return result; } virtual void Schedule(void (*function)(void*), void* arg); virtual void StartThread(void (*function)(void* arg), void* arg); virtual Status GetTestDirectory(std::string* result) { const char* env = getenv("TEST_TMPDIR"); if (env && env[0] != '\0') { *result = env; } else { char buf[100]; snprintf(buf, sizeof(buf), "/tmp/leveldbtest-%d", int(geteuid())); *result = buf; } // Directory may already exist CreateDir(*result); return Status::OK(); } static uint64_t gettid() { pthread_t tid = pthread_self(); uint64_t thread_id = 0; memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid))); return thread_id; } virtual Status NewLogger(const std::string& fname, Logger** result) { FILE* f = fopen(fname.c_str(), "w"); if (f == NULL) { *result = NULL; return IOError(fname, errno); } else { *result = new PosixLogger(f, &PosixEnv::gettid); return Status::OK(); } } virtual uint64_t NowMicros() { struct timeval tv; gettimeofday(&tv, NULL); return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; } virtual void SleepForMicroseconds(int micros) { usleep(micros); } private: void PthreadCall(const char* label, int result) { if (result != 0) { fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); abort(); } } // BGThread() is the body of the background thread void BGThread(); static void* BGThreadWrapper(void* arg) { reinterpret_cast(arg)->BGThread(); return NULL; } pthread_mutex_t mu_; pthread_cond_t bgsignal_; pthread_t bgthread_; bool started_bgthread_; // Entry per Schedule() call struct BGItem { void* arg; void (*function)(void*); }; typedef std::deque BGQueue; BGQueue queue_; PosixLockTable locks_; - MmapLimiter mmap_limit_; + Limiter mmap_limit_; + Limiter fd_limit_; }; -PosixEnv::PosixEnv() : started_bgthread_(false) { +// Return the maximum number of concurrent mmaps. +static int MaxMmaps() { + if (mmap_limit >= 0) { + return mmap_limit; + } + // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. + mmap_limit = sizeof(void*) >= 8 ? 1000 : 0; + return mmap_limit; +} + +// Return the maximum number of read-only files to keep open. +static intptr_t MaxOpenFiles() { + if (open_read_only_file_limit >= 0) { + return open_read_only_file_limit; + } + struct rlimit rlim; + if (getrlimit(RLIMIT_NOFILE, &rlim)) { + // getrlimit failed, fallback to hard-coded default. + open_read_only_file_limit = 50; + } else if (rlim.rlim_cur == RLIM_INFINITY) { + open_read_only_file_limit = std::numeric_limits::max(); + } else { + // Allow use of 20% of available file descriptors for read-only files. + open_read_only_file_limit = rlim.rlim_cur / 5; + } + return open_read_only_file_limit; +} + +PosixEnv::PosixEnv() + : started_bgthread_(false), + mmap_limit_(MaxMmaps()), + fd_limit_(MaxOpenFiles()) { PthreadCall("mutex_init", pthread_mutex_init(&mu_, NULL)); PthreadCall("cvar_init", pthread_cond_init(&bgsignal_, NULL)); } void PosixEnv::Schedule(void (*function)(void*), void* arg) { PthreadCall("lock", pthread_mutex_lock(&mu_)); // Start background thread if necessary if (!started_bgthread_) { started_bgthread_ = true; PthreadCall( "create thread", pthread_create(&bgthread_, NULL, &PosixEnv::BGThreadWrapper, this)); } // If the queue is currently empty, the background thread may currently be // waiting. if (queue_.empty()) { PthreadCall("signal", pthread_cond_signal(&bgsignal_)); } // Add to priority queue queue_.push_back(BGItem()); queue_.back().function = function; queue_.back().arg = arg; PthreadCall("unlock", pthread_mutex_unlock(&mu_)); } void PosixEnv::BGThread() { while (true) { // Wait until there is an item that is ready to run PthreadCall("lock", pthread_mutex_lock(&mu_)); while (queue_.empty()) { PthreadCall("wait", pthread_cond_wait(&bgsignal_, &mu_)); } void (*function)(void*) = queue_.front().function; void* arg = queue_.front().arg; queue_.pop_front(); PthreadCall("unlock", pthread_mutex_unlock(&mu_)); (*function)(arg); } } namespace { struct StartThreadState { void (*user_function)(void*); void* arg; }; } static void* StartThreadWrapper(void* arg) { StartThreadState* state = reinterpret_cast(arg); state->user_function(state->arg); delete state; return NULL; } void PosixEnv::StartThread(void (*function)(void* arg), void* arg) { pthread_t t; StartThreadState* state = new StartThreadState; state->user_function = function; state->arg = arg; PthreadCall("start thread", pthread_create(&t, NULL, &StartThreadWrapper, state)); } } // namespace static pthread_once_t once = PTHREAD_ONCE_INIT; static Env* default_env; static void InitDefaultEnv() { default_env = new PosixEnv; } +void EnvPosixTestHelper::SetReadOnlyFDLimit(int limit) { + assert(default_env == NULL); + open_read_only_file_limit = limit; +} + +void EnvPosixTestHelper::SetReadOnlyMMapLimit(int limit) { + assert(default_env == NULL); + mmap_limit = limit; +} + Env* Env::Default() { pthread_once(&once, InitDefaultEnv); return default_env; } } // namespace leveldb #endif diff --git a/src/leveldb/util/env_posix_test.cc b/src/leveldb/util/env_posix_test.cc new file mode 100644 index 000000000..295f8ae44 --- /dev/null +++ b/src/leveldb/util/env_posix_test.cc @@ -0,0 +1,66 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/env.h" + +#include "port/port.h" +#include "util/testharness.h" +#include "util/env_posix_test_helper.h" + +namespace leveldb { + +static const int kDelayMicros = 100000; +static const int kReadOnlyFileLimit = 4; +static const int kMMapLimit = 4; + +class EnvPosixTest { + public: + Env* env_; + EnvPosixTest() : env_(Env::Default()) { } + + static void SetFileLimits(int read_only_file_limit, int mmap_limit) { + EnvPosixTestHelper::SetReadOnlyFDLimit(read_only_file_limit); + EnvPosixTestHelper::SetReadOnlyMMapLimit(mmap_limit); + } +}; + +TEST(EnvPosixTest, TestOpenOnRead) { + // Write some test data to a single file that will be opened |n| times. + std::string test_dir; + ASSERT_OK(env_->GetTestDirectory(&test_dir)); + std::string test_file = test_dir + "/open_on_read.txt"; + + FILE* f = fopen(test_file.c_str(), "w"); + ASSERT_TRUE(f != NULL); + const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; + fputs(kFileData, f); + fclose(f); + + // Open test file some number above the sum of the two limits to force + // open-on-read behavior of POSIX Env leveldb::RandomAccessFile. + const int kNumFiles = kReadOnlyFileLimit + kMMapLimit + 5; + leveldb::RandomAccessFile* files[kNumFiles] = {0}; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_OK(env_->NewRandomAccessFile(test_file, &files[i])); + } + char scratch; + Slice read_result; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_OK(files[i]->Read(i, 1, &read_result, &scratch)); + ASSERT_EQ(kFileData[i], read_result[0]); + } + for (int i = 0; i < kNumFiles; i++) { + delete files[i]; + } + ASSERT_OK(env_->DeleteFile(test_file)); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + // All tests currently run with the same read-only file limits. + leveldb::EnvPosixTest::SetFileLimits(leveldb::kReadOnlyFileLimit, + leveldb::kMMapLimit); + return leveldb::test::RunAllTests(); +} diff --git a/src/leveldb/util/env_posix_test_helper.h b/src/leveldb/util/env_posix_test_helper.h new file mode 100644 index 000000000..038696059 --- /dev/null +++ b/src/leveldb/util/env_posix_test_helper.h @@ -0,0 +1,28 @@ +// Copyright 2017 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ +#define STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ + +namespace leveldb { + +class EnvPosixTest; + +// A helper for the POSIX Env to facilitate testing. +class EnvPosixTestHelper { + private: + friend class EnvPosixTest; + + // Set the maximum number of read-only files that will be opened. + // Must be called before creating an Env. + static void SetReadOnlyFDLimit(int limit); + + // Set the maximum number of read-only files that will be mapped via mmap. + // Must be called before creating an Env. + static void SetReadOnlyMMapLimit(int limit); +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ diff --git a/src/leveldb/util/env_test.cc b/src/leveldb/util/env_test.cc index b72cb4438..839ae56a1 100644 --- a/src/leveldb/util/env_test.cc +++ b/src/leveldb/util/env_test.cc @@ -1,104 +1,106 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "leveldb/env.h" #include "port/port.h" #include "util/testharness.h" namespace leveldb { static const int kDelayMicros = 100000; +static const int kReadOnlyFileLimit = 4; +static const int kMMapLimit = 4; -class EnvPosixTest { +class EnvTest { private: port::Mutex mu_; std::string events_; public: Env* env_; - EnvPosixTest() : env_(Env::Default()) { } + EnvTest() : env_(Env::Default()) { } }; static void SetBool(void* ptr) { reinterpret_cast(ptr)->NoBarrier_Store(ptr); } -TEST(EnvPosixTest, RunImmediately) { +TEST(EnvTest, RunImmediately) { port::AtomicPointer called (NULL); env_->Schedule(&SetBool, &called); - Env::Default()->SleepForMicroseconds(kDelayMicros); + env_->SleepForMicroseconds(kDelayMicros); ASSERT_TRUE(called.NoBarrier_Load() != NULL); } -TEST(EnvPosixTest, RunMany) { +TEST(EnvTest, RunMany) { port::AtomicPointer last_id (NULL); struct CB { port::AtomicPointer* last_id_ptr; // Pointer to shared slot uintptr_t id; // Order# for the execution of this callback CB(port::AtomicPointer* p, int i) : last_id_ptr(p), id(i) { } static void Run(void* v) { CB* cb = reinterpret_cast(v); void* cur = cb->last_id_ptr->NoBarrier_Load(); ASSERT_EQ(cb->id-1, reinterpret_cast(cur)); cb->last_id_ptr->Release_Store(reinterpret_cast(cb->id)); } }; // Schedule in different order than start time CB cb1(&last_id, 1); CB cb2(&last_id, 2); CB cb3(&last_id, 3); CB cb4(&last_id, 4); env_->Schedule(&CB::Run, &cb1); env_->Schedule(&CB::Run, &cb2); env_->Schedule(&CB::Run, &cb3); env_->Schedule(&CB::Run, &cb4); - Env::Default()->SleepForMicroseconds(kDelayMicros); + env_->SleepForMicroseconds(kDelayMicros); void* cur = last_id.Acquire_Load(); ASSERT_EQ(4, reinterpret_cast(cur)); } struct State { port::Mutex mu; int val; int num_running; }; static void ThreadBody(void* arg) { State* s = reinterpret_cast(arg); s->mu.Lock(); s->val += 1; s->num_running -= 1; s->mu.Unlock(); } -TEST(EnvPosixTest, StartThread) { +TEST(EnvTest, StartThread) { State state; state.val = 0; state.num_running = 3; for (int i = 0; i < 3; i++) { env_->StartThread(&ThreadBody, &state); } while (true) { state.mu.Lock(); int num = state.num_running; state.mu.Unlock(); if (num == 0) { break; } - Env::Default()->SleepForMicroseconds(kDelayMicros); + env_->SleepForMicroseconds(kDelayMicros); } ASSERT_EQ(state.val, 3); } } // namespace leveldb int main(int argc, char** argv) { return leveldb::test::RunAllTests(); } diff --git a/src/leveldb/util/env_win.cc b/src/leveldb/util/env_win.cc index b074b7579..d32c4e676 100644 --- a/src/leveldb/util/env_win.cc +++ b/src/leveldb/util/env_win.cc @@ -1,896 +1,898 @@ // This file contains source that originates from: // http://code.google.com/p/leveldbwin/source/browse/trunk/win32_impl_src/env_win32.h // http://code.google.com/p/leveldbwin/source/browse/trunk/win32_impl_src/port_win32.cc -// Those files dont' have any explict license headers but the +// Those files don't have any explicit license headers but the // project (http://code.google.com/p/leveldbwin/) lists the 'New BSD License' // as the license. #if defined(LEVELDB_PLATFORM_WINDOWS) #include #include "leveldb/env.h" #include "port/port.h" #include "leveldb/slice.h" #include "util/logging.h" #include #include #include #include #include #include #include #ifdef max #undef max #endif #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif #if defined DeleteFile #undef DeleteFile #endif //Declarations namespace leveldb { namespace Win32 { #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ TypeName(const TypeName&); \ void operator=(const TypeName&) std::string GetCurrentDir(); std::wstring GetCurrentDirW(); static const std::string CurrentDir = GetCurrentDir(); static const std::wstring CurrentDirW = GetCurrentDirW(); std::string& ModifyPath(std::string& path); std::wstring& ModifyPath(std::wstring& path); std::string GetLastErrSz(); std::wstring GetLastErrSzW(); size_t GetPageSize(); typedef void (*ScheduleProc)(void*) ; struct WorkItemWrapper { WorkItemWrapper(ScheduleProc proc_,void* content_); ScheduleProc proc; void* pContent; }; DWORD WINAPI WorkItemWrapperProc(LPVOID pContent); class Win32SequentialFile : public SequentialFile { public: friend class Win32Env; virtual ~Win32SequentialFile(); virtual Status Read(size_t n, Slice* result, char* scratch); virtual Status Skip(uint64_t n); BOOL isEnable(); private: BOOL _Init(); void _CleanUp(); Win32SequentialFile(const std::string& fname); std::string _filename; ::HANDLE _hFile; DISALLOW_COPY_AND_ASSIGN(Win32SequentialFile); }; class Win32RandomAccessFile : public RandomAccessFile { public: friend class Win32Env; virtual ~Win32RandomAccessFile(); virtual Status Read(uint64_t offset, size_t n, Slice* result,char* scratch) const; BOOL isEnable(); private: BOOL _Init(LPCWSTR path); void _CleanUp(); Win32RandomAccessFile(const std::string& fname); HANDLE _hFile; const std::string _filename; DISALLOW_COPY_AND_ASSIGN(Win32RandomAccessFile); }; class Win32WritableFile : public WritableFile { public: Win32WritableFile(const std::string& fname, bool append); ~Win32WritableFile(); virtual Status Append(const Slice& data); virtual Status Close(); virtual Status Flush(); virtual Status Sync(); BOOL isEnable(); private: std::string filename_; ::HANDLE _hFile; }; class Win32FileLock : public FileLock { public: friend class Win32Env; virtual ~Win32FileLock(); BOOL isEnable(); private: BOOL _Init(LPCWSTR path); void _CleanUp(); Win32FileLock(const std::string& fname); HANDLE _hFile; std::string _filename; DISALLOW_COPY_AND_ASSIGN(Win32FileLock); }; class Win32Logger : public Logger { public: friend class Win32Env; virtual ~Win32Logger(); virtual void Logv(const char* format, va_list ap); private: explicit Win32Logger(WritableFile* pFile); WritableFile* _pFileProxy; DISALLOW_COPY_AND_ASSIGN(Win32Logger); }; class Win32Env : public Env { public: Win32Env(); virtual ~Win32Env(); virtual Status NewSequentialFile(const std::string& fname, SequentialFile** result); virtual Status NewRandomAccessFile(const std::string& fname, RandomAccessFile** result); virtual Status NewWritableFile(const std::string& fname, WritableFile** result); virtual Status NewAppendableFile(const std::string& fname, WritableFile** result); virtual bool FileExists(const std::string& fname); virtual Status GetChildren(const std::string& dir, std::vector* result); virtual Status DeleteFile(const std::string& fname); virtual Status CreateDir(const std::string& dirname); virtual Status DeleteDir(const std::string& dirname); virtual Status GetFileSize(const std::string& fname, uint64_t* file_size); virtual Status RenameFile(const std::string& src, const std::string& target); virtual Status LockFile(const std::string& fname, FileLock** lock); virtual Status UnlockFile(FileLock* lock); virtual void Schedule( void (*function)(void* arg), void* arg); virtual void StartThread(void (*function)(void* arg), void* arg); virtual Status GetTestDirectory(std::string* path); //virtual void Logv(WritableFile* log, const char* format, va_list ap); virtual Status NewLogger(const std::string& fname, Logger** result); virtual uint64_t NowMicros(); virtual void SleepForMicroseconds(int micros); }; void ToWidePath(const std::string& value, std::wstring& target) { wchar_t buffer[MAX_PATH]; MultiByteToWideChar(CP_ACP, 0, value.c_str(), -1, buffer, MAX_PATH); target = buffer; } void ToNarrowPath(const std::wstring& value, std::string& target) { char buffer[MAX_PATH]; WideCharToMultiByte(CP_ACP, 0, value.c_str(), -1, buffer, MAX_PATH, NULL, NULL); target = buffer; } std::string GetCurrentDir() { CHAR path[MAX_PATH]; ::GetModuleFileNameA(::GetModuleHandleA(NULL),path,MAX_PATH); *strrchr(path,'\\') = 0; return std::string(path); } std::wstring GetCurrentDirW() { WCHAR path[MAX_PATH]; ::GetModuleFileNameW(::GetModuleHandleW(NULL),path,MAX_PATH); *wcsrchr(path,L'\\') = 0; return std::wstring(path); } std::string& ModifyPath(std::string& path) { if(path[0] == '/' || path[0] == '\\'){ path = CurrentDir + path; } std::replace(path.begin(),path.end(),'/','\\'); return path; } std::wstring& ModifyPath(std::wstring& path) { if(path[0] == L'/' || path[0] == L'\\'){ path = CurrentDirW + path; } std::replace(path.begin(),path.end(),L'/',L'\\'); return path; } std::string GetLastErrSz() { LPWSTR lpMsgBuf; FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), 0, // Default language (LPWSTR) &lpMsgBuf, 0, NULL ); std::string Err; ToNarrowPath(lpMsgBuf, Err); LocalFree( lpMsgBuf ); return Err; } std::wstring GetLastErrSzW() { LPVOID lpMsgBuf; FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, GetLastError(), 0, // Default language (LPWSTR) &lpMsgBuf, 0, NULL ); std::wstring Err = (LPCWSTR)lpMsgBuf; LocalFree(lpMsgBuf); return Err; } WorkItemWrapper::WorkItemWrapper( ScheduleProc proc_,void* content_ ) : proc(proc_),pContent(content_) { } DWORD WINAPI WorkItemWrapperProc(LPVOID pContent) { WorkItemWrapper* item = static_cast(pContent); ScheduleProc TempProc = item->proc; void* arg = item->pContent; delete item; TempProc(arg); return 0; } size_t GetPageSize() { SYSTEM_INFO si; GetSystemInfo(&si); return std::max(si.dwPageSize,si.dwAllocationGranularity); } const size_t g_PageSize = GetPageSize(); Win32SequentialFile::Win32SequentialFile( const std::string& fname ) : _filename(fname),_hFile(NULL) { _Init(); } Win32SequentialFile::~Win32SequentialFile() { _CleanUp(); } Status Win32SequentialFile::Read( size_t n, Slice* result, char* scratch ) { Status sRet; DWORD hasRead = 0; if(_hFile && ReadFile(_hFile,scratch,n,&hasRead,NULL) ){ *result = Slice(scratch,hasRead); } else { sRet = Status::IOError(_filename, Win32::GetLastErrSz() ); } return sRet; } Status Win32SequentialFile::Skip( uint64_t n ) { Status sRet; LARGE_INTEGER Move,NowPointer; Move.QuadPart = n; if(!SetFilePointerEx(_hFile,Move,&NowPointer,FILE_CURRENT)){ sRet = Status::IOError(_filename,Win32::GetLastErrSz()); } return sRet; } BOOL Win32SequentialFile::isEnable() { return _hFile ? TRUE : FALSE; } BOOL Win32SequentialFile::_Init() { std::wstring path; ToWidePath(_filename, path); _hFile = CreateFileW(path.c_str(), GENERIC_READ, - FILE_SHARE_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, NULL); + if (_hFile == INVALID_HANDLE_VALUE) + _hFile = NULL; return _hFile ? TRUE : FALSE; } void Win32SequentialFile::_CleanUp() { if(_hFile){ CloseHandle(_hFile); _hFile = NULL; } } Win32RandomAccessFile::Win32RandomAccessFile( const std::string& fname ) : _filename(fname),_hFile(NULL) { std::wstring path; ToWidePath(fname, path); _Init( path.c_str() ); } Win32RandomAccessFile::~Win32RandomAccessFile() { _CleanUp(); } Status Win32RandomAccessFile::Read(uint64_t offset,size_t n,Slice* result,char* scratch) const { Status sRet; OVERLAPPED ol = {0}; ZeroMemory(&ol,sizeof(ol)); ol.Offset = (DWORD)offset; ol.OffsetHigh = (DWORD)(offset >> 32); DWORD hasRead = 0; if(!ReadFile(_hFile,scratch,n,&hasRead,&ol)) sRet = Status::IOError(_filename,Win32::GetLastErrSz()); else *result = Slice(scratch,hasRead); return sRet; } BOOL Win32RandomAccessFile::_Init( LPCWSTR path ) { BOOL bRet = FALSE; if(!_hFile) - _hFile = ::CreateFileW(path,GENERIC_READ,FILE_SHARE_READ,NULL,OPEN_EXISTING, + _hFile = ::CreateFileW(path,GENERIC_READ,FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS,NULL); if(!_hFile || _hFile == INVALID_HANDLE_VALUE ) _hFile = NULL; else bRet = TRUE; return bRet; } BOOL Win32RandomAccessFile::isEnable() { return _hFile ? TRUE : FALSE; } void Win32RandomAccessFile::_CleanUp() { if(_hFile){ ::CloseHandle(_hFile); _hFile = NULL; } } Win32WritableFile::Win32WritableFile(const std::string& fname, bool append) : filename_(fname) { std::wstring path; ToWidePath(fname, path); // NewAppendableFile: append to an existing file, or create a new one // if none exists - this is OPEN_ALWAYS behavior, with // FILE_APPEND_DATA to avoid having to manually position the file // pointer at the end of the file. // NewWritableFile: create a new file, delete if it exists - this is // CREATE_ALWAYS behavior. This file is used for writing only so // use GENERIC_WRITE. _hFile = CreateFileW(path.c_str(), append ? FILE_APPEND_DATA : GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_DELETE|FILE_SHARE_WRITE, NULL, append ? OPEN_ALWAYS : CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); // CreateFileW returns INVALID_HANDLE_VALUE in case of error, always check isEnable() before use } Win32WritableFile::~Win32WritableFile() { if (_hFile != INVALID_HANDLE_VALUE) Close(); } Status Win32WritableFile::Append(const Slice& data) { DWORD r = 0; if (!WriteFile(_hFile, data.data(), data.size(), &r, NULL) || r != data.size()) { return Status::IOError("Win32WritableFile.Append::WriteFile: "+filename_, Win32::GetLastErrSz()); } return Status::OK(); } Status Win32WritableFile::Close() { if (!CloseHandle(_hFile)) { return Status::IOError("Win32WritableFile.Close::CloseHandle: "+filename_, Win32::GetLastErrSz()); } _hFile = INVALID_HANDLE_VALUE; return Status::OK(); } Status Win32WritableFile::Flush() { // Nothing to do here, there are no application-side buffers return Status::OK(); } Status Win32WritableFile::Sync() { if (!FlushFileBuffers(_hFile)) { return Status::IOError("Win32WritableFile.Sync::FlushFileBuffers "+filename_, Win32::GetLastErrSz()); } return Status::OK(); } BOOL Win32WritableFile::isEnable() { return _hFile != INVALID_HANDLE_VALUE; } Win32FileLock::Win32FileLock( const std::string& fname ) : _hFile(NULL),_filename(fname) { std::wstring path; ToWidePath(fname, path); _Init(path.c_str()); } Win32FileLock::~Win32FileLock() { _CleanUp(); } BOOL Win32FileLock::_Init( LPCWSTR path ) { BOOL bRet = FALSE; if(!_hFile) _hFile = ::CreateFileW(path,0,0,NULL,CREATE_ALWAYS,FILE_ATTRIBUTE_NORMAL,NULL); if(!_hFile || _hFile == INVALID_HANDLE_VALUE ){ _hFile = NULL; } else bRet = TRUE; return bRet; } void Win32FileLock::_CleanUp() { ::CloseHandle(_hFile); _hFile = NULL; } BOOL Win32FileLock::isEnable() { return _hFile ? TRUE : FALSE; } Win32Logger::Win32Logger(WritableFile* pFile) : _pFileProxy(pFile) { assert(_pFileProxy); } Win32Logger::~Win32Logger() { if(_pFileProxy) delete _pFileProxy; } void Win32Logger::Logv( const char* format, va_list ap ) { uint64_t thread_id = ::GetCurrentThreadId(); // We try twice: the first time with a fixed-size stack allocated buffer, // and the second time with a much larger dynamically allocated buffer. char buffer[500]; for (int iter = 0; iter < 2; iter++) { char* base; int bufsize; if (iter == 0) { bufsize = sizeof(buffer); base = buffer; } else { bufsize = 30000; base = new char[bufsize]; } char* p = base; char* limit = base + bufsize; SYSTEMTIME st; GetLocalTime(&st); p += snprintf(p, limit - p, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", int(st.wYear), int(st.wMonth), int(st.wDay), int(st.wHour), int(st.wMinute), int(st.wMinute), int(st.wMilliseconds), static_cast(thread_id)); // Print the message if (p < limit) { va_list backup_ap; va_copy(backup_ap, ap); p += vsnprintf(p, limit - p, format, backup_ap); va_end(backup_ap); } // Truncate to available space if necessary if (p >= limit) { if (iter == 0) { continue; // Try again with larger buffer } else { p = limit - 1; } } // Add newline if necessary if (p == base || p[-1] != '\n') { *p++ = '\n'; } assert(p <= limit); DWORD hasWritten = 0; if(_pFileProxy){ _pFileProxy->Append(Slice(base, p - base)); _pFileProxy->Flush(); } if (base != buffer) { delete[] base; } break; } } bool Win32Env::FileExists(const std::string& fname) { std::string path = fname; std::wstring wpath; ToWidePath(ModifyPath(path), wpath); return ::PathFileExistsW(wpath.c_str()) ? true : false; } Status Win32Env::GetChildren(const std::string& dir, std::vector* result) { Status sRet; ::WIN32_FIND_DATAW wfd; std::string path = dir; ModifyPath(path); path += "\\*.*"; std::wstring wpath; ToWidePath(path, wpath); ::HANDLE hFind = ::FindFirstFileW(wpath.c_str() ,&wfd); if(hFind && hFind != INVALID_HANDLE_VALUE){ BOOL hasNext = TRUE; std::string child; while(hasNext){ ToNarrowPath(wfd.cFileName, child); if(child != ".." && child != ".") { result->push_back(child); } hasNext = ::FindNextFileW(hFind,&wfd); } ::FindClose(hFind); } else sRet = Status::IOError(dir,"Could not get children."); return sRet; } void Win32Env::SleepForMicroseconds( int micros ) { ::Sleep((micros + 999) /1000); } Status Win32Env::DeleteFile( const std::string& fname ) { Status sRet; std::string path = fname; std::wstring wpath; ToWidePath(ModifyPath(path), wpath); if(!::DeleteFileW(wpath.c_str())) { sRet = Status::IOError(path, "Could not delete file."); } return sRet; } Status Win32Env::GetFileSize( const std::string& fname, uint64_t* file_size ) { Status sRet; std::string path = fname; std::wstring wpath; ToWidePath(ModifyPath(path), wpath); HANDLE file = ::CreateFileW(wpath.c_str(), - GENERIC_READ,FILE_SHARE_READ,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL); + GENERIC_READ,FILE_SHARE_READ|FILE_SHARE_WRITE,NULL,OPEN_EXISTING,FILE_ATTRIBUTE_NORMAL,NULL); LARGE_INTEGER li; if(::GetFileSizeEx(file,&li)){ *file_size = (uint64_t)li.QuadPart; }else sRet = Status::IOError(path,"Could not get the file size."); CloseHandle(file); return sRet; } Status Win32Env::RenameFile( const std::string& src, const std::string& target ) { Status sRet; std::string src_path = src; std::wstring wsrc_path; ToWidePath(ModifyPath(src_path), wsrc_path); std::string target_path = target; std::wstring wtarget_path; ToWidePath(ModifyPath(target_path), wtarget_path); if(!MoveFileW(wsrc_path.c_str(), wtarget_path.c_str() ) ){ DWORD err = GetLastError(); if(err == 0x000000b7){ if(!::DeleteFileW(wtarget_path.c_str() ) ) sRet = Status::IOError(src, "Could not rename file."); else if(!::MoveFileW(wsrc_path.c_str(), wtarget_path.c_str() ) ) sRet = Status::IOError(src, "Could not rename file."); } } return sRet; } Status Win32Env::LockFile( const std::string& fname, FileLock** lock ) { Status sRet; std::string path = fname; ModifyPath(path); Win32FileLock* _lock = new Win32FileLock(path); if(!_lock->isEnable()){ delete _lock; *lock = NULL; sRet = Status::IOError(path, "Could not lock file."); } else *lock = _lock; return sRet; } Status Win32Env::UnlockFile( FileLock* lock ) { Status sRet; delete lock; return sRet; } void Win32Env::Schedule( void (*function)(void* arg), void* arg ) { QueueUserWorkItem(Win32::WorkItemWrapperProc, new Win32::WorkItemWrapper(function,arg), WT_EXECUTEDEFAULT); } void Win32Env::StartThread( void (*function)(void* arg), void* arg ) { ::_beginthread(function,0,arg); } Status Win32Env::GetTestDirectory( std::string* path ) { Status sRet; WCHAR TempPath[MAX_PATH]; ::GetTempPathW(MAX_PATH,TempPath); ToNarrowPath(TempPath, *path); path->append("leveldb\\test\\"); ModifyPath(*path); return sRet; } uint64_t Win32Env::NowMicros() { #ifndef USE_VISTA_API #define GetTickCount64 GetTickCount #endif return (uint64_t)(GetTickCount64()*1000); } static Status CreateDirInner( const std::string& dirname ) { Status sRet; DWORD attr = ::GetFileAttributes(dirname.c_str()); if (attr == INVALID_FILE_ATTRIBUTES) { // doesn't exist: std::size_t slash = dirname.find_last_of("\\"); if (slash != std::string::npos){ sRet = CreateDirInner(dirname.substr(0, slash)); if (!sRet.ok()) return sRet; } BOOL result = ::CreateDirectory(dirname.c_str(), NULL); if (result == FALSE) { sRet = Status::IOError(dirname, "Could not create directory."); return sRet; } } return sRet; } Status Win32Env::CreateDir( const std::string& dirname ) { std::string path = dirname; if(path[path.length() - 1] != '\\'){ path += '\\'; } ModifyPath(path); return CreateDirInner(path); } Status Win32Env::DeleteDir( const std::string& dirname ) { Status sRet; std::wstring path; ToWidePath(dirname, path); ModifyPath(path); if(!::RemoveDirectoryW( path.c_str() ) ){ sRet = Status::IOError(dirname, "Could not delete directory."); } return sRet; } Status Win32Env::NewSequentialFile( const std::string& fname, SequentialFile** result ) { Status sRet; std::string path = fname; ModifyPath(path); Win32SequentialFile* pFile = new Win32SequentialFile(path); if(pFile->isEnable()){ *result = pFile; }else { delete pFile; sRet = Status::IOError(path, Win32::GetLastErrSz()); } return sRet; } Status Win32Env::NewRandomAccessFile( const std::string& fname, RandomAccessFile** result ) { Status sRet; std::string path = fname; Win32RandomAccessFile* pFile = new Win32RandomAccessFile(ModifyPath(path)); if(!pFile->isEnable()){ delete pFile; *result = NULL; sRet = Status::IOError(path, Win32::GetLastErrSz()); }else *result = pFile; return sRet; } Status Win32Env::NewLogger( const std::string& fname, Logger** result ) { Status sRet; std::string path = fname; // Logs are opened with write semantics, not with append semantics // (see PosixEnv::NewLogger) Win32WritableFile* pMapFile = new Win32WritableFile(ModifyPath(path), false); if(!pMapFile->isEnable()){ delete pMapFile; *result = NULL; sRet = Status::IOError(path,"could not create a logger."); }else *result = new Win32Logger(pMapFile); return sRet; } Status Win32Env::NewWritableFile( const std::string& fname, WritableFile** result ) { Status sRet; std::string path = fname; Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path), false); if(!pFile->isEnable()){ *result = NULL; sRet = Status::IOError(fname,Win32::GetLastErrSz()); }else *result = pFile; return sRet; } Status Win32Env::NewAppendableFile( const std::string& fname, WritableFile** result ) { Status sRet; std::string path = fname; Win32WritableFile* pFile = new Win32WritableFile(ModifyPath(path), true); if(!pFile->isEnable()){ *result = NULL; sRet = Status::IOError(fname,Win32::GetLastErrSz()); }else *result = pFile; return sRet; } Win32Env::Win32Env() { } Win32Env::~Win32Env() { } } // Win32 namespace static port::OnceType once = LEVELDB_ONCE_INIT; static Env* default_env; static void InitDefaultEnv() { default_env = new Win32::Win32Env(); } Env* Env::Default() { port::InitOnce(&once, InitDefaultEnv); return default_env; } } // namespace leveldb #endif // defined(LEVELDB_PLATFORM_WINDOWS) diff --git a/src/leveldb/util/options.cc b/src/leveldb/util/options.cc index 8b618fb1a..b5e622761 100644 --- a/src/leveldb/util/options.cc +++ b/src/leveldb/util/options.cc @@ -1,29 +1,30 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "leveldb/options.h" #include "leveldb/comparator.h" #include "leveldb/env.h" namespace leveldb { Options::Options() : comparator(BytewiseComparator()), create_if_missing(false), error_if_exists(false), paranoid_checks(false), env(Env::Default()), info_log(NULL), write_buffer_size(4<<20), max_open_files(1000), block_cache(NULL), block_size(4096), block_restart_interval(16), + max_file_size(2<<20), compression(kSnappyCompression), reuse_logs(false), filter_policy(NULL) { } } // namespace leveldb