diff --git a/.gitignore b/.gitignore index 1804805..4f10b23 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Fri Dec 2 2013 @@ -89,8 +89,7 @@ stamp-h1 # automakes # Makefile.in -*/Makefile.in - +Makefile # # archive files and backups @@ -98,6 +97,7 @@ Makefile.in *.tgz *.tar.gz *.rpm +*.deb *.swp # @@ -121,16 +121,25 @@ test-driver/* # # Other built objects # -*.pc -Makefile -*/Makefile - -# -# project custom -# RELEASE_VERSION -lib/k2hashversion.cc +COMMIT_HASH +*.pc +*.spec *.k2h +buildutils/control +buildutils/copyright +rpmbuild +rpmbuild/* +debian +debian/* +debian_build +debian_build/* +lib/k2hashversion.cc +tests/results.xml +tests/test-suite.log +tests/test.sh.log +tests/test.sh.trs +tests/test_linetool_noverinfo.log tests/k2hinittest tests/k2hmemtest tests/k2hrwtest diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2f1d237..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "fullock"] - path = fullock - url = https://github.com/yahoojapan/fullock.git diff --git a/.travis.yml b/.travis.yml index c616091..9281e55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ git: before_install: - sudo apt-get update -qq - sudo apt-get install -qq git autoconf autotools-dev gcc g++ make gdb dh-make fakeroot dpkg-dev devscripts libtool pkg-config - - git submodule update --init --recursive + - git clone https://github.com/yahoojapan/fullock.git - cd fullock - ./autogen.sh - ./configure --prefix=/usr diff --git a/AUTHORS b/AUTHORS index a2b7b02..2bf1028 100644 --- a/AUTHORS +++ b/AUTHORS @@ -10,3 +10,6 @@ Wrote k2htouch, k2himport and k2hedit. Porting to multiple languages for k2hash. +4. Hirotaka Wakabayashi + +Technical engineer of k2hash product. diff --git a/ChangeLog b/ChangeLog index 242096d..8ae87b5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,35 @@ +k2hash (1.0.61) trusty; urgency=low + + * avoid static object initialization order problem(SIOF) + + -- Takeshi Nakatani Fri, 29 Jun 2018 14:54:05 +0900 + +k2hash (1.0.60) trusty; urgency=low + + * Fixed build options for pthread + + -- Takeshi Nakatani Tue, 26 Jun 2018 11:32:23 +0900 + +k2hash (1.0.59) trusty; urgency=low + + * Fixed a bug about lead loop + + -- Takeshi Nakatani Thu, 21 Jun 2018 18:27:00 +0900 + +k2hash (1.0.58) trusty; urgency=low + + * Cleanup credit comments in sources + * Supported multi crypt libraries(NSS/gcrypt/nettle) + * Changed build logic to new for rpm/debian packaging + * Fixed build error with openssl 1.1.0 + + -- Takeshi Nakatani Wed, 16 May 2018 10:13:04 +0900 + k2hash (1.0.57) trusty; urgency=low * Fixed type qualifiers ignored on function return type --- Takeshi Nakatani Wed, 17 May 2017 08:19:59 +0900 + -- Takeshi Nakatani Wed, 17 May 2017 08:19:59 +0900 k2hash (1.0.56) trusty; urgency=low diff --git a/Makefile.am b/Makefile.am index 4ce7f78..c3e7bb3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -11,16 +11,16 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Fri Dec 2 2013 # REVISION: # -SUBDIRS=lib docs tests +SUBDIRS=lib docs tests buildutils -EXTRA_DIST=RELEASE_VERSION make_valiables.sh make_release_version_file.sh make_rev.sh +EXTRA_DIST=RELEASE_VERSION # # VIM modelines diff --git a/README b/README index 9dd1b41..33b9b2f 100644 --- a/README +++ b/README @@ -1,7 +1,7 @@ k2hash ------ -K2HASH - NoSQL Key Value Store(KVS) library by Yahoo! JAPAN +K2HASH - NoSQL Key Value Store(KVS) library ### Overview K2HASH is NoSQL(key value store) library and tools for this library by @@ -23,6 +23,7 @@ Yahoo! JAPAN. K2HASH is following specifications. - https://github.com/yahoojapan/k2hash/wiki ### License -This software is released under the MIT License, see the LICENSE file. +This software is released under the MIT License, see the license file. + +Copyright(C) 2013 Yahoo Japan Corporation. -Copyright 2013 Yahoo! JAPAN corporation. diff --git a/README.md b/README.md index 32e4067..d088c26 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ k2hash ------ [![Build Status](https://travis-ci.org/yahoojapan/k2hash.svg?branch=master)](https://travis-ci.org/yahoojapan/k2hash) -K2HASH - NoSQL Key Value Store(KVS) library by Yahoo! JAPAN +K2HASH - NoSQL Key Value Store(KVS) library ### Overview K2HASH is NoSQL(key value store) library and tools for this library by diff --git a/autogen.sh b/autogen.sh index 6a4ddfc..47264cf 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,24 +1,30 @@ #!/bin/sh # -# Automatically generate configuration - autogen.sh +# K2HASH # -# Copyright 2016 Yahoo! JAPAN corporation. +# Utility tools for building configure/packages by AntPickax # -# Templates for customizing screwdriver CPP and autotools. -# This template files are provided by yjcore team. +# Copyright 2018 Yahoo Japan Corporation. # -# This script file is for initializing autoconf/autotools -# files. And make version number information file from -# git tag(release), this file is used by other build tools. +# AntPickax provides utility tools for supporting autotools +# builds. # +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani -# CREATE: Thu, Jun 2 2016 +# CREATE: Fri, Apr 13 2018 # REVISION: # + # # Usage: autogen.sh [-noupdate_version_file] [-no_aclocal_force] [-no_check_ver_diff] # @@ -46,7 +52,7 @@ while [ $# -ne 0 ]; do echo "Usage: ${AUTOGEN_NAME} [-noupdate_version_file] [-no_aclocal_force] [-no_check_ver_diff]" exit 1 else - echo "ERROR: Unkown option $1" + echo "ERROR: Unknown option $1" echo "Usage: ${AUTOGEN_NAME} [-noupdate_version_file] [-no_aclocal_force] [-no_check_ver_diff]" exit 1 fi @@ -57,7 +63,8 @@ done # update RELEASE_VERSION file # if [ "X${NOUPDATE}" = "Xno" ]; then - ${SRCTOP}/make_release_version_file.sh ${PARAMETERS} + echo "--- run make_release_version_file.sh" + ${SRCTOP}/buildutils/make_release_version_file.sh ${PARAMETERS} if [ $? -ne 0 ]; then echo "ERROR: update RELEASE_VERSION file" exit 1 diff --git a/buildutils/Makefile.am b/buildutils/Makefile.am new file mode 100644 index 0000000..a97b9d4 --- /dev/null +++ b/buildutils/Makefile.am @@ -0,0 +1,38 @@ +# +# K2HASH +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +EXTRA_DIST =make_variables.sh \ + make_release_version_file.sh \ + make_commit_hash.sh \ + make_commit_hash_source.sh \ + make_rpm_changelog.sh \ + make_description.sh \ + debian_build.sh + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/buildutils/control.in b/buildutils/control.in new file mode 100644 index 0000000..6dc5421 --- /dev/null +++ b/buildutils/control.in @@ -0,0 +1,24 @@ +Source: @PACKAGE_NAME@ +Section: database +Priority: extra +Maintainer: @DEV_NAME@ <@DEV_EMAIL@> +Build-Depends: debhelper (>= 9), autotools-dev, libfullock-dev (>= 1.0.28) +Standards-Version: 3.9.5 +Homepage: https://@GIT_DOMAIN@/@GIT_ORG@/@GIT_REPO@ +Vcs-Git: git://@GIT_DOMAIN@/@GIT_ORG@/@GIT_REPO@.git +Vcs-Browser: https://@GIT_DOMAIN@/@GIT_ORG@/@GIT_REPO@ + +Package: @PACKAGE_NAME@-dev +Section: devel +Architecture: amd64 +Depends: @PACKAGE_NAME@ (= ${binary:Version}), libfullock-dev (>= 1.0.28) +Description: @SHORTDESC@ (development) + Development package for building with @PACKAGE_NAME@ shared library. + This package has header files and symbols for it. + +Package: @PACKAGE_NAME@ +Section: database +Architecture: amd64 +Depends: ${shlibs:Depends}, ${misc:Depends}, libfullock (>= 1.0.28) +Description: @SHORTDESC@ +@DEBLONGDESC@ diff --git a/buildutils/copyright.in b/buildutils/copyright.in new file mode 100644 index 0000000..61cdf1e --- /dev/null +++ b/buildutils/copyright.in @@ -0,0 +1,8 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: @PACKAGE_NAME@ +Source: https://@GIT_DOMAIN@/@GIT_ORG@/@GIT_REPO@ + +Files: * +Copyright: 2013 Yahoo Japan Corporation +License: @PKGLICENSE@ +@DEBCOPYING@ diff --git a/buildutils/debian_build.sh b/buildutils/debian_build.sh new file mode 100755 index 0000000..6054099 --- /dev/null +++ b/buildutils/debian_build.sh @@ -0,0 +1,356 @@ +#!/bin/sh +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +# +# Autobuid for debian package +# +func_usage() +{ + echo "" + echo "Usage: $1 [-buildnum ] [-nodebuild] [-rootdir] [-product ] [-y] [additional debuild options]" + echo " -buildnum specify build number for packaging(default 1)" + echo " -nodebuild stops before do debuild command." + echo " -rootdir layout \"debian\" directory for packaging under source top directory" + echo " -product specify product name(use PACKAGE_NAME in Makefile s default)" + echo " -y runs no interacitive mode." + echo " additional debuild options this script run debuild with \"-uc -us\", can specify additional options." + echo " -h print help" + echo "" +} +PRGNAME=`basename $0` +MYSCRIPTDIR=`dirname $0` +MYSCRIPTDIR=`cd ${MYSCRIPTDIR}; pwd` +SRCTOP=`cd ${MYSCRIPTDIR}/..; pwd` +BUILDDEBDIR=${SRCTOP}/debian_build + +# +# Check options +# +IS_DEBUILD=1 +IS_INTERACTIVE=1 +IS_ROOTDIR=0 +DH_MAKE_AUTORUN_OPTION="" +BUILD_NUMBER=1 +DEBUILD_OPT="" +while [ $# -ne 0 ]; do + if [ "X$1" = "X" ]; then + break + + elif [ "X$1" = "X-h" -o "X$1" = "X-help" ]; then + func_usage $PRGNAME + exit 0 + + elif [ "X$1" = "X-buildnum" ]; then + shift + if [ $# -eq 0 ]; then + echo "ERROR: -buildnum option needs parameter." 1>&2 + exit 1 + fi + BUILD_NUMBER=$1 + + elif [ "X$1" = "X-nodebuild" ]; then + IS_DEBUILD=0 + BUILD_NUMBER= + + elif [ "X$1" = "X-rootdir" ]; then + IS_ROOTDIR=1 + + elif [ "X$1" = "X-product" ]; then + shift + if [ $# -eq 0 ]; then + echo "ERROR: -product option needs parameter." 1>&2 + exit 1 + fi + PACKAGE_NAME=$1 + + elif [ "X$1" = "X-y" ]; then + IS_INTERACTIVE=0 + DH_MAKE_AUTORUN_OPTION="-y" + + else + if [ "X${DEBUILD_OPT}" != "X" ]; then + DEBUILD_OPT="${DEBUILD_OPT} $1" + else + DEBUILD_OPT="$1" + fi + fi + shift +done + +# +# Package name +# +if [ "X${PACKAGE_NAME}" = "X" ]; then + PACKAGE_NAME=`grep PACKAGE_NAME ${SRCTOP}/Makefile 2>/dev/null | awk '{print $3}' 2>/dev/null` + if [ "X${PACKAGE_NAME}" = "X" ]; then + echo "ERROR: no product name" 1>&2 + exit 1 + fi +fi +PACKAGE_DEV_NAME="${PACKAGE_NAME}-dev" +LIB_BASENAME="lib${PACKAGE_NAME}" + +# +# Welcome message and confirming for interactive mode +# +if [ ${IS_INTERACTIVE} -eq 1 ]; then + echo "---------------------------------------------------------------" + echo " Do you change these file and commit to github?" + echo " - ChangeLog modify / add changes like dch tool format" + echo " - Git TAG stamp git tag for release" + echo "---------------------------------------------------------------" + while true; do + echo -n "Confirm: [y/n] " + read CONFIRM + + if [ "X${CONFIRM}" = "XY" -o "X${CONFIRM}" = "Xy" ]; then + break; + elif [ "X${CONFIRM}" = "XN" -o "X${CONFIRM}" = "Xn" ]; then + echo "Bye..." + exit 1 + fi + done + echo "" +fi + +# +# Make dist package by make dist +# +${SRCTOP}/autogen.sh || exit 1 +${SRCTOP}/configure || exit 1 +PACKAGE_VERSION=`${MYSCRIPTDIR}/make_variables.sh -pkg_version` +PACKAGE_MAJOR_VER=`${MYSCRIPTDIR}/make_variables.sh -major_number` + +echo "===== make dist: start ==============================" +make dist || exit 1 +echo "===== make dist: end ==============================" + +# +# Create debian package directory and change current +# +echo "===== prepare working directory: start =============" + +if [ -f ${BUILDDEBDIR} ]; then + echo "ERROR: debian file exists, could not make debian directory." 1>&2 + exit 1 +fi +if [ -d ${BUILDDEBDIR} ]; then + echo "WANING: debian directory exists, remove and remake it." 1>&2 + rm -rf ${BUILDDEBDIR} || exit 1 +fi +mkdir ${BUILDDEBDIR} || exit 1 +cd ${BUILDDEBDIR} || exit 1 + +# +# copy dist package and expand source files +# +cp ${SRCTOP}/${PACKAGE_NAME}-${PACKAGE_VERSION}.tar.gz . || exit 1 +tar xvfz ${PACKAGE_NAME}-${PACKAGE_VERSION}.tar.gz || exit 1 + +# +# change current directory +# +EXPANDDIR=${BUILDDEBDIR}/${PACKAGE_NAME}-${PACKAGE_VERSION} +cd ${EXPANDDIR} || exit 1 + +# +# initialize debian directory +# +dh_make -f ${BUILDDEBDIR}/${PACKAGE_NAME}-${PACKAGE_VERSION}.tar.gz --createorig -m ${DH_MAKE_AUTORUN_OPTION} || exit 1 + +# +# remove unnecessary template files +# +rm -rf ${EXPANDDIR}/debian/*.ex ${EXPANDDIR}/debian/*.EX ${EXPANDDIR}/debian/${PACKAGE_NAME}-doc.* ${EXPANDDIR}/debian/README.* ${EXPANDDIR}/debian/docs + +# +# adding some lines into rules file +# +echo "" >> ${EXPANDDIR}/debian/rules || exit 1 +echo "# for ${PACKAGE_NAME} and ${PACKAGE_DEV_NAME} packages" >> ${EXPANDDIR}/debian/rules || exit 1 +echo "override_dh_auto_install:" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " dh_auto_install --destdir=debian/${PACKAGE_NAME}" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " dh_auto_install --destdir=debian/${PACKAGE_DEV_NAME}" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " if [ -d debian/${PACKAGE_NAME}/usr/include ]; then rm -rf debian/${PACKAGE_NAME}/usr/include; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " if [ -d debian/${PACKAGE_NAME}/usr/lib/x86_64-linux-gnu/pkgconfig ]; then rm -rf debian/${PACKAGE_NAME}/usr/lib/x86_64-linux-gnu/pkgconfig; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " if [ -d debian/${PACKAGE_DEV_NAME}/usr/bin ]; then rm -rf debian/${PACKAGE_DEV_NAME}/usr/bin; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " if [ -d debian/${PACKAGE_DEV_NAME}/usr/share/man ]; then rm -rf debian/${PACKAGE_DEV_NAME}/usr/share/man; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " ls debian/${PACKAGE_NAME}/usr/lib/x86_64-linux-gnu/*.a ]; >/dev/null 2>&1; if [ $? -eq 0 ]; then rm -rf debian/${PACKAGE_NAME}/usr/lib/x86_64-linux-gnu/*.a; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " ls debian/${PACKAGE_NAME}/usr/lib/x86_64-linux-gnu/*.la ]; >/dev/null 2>&1; if [ $? -eq 0 ]; then rm -rf debian/${PACKAGE_NAME}/usr/lib/x86_64-linux-gnu/*.la; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " ls debian/${PACKAGE_DEV_NAME}/usr/lib/x86_64-linux-gnu/*.a >/dev/null 2>&1; if [ $? -eq 0 ]; then rm -rf debian/${PACKAGE_DEV_NAME}/usr/lib/x86_64-linux-gnu/*.a; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " ls debian/${PACKAGE_DEV_NAME}/usr/lib/x86_64-linux-gnu/*.la >/dev/null 2>&1; if [ $? -eq 0 ]; then rm -rf debian/${PACKAGE_DEV_NAME}/usr/lib/x86_64-linux-gnu/*.la; fi" >> ${EXPANDDIR}/debian/rules || exit 1 +echo " ls debian/${PACKAGE_DEV_NAME}/usr/lib/x86_64-linux-gnu/*.so* >/dev/null 2>&1; if [ $? -eq 0 ]; then rm -rf debian/${PACKAGE_DEV_NAME}/usr/lib/x86_64-linux-gnu/*.so*; fi" >> ${EXPANDDIR}/debian/rules || exit 1 + +# +# create links file for library +# +FOUND_LIB_LINES=`find ./ -name Makefile.am -exec grep ${LIB_BASENAME} {} \; 2>/dev/null` +if [ "X${FOUND_LIB_LINES}" != "X" ]; then + LIBRARY_LIBTOOL_VERSION=`${MYSCRIPTDIR}/make_variables.sh -lib_version_for_link 2>/dev/null` || exit 1 + echo "usr/lib/x86_64-linux-gnu/${LIB_BASENAME}.so.${LIBRARY_LIBTOOL_VERSION} usr/lib/x86_64-linux-gnu/${LIB_BASENAME}.so" > ${EXPANDDIR}/debian/${PACKAGE_NAME}.links || exit 1 + echo "usr/lib/x86_64-linux-gnu/${LIB_BASENAME}.so.${LIBRARY_LIBTOOL_VERSION} usr/lib/x86_64-linux-gnu/${LIB_BASENAME}.so.${PACKAGE_MAJOR_VER}" >> ${EXPANDDIR}/debian/${PACKAGE_NAME}.links || exit 1 +fi + +# +# copy copyright +# +cp ${MYSCRIPTDIR}/copyright ${EXPANDDIR}/debian/copyright || exit 1 + +# +# copy control file +# +cp ${MYSCRIPTDIR}/control ${EXPANDDIR}/debian/control || exit 1 + +# +# copy changelog with converting build number +# +CHLOG_ORG_MENT=`cat ChangeLog | grep "^ --" | head -1` +CHLOG_NEW_MENT=`cat ${EXPANDDIR}/debian/changelog | grep "^ --" | head -1` +if [ "X${BUILD_NUMBER}" = "X" ]; then + cat ChangeLog | sed "s/${CHLOG_ORG_MENT}/${CHLOG_NEW_MENT}/g" > ${EXPANDDIR}/debian/changelog || exit 1 +else + cat ChangeLog | sed "s/${PACKAGE_VERSION}/${PACKAGE_VERSION}-${BUILD_NUMBER}/g" | sed "s/${CHLOG_ORG_MENT}/${CHLOG_NEW_MENT}/g" > ${EXPANDDIR}/debian/changelog || exit 1 +fi + +# +# preinst/postinst/prerm/postrm +# +if [ -f ${MYSCRIPTDIR}/${PACKAGE_NAME}.preinst ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_NAME}.preinst ${EXPANDDIR}/debian/${PACKAGE_NAME}.preinst || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_NAME}.preinst || exit 1 +fi +if [ -f ${MYSCRIPTDIR}/${PACKAGE_NAME}.postinst ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_NAME}.postinst ${EXPANDDIR}/debian/${PACKAGE_NAME}.postinst || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_NAME}.postinst || exit 1 +fi +if [ -f ${MYSCRIPTDIR}/${PACKAGE_NAME}.prerm ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_NAME}.prerm ${EXPANDDIR}/debian/${PACKAGE_NAME}.prerm || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_NAME}.prerm || exit 1 +fi +if [ -f ${MYSCRIPTDIR}/${PACKAGE_NAME}.postrm ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_NAME}.postrm ${EXPANDDIR}/debian/${PACKAGE_NAME}.postrm || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_NAME}.postrm || exit 1 +fi +if [ -f ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.preinst ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.preinst ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.preinst || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.preinst || exit 1 +fi +if [ -f ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.postinst ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.postinst ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.postinst || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.postinst || exit 1 +fi +if [ -f ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.prerm ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.prerm ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.prerm || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.prerm || exit 1 +fi +if [ -f ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.postrm ]; then + cp -p ${MYSCRIPTDIR}/${PACKAGE_DEV_NAME}.postrm ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.postrm || exit 1 + chmod +x ${EXPANDDIR}/debian/${PACKAGE_DEV_NAME}.postrm || exit 1 +fi + +echo "===== prepare working directory: end ===============" + +# +# change debian directory to source top directory +# +if [ ${IS_ROOTDIR} -eq 1 ]; then + if [ -f ${SRCTOP}/debian ]; then + echo "ERROR: ${SRCTOP}/debian file exists, could not make debian directory." 1>&2 + exit 1 + fi + if [ -d ${SRCTOP}/debian ]; then + echo "${SRCTOP}/debian directory exists, remove and remake it..." 1>&2 + rm -rf ${SRCTOP}/debian || exit 1 + fi + cp -rp ${EXPANDDIR}/debian ${SRCTOP}/. || exit 1 + + # + # change current directory + # + cd ${SRCTOP} + + # + # base directory is changed + # + BUILDDEBDIR=${SRCTOP} +fi + +# +# Check stop before debuild(for manually) +# +if [ ${IS_DEBUILD} -ne 1 ]; then + # + # Not run debuild (this means just stop preparing the file) + # + echo "MESSGAE: ${PRGNAME} exits immediately before debuild is executed," + echo " that is, it prepares only files and directories." + echo " By running \"debuild -uc -us(-tc -b)\", you can create" + echo " the debian package manually and find the created package" + echo " in \"${BUILDDEBDIR}/..\" directory." + echo "" + + exit 0 +fi + +# +# Run debuild +# +echo "===== build package: start =========================" +debuild -us -uc || exit 1 +echo "===== build package: end ===========================" + +# +# Check and show debian package +# +ls ${BUILDDEBDIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb >/dev/null 2>&1 +if [ $? -eq 0 ]; then + echo "===== show ${BUILDDEBDIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb package: start =====" + dpkg -c ${BUILDDEBDIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb + echo "" + dpkg -I ${BUILDDEBDIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb + echo "===== show ${BUILDDEBDIR}/${PACKAGE_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb package: end =====" +fi + +ls ${BUILDDEBDIR}/${PACKAGE_DEV_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb >/dev/null 2>&1 +if [ $? -eq 0 ]; then + echo "===== show ${BUILDDEBDIR}/${PACKAGE_DEV_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb package: start =====" + dpkg -c ${BUILDDEBDIR}/${PACKAGE_DEV_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb + echo "" + dpkg -I ${BUILDDEBDIR}/${PACKAGE_DEV_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb + echo "===== show ${BUILDDEBDIR}/${PACKAGE_DEV_NAME}_${PACKAGE_VERSION}-${BUILD_NUMBER}*.deb package: end =====" +fi + +# +# finish +# +echo "" +echo "You can find ${PACKAGE_NAME} ${PACKAGE_VERSION}-${BUILD_NUMBER} version debian package in ${BUILDDEBDIR} directory." +echo "" +exit 0 + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/buildutils/k2hash.spec.in b/buildutils/k2hash.spec.in new file mode 100644 index 0000000..0642d02 --- /dev/null +++ b/buildutils/k2hash.spec.in @@ -0,0 +1,118 @@ +# +# K2HASH +# +# Copyright 2013 Yahoo Japan Corporation. +# +# K2HASH is key-valuew store base libraries. +# K2HASH is made for the purpose of the construction of +# original KVS system and the offer of the library. +# The characteristic is this KVS library which Key can +# layer. And can support multi-processing and multi-thread, +# and is provided safely as available KVS. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri 17 Jul 2016 +# REVISION: +# + +# +# Local macros/define +# +%if %{undefined package_revision} +%global package_revision 1 +%endif + +%if %{undefined make_build} +%global make_build %{__make} %{?_smp_mflags} +%endif + +%if %{undefined autosetup} +%global autosetup %setup -q +%endif + +%global gittag v@VERSION@ + +%if %{undefined make_check} +%global make_check 1 +%endif + +# +# main package +# +Summary: @SHORTDESC@ +Name: @PACKAGE_NAME@ +Version: @VERSION@ +Release: %{package_revision}%{?dist} +License: @PKGLICENSE@ +Group: Applications/Databases +URL: https://@GIT_DOMAIN@/@GIT_ORG@/@PACKAGE_NAME@ +Source0: https://@GIT_DOMAIN@/@GIT_ORG@/@PACKAGE_NAME@/archive/%{gittag}/%{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root +BuildRequires: git-core gcc-c++ make libtool libfullock-devel%{?_isa} >= 1.0.28 +Prefix: %{_prefix} + +%description +@LONGDESC@ + +%prep +%autosetup -n %{name}-%{version} + +%build +./autogen.sh +%configure --disable-static +%make_build + +%install +%make_install +find %{buildroot} -name '*.la' -exec rm -f {} ';' + +%if %{make_check} +%check +%{__make} check +%endif + +%if %{defined ldconfig_scriptlets} +%ldconfig_scriptlets +%else +%post -p /sbin/ldconfig +%postun -p /sbin/ldconfig +%endif + +%if 0%{?rhel} == 6 +%clean +rm -rf %{buildroot} +%endif + +%files +%if 0%{?rhel} == 6 +%defattr(-,root,root) +%endif +%doc COPYING README AUTHORS ChangeLog +%{_libdir}/*.so* +%{_mandir}/man1/* +%{_bindir}/* + +# +# devel package +# +%package devel +Summary: @SHORTDESC@ (development) +Requires: %{name}%{?_isa} = %{version}-%{release}, libfullock-devel%{?_isa} >= 1.0.28 + +%description devel +Development package for building with @PACKAGE_NAME@ shared library. +This package has header files and symbols for it. + +%files devel +%if 0%{?rhel} == 6 +%defattr(-,root,root) +%endif +%doc COPYING README AUTHORS ChangeLog +%{_includedir}/* +%{_libdir}/pkgconfig/*.pc + +%changelog +@RPMCHANGELOG@ diff --git a/buildutils/make_commit_hash.sh b/buildutils/make_commit_hash.sh new file mode 100755 index 0000000..5ffa53e --- /dev/null +++ b/buildutils/make_commit_hash.sh @@ -0,0 +1,221 @@ +#!/bin/sh +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +# +# This script gets commit hash(sha1) from github.com or .git directory +# +func_usage() +{ + echo "" + echo "Usage: $1 [-o org] [-r reponame] [-ep endpoint] [-t tag] [-short]" + echo " -o organization specify organazation name if .git directory is not existed" + echo " -r repository specify repository name if .git directory is not existed" + echo " -ep endpoint if not github.com, specify endpoint for api" + echo " -t tag if counld not found release version tag, use this" + echo " -short get short commit hash(sha1)" + echo " -h print help" + echo "" +} +PRGNAME=`basename $0` +MYSCRIPTDIR=`dirname $0` +SRCTOP=`cd ${MYSCRIPTDIR}/..; pwd` +COMMITHASHFILE="${SRCTOP}/COMMIT_HASH" + +# +# Check options +# +GITDIR=0 +ORGNAME="" +REPONAME="" +DEFAULTENDPOINT="https://api.github.com/repos/" +ENDPOINT=${DEFAULTENDPOINT} +TAGNAME="" +ISSHORT=0 +while [ $# -ne 0 ]; do + if [ "X$1" = "X" ]; then + break + elif [ "X$1" = "X-h" -o "X$1" = "X-help" ]; then + func_usage $PRGNAME + exit 0 + elif [ "X$1" = "X-short" ]; then + ISSHORT=1 + elif [ "X$1" = "X-o" ]; then + if [ "X$ORGNAME" != "X" ]; then + echo "ERROR: already -o option is specified." 1>&2 + echo "unknown" + exit 1 + fi + shift + if [ "X$1" = "X" ]; then + echo "ERROR: -o option needs parameter" 1>&2 + echo "unknown" + exit 1 + fi + ORGNAME=$1 + elif [ "X$1" = "X-r" ]; then + if [ "X$REPONAME" != "X" ]; then + echo "ERROR: already -r option is specified." 1>&2 + echo "unknown" + exit 1 + fi + shift + if [ "X$1" = "X" ]; then + echo "ERROR: -r option needs parameter" 1>&2 + echo "unknown" + exit 1 + fi + REPONAME=$1 + elif [ "X$1" = "X-t" ]; then + if [ "X$TAGNAME" != "X" ]; then + echo "ERROR: already -t option is specified." 1>&2 + echo "unknown" + exit 1 + fi + shift + if [ "X$1" = "X" ]; then + echo "ERROR: -t option needs parameter" 1>&2 + echo "unknown" + exit 1 + fi + TAGNAME=$1 + elif [ "X$1" = "X-ep" ]; then + if [ "X$ENDPOINT" != "X$DEFAULTENDPOINT" ]; then + echo "ERROR: already -ep option is specified." 1>&2 + echo "unknown" + exit 1 + fi + shift + if [ "X$1" = "X" ]; then + echo "ERROR: -ep option needs parameter" 1>&2 + echo "unknown" + exit 1 + fi + ENDPOINT=$1 + else + echo "ERROR: unknown option $1" 1>&2 + echo "unknown" + exit 1 + fi + shift +done + +# +# Check .git directory +# +if [ -d ${SRCTOP}/.git ]; then + GITDIR=1 +fi + +# +# Do +# +if [ $GITDIR -eq 1 ]; then + # + # use git command + # + if [ -f ${COMMITHASHFILE} ]; then + rm -f ${COMMITHASHFILE} + fi + SHORTOPT="" + if [ $ISSHORT -eq 1 ]; then + SHORTOPT="--short" + fi + SHA1RESULT=`git rev-parse ${SHORTOPT} HEAD` + +elif [ -f ${COMMITHASHFILE} ]; then + # + # get from COMMIT_HASH file + # + SHA1RESULT=`cat ${COMMITHASHFILE}` + + if [ $ISSHORT -eq 1 ]; then + SHA1RESULT=`echo ${SHA1RESULT} 2>/dev/null | cut -c 1-7 2>/dev/null` + fi + +else + # + # access to github api + # + if [ "X$ORGNAME" = "X" -o "X$REPONAME" = "X" ]; then + echo "ERROR: need to specify -o and -r options, because you do not have .git directory." 1>&2 + echo "unknown" + exit 1 + fi + if [ "X$TAGNAME" = "X" ]; then + if [ -f ${SRCTOP}/RELEASE_VERSION ]; then + # + # release tag from RELEASE_VERSION file + # + TAGNAME=`cat ${SRCTOP}/RELEASE_VERSION` + else + # + # release tag from srource top directory name(expects source files from tar.gz file on github) + # + SRCTOPNAME=`basename $SRCTOP` + TAGNAME=`echo ${SRCTOPNAME} | sed 's/-/ /g' | awk '{print $NF}'` + fi + fi + + # + # first, access github api with adding "v" to tagname + # + SHA1RESULT=`curl ${ENDPOINT}${ORGNAME}/${REPONAME}/git/refs/tags/v${TAGNAME} 2>/dev/null | grep '"sha":' 2>/dev/null | awk '{print $2}' 2>/dev/null | sed 's/[",]//g' 2>/dev/null` + if [ $? -ne 0 -o "X$SHA1RESULT" = "X" ]; then + # + # retry without "v" + # + SHA1RESULT=`curl ${ENDPOINT}${ORGNAME}/${REPONAME}/git/refs/tags/${TAGNAME} 2>/dev/null | grep '"sha":' 2>/dev/null | awk '{print $2}' 2>/dev/null | sed 's/[",]//g' 2>/dev/null` + if [ $? -ne 0 -o "X$SHA1RESULT" = "X" ]; then + echo "ERROR: Could not get commit hash for ${ORGNAME}/${REPONAME} - ${TAGNAME}" 1>&2 + echo "unknown" + exit 1 + fi + fi + # + # put commit hash file(as long) + # + echo "${SHA1RESULT}" > ${COMMITHASHFILE} + + if [ $ISSHORT -eq 1 ]; then + SHA1RESULT=`echo ${SHA1RESULT} 2>/dev/null | cut -c 1-7 2>/dev/null` + fi +fi + +# +# confirm +# +if [ $? -ne 0 -o "X$SHA1RESULT" = "X" ]; then + echo "ERROR: Could not get commit hash" 1>&2 + echo "unknown" + exit 1 +fi + +echo ${SHA1RESULT} +exit 0 + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/buildutils/make_commit_hash_source.sh b/buildutils/make_commit_hash_source.sh new file mode 100755 index 0000000..673236e --- /dev/null +++ b/buildutils/make_commit_hash_source.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +# +# This script puts git commit hash string to C header file. +# ex) char version[] = "...."; +# +func_usage() +{ + echo "" + echo "Usage: $1 " + echo " file path specify output file path" + echo " value name specify variable name in output C source file" + echo " -h print help" + echo "" +} +PRGNAME=`basename $0` +MYSCRIPTDIR=`dirname $0` +SRCTOP=`cd ${MYSCRIPTDIR}/..; pwd` +COMMITHASH_TOOL="${MYSCRIPTDIR}/make_commit_hash.sh" + +# +# Check options +# +FILEPATH="" +VALUENAME="" +while [ $# -ne 0 ]; do + if [ "X$1" = "X" ]; then + break + elif [ "X$1" = "X-h" -o "X$1" = "X-help" ]; then + func_usage $PRGNAME + exit 0 + else + if [ "X${FILEPATH}" = "X" ]; then + FILEPATH=$1 + elif [ "X${VALUENAME}" = "X" ]; then + VALUENAME=$1 + else + echo "ERROR: unknown option $1" 1>&2 + exit 1 + fi + fi + shift +done +if [ "X${FILEPATH}" = "X" -o "X${VALUENAME}" = "X" ]; then + echo "ERROR: not specify file path and value name" 1>&2 + exit 1 +fi + +# +# Get commit hash value +# +COMMITHASH=`${COMMITHASH_TOOL} -short` +if [ $? -ne 0 ]; then + echo "WARNING: git commit hash code is not found, so set to \"unknown\"." 1>&2 + COMMITHASH="unknown" +fi + +# +# Make code line +# +NEWCODES="char ${VALUENAME}[] = \"${COMMITHASH}\";" + +# +# Put codes to file +# +if [ -f ${FILEPATH} ]; then + # + # The file exists, then we need to check it whichever codes is same. + # + FILECODES=`cat ${FILEPATH}` + + if [ "X${FILECODES}" = "X${NEWCODES}" ]; then + echo "SUCCESS: ${FILEPATH} already has current git commit hash value." 1>&2 + exit 0 + fi +fi + +echo ${NEWCODES} > ${FILEPATH} +if [ $? -ne 0 ]; then + echo "ERROR: Could not put git commit hash value to ${FILEPATH}" 1>&2 + exit 1 +fi + +echo "SUCCESS: ${FILEPATH} is updated with current git commit hash value(${COMMITHASH}) in ${VALUENAME} variable." 1>&2 +exit 0 + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/buildutils/make_description.sh b/buildutils/make_description.sh new file mode 100755 index 0000000..e5d4373 --- /dev/null +++ b/buildutils/make_description.sh @@ -0,0 +1,165 @@ +#!/bin/sh +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +# +# This script gets short/long description from man page. +# +func_usage() +{ + echo "" + echo "Usage: $1 [-short | -long | -esclong | -deblong]" + echo "" + echo " man page file path Input file path for man page file in source directory" + echo " -short return short(summary) description" + echo " -long return long description" + echo " -esclong return long description with escaped LF" + echo " -deblong return long description for debian in debian control file" + echo "" +} +PRGNAME=`basename $0` +MYSCRIPTDIR=`dirname $0` +SRCTOP=`cd ${MYSCRIPTDIR}/..; pwd` + +# +# Check options +# +INFILE="" +ISSHORT=0 +SPACECHAR=0 +ESCLF="" +while [ $# -ne 0 ]; do + if [ "X$1" = "X" ]; then + break; + + elif [ "X$1" = "X-h" -o "X$1" = "X-help" ]; then + func_usage $PRGNAME + exit 0 + + elif [ "X$1" = "X-short" ]; then + ISSHORT=1 + + elif [ "X$1" = "X-long" ]; then + ISSHORT=0 + SPACECHAR=0 + + elif [ "X$1" = "X-esclong" ]; then + ISSHORT=0 + ESCLF="\\n\\" + + elif [ "X$1" = "X-deblong" ]; then + ISSHORT=0 + SPACECHAR=1 + + else + if [ "X${INFILE}" != "X" ]; then + echo "ERROR: already ${INFILE} file is specified." 1>&2 + echo "No description by $PRGNAME with error." + exit 1 + fi + if [ ! -f $1 ]; then + echo "ERROR: $1 file is not existed." 1>&2 + echo "No description by $PRGNAME with error." + exit 1 + fi + INFILE=$1 + fi + shift +done + +# +# put man page formatted by nroff and insert space head. +# +TEMPFILE=/tmp/${PRGNAME}_$$.tmp +nroff -man ${INFILE} 2>/dev/null | col -b 2>/dev/null | sed 's/[0-9][0-9]*m//g' 2>/dev/null | sed 's/^\s/_____/g' > ${TEMPFILE} 2>/dev/null +if [ $? -ne 0 ]; then + echo "ERROR: Could not read ${INFILE} file with converting." 1>&2 + echo "No description by $PRGNAME with error." + rm -f ${TEMPFILE} > /dev/null 2>&1 + exit 1 +fi + +# +# Loop for printing with converting +# +LINELEVEL=0 +LONGDEST_START="no" +while read LINE; do + # + # revert inserted special chars. + # + CHKLINE=`echo ${LINE} | sed 's/^_____//g'` + CHKLINE=`echo ${CHKLINE}` + + if [ $LINELEVEL -eq 0 ]; then + if [ "X$CHKLINE" = "XNAME" ]; then + LINELEVEL=1 + fi + elif [ $LINELEVEL -eq 1 ]; then + if [ $ISSHORT -eq 1 ]; then + echo "${CHKLINE}${ESCLF}" | sed 's/.* - //g' + break + fi + LINELEVEL=2 + + elif [ $LINELEVEL -eq 2 ]; then + if [ "X$CHKLINE" = "XDESCRIPTION" ]; then + LINELEVEL=3 + fi + elif [ $LINELEVEL -eq 3 ]; then + if [ "X$CHKLINE" = "X" ]; then + if [ $SPACECHAR -eq 1 ]; then + echo " .${ESCLF}" + else + echo "${ESCLF}" + fi + else + if [ "X$LINE" = "X$CHKLINE" ]; then + # + # This is new section + # + break + fi + if [ $SPACECHAR -eq 0 ]; then + echo "${CHKLINE}${ESCLF}" + else + if [ "X$LONGDEST_START" = "Xno" ]; then + echo " ${CHKLINE}${ESCLF}" + LONGDEST_START="yes" + else + echo " ${CHKLINE}${ESCLF}" + fi + fi + fi + fi +done < ${TEMPFILE} + +rm -f ${TEMPFILE} > /dev/null 2>&1 + +exit 0 + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/buildutils/make_release_version_file.sh b/buildutils/make_release_version_file.sh new file mode 100755 index 0000000..b2dee6f --- /dev/null +++ b/buildutils/make_release_version_file.sh @@ -0,0 +1,254 @@ +#!/bin/sh +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +# +# Make RELEASE_VERSION file in source top directory +# +# RELEASE_VERSION file is used from configure.ac and other files for building. +# RELEASE_VERSION has version(release) number in it, it is created from Git +# release tag or latest version in ChangeLog file. +# +func_usage() +{ + echo "" + echo "Usage: $1 [-not_use_git] [-no_check_ver_diff] [-f changelog file path]" + echo " -not_use_git specify for not checking git release tag by git tag command" + echo " -no_check_ver_diff specify for not comparing git release tag and changelog" + echo " -f changelog specify changelog file name in source top directory" + echo " -h print help" + echo "" +} +PRGNAME=`basename $0` +MYSCRIPTDIR=`dirname $0` +SRCTOP=`cd ${MYSCRIPTDIR}/..; pwd` +RELEASE_VERSION_FILE="${SRCTOP}/RELEASE_VERSION" +GIT_DIR="${SRCTOP}/.git" + + +# +# Check options +# +NOGIT="no" +NOCHECKVERDIFF="no" +CHANGELOGFILE="${SRCTOP}/ChangeLog" +while [ $# -ne 0 ]; do + if [ "X$1" = "X-h" -o "X$1" = "X-help" ]; then + func_usage $PRGNAME + exit 0 + + elif [ "X$1" = "X-not_use_git" ]; then + NOGIT="yes" + + elif [ "X$1" = "X-no_check_ver_diff" ]; then + NOCHECKVERDIFF="yes" + + elif [ "X$1" = "X-f" ]; then + shift + if [ $# -eq 0 ]; then + echo "ERROR: Must set changelog file name after -f option." 1>&2 + exit 1 + fi + if [ ! -f ${SRCTOP}/$1 ]; then + echo "ERROR: Not found changelog($1) file " 1>&2 + exit 1 + fi + CHANGELOGFILE=${SRCTOP}/$1 + + else + echo "ERROR: Unkown option $1" 1>&2 + exit 1 + fi + shift +done + +# +# Version number from git tag command +# +# get version number from git release tag formatted following: +# "v10", "v 10", "ver10", "ver-10", "version10", "version,10" +# "v10.0.0", "v 10.0", "ver 10.0.0a", "v10.0.0-1", etc +# +# and the last build number is cut.(ex, "v10.0.1-1" -> "10.0.1") +# +if [ -d ${GIT_DIR} ]; then + if [ "X${NOGIT}" = "Xno" ]; then + GIT_RELEASE_VERSION=`git tag | grep '^[v|V]\([e|E][r|R]\([s|S][i|I][o|O][n|N]\)\{0,1\}\)\{0,1\}' | sed 's/^[v|V]\([e|E][r|R]\([s|S][i|I][o|O][n|N]\)\{0,1\}\)\{0,1\}//' | grep -o '[0-9]\+\([\.]\([0-9]\)\+\)\+\(.\)*$' | sed 's/-\(.\)*$//' | sort -t . -n -k 1,1 -k 2,2 -k 3,3 -k 4,4 | uniq | tail -1 | tr -d '\n'` + + if [ "X${GIT_RELEASE_VERSION}" = "X" ]; then + echo "WARNING: Could not get latest release tag from git release tag" 1>&2 + GIT_RELEASE_VERSION= + fi + else + GIT_RELEASE_VERSION= + fi +else + echo "WARNING: ${GIT_DIR} directory is not existed." 1>&2 + GIT_RELEASE_VERSION= +fi + +# +# Version number from ChangeLog +# +# get version number from ChangeLog file formatted like debian. +# and the last build number is cut.(ex, "10.0.1-1" -> "10.0.1") +# +if [ -f ${CHANGELOGFILE} ]; then + CH_RELEASE_VERSION=`grep -o '^.*[(].*[)].*[;].*$' ${CHANGELOGFILE} | grep -o '[(].*[)]' | head -1 | sed 's/[(|)]//g'` + + if [ "X${CH_RELEASE_VERSION}" = "X" ]; then + echo "WARNING: Could not get latest release tag from ChangeLog file ( ${CHANGELOGFILE} )" 1>&2 + CH_RELEASE_VERSION= + fi +else + echo "MESSAGE: not found ChangeLog file ( ${CHANGELOGFILE} )" 1>&2 + CH_RELEASE_VERSION= +fi + +# +# Check version number between git release tag and ChangeLog file +# +# If version number from git release tag is later than one from ChangeLog, +# this script puts error and exits. +# The other case, this script continue to work and puts version number +# to RELEASE_VERION file. +# If there are no version number from git release tag and ChangeLog, this +# script checks RELEASE_VERION file existing. +# +IS_PUT_RELEASE_VERSION_FILE=yes + +if [ "X${GIT_RELEASE_VERSION}" != "X" -a "X${CH_RELEASE_VERSION}" != "X" ]; then + if [ "X${NOCHECKVERDIFF}" = "Xno" ]; then + # + # Check latest version + # + GIT_VERS=`echo ${GIT_RELEASE_VERSION} | sed 's/\./ /g'` + CH_VERS=`echo ${CH_RELEASE_VERSION} | sed 's/\./ /g'` + + GIT_VER_PART_CNT=0 + LATEST_VER_TYPE= + for git_ver_part in ${GIT_VERS}; do + ch_ver_part= + + CH_VER_PART_CNT=0 + for ver_tmp in ${CH_VERS}; do + ch_ver_part=`echo ${ver_tmp}` + CH_VER_PART_CNT=`expr ${CH_VER_PART_CNT} + 1` + + if [ ${GIT_VER_PART_CNT} -lt ${CH_VER_PART_CNT} ]; then + break + fi + done + + if [ "X${ch_ver_part}" != "X" ]; then + if [ ${git_ver_part} -gt ${ch_ver_part} ]; then + LATEST_VER_TYPE=gitver + break + elif [ ${git_ver_part} -lt ${ch_ver_part} ]; then + LATEST_VER_TYPE=chver + break + fi + else + LATEST_VER_TYPE=gitver + break + fi + + GIT_VER_PART_CNT=`expr ${GIT_VER_PART_CNT} + 1` + done + + if [ "X${LATEST_VER_TYPE}" = "X" ]; then + GIT_VER_PART_CNT=0 + for git_ver_part in ${GIT_VERS}; do + GIT_VER_PART_CNT=`expr ${GIT_VER_PART_CNT} + 1` + done + + CH_VER_PART_CNT=0 + for ver_tmp in ${CH_VERS}; do + CH_VER_PART_CNT=`expr ${CH_VER_PART_CNT} + 1` + done + + if [ ${GIT_VER_PART_CNT} -lt ${CH_VER_PART_CNT} ]; then + LATEST_VER_TYPE=chver + fi + fi + + if [ "X${LATEST_VER_TYPE}" = "Xgitver" ]; then + echo "ERROR: git release tag ( ${GIT_RELEASE_VERSION} ) is later than ChangeLog file ( ${CHANGELOGFILE} ) version ( ${CH_RELEASE_VERSION} )." 1>&2 + exit 1 + + elif [ "X${LATEST_VER_TYPE}" = "Xchver" ]; then + echo "WARNING: ChangeLog file ( ${CHANGELOGFILE} ) version ( ${CH_RELEASE_VERSION} ) is later than git release tag ( ${GIT_RELEASE_VERSION} )." 1>&2 + echo " Then RELEASE_VERSION file is put git release tag ( ${GIT_RELEASE_VERSION} )" 1>&2 + + RELEASE_VERSION=${GIT_RELEASE_VERSION} + + else + # LATEST_VER_TYPE is not set, this means same version. + + RELEASE_VERSION=${GIT_RELEASE_VERSION} + fi + + else + # + # Not check version number, so only use it from git + # + RELEASE_VERSION=${GIT_RELEASE_VERSION} + fi + +elif [ "X${GIT_RELEASE_VERSION}" != "X" ]; then + RELEASE_VERSION=${GIT_RELEASE_VERSION} + +elif [ "X${CH_RELEASE_VERSION}" != "X" ]; then + RELEASE_VERSION=${CH_RELEASE_VERSION} + +elif [ -f ${RELEASE_VERSION_FILE} ]; then + RELEASE_VERSION=`cat ${RELEASE_VERSION_FILE}` + IS_PUT_RELEASE_VERSION_FILE=no + +else + echo "ERROR: There is no version number information." 1>&2 + echo " The version number must be given by git release tag" 1>&2 + echo " or ChangeLog file or RELEASE_VERSION file." 1>&2 + exit 1 +fi + +# +# Make RELEASE_VERSION file +# +if [ "X${IS_PUT_RELEASE_VERSION_FILE}" = "Xyes" ]; then + echo "MESSAGE: Put version number ${RELEASE_VERSION} to RELEASE_VERSION file" 1>&2 + echo -n ${RELEASE_VERSION} > ${RELEASE_VERSION_FILE} +fi + + +# +# finish +# +echo "SUCCEED: Version number is ${RELEASE_VERSION}" 1>&2 +exit 0 + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/buildutils/make_rpm_changelog.sh b/buildutils/make_rpm_changelog.sh new file mode 100755 index 0000000..2feb3e2 --- /dev/null +++ b/buildutils/make_rpm_changelog.sh @@ -0,0 +1,124 @@ +#!/bin/sh +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +# +# Convert ChangeLog to use it in spec file for rpm +# +func_usage() +{ + echo "" + echo "Usage: $1 [-h]" + echo " specify ChnageLog file path. if not specify, use ChangeLog file in top directory as default." + echo " -h(help) print help." + echo "" +} +PRGNAME=`basename $0` +MYSCRIPTDIR=`dirname $0` +SRCTOP=`cd ${MYSCRIPTDIR}/..; pwd` + +# +# Check options +# +CHANGELOG_FILE="${SRCTOP}/ChangeLog" +ISSETCHANGELOG=0 +while [ $# -ne 0 ]; do + if [ "X$1" = "X" ]; then + break; + + elif [ "X$1" = "X-h" -o "X$1" = "X-help" ]; then + func_usage $PRGNAME + exit 0 + + else + if [ ${ISSETCHANGELOG} -ne 0 ]; then + echo "ERROR: already ${CHANGELOG_FILE} file is specified." 1>&2 + echo "No changelog by ${CHANGELOG_FILE} with error." + exit 1 + fi + if [ ! -f $1 ]; then + echo "ERROR: $1 file is not existed." 1>&2 + echo "No changelog by ${CHANGELOG_FILE} with error." + exit 1 + fi + CHANGELOG_FILE=$1 + ISSETCHANGELOG=1 + fi + shift +done + +# +# convert ChangeLog to spec file format for rpm +# +INONEVER=0 +DETAILS="" +ALLLINES="" +while read oneline; do + oneline=`echo "${oneline}"` + if [ "X${oneline}" = "X" ]; then + continue + fi + + if [ ${INONEVER} -eq 0 ]; then + PKG_VERSION=`echo "${oneline}" | grep '^.*[(].*\..*[)].*[;].*$' | grep -o '[(].*[)]' | sed 's/[(|)]//g'` + PKG_VERSION=`echo "${PKG_VERSION}"` + if [ "X${PKG_VERSION}" != "X" ]; then + INONEVER=1 + DETAILS="" + fi + else + TEST_CONTENTS=`echo "${oneline}" | grep '^[-][-].*[ ][ ].*$'` + PKG_RF2822=`echo "${TEST_CONTENTS}" | grep -o '[ ][ ].*'` + PKG_RF2822=`echo ${PKG_RF2822}` + PKG_COMITTER=`echo "${TEST_CONTENTS}" | grep -o '.*[ ][ ]' | sed 's/^[-][-][ ]//'` + if [ "X${PKG_RF2822}" != "X" -a "X${PKG_COMITTER}" != "X" ]; then + INONEVER=0 + PKG_DATE=`echo ${PKG_RF2822} | sed 's/,/ /g' | awk '{print $1" "$3" "$2" "$4}'` + PKG_LINE="* ${PKG_DATE} ${PKG_COMITTER} ${PKG_VERSION}${DETAILS}" + if [ "X${ALLLINES}" != "X" ]; then + ALLLINES="${ALLLINES}\\n\\n${PKG_LINE}" + else + ALLLINES="${PKG_LINE}" + fi + else + ONEDETAIL=`echo "$oneline" | grep '^[\*][ ].*' | sed 's/^[\*]//g'` + ONEDETAIL=`echo ${ONEDETAIL}` + if [ "X${ONEDETAIL}" != "X" ]; then + DETAILS="${DETAILS}\\n- ${ONEDETAIL}" + fi + fi + fi +done < ${CHANGELOG_FILE} + +# +# print changelog +# +echo -e "${ALLLINES}" + +exit 0 + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/buildutils/make_variables.sh b/buildutils/make_variables.sh new file mode 100755 index 0000000..9c0f857 --- /dev/null +++ b/buildutils/make_variables.sh @@ -0,0 +1,141 @@ +#!/bin/sh +# +# Utility tools for building configure/packages by AntPickax +# +# Copyright 2018 Yahoo Japan Corporation. +# +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# +# For the full copyright and license information, please view +# the license file that was distributed with this source code. +# +# AUTHOR: Takeshi Nakatani +# CREATE: Fri, Apr 13 2018 +# REVISION: +# + +# +# Puts project version/revision/age/etc variables for building +# +func_usage() +{ + echo "" + echo "Usage: $1 [-pkg_version | -lib_version_info | -lib_version_for_link | -major_number]" + echo " -pkg_version returns package version." + echo " -lib_version_info returns library libtools revision" + echo " -lib_version_for_link return library version for symbolic link" + echo " -major_number return major version number" + echo " -h(help) print help." + echo "" +} +PRGNAME=`basename $0` +MYSCRIPTDIR=`dirname $0` +SRCTOP=`cd ${MYSCRIPTDIR}/..; pwd` +RELEASE_VERSION_FILE="${SRCTOP}/RELEASE_VERSION" + +# +# Check options +# +PRGMODE="" +while [ $# -ne 0 ]; do + if [ "X$1" = "X" ]; then + break; + + elif [ "X$1" = "X-h" -o "X$1" = "X-help" ]; then + func_usage $PRGNAME + exit 0 + + elif [ "X$1" = "X-pkg_version" ]; then + PRGMODE="PKG" + + elif [ "X$1" = "X-lib_version_info" ]; then + PRGMODE="LIB" + + elif [ "X$1" = "X-lib_version_for_link" ]; then + PRGMODE="LINK" + + elif [ "X$1" = "X-major_number" ]; then + PRGMODE="MAJOR" + + else + echo "ERROR: unknown option $1" 1>&2 + echo -n "0" + exit 1 + fi + shift +done +if [ "X${PRGMODE}" = "X" ]; then + echo "ERROR: option is not specified." 1>&2 + echo -n "0" + exit 1 +fi + +# +# Make result +# +if [ ${PRGMODE} = "PKG" ]; then + RESULT=`cat ${RELEASE_VERSION_FILE}` + +elif [ ${PRGMODE} = "LIB" -o ${PRGMODE} = "LINK" ]; then + MAJOR_VERSION=`cat ${RELEASE_VERSION_FILE} | sed 's/["|\.]/ /g' | awk '{print $1}'` + MID_VERSION=`cat ${RELEASE_VERSION_FILE} | sed 's/["|\.]/ /g' | awk '{print $2}'` + LAST_VERSION=`cat ${RELEASE_VERSION_FILE} | sed 's/["|\.]/ /g' | awk '{print $3}'` + + # check version number + expr "${MAJOR_VERSION}" + 1 >/dev/null 2>&1 + if [ $? -ge 2 ]; then + echo "ERROR: wrong version number in RELEASE_VERSION file" 1>&2 + echo -n "0" + exit 1 + fi + expr "${MID_VERSION}" + 1 >/dev/null 2>&1 + if [ $? -ge 2 ]; then + echo "ERROR: wrong version number in RELEASE_VERSION file" 1>&2 + echo -n "0" + exit 1 + fi + expr "${LAST_VERSION}" + 1 >/dev/null 2>&1 + if [ $? -ge 2 ]; then + echo "ERROR: wrong version number in RELEASE_VERSION file" 1>&2 + echo -n "0" + exit 1 + fi + + # make library revision number + if [ ${MID_VERSION} -gt 0 ]; then + REV_VERSION=`expr ${MID_VERSION} \* 100` + REV_VERSION=`expr ${LAST_VERSION} + ${REV_VERSION}` + else + REV_VERSION=${LAST_VERSION} + fi + + if [ ${PRGMODE} = "LIB" ]; then + RESULT="${MAJOR_VERSION}:${REV_VERSION}:0" + else + RESULT="${MAJOR_VERSION}.0.${REV_VERSION}" + fi + +elif [ ${PRGMODE} = "MAJOR" ]; then + RESULT=`cat ${RELEASE_VERSION_FILE} | sed 's/["|\.]/ /g' | awk '{print $1}'` +fi + +# +# Output result +# +echo -n $RESULT + +exit 0 + +# +# VIM modelines +# +# vim:set ts=4 fenc=utf-8: +# diff --git a/configure.ac b/configure.ac index 3339ad3..0730a29 100644 --- a/configure.ac +++ b/configure.ac @@ -1,20 +1,25 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Utility tools for building configure/packages by AntPickax # -# K2HASH is key-valuew store base libraries. -# K2HASH is made for the purpose of the construction of -# original KVS system and the offer of the library. -# The characteristic is this KVS library which Key can -# layer. And can support multi-processing and multi-thread, -# and is provided safely as available KVS. +# Copyright 2018 Yahoo Japan corporation. # +# AntPickax provides utility tools for supporting autotools +# builds. +# +# These tools retrieve the necessary information from the +# repository and appropriately set the setting values of +# configure, Makefile, spec,etc file and so on. +# These tools were recreated to reduce the number of fixes and +# reduce the workload of developers when there is a change in +# the project configuration. +# # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani -# CREATE: Fri Dec 2 2013 +# CREATE: Fri, Apr 13 2018 # REVISION: # @@ -29,7 +34,9 @@ LT_INIT() AC_CONFIG_SRCDIR([config.h.in]) AC_CONFIG_HEADERS([config.h]) +# # Checks for programs. +# AC_PROG_CXX AC_PROG_AWK AC_PROG_CC @@ -40,14 +47,19 @@ AC_PROG_MAKE_SET AC_PROG_RANLIB AC_PROG_LIBTOOL +# # Checks for libraries. +# AC_CHECK_LIB(dl, dlopen) -#AC_CHECK_LIB([k2hash], [main]) +# # Checks for header files. -AC_CHECK_HEADERS([fcntl.h stdint.h stdlib.h string.h strings.h sys/time.h termios.h unistd.h netdb.h]) +# +AC_CHECK_HEADERS([fcntl.h stdint.h stdlib.h string.h strings.h sys/time.h termios.h unistd.h netdb.h endian.h sys/endian.h]) +# # Checks for typedefs, structures, and compiler characteristics. +# AC_HEADER_STDBOOL AC_C_INLINE AC_TYPE_OFF_T @@ -55,9 +67,13 @@ AC_TYPE_PID_T AC_TYPE_SIZE_T AC_TYPE_SSIZE_T AC_TYPE_UINT64_T +AC_TYPE_INT32_T AC_TYPE_UID_T +AC_TYPE_MODE_T +# # Checks for library functions. +# AC_FUNC_MALLOC AC_FUNC_MMAP AC_FUNC_REALLOC @@ -66,43 +82,175 @@ AC_CHECK_FUNCS([fdatasync ftruncate gettimeofday memset munmap realpath strcasec AC_CONFIG_MACRO_DIR([m4]) -AC_CONFIG_FILES([Makefile - docs/Makefile - docs/examples/Makefile - lib/Makefile - lib/libk2hash.pc - tests/Makefile]) +# +# Symbols for buildutils +# +AC_SUBST([GIT_DOMAIN], "github.com") +AC_SUBST([GIT_ORG], "yahoojapan") +AC_SUBST([GIT_REPO], "k2hash") +AC_SUBST([CURRENTREV], "`$(pwd)/buildutils/make_commit_hash.sh -o yahoojapan -r k2hash -short`") +AC_SUBST([DEV_EMAIL], "`echo ${DEBEMAIL:-antpickax@mail.yahoo.co.jp}`") +AC_SUBST([DEV_NAME], "`echo ${DEBFULLNAME:-K2HASH_DEVELOPER}`") -# CFLAGS/CXXFLAGS -CFLAGS="-Wall $CFLAGS" -CXXFLAGS="-Wall $CXXFLAGS" -AC_ARG_ENABLE(expanding-debug, - [ --enable-expanding-debug Turn on depanging area debugging], +AC_SUBST([RPMCHANGELOG], "`$(pwd)/buildutils/make_rpm_changelog.sh $(pwd)/ChangeLog`") +AC_SUBST([SHORTDESC], "`$(pwd)/buildutils/make_description.sh $(pwd)/docs/k2hash.1 -short`") +AC_SUBST([LONGDESC], "`$(pwd)/buildutils/make_description.sh $(pwd)/docs/k2hash.1 -long`") +AC_SUBST([ESCLONGDESC], "`$(pwd)/buildutils/make_description.sh $(pwd)/docs/k2hash.1 -esclong`") +AC_SUBST([DEBLONGDESC], "`$(pwd)/buildutils/make_description.sh $(pwd)/docs/k2hash.1 -deblong`") +AC_SUBST([PKG_VERSION], "`$(pwd)/buildutils/make_variables.sh -pkg_version`") +AC_SUBST([LIB_VERSION_INFO], "`$(pwd)/buildutils/make_variables.sh -lib_version_info`") +AC_SUBST([LIB_VERSION_LINK], "`$(pwd)/buildutils/make_variables.sh -lib_version_for_link`") +AC_SUBST([LIB_MAJOR_NUMBER], "`$(pwd)/buildutils/make_variables.sh -major_number`") +AC_SUBST([PKGLICENSE], "`grep License COPYING | sed 's/ License//g'`") +AC_SUBST([DEBCOPYING], "`tail -n +5 COPYING | sed 's/^$/./g' | sed 's/^/ /g'`") +AM_SUBST_NOTMAKE([CURRENTREV]) +AM_SUBST_NOTMAKE([RPMCHANGELOG]) +AM_SUBST_NOTMAKE([SHORTDESC]) +AM_SUBST_NOTMAKE([LONGDESC]) +AM_SUBST_NOTMAKE([ESCLONGDESC]) +AM_SUBST_NOTMAKE([DEBLONGDESC]) +AM_SUBST_NOTMAKE([DEBCOPYING]) + +# +# pkg-config for old version +# +AC_PATH_PROG(PKG_CONFIG, pkg-config, [AC_MSG_RESULT(no)]) +AS_IF([test "$PKG_CONFIG" = "no"], [AC_MSG_ERROR(You have to install pkg-config to compile $PACKAGE_NAME v$PACKAGE_VERSION)]) + +# +# Option for crypt libraries +# +crypt_lib=openssl + +AC_MSG_CHECKING([for crypt library option with gcrypt(GnuTLS)]) +AC_ARG_WITH(gcrypt, + [AS_HELP_STRING([--with-gcrypt], [build with gcrypt(GnuTLS)])], + [case "${withval}" in + yes) crypt_lib=gcrypt;; + *) ;; + esac] +) +AS_IF([test ${crypt_lib} = gcrypt], [AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)]) + +AC_MSG_CHECKING([for crypt library option with nettle(GnuTLS)]) +AC_ARG_WITH(nettle, + [AS_HELP_STRING([--with-nettle], [build with nettle(GnuTLS)])], + [case "${withval}" in + yes) crypt_lib=nettle;; + *) ;; + esac] +) +AS_IF([test ${crypt_lib} = nettle], [AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)]) + +AC_MSG_CHECKING([for crypt library option with NSS]) +AC_ARG_WITH(nss, + [AS_HELP_STRING([--with-nss], [build with NSS])], + [case "${withval}" in + yes) crypt_lib=nss;; + *) ;; + esac] +) +AS_IF([test ${crypt_lib} = nss], [AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)]) + +AC_MSG_CHECKING([for crypt library option with OpenSSL(default)]) +AC_ARG_WITH(openssl, + [AS_HELP_STRING([--with-openssl], [build with OpenSSL(default)])], + [case "${withval}" in + yes) crypt_lib=openssl;; + *) ;; + esac] +) +AS_IF([test ${crypt_lib} = openssl], [AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)]) + +# +# Checking crypt libraries +# +# [NOTE] +# Although gcrypt does not provide a pkg-config file, it provides +# libgcrypt-config and we check it with libgcrypt-config. +# +AC_MSG_CHECKING([for using crypt library type]) +case "${crypt_lib}" in + openssl) + AC_MSG_RESULT(OpenSSL) + PKG_CHECK_MODULES([CRYPT], [libcrypto >= 1.0.0 ]) + AC_SUBST([CRYPT_LIB_SOURCE], "k2hcryptopenssl.cc") + ;; + nss) + AC_MSG_RESULT(NSS) + PKG_CHECK_MODULES([CRYPT], [nss >= 3.28.0 ]) + AC_SUBST([CRYPT_LIB_SOURCE], "k2hcryptnss.cc") + ;; + nettle) + AC_MSG_RESULT(nettle) + PKG_CHECK_MODULES([CRYPT], [nettle >= 3.2 ]) + AC_SUBST([CRYPT_LIB_SOURCE], "k2hcryptnettle.cc") + ;; + gcrypt) + AC_MSG_RESULT(gcrypt) + AC_CHECK_PROG(gcrypt_config, [libgcrypt-config], [yes], [no]) + AS_IF([test ${gcrypt_config} = no], [AC_MSG_ERROR([libgcrypt-config is not found])]) + CRYPT_VERSION=`libgcrypt-config --version` + CRYPT_CFLAGS=`libgcrypt-config --cflags` + CRYPT_LIBS=`libgcrypt-config --libs` + AC_MSG_CHECKING([for libgcrypt-config version]) + AC_MSG_RESULT($CRYPT_VERSION) + AC_SUBST([CRYPT_LIB_SOURCE], "k2hcryptgcrypt.cc") + ;; + *) + AC_MSG_ERROR([unknown crypt library type.]) + ;; +esac + +AM_CONDITIONAL([CRYPT_LIB_OPENSSL], [test "$crypt_lib" = openssl]) +AM_CONDITIONAL([CRYPT_LIB_NSS], [test "$crypt_lib" = nss]) +AM_CONDITIONAL([CRYPT_LIB_NETTLE], [test "$crypt_lib" = nettle]) +AM_CONDITIONAL([CRYPT_LIB_GCRYPT], [test "$crypt_lib" = gcrypt]) + +# +# Checking Libraries +# +check_depend_libs=1 +AC_MSG_CHECKING([for checking dependent libraries]) +AC_ARG_ENABLE(check-depend-libs, + [AS_HELP_STRING([--disable-check-depend-libs], [disable checking dependent libraries])], [case "${enableval}" in - yes) CFLAGS="-DDEBUG_FOR_EXPANDING $CFLAGS" - CXXFLAGS="-DDEBUG_FOR_EXPANDING $CXXFLAGS" - ;; + yes) check_depend_libs=1;; + *) check_depend_libs=0;; esac] ) +AS_IF([test ${check_depend_libs} = 1], [AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)]) +AS_IF([test ${check_depend_libs} = 1], [PKG_CHECK_MODULES([fullock], [libfullock >= 1.0.28], [], [AC_MSG_ERROR(not found libfullock package)])]) +# +# CFLAGS/CXXFLAGS +# +CFLAGS="-Wall $CFLAGS $CRYPT_CFLAGS $fullock_CFLAGS" +CXXFLAGS="-Wall $CXXFLAGS $CRYPT_CFLAGS $fullock_CFLAGS" + +# +# for STD::FNV-1A +# +std_fnv_function=0 +AC_MSG_CHECKING([for using std fnv-1a hash function]) AC_ARG_ENABLE(std-fnv-function, - [ --enable-std-fnv-function Use STD::FNV-1A hash function], + [AS_HELP_STRING([--enable-std-fnv-function], [using STD::FNV-1A hash function])], [case "${enableval}" in - yes) CFLAGS="-DUSE_STD_FNV_HASH_FUNCTION $CFLAGS" - CXXFLAGS="-DUSE_STD_FNV_HASH_FUNCTION $CXXFLAGS" - ;; + yes) std_fnv_function=1;; + *) std_fnv_function=0;; esac] ) +AS_IF([test ${std_fnv_function} = 1], [AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)]) +AS_IF([test ${std_fnv_function} = 1], + [ + CFLAGS="-DUSE_STD_FNV_HASH_FUNCTION $CFLAGS" + CXXFLAGS="-DUSE_STD_FNV_HASH_FUNCTION $CXXFLAGS" + ] +) -# chek for LIBFULLOCK -if test "x${CHECK_LIBFULLOCK}" != "xno"; then - PKG_CHECK_MODULES([fullock], [libfullock >= 1.0.24], [have_fullock=yes], [have_fullock=no]) - if test "x${have_fullock}" = "xno"; then - AC_MSG_ERROR([not found libfullock-dev package.]) - fi -fi - +# # for MAP_ANONYMOUS +# AC_MSG_CHECKING(for MAP_ANONYMOUS symbol) AC_CACHE_VAL(ac_cv_map_anonymous, AC_TRY_COMPILE([ @@ -117,9 +265,19 @@ if test $ac_cv_map_anonymous = no; then AC_DEFINE(MAP_ANONYMOUS, MAP_ANON, [Define to MAP_ANON if you do not have MAP_ANONYMOUS symbol.]) fi -# LIB_VERSION_INFO for libtool library version info -LIB_VERSION_INFO=`$(pwd)/make_valiables.sh -lib_version_info` -AC_SUBST([LIB_VERSION_INFO]) +# +# Config files +# +AC_CONFIG_FILES([Makefile + docs/Makefile + docs/examples/Makefile + lib/Makefile + lib/libk2hash.pc + tests/Makefile + buildutils/Makefile + buildutils/control + buildutils/copyright + buildutils/k2hash.spec]) AC_OUTPUT diff --git a/docs/Makefile.am b/docs/Makefile.am index 144245b..48fbbcd 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of diff --git a/docs/k2hash.1 b/docs/k2hash.1 index f518674..bc7292b 100644 --- a/docs/k2hash.1 +++ b/docs/k2hash.1 @@ -1,6 +1,6 @@ .TH K2HASH "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HASH \- NoSQL Key Value Store(KVS) library by Yahoo! JAPAN +K2HASH \- NoSQL Key Value Store(KVS) library .SH SYNOPSIS .SS library .TP @@ -60,8 +60,17 @@ K2HASH can put transaction and archive all of data in file. If you need to make \- The other is that you can load your original DSO library which includes same prototype of hash function. .PP You can know how to implement replacing transaction function, please see k2htransfunc.h file. +.PP +.SH FUNCTIONS +.PP +You can develop programs using the interface for C/C++ language provided by the K2HASH library. +You can see following C language function prototypes in k2hash.h. +.br +K2HASH C++ Language interface is implemented almost in k2shm.h. +You can use public K2HShm class methods like C Language interface. .TP 2 K2HASH C Language interface function: +.br void k2h_bump_debug_level(...) .br void k2h_set_debug_level_silent(...) @@ -502,10 +511,8 @@ bool k2h_dump_full(...) bool k2h_print_state(...) .br void k2h_print_version(...) +.br .PP -You can see these function prototypes in k2hash.h -.TP -K2HASH C++ Language interface is implemented almost in k2shm.h. You can use public K2HShm class methods like C Language interface. .SH ENVIRONMENT .TP 2 K2HASH use some environments automatically, plese take care these environments. @@ -530,6 +537,6 @@ k2hlinetool, k2hreplace, k2hcompress, k2htouch, k2hedit is tool for k2hash file/ k2hlinetool(1), k2hreplace(1), k2hcompress(1), k2htouch(1), k2hedit(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2hash has been written by Takeshi Nakatani . diff --git a/docs/k2hbench.1 b/docs/k2hbench.1 index e1f2705..1b3659b 100644 --- a/docs/k2hbench.1 +++ b/docs/k2hbench.1 @@ -1,6 +1,6 @@ .TH K2HBENCH "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HBENCH \- Bench mark Tool for K2HASH by Yahoo! JAPAN +K2HBENCH \- Bench mark Tool for K2HASH .SH SYNOPSIS .B k2hbench [ \-f filename | \-t filename | \-m | \-h ] [OPTIONS] @@ -75,6 +75,6 @@ K2HDBGFILE environment sets the file which is debugging message, this is as same k2hash(1), k2hreplace(1), k2hcompress(1), k2htouch(1), k2hedit(1), k2hlinetool(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2hash has been written by Takeshi Nakatani . diff --git a/docs/k2hcompress.1 b/docs/k2hcompress.1 index 9eead25..832e25e 100644 --- a/docs/k2hcompress.1 +++ b/docs/k2hcompress.1 @@ -1,6 +1,6 @@ .TH K2HCOMPRESS "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HCOMPRESS \- Utility Tool for K2HASH by Yahoo! JAPAN +K2HCOMPRESS \- Utility Tool for K2HASH .SH SYNOPSIS .B k2hcompress [ \-replace | \-direct | \-print ] [ OPTIONS ] FILE @@ -54,6 +54,6 @@ K2HDBGFILE environment sets the file which is debugging message, this is as same k2hash(1), k2hlinetool(1), k2hreplace(1), k2htouch(1), k2hedit(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2hash has been written by Takeshi Nakatani . diff --git a/docs/k2hedit.1 b/docs/k2hedit.1 index e9cae20..c04160f 100644 --- a/docs/k2hedit.1 +++ b/docs/k2hedit.1 @@ -1,6 +1,6 @@ .TH K2HEDIT "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HEDIT \- Utility Tool for K2HASH by Yahoo! JAPAN +K2HEDIT \- Utility Tool for K2HASH .SH SYNOPSIS .B k2hedit [ \-h ] @@ -29,6 +29,6 @@ If EDITOR environment is set, k2hedit uses it. k2hash(1), k2hlinetool(1), k2hreplace(1), k2hcompress(1), k2htouch(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2hash has been written by Tetsuya Mochizuki . diff --git a/docs/k2himport.1 b/docs/k2himport.1 index 863f780..3c3fe5e 100644 --- a/docs/k2himport.1 +++ b/docs/k2himport.1 @@ -1,6 +1,6 @@ .TH K2HIMPORT "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HIMPORT \- Utility Tool for K2HASH by Yahoo! JAPAN +K2HIMPORT \- Utility Tool for K2HASH .SH SYNOPSIS .B k2himport [ \-h ] @@ -23,6 +23,6 @@ import from mdbm_export format text file. k2hash(1), k2hlinetool(1), k2hreplace(1), k2hcompress(1), k2hedit(1) , k2htouch(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2himport has been written by Tetsuya Mochizuki . diff --git a/docs/k2hlinetool.1 b/docs/k2hlinetool.1 index bed30ce..721a708 100644 --- a/docs/k2hlinetool.1 +++ b/docs/k2hlinetool.1 @@ -1,6 +1,6 @@ .TH K2HLINETOOL "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HLINETOOL \- Interactive Command Line Tool for K2HASH by Yahoo! JAPAN +K2HLINETOOL \- Interactive Command Line Tool for K2HASH .SH SYNOPSIS .B k2hlinetool [OPTIONS] [ \-f filename | \-t filename | \-m | \-h ] @@ -78,6 +78,6 @@ K2HDBGFILE environment sets the file which is debugging message, this is as same k2hash(1), k2hreplace(1), k2hcompress(1), k2htouch(1), k2hedit(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2hash has been written by Takeshi Nakatani . diff --git a/docs/k2hreplace.1 b/docs/k2hreplace.1 index 22e97ad..2101865 100644 --- a/docs/k2hreplace.1 +++ b/docs/k2hreplace.1 @@ -1,6 +1,6 @@ .TH K2HREPLACE "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HREPLACE \- Utility Tool for K2HASH by Yahoo! JAPAN +K2HREPLACE \- Utility Tool for K2HASH .SH SYNOPSIS .B k2hreplace [ \-d ] SOURCE_FILE DESTINATION_FILE @@ -30,6 +30,6 @@ K2HDBGFILE environment sets the file which is debugging message, this is as same k2hash(1), k2hlinetool(1), k2hcompress(1), k2htouch(1), k2hedit(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2hash has been written by Takeshi Nakatani . diff --git a/docs/k2htouch.1 b/docs/k2htouch.1 index 6b79a20..5a6f4f7 100644 --- a/docs/k2htouch.1 +++ b/docs/k2htouch.1 @@ -1,6 +1,6 @@ .TH K2HTOUCH "1" "February 2014" "K2HASH" "NoSQL(KVS) Library" .SH NAME -K2HTOUCH \- Utility Tool for K2HASH by Yahoo! JAPAN +K2HTOUCH \- Utility Tool for K2HASH .SH SYNOPSIS .B k2htouch [ \-h ] @@ -85,6 +85,6 @@ K2HDBGFILE environment sets the file which is debugging message, this is as same k2hash(1), k2hlinetool(1), k2hreplace(1), k2hcompress(1), k2hedit(1) .SH BUGS .TP -Please report owner and comitter. +Please report owner and committers. .SH AUTHOR k2hash has been written by Tetsuya Mochizuki . diff --git a/fullock b/fullock deleted file mode 160000 index eea4e33..0000000 --- a/fullock +++ /dev/null @@ -1 +0,0 @@ -Subproject commit eea4e33e880be5c364fd62a4e0d9b2d6d04a3673 diff --git a/lib/Makefile.am b/lib/Makefile.am index e59a6bd..3e697ef 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -11,39 +11,129 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Fri Dec 2 2013 # REVISION: # -## AUTOMAKE_OPTIONS = +pkginclude_HEADERS = k2hash.h \ + k2harchive.h \ + k2hashfunc.h \ + k2hcommand.h \ + k2hcommon.h \ + k2hdbg.h \ + k2hfind.h \ + k2hlock.h \ + k2hmmapinfo.h \ + k2hpagefile.h \ + k2hpage.h \ + k2hpagemem.h \ + k2hshm.h \ + k2hshmdirect.h \ + k2hstructure.h \ + k2hsubkeys.h \ + k2hattrs.h \ + k2htransfunc.h \ + k2htrans.h \ + k2hutil.h \ + k2hdaccess.h \ + k2hstream.h \ + k2hfilemonitor.h \ + k2hshmupdater.h \ + k2hqueue.h \ + k2hattrop.h \ + k2hattrbuiltin.h \ + k2hcryptcommon.h \ + k2hattropsman.h \ + k2hattrplugin.h \ + k2hattrfunc.h -pkginclude_HEADERS = k2hash.h k2harchive.h k2hashfunc.h k2hcommand.h k2hcommon.h k2hdbg.h k2hfind.h k2hlock.h k2hmmapinfo.h k2hpagefile.h k2hpage.h k2hpagemem.h k2hshm.h k2hshmdirect.h k2hstructure.h k2hsubkeys.h k2hattrs.h k2htransfunc.h k2htrans.h k2hutil.h k2hdaccess.h k2hstream.h k2hfilemonitor.h k2hshmupdater.h k2hqueue.h k2hattrop.h k2hattrbuiltin.h k2hattropsman.h k2hattrplugin.h k2hattrfunc.h -pkgincludedir = $(includedir)/k2hash +pkgincludedir = $(includedir)/k2hash +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = libk2hash.pc -EXTRA_DIST = +DISTCLEANFILES = $(pkgconfig_DATA) -pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = libk2hash.pc -DISTCLEANFILES = $(pkgconfig_DATA) +lib_LTLIBRARIES = libk2hash.la -lib_LTLIBRARIES = libk2hash.la -libk2hash_la_SOURCES = k2hash.cc k2hdbg.cc k2hmmapinfo.cc k2hpage.cc k2hpagefile.cc k2hpagemem.cc k2hshm.cc k2hshmdirect.cc k2hshminit.cc k2hshmdump.cc k2hshmcomp.cc k2hashfunc.cc k2hutil.cc k2hfind.cc k2hsubkeys.cc k2hattrs.cc k2hlock.cc k2hcommand.cc k2htrans.cc k2htransfunc.cc k2harchive.cc k2hdaccess.cc k2hfilemonitor.cc k2hshmupdater.cc k2hashversion.cc k2hshmque.cc k2hqueue.cc k2hattrop.cc k2hattrbuiltin.cc k2hattropsman.cc k2hattrplugin.cc -libk2hash_la_LDFLAGS = -version-info $(LIB_VERSION_INFO) -libk2hash_la_LIBADD = $(fullock_LIBS) -lrt -lpthread -lssl -lcrypto +libk2hash_la_SOURCES= k2hash.cc \ + k2hdbg.cc \ + k2hmmapinfo.cc \ + k2hpage.cc \ + k2hpagefile.cc \ + k2hpagemem.cc \ + k2hshm.cc \ + k2hshmdirect.cc \ + k2hshminit.cc \ + k2hshmdump.cc \ + k2hshmcomp.cc \ + k2hashfunc.cc \ + k2hutil.cc \ + k2hfind.cc \ + k2hsubkeys.cc \ + k2hattrs.cc \ + k2hlock.cc \ + k2hcommand.cc \ + k2htrans.cc \ + k2htransfunc.cc \ + k2harchive.cc \ + k2hdaccess.cc \ + k2hfilemonitor.cc \ + k2hshmupdater.cc \ + k2hashversion.cc \ + k2hshmque.cc \ + k2hqueue.cc \ + k2hattrop.cc \ + k2hattrbuiltin.cc \ + k2hcryptcommon.cc \ + k2hattropsman.cc \ + k2hattrplugin.cc -AM_CFLAGS = -DK2HASH -AM_CPPFLAGS = -DK2HASH +### Source code for crypt +# +# [NOTE] +# The crypt type related source code has different source code +# linked depending on the option passed to configure. And only +# object built from selected crypt source code is linked. +# The distributed source codes(tgz) by 'make dist' must have +# only selected source which is selected crypt type. Because +# certain crypt libraries have problems that can not distribute +# the relevant source code due to licensing issues. +# Then source codes for crypt are declared with 'nodist', and +# they are defined in 'EXTRA_DIST' macro. +# By defining 'nodist' and 'EXTRA_DIST', we realize the +# distribution of selective source code. +# +EXTRA_DIST = $(CRYPT_LIB_SOURCE) + +if CRYPT_LIB_OPENSSL +nodist_libk2hash_la_SOURCES = k2hcryptopenssl.cc +endif +if CRYPT_LIB_NSS +nodist_libk2hash_la_SOURCES = k2hcryptnss.cc +endif +if CRYPT_LIB_NETTLE +nodist_libk2hash_la_SOURCES = k2hcryptnettle.cc +endif +if CRYPT_LIB_GCRYPT +nodist_libk2hash_la_SOURCES = k2hcryptgcrypt.cc +endif + +libk2hash_la_LDFLAGS= -version-info $(LIB_VERSION_INFO) +libk2hash_la_LIBADD = $(fullock_LIBS) $(CRYPT_LIBS) -lrt -lpthread + +AM_CFLAGS = -DK2HASH +AM_CPPFLAGS = -DK2HASH -ACLOCAL_AMFLAGS = -I m4 +ACLOCAL_AMFLAGS = -I m4 ### version(commit hash) -.PHONY: k2hashversion +.PHONY: k2hashversion -k2hashversion.cc: k2hashversion - @../make_rev.sh k2hashversion.cc k2h_commit_hash +k2hashversion.cc: k2hashversion + @../buildutils/make_commit_hash_source.sh k2hashversion.cc k2h_commit_hash # # VIM modelines diff --git a/lib/k2harchive.cc b/lib/k2harchive.cc index 7b364c1..e9b6dbd 100644 --- a/lib/k2harchive.cc +++ b/lib/k2harchive.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 10 2014 diff --git a/lib/k2harchive.h b/lib/k2harchive.h index 74b5f7f..f90c643 100644 --- a/lib/k2harchive.h +++ b/lib/k2harchive.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 10 2014 diff --git a/lib/k2hash.cc b/lib/k2hash.cc index 097852f..e8cd2f8 100644 --- a/lib/k2hash.cc +++ b/lib/k2hash.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 @@ -33,6 +33,7 @@ #include "k2htrans.h" #include "k2harchive.h" #include "k2hqueue.h" +#include "k2hcryptcommon.h" using namespace std; @@ -2726,9 +2727,9 @@ void k2h_print_version(FILE* stream) { static const char format[] = "\n" - "K2HASH library Version %s (commit: %s)\n" + "K2HASH library Version %s (commit: %s) with %s\n" "\n" - "Copyright 2013 Yahoo! JAPAN corporation.\n" + "Copyright(C) 2013 Yahoo Japan Corporation.\n" "\n" "K2HASH is key-valuew store base libraries. K2HASH is made for\n" "the purpose of the construction of original KVS system and the\n" @@ -2740,7 +2741,7 @@ void k2h_print_version(FILE* stream) if(!stream){ stream = stdout; } - fprintf(stream, format, VERSION, k2h_commit_hash); + fprintf(stream, format, VERSION, k2h_commit_hash, k2h_crypt_lib_name()); fullock_print_version(stream); } diff --git a/lib/k2hash.h b/lib/k2hash.h index fc8af13..89b7189 100644 --- a/lib/k2hash.h +++ b/lib/k2hash.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 diff --git a/lib/k2hashfunc.cc b/lib/k2hashfunc.cc index 560728b..05cf26a 100644 --- a/lib/k2hashfunc.cc +++ b/lib/k2hashfunc.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 @@ -21,7 +21,6 @@ #include #include -#include #include #include "k2hashfunc.h" @@ -99,20 +98,21 @@ const char* k2h_hash_version(void) //--------------------------------------------------------- // K2HashDynLib Class //--------------------------------------------------------- -K2HashDynLib K2HashDynLib::Singleton; +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +K2HashDynLib* K2HashDynLib::get(void) +{ + static K2HashDynLib hashlib; // singleton + return &hashlib; +} K2HashDynLib::K2HashDynLib() : hDynLib(NULL), fp_k2h_hash(NULL), fp_k2h_second_hash(NULL), fp_k2h_hash_version(NULL) { - if(this != K2HashDynLib::get()){ - assert(false); - } } K2HashDynLib::~K2HashDynLib() { - if(this != K2HashDynLib::get()){ - assert(false); - } Unload(); } diff --git a/lib/k2hashfunc.h b/lib/k2hashfunc.h index dd14db5..b0fa690 100644 --- a/lib/k2hashfunc.h +++ b/lib/k2hashfunc.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 @@ -97,9 +97,6 @@ DECL_EXTERN_C_END // extern "C" - end //--------------------------------------------------------- class K2HashDynLib { - private: - static K2HashDynLib Singleton; - private: void* hDynLib; Tfp_k2h_hash fp_k2h_hash; @@ -107,7 +104,7 @@ class K2HashDynLib Tfp_k2h_hash_version fp_k2h_hash_version; public: - static K2HashDynLib* get(void) { return &K2HashDynLib::Singleton; } + static K2HashDynLib* get(void); K2HashDynLib(); virtual ~K2HashDynLib(); diff --git a/lib/k2hattrbuiltin.cc b/lib/k2hattrbuiltin.cc index 590c4ce..c47948d 100644 --- a/lib/k2hattrbuiltin.cc +++ b/lib/k2hattrbuiltin.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 18 2015 @@ -19,17 +19,12 @@ * */ -#include -#include -#include -#include -#include -#include #include #include #include "k2hcommon.h" +#include "k2hcryptcommon.h" #include "k2hattrbuiltin.h" #include "k2hattrs.h" #include "k2hutil.h" @@ -38,389 +33,88 @@ using namespace std; //--------------------------------------------------------- -// Utilities for MD5 +// Symbols //--------------------------------------------------------- -// [TODO] -// -// In the future, following functions might be moved to a other -// file or other library. In particular, these function should -// not dependent on only OpenSSL, should be configured with -// GnuTLS and NSS. At present, we are left these functions in -// this file so only k2hash uses it. -// -#define K2H_CVT_MD_PARTSIZE static_cast(512) - -static string to_base64(const unsigned char* data, size_t length) -{ - static const char* composechar = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; - - unsigned char block[4]; - string result; - - for(size_t readpos = 0; readpos < length; readpos += 3){ - block[0] = (data[readpos] & 0xfc) >> 2; - block[1] = ((data[readpos] & 0x03) << 4) | ((((readpos + 1) < length ? data[readpos + 1] : 0x00) & 0xf0) >> 4); - block[2] = (readpos + 1) < length ? (((data[readpos + 1] & 0x0f) << 2) | ((((readpos + 2) < length ? data[readpos + 2] : 0x00) & 0xc0) >> 6)) : 0x40; - block[3] = (readpos + 2) < length ? (data[readpos + 2] & 0x3f) : 0x40; - - result += composechar[block[0]]; - result += composechar[block[1]]; - result += composechar[block[2]]; - result += composechar[block[3]]; - } - return result; -} - -// [NOTE] -// Not include end of string NULL. -// -static string to_md5_string(const char* str) -{ - MD5_CTX md5ctx; - unsigned char md5hex[MD5_DIGEST_LENGTH]; - - // md5 - MD5_Init(&md5ctx); - for(size_t length = strlen(str); !ISEMPTYSTR(str) && 0 < length; length -= min(length, K2H_CVT_MD_PARTSIZE), str = &str[min(length, K2H_CVT_MD_PARTSIZE)]){ - MD5_Update(&md5ctx, str, min(length, K2H_CVT_MD_PARTSIZE)); - } - MD5_Final(md5hex, &md5ctx); - - // base64 - return to_base64(md5hex, MD5_DIGEST_LENGTH); -} - -// -// binary to sha256 base64'ed string -// -static string to_sha256_string(const unsigned char* bin, size_t length) -{ - EVP_MD_CTX* sha256ctx = EVP_MD_CTX_create(); - unsigned int digest_len = SHA256_DIGEST_LENGTH; - unsigned char sha256hex[SHA256_DIGEST_LENGTH]; - - if(!sha256ctx){ - ERR_K2HPRN("Could not create context for sha256."); - return string(""); - } - EVP_MD_CTX_set_flags(sha256ctx, EVP_MD_CTX_FLAG_ONESHOT); - - // sha256 - if(1 != EVP_DigestInit_ex(sha256ctx, EVP_sha256(), NULL)){ - ERR_K2HPRN("Could not initialize context for sha256 digest."); - EVP_MD_CTX_destroy(sha256ctx); - return string(""); - } - - for(size_t onelength = 0; 0 < length; length -= onelength, bin = &bin[onelength]){ - onelength = min(length, K2H_CVT_MD_PARTSIZE); - if(1 != EVP_DigestUpdate(sha256ctx, bin, onelength)){ - ERR_K2HPRN("Could not update context for sha256 digest."); - EVP_MD_CTX_destroy(sha256ctx); - return string(""); - } - } - if(1 != EVP_DigestFinal_ex(sha256ctx, sha256hex, &digest_len)){ - ERR_K2HPRN("Could not final context for sha256 digest."); - EVP_MD_CTX_destroy(sha256ctx); - return string(""); - } - EVP_MD_CTX_destroy(sha256ctx); - - // base64 - return to_base64(sha256hex, static_cast(digest_len)); -} - -// [NOTE] -// This function generates system-wide unique ID. -// If gerarated ID does not unique, but we do not care for it. -// Because k2hash needs unique ID for only one key's history. -// This means that it may be a unique only to one key. -// -// We generates it by following seed's data. -// unique ID = -// base64( -// sha256( -// bytes array( -// 8 bytes - nano seconds(by clock_gettime) at calling this. -// 8 bytes - seconds(by clock_gettime) at calling this. -// 8 bytes - thread id on the box. -// 8 bytes - random value by rand() function. -// n bytes - hostname -// ) -// ) -// ); -// -static string k2h_get_uniqid_for_history(const struct timespec& rtime) -{ - static unsigned int seed = 0; - static char hostname[NI_MAXHOST]; - static size_t hostnamelen; - static bool init = false; - if(!init){ - // seed for rand() - struct timespec rtime = {0, 0}; - if(-1 == clock_gettime(CLOCK_REALTIME_COARSE, &rtime)){ - WAN_K2HPRN("could not get clock time by errno(%d), so unix time is instead of it.", errno); - seed = static_cast(time(NULL)); // base is sec - }else{ - seed = static_cast(rtime.tv_nsec / 1000); // base is us - } - - // local hostname - if(0 != gethostname(hostname, sizeof(hostname))){ - WAN_K2HPRN("Could not get localhost name by errno(%d), so \"localhost\" is set.", errno); - strcpy(hostname, "localhost"); - } - hostnamelen = strlen(hostname); - - init = true; - } +#define K2HATTR_BUILTIN_VERSION "K2H ATTR BUILTIN" +#define K2HATTR_ENV_MTIME "K2HATTR_MTIME" +#define K2HATTR_ENV_HISTORY "K2HATTR_HISTORY" +#define K2HATTR_ENV_EXPIRE_SEC "K2HATTR_EXPIRE_SEC" +#define K2HATTR_ENV_DEFAULT_ENC "K2HATTR_DEFENC" +#define K2HATTR_ENV_ENCFILE "K2HATTR_ENCFILE" +#define K2HATTR_ENV_ENC_TYPE "K2HATTR_ENC_TYPE" +#define K2HATTR_ENV_ENC_ITER "K2HATTR_ENC_ITER" - // set datas - uint64_t ard64[4]; // [1]->nsec, [2]->sec, [3]->tid, [4]->rand - ard64[0] = static_cast(rtime.tv_nsec); - ard64[1] = static_cast(rtime.tv_sec); - ard64[2] = static_cast(gettid()); - ard64[3] = static_cast(rand_r(&seed)); // [TODO] should use MAC address instead of rand(). - seed++; // not need barrier +#define K2HATTR_ENV_VAL_NO "NO" +#define K2HATTR_ENV_VAL_OFF "OFF" +#define K2HATTR_ENV_VAL_YES "YES" +#define K2HATTR_ENV_VAL_ON "ON" +#define K2HATTR_ENV_VAL_ENCTYPE_AES256_PBKDF1 "AES256_PBKDF1" +#define K2HATTR_ENV_VAL_ENCTYPE_AES256_PBKDF2 "AES256_PBKDF2" - // set to binary array - unsigned char bindata[NI_MAXHOST + (sizeof(uint64_t) * 4)]; - size_t setpos; - for(setpos = 0; setpos < (sizeof(uint64_t) * 4); ++setpos){ - bindata[setpos] = static_cast(((setpos % 8) ? (ard64[setpos / 8] >> ((setpos % 8) * 8)) : ard64[setpos / 8]) & 0xff); - } - memcpy(&bindata[setpos], hostname, hostnamelen); - setpos += hostnamelen; +#define K2HATTR_ENCFILE_COMMENT_CHAR '#' - // SHA256 - return to_sha256_string(bindata, setpos); -} +#define K2HATTR_COMMON_MTIME "mtime" +#define K2HATTR_COMMON_EXPIRE "expire" +#define K2HATTR_COMMON_AES256_MD5 "aes256md5" +#define K2HATTR_COMMON_AES256_PBKDF2 "aes256_cbc_pad.pbkdf2.sha256" // .. +#define K2HATTR_COMMON_UNIQ_ID "uniqid" +#define K2HATTR_COMMON_PARENT_UNIQ_ID "parentuniqid" +#define K2HATTR_COMMON_HISTORY_MARKER "hismark" //--------------------------------------------------------- -// Utilities for AES256 +// K2hCryptContext Class //--------------------------------------------------------- -// [TODO] +// This class initializes the Crypt library when the k2hash +// library is loaded and discards it at the timing of unloading. // -// In the future, following functions might be moved to a other -// file or other library. In particular, these function should -// not dependent on only OpenSSL, should be configured with -// GnuTLS and NSS. At present, we are left these functions in -// this file so only k2hash uses it. +// This class is used in K2hAttrBuiltin class as singleton. +// This function provides for initializing crypt library when +// starting k2hash library and forking in child process. // - // [NOTE] -// We might should be used such as std::random rather than rand(). -// However, this function is performed only generation of salt, -// salt need not be strictly random. And salt is not a problem -// even if collision between threads. Thus, we use the rand. -// However, this function is to be considerate of another functions -// using rand, so this uses rand_r for not changing seed. +// To avoid static object initialization order problem(SIOF) // -static void k2h_pkcs5_salt(unsigned char* salt, size_t length) +class K2hCryptContext { - static unsigned int seed = 0; - static bool init = false; - if(!init){ - struct timespec rtime = {0, 0}; - if(-1 == clock_gettime(CLOCK_REALTIME_COARSE, &rtime)){ - WAN_K2HPRN("could not get clock time by errno(%d), so unix time is instead of it.", errno); - seed = static_cast(time(NULL)); // base is sec - }else{ - seed = static_cast(rtime.tv_nsec / 1000); // base is us - } - init = true; - } + protected: + static void PreforkHandler(void); - for(size_t cnt = 0; cnt < length; ++cnt){ - seed = static_cast(rand_r(&seed)); - salt[cnt] = static_cast(seed & 0xFF); - } -} + public: + K2hCryptContext(void); + virtual ~K2hCryptContext(void); +}; -// [NOTE] -// -// Encrypted binary data is formatted following: -// "Salted__xxxxxxxxEEEE....EEEEPPPP..." -// -// SALT Prefix: start with "Salted__" with salt(8byte) -// Encrypted data: EEEE....EEEE, it is same original data length. -// Padding: Encrypted data is 16byte block(AES256 CBC), then -// max 31 bytes for padding.(== 32 bytes) -// -#define K2H_ENCRYPT_SALT_PREFIX "Salted__" -#define K2H_ENCRYPT_SALT_PREFIX_LENGTH 8 -#define K2H_ENCRYPT_SALT_LENGTH PKCS5_SALT_LEN -#define K2H_ENCRYPT_MAX_PADDING_LENGTH 32 // max 31 bytes -#define K2H_ENCRYPTED_DATA_EX_LENGTH (K2H_ENCRYPT_SALT_PREFIX_LENGTH + PKCS5_SALT_LEN + K2H_ENCRYPT_MAX_PADDING_LENGTH) - -static unsigned char* k2h_encrypt_aes256_cbc(const char* pass, const unsigned char* orgdata, size_t orglen, size_t& enclen) +void K2hCryptContext::PreforkHandler(void) { - static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; - - if(ISEMPTYSTR(pass) || !orgdata || 0 == orglen){ - ERR_K2HPRN("parameters are wrong."); - return NULL; - } - - EVP_CIPHER_CTX cictx; - const EVP_CIPHER* cipher = EVP_aes_256_cbc(); - unsigned char key[EVP_MAX_KEY_LENGTH]; - unsigned char iv[EVP_MAX_IV_LENGTH]; - unsigned char* encryptdata; - unsigned char* saltpos; - unsigned char* setdatapos; - int encbodylen = 0; - int enclastlen = 0; - - // allocated for encrypted data area - if(NULL == (encryptdata = reinterpret_cast(malloc(orglen + K2H_ENCRYPTED_DATA_EX_LENGTH)))){ - ERR_K2HPRN("Could not allcation memory."); - return NULL; - } - - // copy salt prefix - memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); - saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; - - // make salt - k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH); - setdatapos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; - - // make common encryption key(iteration count = 1) - if(0 == EVP_BytesToKey(cipher, EVP_md5(), saltpos, reinterpret_cast(pass), strlen(pass), 1, key, iv)){ - ERR_K2HPRN("Failed to make AES256 key from pass."); - K2H_Free(encryptdata); - return NULL; - } - - // initialize - EVP_CIPHER_CTX_init(&cictx); - - // initialize context - if(1 != EVP_EncryptInit_ex(&cictx, cipher, NULL, key, iv)){ // type is normally supplied by a function such as EVP_aes_256_cbc(). - ERR_K2HPRN("Could not initialize EVP context."); - K2H_Free(encryptdata); - return NULL; - } - - // do encrypt - if(1 != EVP_EncryptUpdate(&cictx, setdatapos, &encbodylen, orgdata, static_cast(orglen))){ - ERR_K2HPRN("Failed to AES256 CBC encrypt."); - K2H_Free(encryptdata); - return NULL; - } - // last encrypt - if(1 != EVP_EncryptFinal_ex(&cictx, &setdatapos[encbodylen], &enclastlen)){ // padding is on as default - ERR_K2HPRN("Failed to AES256 CBC encrypt finally."); - K2H_Free(encryptdata); - return NULL; - } - - // destroy context - if(1 != EVP_CIPHER_CTX_cleanup(&cictx)){ - ERR_K2HPRN("Failed to destroy EVP context."); - K2H_Free(encryptdata); - return NULL; + if(!k2h_crypt_lib_initialize()){ + ERR_K2HPRN("Something error occurred in initializing crypt library at child process in fork()."); + }else{ + MSG_K2HPRN("Succeed initializing crypt library at child process in fork()."); } - - // length - enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + encbodylen + enclastlen); - - return encryptdata; } -static unsigned char* k2h_decrypt_aes256_cbc(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +K2hCryptContext::K2hCryptContext(void) { - if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ - ERR_K2HPRN("parameters are wrong."); - return NULL; - } - - EVP_CIPHER_CTX cictx; - const EVP_CIPHER* cipher = EVP_aes_256_cbc(); - unsigned char key[EVP_MAX_KEY_LENGTH]; - unsigned char iv[EVP_MAX_IV_LENGTH]; - unsigned char* decryptdata; - const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; - const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; - int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH)); - int decbodylen = 0; - int declastlen = 0; - - // allocated for decrypted data area - if(NULL == (decryptdata = reinterpret_cast(malloc(enclen)))){ // declen < enclen - ERR_K2HPRN("Could not allcation memory."); - return NULL; + int result; + if(0 != (result = pthread_atfork(NULL, NULL, K2hCryptContext::PreforkHandler))){ + ERR_K2HPRN("Failed to set handler for forking(errno=%d), but continue...", result); } - - // make common encryption key(iteration count = 1) - if(0 == EVP_BytesToKey(cipher, EVP_md5(), saltpos, reinterpret_cast(pass), strlen(pass), 1, key, iv)){ - ERR_K2HPRN("Failed to make AES256 key from pass."); - K2H_Free(decryptdata); - return NULL; - } - - // initialize - EVP_CIPHER_CTX_init(&cictx); - - // initialize context - if(1 != EVP_DecryptInit_ex(&cictx, cipher, NULL, key, iv)){ // type is normally supplied by a function such as EVP_aes_256_cbc(). - ERR_K2HPRN("Could not initialize EVP context."); - K2H_Free(decryptdata); - return NULL; - } - - // do decrypt - if(1 != EVP_DecryptUpdate(&cictx, decryptdata, &decbodylen, encbodypos, encbodylen)){ - ERR_K2HPRN("Failed to AES256 CBC decrypt."); - K2H_Free(decryptdata); - return NULL; - } - // last decrypt - if(1 != EVP_DecryptFinal_ex(&cictx, &decryptdata[decbodylen], &declastlen)){ - ERR_K2HPRN("Failed to AES256 CBC decrypt finally."); - K2H_Free(decryptdata); - return NULL; + if(!k2h_crypt_lib_initialize()){ + ERR_K2HPRN("Something error occurred in initializing crypt library."); + }else{ + MSG_K2HPRN("Succeed initializing crypt library."); } +} - // destroy context - if(1 != EVP_CIPHER_CTX_cleanup(&cictx)){ - ERR_K2HPRN("Failed to destroy EVP context."); - K2H_Free(decryptdata); - return NULL; +K2hCryptContext::~K2hCryptContext(void) +{ + if(!k2h_crypt_lib_terminate()){ + ERR_K2HPRN("Something error occurred in terminating crypt library."); + }else{ + MSG_K2HPRN("Succeed terminating crypt library."); } - - // length - declen = static_cast(decbodylen + declastlen); - - return decryptdata; } -//--------------------------------------------------------- -// Symbols -//--------------------------------------------------------- -#define K2HATTR_BUILTIN_VERSION "K2H ATTR BUILTIN" -#define K2HATTR_ENV_MTIME "K2HATTR_MTIME" -#define K2HATTR_ENV_HISTORY "K2HATTR_HISTORY" -#define K2HATTR_ENV_EXPIRE_SEC "K2HATTR_EXPIRE_SEC" -#define K2HATTR_ENV_DEFAULT_ENC "K2HATTR_DEFENC" -#define K2HATTR_ENV_ENCFILE "K2HATTR_ENCFILE" - -#define K2HATTR_ENV_VAL_NO "NO" -#define K2HATTR_ENV_VAL_OFF "OFF" -#define K2HATTR_ENV_VAL_YES "YES" -#define K2HATTR_ENV_VAL_ON "ON" -#define K2HATTR_ENCFILE_COMMENT_CHAR '#' - -#define K2HATTR_COMMON_MTIME "mtime" -#define K2HATTR_COMMON_EXPIRE "expire" -#define K2HATTR_COMMON_AES256_MD5 "aes256md5" -#define K2HATTR_COMMON_UNIQ_ID "uniqid" -#define K2HATTR_COMMON_PARENT_UNIQ_ID "parentuniqid" -#define K2HATTR_COMMON_HISTORY_MARKER "hismark" - //--------------------------------------------------------- // K2hAttrBuiltin Class varialbles //--------------------------------------------------------- @@ -429,6 +123,8 @@ const char* K2hAttrBuiltin::ATTR_ENV_HISTORY = K2HATTR_ENV_HISTORY; const char* K2hAttrBuiltin::ATTR_ENV_EXPIRE_SEC = K2HATTR_ENV_EXPIRE_SEC; const char* K2hAttrBuiltin::ATTR_ENV_DEFAULT_ENC= K2HATTR_ENV_DEFAULT_ENC; const char* K2hAttrBuiltin::ATTR_ENV_ENCFILE = K2HATTR_ENV_ENCFILE; +const char* K2hAttrBuiltin::ATTR_ENV_ENC_TYPE = K2HATTR_ENV_ENC_TYPE; +const char* K2hAttrBuiltin::ATTR_ENV_ENC_ITER = K2HATTR_ENV_ENC_ITER; const int K2hAttrBuiltin::TYPE_ATTRBUILTIN; const char* K2hAttrBuiltin::ATTR_BUILTIN_VERSION= K2HATTR_BUILTIN_VERSION; @@ -438,6 +134,7 @@ const char* K2hAttrBuiltin::ATTR_EXPIRE = K2HATTR_COMMON_EXPIRE; const char* K2hAttrBuiltin::ATTR_UNIQID = K2HATTR_COMMON_UNIQ_ID; const char* K2hAttrBuiltin::ATTR_PUNIQID = K2HATTR_COMMON_PARENT_UNIQ_ID; const char* K2hAttrBuiltin::ATTR_AES256_MD5 = K2HATTR_COMMON_AES256_MD5; +const char* K2hAttrBuiltin::ATTR_AES256_PBKDF2 = K2HATTR_COMMON_AES256_PBKDF2; const char* K2hAttrBuiltin::ATTR_HISMARK = K2HATTR_COMMON_HISTORY_MARKER; const time_t K2hAttrBuiltin::NOT_EXPIRE; @@ -448,8 +145,6 @@ const int K2hAttrBuiltin::ATTR_MASK_HISTORY; const int K2hAttrBuiltin::ATTR_MASK_EXPIRE; const int K2hAttrBuiltin::ATTR_MASK_EXPIRE_KP; -k2hbapackmap_t K2hAttrBuiltin::AttrPackMap; - //--------------------------------------------------------- // Utility macros //--------------------------------------------------------- @@ -466,14 +161,33 @@ k2hbapackmap_t K2hAttrBuiltin::AttrPackMap; //--------------------------------------------------------- // K2hAttrBuiltin Class Methods //--------------------------------------------------------- +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +K2hCryptContext& K2hAttrBuiltin::GetCryptLibContext(void) +{ + static K2hCryptContext cryptlibcontext; // singleton + return cryptlibcontext; +} + +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +k2hbapackmap_t& K2hAttrBuiltin::GetAttrPackMap(void) +{ + static k2hbapackmap_t AttrPackMap; // singleton + (void)K2hAttrBuiltin::GetCryptLibContext(); // for initialize singleton if not initializing(this method call in K2hAttrBuiltin::Initialize) + return AttrPackMap; +} + PK2HBATTRPACK K2hAttrBuiltin::GetBuiltinAttrPack(const K2HShm* pshm) { if(!pshm){ ERR_K2HPRN("Parameter is wrong."); return NULL; } - k2hbapackmap_t::iterator iter = K2hAttrBuiltin::AttrPackMap.find(pshm); - if(K2hAttrBuiltin::AttrPackMap.end() == iter){ + k2hbapackmap_t::iterator iter = K2hAttrBuiltin::GetAttrPackMap().find(pshm); + if(K2hAttrBuiltin::GetAttrPackMap().end() == iter){ return NULL; } return iter->second; @@ -486,15 +200,15 @@ bool K2hAttrBuiltin::CleanAttrBuiltin(const K2HShm* pshm) return false; } - k2hbapackmap_t::iterator iter = K2hAttrBuiltin::AttrPackMap.find(pshm); - if(K2hAttrBuiltin::AttrPackMap.end() == iter){ + k2hbapackmap_t::iterator iter = K2hAttrBuiltin::GetAttrPackMap().find(pshm); + if(K2hAttrBuiltin::GetAttrPackMap().end() == iter){ WAN_K2HPRN("There is no builtin attribute pack structure in map."); return true; // already unload } PK2HBATTRPACK pPack = iter->second; K2H_Delete(pPack); - K2hAttrBuiltin::AttrPackMap.erase(iter); + K2hAttrBuiltin::GetAttrPackMap().erase(iter); return true; } @@ -623,6 +337,28 @@ bool K2hAttrBuiltin::RawInitializeEnv(PK2HBATTRPACK pPack) return false; } } + if(k2h_getenv(K2hAttrBuiltin::ATTR_ENV_ENC_TYPE, value)){ + if(0 == strcasecmp(value.c_str(), K2HATTR_ENV_VAL_ENCTYPE_AES256_PBKDF1)){ + pPack->EncType = K2H_ENC_AES256_PBKDF1; + pPack->IterCount = 1; // This case is always 1 for compatibility + }else if(0 == strcasecmp(value.c_str(), K2HATTR_ENV_VAL_ENCTYPE_AES256_PBKDF2)){ + pPack->EncType = K2H_ENC_AES256_PBKDF2; + }else{ + WAN_K2HPRN("environment %s has unknown value(%s), but continue...", K2hAttrBuiltin::ATTR_ENV_ENC_TYPE, value.c_str()); + } + } + if(k2h_getenv(K2hAttrBuiltin::ATTR_ENV_ENC_ITER, value)){ + if(K2H_ENC_AES256_PBKDF1 == pPack->EncType){ + WAN_K2HPRN("environment %s is specified, but encrypt type is %s. this type is always iteration count = 1.", K2hAttrBuiltin::ATTR_ENV_ENC_ITER, K2HATTR_ENV_VAL_ENCTYPE_AES256_PBKDF1); + }else{ + int count = atoi(value.c_str()); + if(count < 1){ + WAN_K2HPRN("environment %s is wrong value(%d), iteration count must be 1 - 0x7fffffff", K2hAttrBuiltin::ATTR_ENV_ENC_ITER, count); + }else{ + pPack->IterCount = count; + } + } + } return true; } @@ -671,7 +407,7 @@ bool K2hAttrBuiltin::Initialize(const K2HShm* pshm, const bool* is_mtime, const PK2HBATTRPACK pPack = K2hAttrBuiltin::GetBuiltinAttrPack(pshm); if(!pPack){ pPack = new K2HBATTRPACK; - K2hAttrBuiltin::AttrPackMap[pshm] = pPack; + K2hAttrBuiltin::GetAttrPackMap()[pshm] = pPack; } if(!K2hAttrBuiltin::RawInitializeEnv(pPack)){ ERR_K2HPRN("Failed to load preset attributes from environment."); @@ -744,6 +480,7 @@ bool K2hAttrBuiltin::AddCryptPass(const K2HShm* pshm, const char* pPass, bool is //--------------------------------------------------------- K2hAttrBuiltin::K2hAttrBuiltin(void) : K2hAttrOpsBase(), pBuiltinAttrPack(NULL), EncPass(""), ExpireSec(K2hAttrBuiltin::NOT_EXPIRE), OldUniqID(""), AttrMask(K2hAttrBuiltin::ATTR_MASK_NO) { + (void)K2hAttrBuiltin::GetCryptLibContext(); // for initialize singleton if not initializing VerInfo = K2hAttrBuiltin::ATTR_BUILTIN_VERSION; } @@ -799,11 +536,12 @@ bool K2hAttrBuiltin::IsHandleAttr(const unsigned char* key, size_t keylen) const MSG_K2HPRN("target key name for attribute is empty."); return false; } - if( ((strlen(K2hAttrBuiltin::ATTR_MTIME) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_MTIME, key, keylen)) || - ((strlen(K2hAttrBuiltin::ATTR_EXPIRE) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_EXPIRE, key, keylen)) || - ((strlen(K2hAttrBuiltin::ATTR_UNIQID) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_UNIQID, key, keylen)) || - ((strlen(K2hAttrBuiltin::ATTR_PUNIQID) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_PUNIQID, key, keylen)) || - ((strlen(K2hAttrBuiltin::ATTR_AES256_MD5) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_AES256_MD5, key, keylen)) ) + if( ((strlen(K2hAttrBuiltin::ATTR_MTIME) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_MTIME, key, keylen)) || + ((strlen(K2hAttrBuiltin::ATTR_EXPIRE) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_EXPIRE, key, keylen)) || + ((strlen(K2hAttrBuiltin::ATTR_UNIQID) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_UNIQID, key, keylen)) || + ((strlen(K2hAttrBuiltin::ATTR_PUNIQID) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_PUNIQID, key, keylen)) || + ((strlen(K2hAttrBuiltin::ATTR_AES256_PBKDF2)+ 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_AES256_PBKDF2, key, keylen)) || + ((strlen(K2hAttrBuiltin::ATTR_AES256_MD5) + 1) == keylen && 0 == memcmp(K2hAttrBuiltin::ATTR_AES256_MD5, key, keylen)) ) { return true; } @@ -895,8 +633,13 @@ bool K2hAttrBuiltin::UpdateAttr(K2HAttrs& attrs) // encrypt unsigned char* encValue; size_t encValLen = 0; - if(NULL == (encValue = k2h_encrypt_aes256_cbc(pass.c_str(), byValue, ValLen, encValLen))){ - ERR_K2HPRN("Could not encrypt value by AES256."); + if(K2H_ENC_AES256_PBKDF1 == pBuiltinAttrPack->EncType){ + encValue = k2h_encrypt_aes256_cbc(pass.c_str(), byValue, ValLen, encValLen); + }else{ // K2H_ENC_AES256_PBKDF2 == pBuiltinAttrPack->EncType + encValue = k2h_encrypt_aes256_cbc_pbkdf2(pass.c_str(), pBuiltinAttrPack->IterCount, byValue, ValLen, encValLen); + } + if(!encValue){ + ERR_K2HPRN("Could not encrypt value by AES256 CBC PAD(PBKDF1 or PBKDF2)."); return false; } @@ -910,12 +653,12 @@ bool K2hAttrBuiltin::UpdateAttr(K2HAttrs& attrs) IsCahnged = true; // set md5 - if(!SetAttr(attrs, K2hAttrBuiltin::ATTR_AES256_MD5, strmd5.c_str())){ - ERR_K2HPRN("Could not set %s attribute.", K2hAttrBuiltin::ATTR_AES256_MD5); + if(!SetAttr(attrs, (K2H_ENC_AES256_PBKDF1 == pBuiltinAttrPack->EncType ? K2hAttrBuiltin::ATTR_AES256_MD5 : K2hAttrBuiltin::ATTR_AES256_PBKDF2), strmd5.c_str())){ + ERR_K2HPRN("Could not set %s attribute.", (K2H_ENC_AES256_PBKDF1 == pBuiltinAttrPack->EncType ? K2hAttrBuiltin::ATTR_AES256_MD5 : K2hAttrBuiltin::ATTR_AES256_PBKDF2)); return false; } }else{ - RemoveAttr(attrs, K2hAttrBuiltin::ATTR_AES256_MD5); + RemoveAttr(attrs, (K2H_ENC_AES256_PBKDF1 == pBuiltinAttrPack->EncType ? K2hAttrBuiltin::ATTR_AES256_MD5 : K2hAttrBuiltin::ATTR_AES256_PBKDF2)); } // history @@ -1144,9 +887,11 @@ bool K2hAttrBuiltin::GetUniqId(K2HAttrs& attrs, bool is_parent, string& uniqid) bool K2hAttrBuiltin::GetEncryptKeyMd5(K2HAttrs& attrs, string* enckeymd5) const { const char* pval = NULL; - if(!GetAttr(attrs, K2hAttrBuiltin::ATTR_AES256_MD5, &pval) || !pval){ - // attr does not have key - return false; + if(!GetAttr(attrs, K2hAttrBuiltin::ATTR_AES256_PBKDF2, &pval) || !pval){ // check AES256 CBC PBKDF2 type + if(!GetAttr(attrs, K2hAttrBuiltin::ATTR_AES256_MD5, &pval) || !pval){ // check AES256 CBC PBKDF1 type + // attr does not have key + return false; + } } if(enckeymd5){ *enckeymd5 = pval; @@ -1154,6 +899,20 @@ bool K2hAttrBuiltin::GetEncryptKeyMd5(K2HAttrs& attrs, string* enckeymd5) const return true; } +bool K2hAttrBuiltin::GetEncryptType(K2HAttrs& attrs, K2HATTR_ENC_TYPE& enctype) const +{ + const char* pval = NULL; + bool result = false; + if(GetAttr(attrs, K2hAttrBuiltin::ATTR_AES256_PBKDF2, &pval) && pval){ // AES256 CBC PBKDF2 type + enctype = K2H_ENC_AES256_PBKDF2; + result = true; + }else if(GetAttr(attrs, K2hAttrBuiltin::ATTR_AES256_MD5, &pval) && pval){ // AES256 CBC PBKDF1 type + enctype = K2H_ENC_AES256_PBKDF1; + result = true; + } + return result; +} + // // This method returns copied the Value if the value is not encrypted. // If the value is encrypted, there is following perttern. @@ -1176,8 +935,9 @@ unsigned char* K2hAttrBuiltin::GetDecryptValue(K2HAttrs& attrs, const char* encp return presult; } - string enckeymd5; - if(GetEncryptKeyMd5(attrs, enckeymd5)){ + K2HATTR_ENC_TYPE enctype = K2H_ENC_AES256_PBKDF2; + string enckeymd5; + if(GetEncryptType(attrs, enctype) && GetEncryptKeyMd5(attrs, enckeymd5)){ // value is encrypted string strPass; if(ISEMPTYSTR(encpass)){ @@ -1202,7 +962,12 @@ unsigned char* K2hAttrBuiltin::GetDecryptValue(K2HAttrs& attrs, const char* encp } // do decrypt - if(NULL == (presult = k2h_decrypt_aes256_cbc(strPass.c_str(), byValue, ValLen, declen))){ + if(K2H_ENC_AES256_PBKDF1 == enctype){ + presult = k2h_decrypt_aes256_cbc(strPass.c_str(), byValue, ValLen, declen); + }else{ // K2H_ENC_AES256_PBKDF2 == enctype + presult = k2h_decrypt_aes256_cbc_pbkdf2(strPass.c_str(), byValue, ValLen, declen); + } + if(!presult){ ERR_K2HPRN("Failed to decrypt value by pass."); return presult; } diff --git a/lib/k2hattrbuiltin.h b/lib/k2hattrbuiltin.h index 74a0bc6..3d40abe 100644 --- a/lib/k2hattrbuiltin.h +++ b/lib/k2hattrbuiltin.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 18 2015 @@ -29,6 +29,13 @@ //--------------------------------------------------------- // Structure //--------------------------------------------------------- +#define K2H_PPBKDF2_KEY_ITERATION_COUNT 1000 // default iteration count + +typedef enum _k2hattr_enc_type{ // k2hattr encrypt type + K2H_ENC_AES256_PBKDF1 = 0, // AES256 CBC PAD with PBKDF1 + K2H_ENC_AES256_PBKDF2 = 1 // AES256 CBC PAD with PBKDF2 +}K2HATTR_ENC_TYPE; + // [NOTE] // In the future, k2hash could also support encryption of not AES256. // In that case, it would be better to add the type of encryption to @@ -45,14 +52,16 @@ typedef struct k2h_encrypt_pass{ typedef std::map k2hepmap_t; typedef struct k2h_builtin_attr_pack{ - bool IsAttrMTime; // whether stamp last modified time - bool IsAttrHistory; // whether build versioning - time_t AttrExpireSec; // expire second, -1 means not set - bool IsDefaultEncrypt; // whether encode as default - std::string DefaultPassMD5; // md5 for default encrypt pass - k2hepmap_t EncPassMap; // map for encrypt pass - - k2h_builtin_attr_pack(void) : IsAttrMTime(false), IsAttrHistory(false), AttrExpireSec(-1), IsDefaultEncrypt(false), DefaultPassMD5("") { } // -1 = K2hAttrBuiltin::NOT_EXPIRE + bool IsAttrMTime; // whether stamp last modified time + bool IsAttrHistory; // whether build versioning + time_t AttrExpireSec; // expire second, -1 means not set + bool IsDefaultEncrypt; // whether encode as default + std::string DefaultPassMD5; // md5 for default encrypt pass + int IterCount; // iteration count for PBKDF2 key + K2HATTR_ENC_TYPE EncType; // encrypt type(default AES256 CBC with PBKDF2) + k2hepmap_t EncPassMap; // map for encrypt pass + + k2h_builtin_attr_pack(void) : IsAttrMTime(false), IsAttrHistory(false), AttrExpireSec(-1), IsDefaultEncrypt(false), DefaultPassMD5(""), IterCount(K2H_PPBKDF2_KEY_ITERATION_COUNT), EncType(K2H_ENC_AES256_PBKDF2) { } // -1 = K2hAttrBuiltin::NOT_EXPIRE ~k2h_builtin_attr_pack(void) { } }K2HBATTRPACK, *PK2HBATTRPACK; @@ -61,6 +70,8 @@ typedef std::map k2hbapackmap_t; //--------------------------------------------------------- // Class K2hAttrBuiltin //--------------------------------------------------------- +class K2hCryptContext; + class K2hAttrBuiltin : public K2hAttrOpsBase { protected: @@ -70,6 +81,8 @@ class K2hAttrBuiltin : public K2hAttrOpsBase static const char* ATTR_ENV_EXPIRE_SEC; static const char* ATTR_ENV_DEFAULT_ENC; static const char* ATTR_ENV_ENCFILE; + static const char* ATTR_ENV_ENC_TYPE; + static const char* ATTR_ENV_ENC_ITER; static const time_t NOT_EXPIRE = -1; @@ -84,6 +97,7 @@ class K2hAttrBuiltin : public K2hAttrOpsBase static const char* ATTR_UNIQID; static const char* ATTR_PUNIQID; static const char* ATTR_AES256_MD5; + static const char* ATTR_AES256_PBKDF2; static const char* ATTR_HISMARK; // Mask values @@ -103,15 +117,16 @@ class K2hAttrBuiltin : public K2hAttrOpsBase static const int ATTR_MASK_EXPIRE_KP = 1 << 4; // Special mask protected: - static k2hbapackmap_t AttrPackMap; // builtin attribute setting map for each shm - PK2HBATTRPACK pBuiltinAttrPack; std::string EncPass; time_t ExpireSec; - std::string OldUniqID; // Unique ID for old value after updating, it means to need to make history. - int AttrMask; // attribute mask for temporary on the object + std::string OldUniqID; // Unique ID for old value after updating, it means to need to make history. + int AttrMask; // attribute mask for temporary on the object protected: + static K2hCryptContext& GetCryptLibContext(void); // singleton for crypt library(initializer/destructor) + static k2hbapackmap_t& GetAttrPackMap(void); // builtin attribute setting map for each shm + static PK2HBATTRPACK GetBuiltinAttrPack(const K2HShm* pshm); static bool ClearEncryptPassMap(PK2HBATTRPACK pPack); static int LoadEncryptPassMap(PK2HBATTRPACK pPack, const char* pfile); @@ -124,6 +139,7 @@ class K2hAttrBuiltin : public K2hAttrOpsBase bool GetTime(K2HAttrs& attrs, const char* key, struct timespec& time) const; bool GetUniqId(K2HAttrs& attrs, bool is_parent, std::string& uniqid) const; bool GetEncryptKeyMd5(K2HAttrs& attrs, std::string* enckeymd5) const; + bool GetEncryptType(K2HAttrs& attrs, K2HATTR_ENC_TYPE& enctype) const; public: static bool CleanAttrBuiltin(const K2HShm* pshm); diff --git a/lib/k2hattrfunc.h b/lib/k2hattrfunc.h index 10784c0..296ab50 100644 --- a/lib/k2hattrfunc.h +++ b/lib/k2hattrfunc.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Dec 21 2015 diff --git a/lib/k2hattrop.cc b/lib/k2hattrop.cc index d394692..7735a72 100644 --- a/lib/k2hattrop.cc +++ b/lib/k2hattrop.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 18 2015 diff --git a/lib/k2hattrop.h b/lib/k2hattrop.h index 35a95de..3438237 100644 --- a/lib/k2hattrop.h +++ b/lib/k2hattrop.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 18 2015 diff --git a/lib/k2hattropsman.cc b/lib/k2hattropsman.cc index ae5712a..aba1b8b 100644 --- a/lib/k2hattropsman.cc +++ b/lib/k2hattropsman.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Dec 22 2015 @@ -43,14 +43,18 @@ inline int GetBuiltinMaskValue(K2hAttrOpsMan::ATTRINITTYPE type) return K2hAttrBuiltin::ATTR_MASK_NO; } -//--------------------------------------------------------- -// K2hAttrOpsMan Class Variable -//--------------------------------------------------------- -k2hattrlibmap_t K2hAttrOpsMan::PluginLibs; - //--------------------------------------------------------- // K2hAttrOpsMan Class Methods //--------------------------------------------------------- +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +k2hattrlibmap_t& K2hAttrOpsMan::GetLibMap(void) +{ + static k2hattrlibmap_t PluginLibs; // singleton + return PluginLibs; +} + bool K2hAttrOpsMan::InitializeCommonAttr(const K2HShm* pshm, const bool* is_mtime, const bool* is_defenc, const char* passfile, const bool* is_history, const time_t* expire, const strarr_t* pluginlibs) { if(!K2hAttrBuiltin::Initialize(pshm, is_mtime, is_defenc, passfile, is_history, expire)){ @@ -86,13 +90,13 @@ bool K2hAttrOpsMan::AddPluginLib(const K2HShm* pshm, const char* path) return false; } - k2hattrlibmap_t::const_iterator miter = K2hAttrOpsMan::PluginLibs.find(pshm); - if(K2hAttrOpsMan::PluginLibs.end() == miter){ + k2hattrlibmap_t::const_iterator miter = K2hAttrOpsMan::GetLibMap().find(pshm); + if(K2hAttrOpsMan::GetLibMap().end() == miter){ // need to new list k2hattrliblist_t* plist = new k2hattrliblist_t; plist->push_back(pLib); - K2hAttrOpsMan::PluginLibs[pshm] = plist; + K2hAttrOpsMan::GetLibMap()[pshm] = plist; }else{ k2hattrliblist_t* plist = miter->second; @@ -128,7 +132,7 @@ bool K2hAttrOpsMan::RemovePluginLib(k2hattrliblist_t* plist) bool K2hAttrOpsMan::RemovePluginLib(const K2HShm* pshm) { if(!pshm){ - for(k2hattrlibmap_t::iterator miter = K2hAttrOpsMan::PluginLibs.begin(); miter != K2hAttrOpsMan::PluginLibs.end(); K2hAttrOpsMan::PluginLibs.erase(miter++)){ + for(k2hattrlibmap_t::iterator miter = K2hAttrOpsMan::GetLibMap().begin(); miter != K2hAttrOpsMan::GetLibMap().end(); K2hAttrOpsMan::GetLibMap().erase(miter++)){ k2hattrliblist_t* plist = miter->second; // remove all libraries in list @@ -141,8 +145,8 @@ bool K2hAttrOpsMan::RemovePluginLib(const K2HShm* pshm) } }else{ // search - k2hattrlibmap_t::iterator miter = K2hAttrOpsMan::PluginLibs.find(pshm); - if(K2hAttrOpsMan::PluginLibs.end() == miter){ + k2hattrlibmap_t::iterator miter = K2hAttrOpsMan::GetLibMap().find(pshm); + if(K2hAttrOpsMan::GetLibMap().end() == miter){ WAN_K2HPRN("There is no library list for shm."); return true; // OK } @@ -157,7 +161,7 @@ bool K2hAttrOpsMan::RemovePluginLib(const K2HShm* pshm) K2H_Delete(plist); // retreive list from map - K2hAttrOpsMan::PluginLibs.erase(miter); + K2hAttrOpsMan::GetLibMap().erase(miter); } return true; } @@ -192,8 +196,8 @@ bool K2hAttrOpsMan::GetVersionInfos(const K2HShm* pshm, strarr_t& verinfos) } // plugin libs - k2hattrlibmap_t::const_iterator miter = K2hAttrOpsMan::PluginLibs.find(pshm); - if(K2hAttrOpsMan::PluginLibs.end() != miter){ + k2hattrlibmap_t::const_iterator miter = K2hAttrOpsMan::GetLibMap().find(pshm); + if(K2hAttrOpsMan::GetLibMap().end() != miter){ const k2hattrliblist_t* plist = miter->second; for(k2hattrliblist_t::const_iterator liter = plist->begin(); liter != plist->end(); ++liter){ const K2hAttrPluginLib* ploaded = *liter; @@ -241,8 +245,8 @@ bool K2hAttrOpsMan::Initialize(const K2HShm* pshm, const unsigned char* pkey, si if(K2hAttrOpsMan::OPSMAN_MASK_ALL != type){ // if has lib list in map, make attr plugin objects from plugin library list, and push objct list. - k2hattrlibmap_t::const_iterator miter = K2hAttrOpsMan::PluginLibs.find(pshm); - if(K2hAttrOpsMan::PluginLibs.end() != miter){ + k2hattrlibmap_t::const_iterator miter = K2hAttrOpsMan::GetLibMap().find(pshm); + if(K2hAttrOpsMan::GetLibMap().end() != miter){ const k2hattrliblist_t* plist = miter->second; for(k2hattrliblist_t::const_iterator liter = plist->begin(); liter != plist->end(); ++liter){ diff --git a/lib/k2hattropsman.h b/lib/k2hattropsman.h index 81ef891..3c857de 100644 --- a/lib/k2hattropsman.h +++ b/lib/k2hattropsman.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Dec 22 2015 @@ -32,15 +32,13 @@ class K2hAttrOpsMan { protected: - static k2hattrlibmap_t PluginLibs; // all libraries list map by each shm - - k2hattroplist_t attroplist; // libraries list - const unsigned char* byKey; - size_t KeyLen; - const unsigned char* byValue; - size_t ValLen; - const unsigned char* byUpdateValue; - size_t UpdateValLen; + k2hattroplist_t attroplist; // libraries list + const unsigned char* byKey; + size_t KeyLen; + const unsigned char* byValue; + size_t ValLen; + const unsigned char* byUpdateValue; + size_t UpdateValLen; public: // [NOTE] @@ -59,6 +57,8 @@ class K2hAttrOpsMan }ATTRINITTYPE; protected: + static k2hattrlibmap_t& GetLibMap(void); // all libraries list map by each shm + static bool RemoveBuiltinAttr(const K2HShm* pshm); static bool RemovePluginLib(k2hattrliblist_t* plist); static bool RemovePluginLib(const K2HShm* pshm); diff --git a/lib/k2hattrplugin.cc b/lib/k2hattrplugin.cc index efafa42..c37dcff 100644 --- a/lib/k2hattrplugin.cc +++ b/lib/k2hattrplugin.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Dec 21 2015 diff --git a/lib/k2hattrplugin.h b/lib/k2hattrplugin.h index 063f29c..44f31ce 100644 --- a/lib/k2hattrplugin.h +++ b/lib/k2hattrplugin.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Dec 21 2015 diff --git a/lib/k2hattrs.cc b/lib/k2hattrs.cc index 2b84a84..5339570 100644 --- a/lib/k2hattrs.cc +++ b/lib/k2hattrs.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Wed Nov 25 2015 diff --git a/lib/k2hattrs.h b/lib/k2hattrs.h index f1fa4a0..8bb9c9c 100644 --- a/lib/k2hattrs.h +++ b/lib/k2hattrs.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Wed Nov 25 2015 diff --git a/lib/k2hcommand.cc b/lib/k2hcommand.cc index e1146a4..7372584 100644 --- a/lib/k2hcommand.cc +++ b/lib/k2hcommand.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 10 2014 diff --git a/lib/k2hcommand.h b/lib/k2hcommand.h index 4d424d2..6e8e84d 100644 --- a/lib/k2hcommand.h +++ b/lib/k2hcommand.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 10 2014 diff --git a/lib/k2hcommon.h b/lib/k2hcommon.h index 97fc552..2003d64 100644 --- a/lib/k2hcommon.h +++ b/lib/k2hcommon.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,13 +11,14 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 * REVISION: * */ + #ifndef K2HCOMMON_H #define K2HCOMMON_H @@ -84,6 +85,26 @@ template inline bool ISEMPTYSTR(const T& pstr) #define __STDC_FORMAT_MACROS #include +//--------------------------------------------------------- +// For endian +//--------------------------------------------------------- +#ifndef _BSD_SOURCE +#define _BSD_SOURCE +#define SET_LOCAL_BSD_SOURCE 1 +#endif + +#ifdef HAVE_ENDIAN_H +#include +#else +#ifdef HAVE_SYS_ENDIAN_H +#include +#endif +#endif + +#ifdef SET_LOCAL_BSD_SOURCE +#undef _BSD_SOURCE +#endif + /* * VIM modelines * diff --git a/lib/k2hcryptcommon.cc b/lib/k2hcryptcommon.cc new file mode 100644 index 0000000..557d240 --- /dev/null +++ b/lib/k2hcryptcommon.cc @@ -0,0 +1,255 @@ +/* + * K2HASH + * + * Copyright 2018 Yahoo Japan Corporation. + * + * K2HASH is key-valuew store base libraries. + * K2HASH is made for the purpose of the construction of + * original KVS system and the offer of the library. + * The characteristic is this KVS library which Key can + * layer. And can support multi-processing and multi-thread, + * and is provided safely as available KVS. + * + * For the full copyright and license information, please view + * the license file that was distributed with this source code. + * + * AUTHOR: Takeshi Nakatani + * CREATE: Mon May 7 2018 + * REVISION: + * + */ + +#include +#include +#include + +#include "k2hcommon.h" +#include "k2hcryptcommon.h" +#include "k2hutil.h" +#include "k2hdbg.h" + +using namespace std; + +//--------------------------------------------------------- +// Common Utilities +//--------------------------------------------------------- +// +// Get unique ID +// +// [NOTE] +// This function generates system-wide unique ID. +// If gerarated ID does not unique, but we do not care for it. +// Because k2hash needs unique ID for only one key's history. +// This means that it may be a unique only to one key. +// +// We generates it by following seed's data. +// unique ID = +// base64( +// sha256( +// bytes array( +// 8 bytes - nano seconds(by clock_gettime) at calling this. +// 8 bytes - seconds(by clock_gettime) at calling this. +// 8 bytes - thread id on the box. +// 8 bytes - random value by rand() function. +// n bytes - hostname +// ) +// ) +// ); +// +string k2h_get_uniqid_for_history(const struct timespec& rtime) +{ + static unsigned int seed = 0; + static char hostname[NI_MAXHOST]; + static size_t hostnamelen; + static bool init = false; + if(!init){ + // seed for rand() + struct timespec rtime = {0, 0}; + if(-1 == clock_gettime(CLOCK_REALTIME_COARSE, &rtime)){ + WAN_K2HPRN("could not get clock time by errno(%d), so unix time is instead of it.", errno); + seed = static_cast(time(NULL)); // base is sec + }else{ + seed = static_cast(rtime.tv_nsec / 1000); // base is us + } + + // local hostname + if(0 != gethostname(hostname, sizeof(hostname))){ + WAN_K2HPRN("Could not get localhost name by errno(%d), so \"localhost\" is set.", errno); + strcpy(hostname, "localhost"); + } + hostnamelen = strlen(hostname); + + init = true; + } + + // set datas + uint64_t ard64[4]; // [1]->nsec, [2]->sec, [3]->tid, [4]->rand + ard64[0] = static_cast(rtime.tv_nsec); + ard64[1] = static_cast(rtime.tv_sec); + ard64[2] = static_cast(gettid()); + ard64[3] = static_cast(rand_r(&seed)); // [TODO] should use MAC address instead of rand(). + seed++; // not need barrier + + // set to binary array + unsigned char bindata[NI_MAXHOST + (sizeof(uint64_t) * 4)]; + size_t setpos; + for(setpos = 0; setpos < (sizeof(uint64_t) * 4); ++setpos){ + bindata[setpos] = static_cast(((setpos % 8) ? (ard64[setpos / 8] >> ((setpos % 8) * 8)) : ard64[setpos / 8]) & 0xff); + } + memcpy(&bindata[setpos], hostname, hostnamelen); + setpos += hostnamelen; + + // SHA256 + return to_sha256_string(bindata, setpos); +} + +// +// Utilities for AES256 Salt +// +// [NOTE] +// We might should be used such as std::random rather than rand(). +// However, this function is performed only generation of salt, +// salt need not be strictly random. And salt is not a problem +// even if collision between threads. Thus, we use the rand. +// However, this function is to be considerate of another functions +// using rand, so this uses rand_r for not changing seed. +// +bool k2h_pkcs5_salt(unsigned char* salt, size_t length) +{ + static unsigned int seed = 0; + static bool init = false; + if(!init){ + struct timespec rtime = {0, 0}; + if(-1 == clock_gettime(CLOCK_REALTIME_COARSE, &rtime)){ + WAN_K2HPRN("could not get clock time by errno(%d), so unix time is instead of it.", errno); + seed = static_cast(time(NULL)); // base is sec + }else{ + seed = static_cast(rtime.tv_nsec / 1000); // base is us + } + init = true; + } + if(!salt || length <= 0){ + ERR_K2HPRN("parameters are wrong."); + return false; + } + for(size_t cnt = 0; cnt < length; ++cnt){ + seed = static_cast(rand_r(&seed)); + salt[cnt] = static_cast(seed & 0xFF); + } + return true; +} + +// +// Utilities for AES256 IV +// +// [NOTE] +// Initial Vector(IV) should be generated using functions of each +// crypt library. However, it is complicated with OpenSSL(RAND_bytes)/ +// gcrypt(gcry_randomize)/NSS(PK11_GenerateRandom)/nettle, and random +// functions are not found with nettle alone. +// As a result, nettle uses /dev/(u)random etc and uses random/rand +// function. Therefore, we make initial seed as random as possible, +// and make IV with this function like generating salt. +// +bool k2h_generate_iv(unsigned char* iv, size_t length) +{ + static unsigned int seed = 0; + static bool init = false; + if(!init){ + unsigned char tid = static_cast(gettid()); // use thread id + struct timespec rtime = {0, 0}; + if(-1 == clock_gettime(CLOCK_REALTIME_COARSE, &rtime)){ + WAN_K2HPRN("could not get clock time by errno(%d), so unix time is instead of it.", errno); + seed = static_cast(time(NULL)); // base is sec + }else{ + seed = static_cast(rtime.tv_nsec / 1000); // base is us + } + seed = ((seed << 8) | (static_cast(tid) & 0xff)); // merge + init = true; + } + if(!iv || length <= 0){ + ERR_K2HPRN("parameters are wrong."); + return false; + } + for(size_t cnt = 0; cnt < length; ++cnt){ + seed = static_cast(rand_r(&seed)); + iv[cnt] = static_cast(seed & 0xFF); + } + return true; +} + +// +// Utilities for Iteration count +// +bool k2h_copy_iteration_count(unsigned char* setpos, int iter, size_t count) +{ + if(!setpos || iter <= 0 || count < 4){ + ERR_K2HPRN("parameters are wrong."); + return false; + } + // force 4 bytes big endian + int32_t tmpiter = static_cast(iter); + tmpiter = htobe32(tmpiter); + + // 4 byte is iteration count(big endian) + setpos[0] = static_cast((tmpiter >> 24) & 0xff); + setpos[1] = static_cast((tmpiter >> 16) & 0xff); + setpos[2] = static_cast((tmpiter >> 8) & 0xff); + setpos[3] = static_cast(tmpiter & 0xff); + + // set rest bytes as dummy + if(4 < count && !k2h_generate_iv(&setpos[4], (count - 4))){ + return false; + } + return true; +} + +// +// Utilities for Iteration count +// +int k2h_get_iteration_count(const unsigned char* getpos) +{ + if(!getpos){ + ERR_K2HPRN("parameters are wrong."); + return -1; + } + int32_t tmpiter = 0; + tmpiter |= (static_cast(getpos[0]) << 24) & 0xff000000; + tmpiter |= (static_cast(getpos[1]) << 16) & 0xff0000; + tmpiter |= (static_cast(getpos[2]) << 8) & 0xff00; + tmpiter |= static_cast(getpos[3]) & 0xff; + + // to host byte order + tmpiter = be32toh(tmpiter); + return static_cast(tmpiter); +} + +// +// Base64 +// +string to_base64(const unsigned char* data, size_t length) +{ + static const char* composechar = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; + + unsigned char block[4]; + string result; + + for(size_t readpos = 0; readpos < length; readpos += 3){ + block[0] = (data[readpos] & 0xfc) >> 2; + block[1] = ((data[readpos] & 0x03) << 4) | ((((readpos + 1) < length ? data[readpos + 1] : 0x00) & 0xf0) >> 4); + block[2] = (readpos + 1) < length ? (((data[readpos + 1] & 0x0f) << 2) | ((((readpos + 2) < length ? data[readpos + 2] : 0x00) & 0xc0) >> 6)) : 0x40; + block[3] = (readpos + 2) < length ? (data[readpos + 2] & 0x3f) : 0x40; + + result += composechar[block[0]]; + result += composechar[block[1]]; + result += composechar[block[2]]; + result += composechar[block[3]]; + } + return result; +} + +/* + * VIM modelines + * + * vim:set ts=4 fenc=utf-8: + */ diff --git a/lib/k2hcryptcommon.h b/lib/k2hcryptcommon.h new file mode 100644 index 0000000..4d290e4 --- /dev/null +++ b/lib/k2hcryptcommon.h @@ -0,0 +1,73 @@ +/* + * K2HASH + * + * Copyright 2018 Yahoo Japan Corporation. + * + * K2HASH is key-valuew store base libraries. + * K2HASH is made for the purpose of the construction of + * original KVS system and the offer of the library. + * The characteristic is this KVS library which Key can + * layer. And can support multi-processing and multi-thread, + * and is provided safely as available KVS. + * + * For the full copyright and license information, please view + * the license file that was distributed with this source code. + * + * AUTHOR: Takeshi Nakatani + * CREATE: Mon May 7 2018 + * REVISION: + * + */ + +#ifndef K2HCRYPTCOMMON_H +#define K2HCRYPTCOMMON_H + +#include +#include + +//--------------------------------------------------------- +// Symbols +//--------------------------------------------------------- +#define K2H_CVT_MD_PARTSIZE static_cast(512) + +#define K2H_PKCS5_SALT_LENGTH 8 // [NOTE] OpenSSL has this symbol as PKCS5_SALT_LEN, but others do not have this value. +#define K2H_ENCRYPT_SALT_PREFIX "Salted__" // prefix for salt +#define K2H_ENCRYPT_SALT_PREFIX_LENGTH 8 // = strlen(K2H_ENCRYPT_SALT_PREFIX) +#define K2H_ENCRYPT_SALT_LENGTH K2H_PKCS5_SALT_LENGTH +#define K2H_ENCRYPT_IV_LENGTH 16 // [NOTE] this value should be 16 byte for all crypt libraries +#define K2H_ENCRYPT_ITER_LENGTH 16 // +#define K2H_ENCRYPT_MAX_PADDING_LENGTH 32 // max 31 bytes + +// For PBKDF1 +#define K2H_ENCRYPTED_DATA_EX_LENGTH (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_MAX_PADDING_LENGTH) + +// For PBKDF2 +#define K2H_ENCRYPTED_DATA_EX2_LENGTH (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH + K2H_ENCRYPT_MAX_PADDING_LENGTH) + +//--------------------------------------------------------- +// Functions +//--------------------------------------------------------- +std::string k2h_get_uniqid_for_history(const struct timespec& rtime); +bool k2h_pkcs5_salt(unsigned char* salt, size_t length); +bool k2h_generate_iv(unsigned char* iv, size_t length); +bool k2h_copy_iteration_count(unsigned char* setpos, int iter, size_t count); +int k2h_get_iteration_count(const unsigned char* getpos); + +const char* k2h_crypt_lib_name(void); +bool k2h_crypt_lib_initialize(void); +bool k2h_crypt_lib_terminate(void); +std::string to_base64(const unsigned char* data, size_t length); +std::string to_md5_string(const char* str); +std::string to_sha256_string(const unsigned char* bin, size_t length); +unsigned char* k2h_encrypt_aes256_cbc(const char* pass, const unsigned char* orgdata, size_t orglen, size_t& enclen); +unsigned char* k2h_decrypt_aes256_cbc(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen); +unsigned char* k2h_encrypt_aes256_cbc_pbkdf2(const char* pass, int iter, const unsigned char* orgdata, size_t orglen, size_t& enclen); +unsigned char* k2h_decrypt_aes256_cbc_pbkdf2(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen); + +#endif // K2HCRYPTCOMMON_H + +/* + * VIM modelines + * + * vim:set ts=4 fenc=utf-8: + */ diff --git a/lib/k2hcryptgcrypt.cc b/lib/k2hcryptgcrypt.cc new file mode 100644 index 0000000..2b0b671 --- /dev/null +++ b/lib/k2hcryptgcrypt.cc @@ -0,0 +1,684 @@ +/* + * K2HASH + * + * Copyright 2018 Yahoo Japan Corporation. + * + * K2HASH is key-valuew store base libraries. + * K2HASH is made for the purpose of the construction of + * original KVS system and the offer of the library. + * The characteristic is this KVS library which Key can + * layer. And can support multi-processing and multi-thread, + * and is provided safely as available KVS. + * + * For the full copyright and license information, please view + * the license file that was distributed with this source code. + * + * AUTHOR: Takeshi Nakatani + * CREATE: Mon May 7 2018 + * REVISION: + * + */ + +#include +#include + +#include "k2hcommon.h" +#include "k2hcryptcommon.h" +#include "k2hutil.h" +#include "k2hdbg.h" + +using namespace std; + +//--------------------------------------------------------- +// Symbols +//--------------------------------------------------------- +#define MD5_DIGEST_SIZE 16 +#define SHA256_DIGEST_SIZE 32 +#define GCRY_PBKDF1_KEY_LENGTH 32 +#define GCRY_PBKDF2_KEY_LENGTH 32 + +//--------------------------------------------------------- +// Get crypt library name +//--------------------------------------------------------- +const char* k2h_crypt_lib_name(void) +{ + static const char name[] = "gcrypt"; + return name; +} + +//--------------------------------------------------------- +// Initialize/Terminate +//--------------------------------------------------------- +bool k2h_crypt_lib_initialize(void) +{ + return true; +} + +bool k2h_crypt_lib_terminate(void) +{ + return true; +} + +//--------------------------------------------------------- +// MD5 +//--------------------------------------------------------- +string to_md5_string(const char* str) +{ + gcry_md_hd_t ctx_md5; + gcry_error_t err; + unsigned char md5hex[MD5_DIGEST_SIZE]; + + // md5 + if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_md5, GCRY_MD_MD5, 0))){ + ERR_K2HPRN("MD5 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); + return string(""); + } + + for(size_t length = strlen(str); !ISEMPTYSTR(str) && 0 < length; length -= min(length, K2H_CVT_MD_PARTSIZE), str = &str[min(length, K2H_CVT_MD_PARTSIZE)]){ + gcry_md_write(ctx_md5, reinterpret_cast(str), min(length, K2H_CVT_MD_PARTSIZE)); + } + memcpy(md5hex, gcry_md_read(ctx_md5, 0), MD5_DIGEST_SIZE); + gcry_md_close(ctx_md5); + + // base64 + return to_base64(md5hex, MD5_DIGEST_SIZE); +} + +//--------------------------------------------------------- +// SHA256 +//--------------------------------------------------------- +string to_sha256_string(const unsigned char* bin, size_t length) +{ + gcry_md_hd_t ctx_sha256; + gcry_error_t err; + unsigned char sha256hex[SHA256_DIGEST_SIZE]; + + // sha256 + if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ + ERR_K2HPRN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); + return string(""); + } + + for(size_t onelength = 0; 0 < length; length -= onelength, bin = &bin[onelength]){ + onelength = min(length, K2H_CVT_MD_PARTSIZE); + gcry_md_write(ctx_sha256, bin, onelength); + } + memcpy(sha256hex, gcry_md_read(ctx_sha256, 0), SHA256_DIGEST_SIZE); + gcry_md_close(ctx_sha256); + + // base64 + return to_base64(sha256hex, SHA256_DIGEST_SIZE); +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF1(PCKS#5 v1.5) +//--------------------------------------------------------- +// +// gcry_cipher_encrypt is not padding for CBC, then we padding as PCKS#7 here. +// +static unsigned char* k2h_pcks7_padding(const unsigned char* orgdata, size_t orglen, size_t& padlen, size_t blocksize) +{ + // [NOTE] + // If original length is as same as blocksize, adding blocksize. + // + padlen = ((orglen / blocksize) * blocksize) + blocksize; + + unsigned char data = static_cast(padlen - orglen) & 0xff; + unsigned char* paddata; + if(NULL == (paddata = reinterpret_cast(malloc(padlen)))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + memcpy(paddata, orgdata, orglen); + for(size_t fill = 0; (fill + orglen) < padlen; ++fill){ + paddata[orglen + fill] = data; + } + return paddata; +} + +// +// gcry_cipher_encrypt is not padding for CBC, then we padding as PCKS#7 here. +// +bool k2h_pcks7_remove_padding(const unsigned char* data, int& length) +{ + if(!data || length <= 0){ + ERR_K2HPRN("parameters are wrong."); + return false; + } + + unsigned char lastdata = data[length - 1]; + if(length < static_cast(lastdata)){ + ERR_K2HPRN("padding data length is over all data length."); + return false; + } + length = length - static_cast(lastdata); + + return true; +} + +// +// Create Key and IV for PBKDF1(PCKS#5 v1.5) +// +// [NOTE] +// This function calculates and returns Key and IV value according to the +// argument. gcrypt calculates Key and IV values as PCKS#5 v2(PBKDF2) and +// uses those randomly for AES256. +// However, K2HASH was originally implemented only in OpenSSL, using the +// EVP_BytesToKey function which is not recommended. +// That function calculates Key and IV value from salt and returns those values +// as PCKS#5 v1.5(PBKDF1). Therefore, we also use those Key and IV values +// calculated equivalently from salt for gcrypt and implement so that data created +// with K2HASH of OpenSSL version can be read out. +// Thus, this function mimics a part of OpenSSL EVP_BytesToKey(PBKDF1) for gcrypt. +// In the future, we will move to PCKS#5 v2(PBKDF2) soon. +// For future encryption, this function will no longer be used.(For decoding, +// it is used for compatibility, but it will not be supported for a long time.) +// +// On gcrypt, we should use gcry_kdf_derive with GCRY_KDF_PBKDF1, but we need +// to generate IV and key from passphrase and salt. That function does not generate +// IV, then we use following function. +// +bool create_pbkdf1_key_iv(const unsigned char* pSalt, const unsigned char* pPass, size_t lenPass, unsigned int iteratecnt, unsigned char* pKey, unsigned int lenCipherKey, unsigned char* pIV, int lenIV) +{ + bool result = false; + gcry_md_hd_t md5ctx; + gcry_error_t err; + unsigned char md5buf[MD5_DIGEST_SIZE * 4]; // == 64(for mergin) + unsigned int md5len = MD5_DIGEST_SIZE; // == 16 + bool is_first_loop; + unsigned int cnt; + + if(!pSalt || !pPass || lenPass <= 0 || !pKey || lenCipherKey == 0 || !pIV || lenIV <= 0){ + ERR_K2HPRN("parameters are wrong."); + return result; + } + + for(result = true, is_first_loop = true; true; is_first_loop = false){ + if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&md5ctx, GCRY_MD_MD5, 0))){ // MD5 + ERR_K2HPRN("MD5 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); + result = false; + break; + } + + if(!is_first_loop){ + gcry_md_write(md5ctx, &(md5buf[0]), md5len); + } + gcry_md_write(md5ctx, pPass, lenPass); + gcry_md_write(md5ctx, pSalt, K2H_ENCRYPT_SALT_LENGTH); // K2H_ENCRYPT_SALT_LENGTH = 8 + memcpy(md5buf, gcry_md_read(md5ctx, 0), md5len); + + for(cnt = 1; cnt < iteratecnt; ++cnt){ + gcry_md_write(md5ctx, &(md5buf[0]), md5len); + memcpy(md5buf, gcry_md_read(md5ctx, 0), md5len); + } + gcry_md_close(md5ctx); + if(!result){ + break; + } + + for(cnt = 0; 0 < lenCipherKey && cnt != md5len; --lenCipherKey, ++cnt){ + *(pKey++) = md5buf[cnt]; + } + for(; 0 < lenIV && cnt != md5len; --lenIV, ++cnt){ + *(pIV++) = md5buf[cnt]; + } + + if(0 == lenCipherKey && 0 == lenIV){ + break; + } + } + return result; +} + +// These provide encryption of AES256 CBC PADDING. +// However, the common key is created with PBKDF1(PCKS#5 v1.5). +// These functions are the old type of encryption with K2HASH, +// and now the type of PBKDF2 is used. +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc(const char* pass, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + int cipher = GCRY_CIPHER_AES256; + size_t cipherBlock = gcry_cipher_get_algo_blklen(cipher); + int keySize = gcry_cipher_get_algo_keylen(cipher); + gcry_cipher_hd_t gcryCipherHd; + gcry_error_t gcryError; + unsigned char key[GCRY_PBKDF1_KEY_LENGTH]; // always key size is GCRY_PBKDF1_KEY_LENGTH(32) on this case + unsigned char iv[K2H_ENCRYPT_IV_LENGTH]; // always IV size is K2H_ENCRYPT_IV_LENGTH(16). + unsigned char* newdata; + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* setdatapos; + size_t encbodylen; + + // check key size + if(keySize <= 0 || GCRY_PBKDF1_KEY_LENGTH < keySize){ + ERR_K2HPRN("key size is wrong(%d)", keySize); + return NULL; + } + + // Do padding(PCKS#7) + if(NULL == (newdata = k2h_pcks7_padding(orgdata, orglen, encbodylen, cipherBlock))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // allocated for encrypted data area(size is after padding) + if(NULL == (encryptdata = reinterpret_cast(malloc(encbodylen + K2H_ENCRYPTED_DATA_EX_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + K2H_Free(newdata); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + setdatapos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // + // generate PBKDF1 key and iv + // + if(!create_pbkdf1_key_iv(saltpos, reinterpret_cast(pass), strlen(pass), 1, key, keySize, iv, K2H_ENCRYPT_IV_LENGTH)){ + ERR_K2HPRN("could not create IV."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // initialize + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_open(&gcryCipherHd, cipher, GCRY_CIPHER_MODE_CBC, 0))){ + ERR_K2HPRN("Failed to open gcrypt cipher by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // set symmetric key + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setkey(gcryCipherHd, key, GCRY_PBKDF2_KEY_LENGTH))){ + ERR_K2HPRN("Failed to set symmetric key by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // set IV + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setiv(gcryCipherHd, iv, K2H_ENCRYPT_IV_LENGTH))){ + ERR_K2HPRN("Failed to set IV by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // do encrypt + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_encrypt(gcryCipherHd, setdatapos, static_cast(encbodylen), newdata, static_cast(encbodylen)))){ + ERR_K2HPRN("Failed to AES256 CBC encrypt by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + K2H_Free(newdata); + + // destroy context + gcry_cipher_close(gcryCipherHd); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + encbodylen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + int cipher = GCRY_CIPHER_AES256; + int keySize = gcry_cipher_get_algo_keylen(cipher); + gcry_cipher_hd_t gcryCipherHd; + gcry_error_t gcryError; + unsigned char key[GCRY_PBKDF1_KEY_LENGTH]; // always key size is GCRY_PBKDF1_KEY_LENGTH(32) on this case + unsigned char iv[K2H_ENCRYPT_IV_LENGTH]; // always IV size is K2H_ENCRYPT_IV_LENGTH(16). + unsigned char* decryptdata; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH)); + int decbodylen = encbodylen; + + // check key size + if(keySize <= 0 || GCRY_PBKDF1_KEY_LENGTH < keySize){ + ERR_K2HPRN("key size is wrong(%d)", keySize); + return NULL; + } + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(decbodylen)))){ // decbodylen < encbodylen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // + // generate PBKDF1 key and iv + // + if(!create_pbkdf1_key_iv(saltpos, reinterpret_cast(pass), strlen(pass), 1, key, keySize, iv, K2H_ENCRYPT_IV_LENGTH)){ + ERR_K2HPRN("could not create IV."); + K2H_Free(decryptdata); + return NULL; + } + + // initialize + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_open(&gcryCipherHd, cipher, GCRY_CIPHER_MODE_CBC, 0))){ + ERR_K2HPRN("Failed to open gcrypt cipher by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + K2H_Free(decryptdata); + return NULL; + } + + // set symmetric key + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setkey(gcryCipherHd, key, GCRY_PBKDF2_KEY_LENGTH))){ + ERR_K2HPRN("Failed to set symmetric key by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(decryptdata); + return NULL; + } + + // set IV + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setiv(gcryCipherHd, iv, K2H_ENCRYPT_IV_LENGTH))){ + ERR_K2HPRN("Failed to set IV by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(decryptdata); + return NULL; + } + + // do decrypt + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_decrypt(gcryCipherHd, decryptdata, decbodylen, encbodypos, encbodylen))){ + ERR_K2HPRN("Failed to AES256 CBC decrypt by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(decryptdata); + return NULL; + } + + // destroy context + gcry_cipher_close(gcryCipherHd); + + // remove padding data + if(!k2h_pcks7_remove_padding(decryptdata, decbodylen)){ + K2H_Free(decryptdata); + return NULL; + } + + // length + declen = static_cast(decbodylen); + + return decryptdata; +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF2(PCKS#5 v2) +//--------------------------------------------------------- +// These provide encryption of AES256 CBC PADDING and the common +// key is created with PBKDF2(PCKS#5 v2). +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// IV value: initialize vector - IV(16byte) +// Iteration: iteration count value(16byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc_pbkdf2(const char* pass, int iter, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || iter < 1 || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + int cipher = GCRY_CIPHER_AES256; + size_t cipherBlock = gcry_cipher_get_algo_blklen(cipher); + int keySize = gcry_cipher_get_algo_keylen(cipher); + gcry_cipher_hd_t gcryCipherHd; + gcry_error_t gcryError; + unsigned char* newdata; + unsigned char key[GCRY_PBKDF2_KEY_LENGTH]; + unsigned char iv[K2H_ENCRYPT_IV_LENGTH]; + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* ivpos; + unsigned char* iterpos; + unsigned char* setdatapos; + size_t encbodylen; + + // check key size + if(keySize <= 0 || GCRY_PBKDF2_KEY_LENGTH < keySize){ + ERR_K2HPRN("key size is wrong(%d)", keySize); + return NULL; + } + + // Do padding(PCKS#7) + if(NULL == (newdata = k2h_pcks7_padding(orgdata, orglen, encbodylen, cipherBlock))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // allocated for encrypted data area(size is after padding) + if(NULL == (encryptdata = reinterpret_cast(malloc(encbodylen + K2H_ENCRYPTED_DATA_EX2_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + K2H_Free(newdata); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + ivpos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // make iv and copy it + if(!k2h_generate_iv(iv, K2H_ENCRYPT_IV_LENGTH)){ // always IV size is K2H_ENCRYPT_IV_LENGTH(16). + ERR_K2HPRN("Could not make iv."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + memcpy(ivpos, iv, K2H_ENCRYPT_IV_LENGTH); // must be K2H_ENCRYPT_IV_LENGTH = 16 + iterpos = &ivpos[K2H_ENCRYPT_IV_LENGTH]; + + // copy iter + if(!k2h_copy_iteration_count(iterpos, iter, K2H_ENCRYPT_ITER_LENGTH)){ + ERR_K2HPRN("Could not save iteration count."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + setdatapos = &iterpos[K2H_ENCRYPT_ITER_LENGTH]; + + // create key from pass/salt/iteration + if(GPG_ERR_NO_ERROR != (gcryError = gcry_kdf_derive(pass, strlen(pass), GCRY_KDF_PBKDF2, GCRY_MD_SHA512, saltpos, K2H_ENCRYPT_SALT_LENGTH, iter, keySize, key))){ + ERR_K2HPRN("Failed to make PKBDF2 key from pass/salt/iteration by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // initialize + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_open(&gcryCipherHd, cipher, GCRY_CIPHER_MODE_CBC, 0))){ + ERR_K2HPRN("Failed to open gcrypt cipher by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // set symmetric key + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setkey(gcryCipherHd, key, keySize))){ + ERR_K2HPRN("Failed to set symmetric key by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // set IV + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setiv(gcryCipherHd, iv, K2H_ENCRYPT_IV_LENGTH))){ + ERR_K2HPRN("Failed to set IV by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // do encrypt + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_encrypt(gcryCipherHd, setdatapos, static_cast(encbodylen), newdata, static_cast(encbodylen)))){ + ERR_K2HPRN("Failed to AES256 CBC encrypt by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + K2H_Free(newdata); + + // destroy context + gcry_cipher_close(gcryCipherHd); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH + encbodylen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc_pbkdf2(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + int cipher = GCRY_CIPHER_AES256; + int keySize = gcry_cipher_get_algo_keylen(cipher); + gcry_cipher_hd_t gcryCipherHd; + gcry_error_t gcryError; + unsigned char key[GCRY_PBKDF2_KEY_LENGTH]; + unsigned char* decryptdata; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* iv = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + const unsigned char* iterpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH)); + int decbodylen = encbodylen; + int iter; + + // check key size + if(keySize <= 0 || GCRY_PBKDF2_KEY_LENGTH < keySize){ + ERR_K2HPRN("key size is wrong(%d)", keySize); + return NULL; + } + + // get iteration count + if(-1 == (iter = k2h_get_iteration_count(iterpos))){ + ERR_K2HPRN("Could not get iteration count."); + return NULL; + } + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(decbodylen)))){ // decbodylen < encbodylen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // create key from pass/salt/iteration + if(GPG_ERR_NO_ERROR != (gcryError = gcry_kdf_derive(pass, strlen(pass), GCRY_KDF_PBKDF2, GCRY_MD_SHA512, saltpos, K2H_ENCRYPT_SALT_LENGTH, iter, keySize, key))){ + ERR_K2HPRN("Failed to make PKBDF2 key from pass/salt/iteration by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + K2H_Free(decryptdata); + return NULL; + } + + // initialize + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_open(&gcryCipherHd, cipher, GCRY_CIPHER_MODE_CBC, 0))){ + ERR_K2HPRN("Failed to open gcrypt cipher by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + K2H_Free(decryptdata); + return NULL; + } + + // set symmetric key + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setkey(gcryCipherHd, key, keySize))){ + ERR_K2HPRN("Failed to set symmetric key by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(decryptdata); + return NULL; + } + + // set IV + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_setiv(gcryCipherHd, iv, K2H_ENCRYPT_IV_LENGTH))){ + ERR_K2HPRN("Failed to set IV by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(decryptdata); + return NULL; + } + + // do decrypt + if(GPG_ERR_NO_ERROR != (gcryError = gcry_cipher_decrypt(gcryCipherHd, decryptdata, decbodylen, encbodypos, encbodylen))){ + ERR_K2HPRN("Failed to AES256 CBC decrypt by %s/%s", gcry_strsource(gcryError), gcry_strerror(gcryError)); + gcry_cipher_close(gcryCipherHd); + K2H_Free(decryptdata); + return NULL; + } + + // destroy context + gcry_cipher_close(gcryCipherHd); + + // remove padding data + if(!k2h_pcks7_remove_padding(decryptdata, decbodylen)){ + K2H_Free(decryptdata); + return NULL; + } + + // length + declen = static_cast(decbodylen); + + return decryptdata; +} + +/* + * VIM modelines + * + * vim:set ts=4 fenc=utf-8: + */ diff --git a/lib/k2hcryptnettle.cc b/lib/k2hcryptnettle.cc new file mode 100644 index 0000000..aebe7a7 --- /dev/null +++ b/lib/k2hcryptnettle.cc @@ -0,0 +1,514 @@ +/* + * K2HASH + * + * Copyright 2018 Yahoo Japan Corporation. + * + * K2HASH is key-valuew store base libraries. + * K2HASH is made for the purpose of the construction of + * original KVS system and the offer of the library. + * The characteristic is this KVS library which Key can + * layer. And can support multi-processing and multi-thread, + * and is provided safely as available KVS. + * + * For the full copyright and license information, please view + * the license file that was distributed with this source code. + * + * AUTHOR: Takeshi Nakatani + * CREATE: Mon May 7 2018 + * REVISION: + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "k2hcommon.h" +#include "k2hcryptcommon.h" +#include "k2hutil.h" +#include "k2hdbg.h" + +using namespace std; + +//--------------------------------------------------------- +// Structure for CBC +//--------------------------------------------------------- +struct local_cbc_ctx CBC_CTX(struct aes256_ctx, AES_BLOCK_SIZE); // IV size is as same as AES_BLOCK_SIZE = K2H_ENCRYPT_IV_LENGTH(16) + +//--------------------------------------------------------- +// Get crypt library name +//--------------------------------------------------------- +const char* k2h_crypt_lib_name(void) +{ + static const char name[] = "nettle"; + return name; +} + +//--------------------------------------------------------- +// Initialize/Terminate +//--------------------------------------------------------- +bool k2h_crypt_lib_initialize(void) +{ + return true; +} + +bool k2h_crypt_lib_terminate(void) +{ + return true; +} + +//--------------------------------------------------------- +// MD5 +//--------------------------------------------------------- +string to_md5_string(const char* str) +{ + struct md5_ctx ctx_md5; + unsigned char md5hex[MD5_DIGEST_SIZE]; + + // md5 + md5_init(&ctx_md5); + for(size_t length = strlen(str); !ISEMPTYSTR(str) && 0 < length; length -= min(length, K2H_CVT_MD_PARTSIZE), str = &str[min(length, K2H_CVT_MD_PARTSIZE)]){ + md5_update(&ctx_md5, min(length, K2H_CVT_MD_PARTSIZE), reinterpret_cast(str)); + } + md5_digest(&ctx_md5, MD5_DIGEST_SIZE, md5hex); + + // base64 + return to_base64(md5hex, MD5_DIGEST_SIZE); +} + +//--------------------------------------------------------- +// SHA256 +//--------------------------------------------------------- +string to_sha256_string(const unsigned char* bin, size_t length) +{ + struct sha256_ctx ctx_sha256; + unsigned char buf[K2H_CVT_MD_PARTSIZE]; + unsigned int digest_len = SHA256_DIGEST_SIZE; + unsigned char sha256hex[SHA256_DIGEST_SIZE]; + + memset(buf, 0, K2H_CVT_MD_PARTSIZE); + + // sha256 + sha256_init(&ctx_sha256); + + for(size_t onelength = 0; 0 < length; length -= onelength, bin = &bin[onelength]){ + onelength = min(length, K2H_CVT_MD_PARTSIZE); + sha256_update(&ctx_sha256, onelength, bin); + memset(buf, 0, K2H_CVT_MD_PARTSIZE); + } + sha256_digest(&ctx_sha256, digest_len, sha256hex); + + // base64 + return to_base64(sha256hex, static_cast(digest_len)); +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF1(PCKS#5 v1.5) +//--------------------------------------------------------- +// +// CBC_ENCRYPT is not padding for CBC, then we padding as PCKS#7 here. +// +static unsigned char* k2h_pcks7_padding(const unsigned char* orgdata, size_t orglen, size_t& padlen, size_t blocksize) +{ + // [NOTE] + // If original length is as same as blocksize, adding blocksize. + // + padlen = ((orglen / blocksize) * blocksize) + blocksize; + + unsigned char data = static_cast(padlen - orglen) & 0xff; + unsigned char* paddata; + if(NULL == (paddata = reinterpret_cast(malloc(padlen)))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + memcpy(paddata, orgdata, orglen); + for(size_t fill = 0; (fill + orglen) < padlen; ++fill){ + paddata[orglen + fill] = data; + } + return paddata; +} + +// +// [NOTE] +// CBC_DECRYPT is not padding for CBC, then we padding as PCKS#7 here. +// +bool k2h_pcks7_remove_padding(const unsigned char* data, int& length) +{ + if(!data || length <= 0){ + ERR_K2HPRN("parameters are wrong."); + return false; + } + + unsigned char lastdata = data[length - 1]; + if(length < static_cast(lastdata)){ + ERR_K2HPRN("padding data length is over all data length."); + return false; + } + length = length - static_cast(lastdata); + + return true; +} + +// +// Create Key and IV for PBKDF1(PCKS#5 v1.5) +// +// [NOTE] +// This function calculates and returns Key and IV value according to the +// argument. nettle calculates Key and IV values as PCKS#5 v2(PBKDF2) and +// uses those randomly for AES256. +// However, K2HASH was originally implemented only in OpenSSL, using the +// EVP_BytesToKey function which is not recommended. +// That function calculates Key and IV value from salt and returns those values +// as PCKS#5 v1.5(PBKDF1). Therefore, we also use those Key and IV values +// calculated equivalently from salt for nettle and implement so that data created +// with K2HASH of OpenSSL version can be read out. +// Thus, this function mimics a part of OpenSSL EVP_BytesToKey(PBKDF1) for nettle. +// In the future, we will move to PCKS#5 v2(PBKDF2) soon. +// For future encryption, this function will no longer be used.(For decoding, +// it is used for compatibility, but it will not be supported for a long time.) +// +bool create_pbkdf1_key_iv(const unsigned char* pSalt, const unsigned char* pPass, size_t lenPass, unsigned int iteratecnt, unsigned char* pKey, unsigned int lenCipherKey, unsigned char* pIV, int lenIV) +{ + bool result = false; + struct md5_ctx md5ctx; + unsigned char md5buf[MD5_DIGEST_SIZE * 4]; // == 64(for mergin) + unsigned int md5len = MD5_DIGEST_SIZE; // == 16 + bool is_first_loop; + unsigned int cnt; + + if(!pSalt || !pPass || lenPass <= 0 || !pKey || lenCipherKey == 0 || !pIV || lenIV <= 0){ + ERR_K2HPRN("parameters are wrong."); + return result; + } + + for(result = true, is_first_loop = true; true; is_first_loop = false){ + md5_init(&md5ctx); + + if(!is_first_loop){ + md5_update(&md5ctx, md5len, &(md5buf[0])); + } + md5_update(&md5ctx, lenPass, pPass); + md5_update(&md5ctx, K2H_ENCRYPT_SALT_LENGTH, pSalt); // K2H_ENCRYPT_SALT_LENGTH = 8 + md5_digest(&md5ctx, md5len, md5buf); + + for(cnt = 1; cnt < iteratecnt; ++cnt){ + md5_update(&md5ctx, md5len, &(md5buf[0])); + md5_digest(&md5ctx, md5len, md5buf); + } + if(!result){ + break; + } + + for(cnt = 0; 0 < lenCipherKey && cnt != md5len; --lenCipherKey, ++cnt){ + *(pKey++) = md5buf[cnt]; + } + for(; 0 < lenIV && cnt != md5len; --lenIV, ++cnt){ + *(pIV++) = md5buf[cnt]; + } + + if(0 == lenCipherKey && 0 == lenIV){ + break; + } + } + return result; +} + +// These provide encryption of AES256 CBC PADDING. +// However, the common key is created with PBKDF1(PCKS#5 v1.5). +// These functions are the old type of encryption with K2HASH, +// and now the type of PBKDF2 is used. +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc(const char* pass, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + struct local_cbc_ctx aes256_cbc_ctx; + unsigned char key[AES256_KEY_SIZE]; // key size is 32 + unsigned char iv[AES_BLOCK_SIZE]; // AES_BLOCK_SIZE = K2H_ENCRYPT_IV_LENGTH(16) + unsigned char* newdata; + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* setdatapos; + size_t encbodylen = 0; + + // Do padding(PCKS#7) + if(NULL == (newdata = k2h_pcks7_padding(orgdata, orglen, encbodylen, AES_BLOCK_SIZE))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // allocated for encrypted data area + if(NULL == (encryptdata = reinterpret_cast(malloc(encbodylen + K2H_ENCRYPTED_DATA_EX_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + K2H_Free(newdata); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + setdatapos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // + // generate PBKDF1 key and iv + // + if(!create_pbkdf1_key_iv(saltpos, reinterpret_cast(pass), strlen(pass), 1, key, AES256_KEY_SIZE, iv, AES_BLOCK_SIZE)){ + ERR_K2HPRN("could not create IV."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + + // Context + aes256_set_encrypt_key(&(aes256_cbc_ctx.ctx), key); + CBC_SET_IV(&aes256_cbc_ctx, iv); + + // do encrypt + CBC_ENCRYPT(&aes256_cbc_ctx, aes256_encrypt, encbodylen, setdatapos, newdata); + + K2H_Free(newdata); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + encbodylen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + struct local_cbc_ctx aes256_cbc_ctx; + unsigned char key[AES256_KEY_SIZE]; // key size is 32 + unsigned char iv[AES_BLOCK_SIZE]; // AES_BLOCK_SIZE = K2H_ENCRYPT_IV_LENGTH(16) + unsigned char* decryptdata; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH)); + int decbodylen = encbodylen; + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(decbodylen)))){ // decbodylen < encbodylen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // + // generate PBKDF1 key and iv + // + if(!create_pbkdf1_key_iv(saltpos, reinterpret_cast(pass), strlen(pass), 1, key, AES256_KEY_SIZE, iv, AES_BLOCK_SIZE)){ + ERR_K2HPRN("could not create IV."); + K2H_Free(decryptdata); + return NULL; + } + + // Context + aes256_set_decrypt_key(&(aes256_cbc_ctx.ctx), key); + CBC_SET_IV(&aes256_cbc_ctx, iv); + + // do encrypt + CBC_DECRYPT(&aes256_cbc_ctx, aes256_decrypt, decbodylen, decryptdata, encbodypos); + + // remove padding data + if(!k2h_pcks7_remove_padding(decryptdata, decbodylen)){ + K2H_Free(decryptdata); + return NULL; + } + + // length + declen = static_cast(decbodylen); + + return decryptdata; +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF2(PCKS#5 v2) +//--------------------------------------------------------- +// These provide encryption of AES256 CBC PADDING and the common +// key is created with PBKDF2(PCKS#5 v2). +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// IV value: initialize vector - IV(16byte) +// Iteration: iteration count value(16byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc_pbkdf2(const char* pass, int iter, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || iter < 1 || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + struct hmac_sha512_ctx sha512_ctx; + struct local_cbc_ctx aes256_cbc_ctx; + unsigned char key[AES256_KEY_SIZE]; // key size is 32 + unsigned char iv[AES_BLOCK_SIZE]; // AES_BLOCK_SIZE = K2H_ENCRYPT_IV_LENGTH(16) + unsigned char* newdata; + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* ivpos; + unsigned char* iterpos; + unsigned char* setdatapos; + size_t encbodylen = 0; + + // Do padding(PCKS#7) + if(NULL == (newdata = k2h_pcks7_padding(orgdata, orglen, encbodylen, AES_BLOCK_SIZE))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // allocated for encrypted data area + if(NULL == (encryptdata = reinterpret_cast(malloc(encbodylen + K2H_ENCRYPTED_DATA_EX2_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + K2H_Free(newdata); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + ivpos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // make iv and copy it + if(!k2h_generate_iv(iv, AES_BLOCK_SIZE)){ // always IV size is AES_BLOCK_SIZE = K2H_ENCRYPT_IV_LENGTH(16). + ERR_K2HPRN("Could not make iv."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + memcpy(ivpos, iv, K2H_ENCRYPT_IV_LENGTH); // must be K2H_ENCRYPT_IV_LENGTH = AES_BLOCK_SIZE(16) + iterpos = &ivpos[K2H_ENCRYPT_IV_LENGTH]; + + // copy iter + if(!k2h_copy_iteration_count(iterpos, iter, K2H_ENCRYPT_ITER_LENGTH)){ + ERR_K2HPRN("Could not save iteration count."); + K2H_Free(encryptdata); + K2H_Free(newdata); + return NULL; + } + setdatapos = &iterpos[K2H_ENCRYPT_ITER_LENGTH]; + + // make key + hmac_sha512_set_key(&sha512_ctx, strlen(pass), reinterpret_cast(pass)); + PBKDF2(&sha512_ctx, hmac_sha512_update, hmac_sha512_digest, SHA512_DIGEST_SIZE, iter, K2H_ENCRYPT_SALT_LENGTH, saltpos, AES256_KEY_SIZE, key); + + // Context + aes256_set_encrypt_key(&(aes256_cbc_ctx.ctx), key); + CBC_SET_IV(&aes256_cbc_ctx, iv); + + // do encrypt + CBC_ENCRYPT(&aes256_cbc_ctx, aes256_encrypt, encbodylen, setdatapos, newdata); + + K2H_Free(newdata); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH + encbodylen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc_pbkdf2(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + struct hmac_sha512_ctx sha512_ctx; + struct local_cbc_ctx aes256_cbc_ctx; + unsigned char key[AES256_KEY_SIZE]; // key size is 32 + unsigned char* decryptdata; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* iv = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + const unsigned char* iterpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH)); + int decbodylen = encbodylen; + int iter; + + // get iteration count + if(-1 == (iter = k2h_get_iteration_count(iterpos))){ + ERR_K2HPRN("Could not get iteration count."); + return NULL; + } + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(encbodylen)))){ // decbodylen < encbodylen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // make key + hmac_sha512_set_key(&sha512_ctx, strlen(pass), reinterpret_cast(pass)); + PBKDF2(&sha512_ctx, hmac_sha512_update, hmac_sha512_digest, SHA512_DIGEST_SIZE, iter, K2H_ENCRYPT_SALT_LENGTH, saltpos, AES256_KEY_SIZE, key); + + // Context + aes256_set_decrypt_key(&(aes256_cbc_ctx.ctx), key); + CBC_SET_IV(&aes256_cbc_ctx, iv); + + // do encrypt + CBC_DECRYPT(&aes256_cbc_ctx, aes256_decrypt, decbodylen, decryptdata, encbodypos); + + // remove padding data + if(!k2h_pcks7_remove_padding(decryptdata, decbodylen)){ + K2H_Free(decryptdata); + return NULL; + } + + // length + declen = static_cast(decbodylen); + + return decryptdata; +} + +/* + * VIM modelines + * + * vim:set ts=4 fenc=utf-8: + */ diff --git a/lib/k2hcryptnss.cc b/lib/k2hcryptnss.cc new file mode 100644 index 0000000..473f533 --- /dev/null +++ b/lib/k2hcryptnss.cc @@ -0,0 +1,1003 @@ +/* + * K2HASH + * + * Copyright 2018 Yahoo Japan Corporation. + * + * K2HASH is key-valuew store base libraries. + * K2HASH is made for the purpose of the construction of + * original KVS system and the offer of the library. + * The characteristic is this KVS library which Key can + * layer. And can support multi-processing and multi-thread, + * and is provided safely as available KVS. + * + * For the full copyright and license information, please view + * the license file that was distributed with this source code. + * + * AUTHOR: Takeshi Nakatani + * CREATE: Mon May 7 2018 + * REVISION: + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "k2hcommon.h" +#include "k2hcryptcommon.h" +#include "k2hutil.h" +#include "k2hdbg.h" + +using namespace std; + +//--------------------------------------------------------- +// Get crypt library name +//--------------------------------------------------------- +const char* k2h_crypt_lib_name(void) +{ + static const char name[] = "NSS"; + return name; +} + +//--------------------------------------------------------- +// Initialize/Terminate +//--------------------------------------------------------- +bool k2h_crypt_lib_initialize(void) +{ + if(NSS_IsInitialized()){ + WAN_K2HPRN("Already initialized NSS"); + return false; + } + if(SECSuccess != NSS_NoDB_Init(NULL)){ + ERR_K2HPRN("Failed NSS_NoDB_Init"); + return false; + } + return true; +} + +bool k2h_crypt_lib_terminate(void) +{ + if(SECSuccess != NSS_Shutdown()){ + ERR_K2HPRN("Failed NSS_Shutdown, probabry anyone handles NSS context or objects."); + return false; + } + return true; +} + +//--------------------------------------------------------- +// MD5 +//--------------------------------------------------------- +string to_md5_string(const char* str) +{ + PK11Context* md5ctx; + unsigned char md5hex[MD5_LENGTH]; + unsigned int md5outlen; + + // md5 + if(NULL == (md5ctx = PK11_CreateDigestContext(SEC_OID_MD5))){ + ERR_K2HPRN("Could not create context for md5."); + return string(""); + } + + if(SECSuccess != PK11_DigestBegin(md5ctx)){ + ERR_K2HPRN("Could not begin digest for MD5."); + PK11_DestroyContext(md5ctx, PR_TRUE); + return string(""); + } + for(size_t length = strlen(str); !ISEMPTYSTR(str) && 0 < length; length -= min(length, K2H_CVT_MD_PARTSIZE), str = &str[min(length, K2H_CVT_MD_PARTSIZE)]){ + PK11_DigestOp(md5ctx, reinterpret_cast(str), min(length, K2H_CVT_MD_PARTSIZE)); + } + PK11_DigestFinal(md5ctx, md5hex, &md5outlen, MD5_LENGTH); + PK11_DestroyContext(md5ctx, PR_TRUE); + + // base64 + return to_base64(md5hex, MD5_LENGTH); +} + +//--------------------------------------------------------- +// SHA256 +//--------------------------------------------------------- +string to_sha256_string(const unsigned char* bin, size_t length) +{ + PK11Context* sha256ctx; + unsigned int sha256outlen; + unsigned int digest_len = SHA256_LENGTH; + unsigned char sha256hex[SHA256_LENGTH]; + + // sha256 + if(NULL == (sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256))){ + ERR_K2HPRN("Could not create context for sha256."); + return string(""); + } + + if(SECSuccess != PK11_DigestBegin(sha256ctx)){ + ERR_K2HPRN("Could not begin digest for sha256."); + PK11_DestroyContext(sha256ctx, PR_TRUE); + return string(""); + } + for(size_t onelength = 0; 0 < length; length -= onelength, bin = &bin[onelength]){ + onelength = min(length, K2H_CVT_MD_PARTSIZE); + if(SECSuccess != PK11_DigestOp(sha256ctx, bin, onelength)){ + ERR_K2HPRN("Could not update context for sha256 digest."); + PK11_DestroyContext(sha256ctx, PR_TRUE); + return string(""); + } + } + if(SECSuccess != PK11_DigestFinal(sha256ctx, sha256hex, &sha256outlen, digest_len)){ + ERR_K2HPRN("Could not final context for sha256 digest."); + PK11_DestroyContext(sha256ctx, PR_TRUE); + return string(""); + } + + PK11_DestroyContext(sha256ctx, PR_TRUE); + + // base64 + return to_base64(sha256hex, static_cast(digest_len)); +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF1(PCKS#5 v1.5) +//--------------------------------------------------------- +// +// Create Key and IV for PBKDF1(PCKS#5 v1.5) +// +// [NOTE] +// This function calculates and returns Key and IV value according to the +// argument. NSS calculates Key and IV values as PCKS#5 v2(PBKDF2) and +// uses those randomly for AES256. +// However, K2HASH was originally implemented only in OpenSSL, using the +// EVP_BytesToKey function which is not recommended. +// That function calculates Key and IV value from salt and returns those values +// as PCKS#5 v1.5(PBKDF1). Therefore, we also use those Key and IV values +// calculated equivalently from salt for NSS and implement so that data created +// with K2HASH of OpenSSL version can be read out. +// Thus, this function mimics a part of OpenSSL EVP_BytesToKey(PBKDF1) for NSS. +// In the future, we will move to PCKS#5 v2(PBKDF2) soon. +// For future encryption, this function will no longer be used.(For decoding, +// it is used for compatibility, but it will not be supported for a long time.) +// +static bool create_pbkdf1_key_iv(const unsigned char* pSalt, const unsigned char* pPass, size_t lenPass, unsigned int iteratecnt, unsigned char* pKey, unsigned int lenCipherKey, unsigned char* pIV, int lenIV) +{ + bool result = false; + PK11Context* md5ctx; + unsigned char md5buf[MD5_BLOCK_LENGTH]; // == SHA512_LENGTH + unsigned int md5len = 0; + bool is_first_loop; + unsigned int cnt; + + if(!pSalt || !pPass || lenPass <= 0 || !pKey || lenCipherKey == 0 || !pIV || lenIV <= 0){ + ERR_K2HPRN("parameters are wrong."); + return result; + } + if(NULL == (md5ctx = PK11_CreateDigestContext(SEC_OID_MD5))){ // MD5 + ERR_K2HPRN("Could not create digest context."); + return result; + } + + for(result = true, is_first_loop = true; true; is_first_loop = false){ + if(SECSuccess != PK11_DigestBegin(md5ctx)){ + PK11_DestroyContext(md5ctx, PR_TRUE); + result = false; + break; + } + + if(!is_first_loop){ + PK11_DigestOp(md5ctx, &(md5buf[0]), md5len); + } + PK11_DigestOp(md5ctx, pPass, lenPass); + PK11_DigestOp(md5ctx, pSalt, K2H_ENCRYPT_SALT_LENGTH); // K2H_ENCRYPT_SALT_LENGTH = 8 + PK11_DigestFinal(md5ctx, &(md5buf[0]), &md5len, sizeof(md5buf)); + + for(cnt = 1; cnt < iteratecnt; ++cnt){ + PK11_DigestOp(md5ctx, &(md5buf[0]), md5len); + PK11_DigestFinal(md5ctx, &(md5buf[0]), &md5len, sizeof(md5buf)); + } + if(!result){ + break; + } + + for(cnt = 0; 0 < lenCipherKey && cnt != md5len; --lenCipherKey, ++cnt){ + *(pKey++) = md5buf[cnt]; + } + for(; 0 < lenIV && cnt != md5len; --lenIV, ++cnt){ + *(pIV++) = md5buf[cnt]; + } + + if(0 == lenCipherKey && 0 == lenIV){ + break; + } + } + PK11_DestroyContext(md5ctx, PR_TRUE); + return result; +} + +// These provide encryption of AES256 CBC PADDING. +// However, the common key is created with PBKDF1(PCKS#5 v1.5). +// These functions are the old type of encryption with K2HASH, +// and now the type of PBKDF2 is used. +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc(const char* pass, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* setdatapos; + int encbodylen = 0; + unsigned int enclastlen = 0; + + // allocated for encrypted data area + if(NULL == (encryptdata = reinterpret_cast(malloc(orglen + K2H_ENCRYPTED_DATA_EX_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + return NULL; + } + setdatapos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // SEC OID Tag & Cipher + SECOidTag cipherOid = SEC_OID_AES_256_CBC; + CK_MECHANISM_TYPE cipherMech; + if(CKM_INVALID_MECHANISM == (cipherMech = PK11_AlgtagToMechanism(cipherOid))){ + ERR_K2HPRN("could not get SEC_OID_AES_256_CBC Cipher."); + K2H_Free(encryptdata); + return NULL; + }else{ + CK_MECHANISM_TYPE paddedMech = PK11_GetPadMechanism(cipherMech); + if(CKM_INVALID_MECHANISM == paddedMech || cipherMech == paddedMech){ + ERR_K2HPRN("could not get Padding SEC_OID_AES_256_CBC Cipher."); + K2H_Free(encryptdata); + return NULL; + } + cipherMech = paddedMech; + } + + // Passphrase(key) and Salt Items + SECItem keyItem; + keyItem.type = siBuffer; + keyItem.data = const_cast(reinterpret_cast(pass)); + keyItem.len = strlen(pass); + + SECItem saltItem; + saltItem.type = siBuffer; + saltItem.data = saltpos; + saltItem.len = K2H_ENCRYPT_SALT_LENGTH; + + // Algorithm ID + SECAlgorithmID* algid; + if(NULL == (algid = PK11_CreatePBEV2AlgorithmID(cipherOid, cipherOid, SEC_OID_HMAC_SHA1, 32, 1, &saltItem))){ + ERR_K2HPRN("could not get Algorithm ID."); + K2H_Free(encryptdata); + return NULL; + } + + // Slot + PK11SlotInfo* Slot; + if(NULL == (Slot = PK11_GetBestSlot(cipherMech, NULL))){ + ERR_K2HPRN("could not get PKCS#11 slot."); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // Symmetric key + // + // [NOTE] + // This key acquisition is done solely to obtain the key length for the Cipher. + // This key is temporary key to the last, it is released immediately. + // + PK11SymKey* pKey; + if(NULL == (pKey = PK11_PBEKeyGen(Slot, algid, &keyItem, PR_FALSE, NULL))){ + ERR_K2HPRN("could not get Symmetric Key."); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // key and IV size + unsigned int keySize = PK11_GetKeyLength(pKey); + int ivSize = PK11_GetIVLength(cipherMech); + if(ivSize <= 0 || K2H_ENCRYPT_IV_LENGTH < ivSize){ + ERR_K2HPRN("IV size is wrong(%d)", ivSize); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // + // Get key and IV(for PBKDF1) + // + unsigned char key[32]; // always key size is 32 on this case + unsigned char iv[K2H_ENCRYPT_IV_LENGTH]; // always IV size is 16 == ivSize on this case, this likes EVP_MAX_IV_LENGTH in OpenSSL. + if(!create_pbkdf1_key_iv(saltpos, reinterpret_cast(pass), strlen(pass), 1, key, keySize, iv, ivSize)){ + ERR_K2HPRN("could not create IV."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // switch new key + SECItem newkeyItem; + newkeyItem.type = siBuffer; + newkeyItem.data = key; + newkeyItem.len = keySize; + + PK11SymKey* pNewKey; + if(NULL == (pNewKey = PK11_ImportSymKey(Slot, cipherMech, PK11_OriginUnwrap, CKA_WRAP, &newkeyItem, NULL))){ + ERR_K2HPRN("could not get new Symmetric Key."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + PK11_FreeSymKey(pKey); + pKey = pNewKey; + + // Make SEC parameter + SECItem ivItem; + ivItem.type = siBuffer; + ivItem.data = iv; + ivItem.len = ivSize; + + // SEC Parameter + SECItem* SecParam; + if(NULL == (SecParam = PK11_ParamFromIV(cipherMech, &ivItem))){ + ERR_K2HPRN("could not get SEC Parameter."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // Context + PK11Context* Context; + if(NULL == (Context = PK11_CreateContextBySymKey(cipherMech, CKA_ENCRYPT, pKey, SecParam))){ + ERR_K2HPRN("could not create Context by PK11_CreateContextBySymKey."); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // do encrypt + if(SECSuccess != PK11_CipherOp(Context, setdatapos, &encbodylen, orglen + K2H_ENCRYPT_MAX_PADDING_LENGTH, orgdata, orglen)){ + ERR_K2HPRN("Failed to AES256 CBC encrypt."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // last encrypt + if(SECSuccess != PK11_DigestFinal(Context, (setdatapos + encbodylen), &enclastlen, ((orglen + K2H_ENCRYPT_MAX_PADDING_LENGTH) - encbodylen))){ + ERR_K2HPRN("Failed to AES256 CBC encrypt finally."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + encbodylen + enclastlen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + unsigned char* decryptdata; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH)); + int decbodylen = 0; + unsigned int declastlen = 0; + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(encbodylen)))){ // decbodylen < encbodylen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // SEC OID Tag & Cipher + SECOidTag cipherOid = SEC_OID_AES_256_CBC; + CK_MECHANISM_TYPE cipherMech; + if(CKM_INVALID_MECHANISM == (cipherMech = PK11_AlgtagToMechanism(cipherOid))){ + ERR_K2HPRN("could not get SEC_OID_AES_256_CBC Cipher."); + K2H_Free(decryptdata); + return NULL; + }else{ + CK_MECHANISM_TYPE paddedMech = PK11_GetPadMechanism(cipherMech); + if(CKM_INVALID_MECHANISM == paddedMech || cipherMech == paddedMech){ + ERR_K2HPRN("could not get Padding SEC_OID_AES_256_CBC Cipher."); + K2H_Free(decryptdata); + return NULL; + } + cipherMech = paddedMech; + } + + // Passphrase(key) and Salt Items + SECItem keyItem; + keyItem.type = siBuffer; + keyItem.data = const_cast(reinterpret_cast(pass)); + keyItem.len = strlen(pass); + + SECItem saltItem; + saltItem.type = siBuffer; + saltItem.data = const_cast(saltpos); + saltItem.len = K2H_ENCRYPT_SALT_LENGTH; + + // Algorithm ID + SECAlgorithmID* algid; + if(NULL == (algid = PK11_CreatePBEV2AlgorithmID(cipherOid, cipherOid, SEC_OID_HMAC_SHA1, 32, 1, &saltItem))){ + ERR_K2HPRN("could not get Algorithm ID."); + K2H_Free(decryptdata); + return NULL; + } + + // Slot + PK11SlotInfo* Slot; + if(NULL == (Slot = PK11_GetBestSlot(cipherMech, NULL))){ + ERR_K2HPRN("could not get PKCS#11 slot."); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // Symmetric key + // + // [NOTE] + // This key acquisition is done solely to obtain the key length for the Cipher. + // This key is temporary key to the last, it is released immediately. + // + PK11SymKey* pKey; + if(NULL == (pKey = PK11_PBEKeyGen(Slot, algid, &keyItem, PR_FALSE, NULL))){ + ERR_K2HPRN("could not get Symmetric Key."); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // key and IV size + unsigned int keySize = PK11_GetKeyLength(pKey); + int ivSize = PK11_GetIVLength(cipherMech); + if(ivSize <= 0 || K2H_ENCRYPT_IV_LENGTH < ivSize){ + ERR_K2HPRN("IV size is wrong(0)"); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // + // Get key and IV(for PBKDF1) + // + unsigned char key[32]; // always key size is 32 on this case + unsigned char iv[K2H_ENCRYPT_IV_LENGTH]; // always IV size is 16 == ivSize on this case, this likes EVP_MAX_IV_LENGTH in OpenSSL. + if(!create_pbkdf1_key_iv(saltpos, reinterpret_cast(pass), strlen(pass), 1, key, keySize, iv, ivSize)){ + ERR_K2HPRN("could not create IV."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // switch new key + SECItem newkeyItem; + newkeyItem.type = siBuffer; + newkeyItem.data = key; + newkeyItem.len = keySize; + + PK11SymKey* pNewKey; + if(NULL == (pNewKey = PK11_ImportSymKey(Slot, cipherMech, PK11_OriginUnwrap, CKA_WRAP, &newkeyItem, NULL))){ + ERR_K2HPRN("could not get new Symmetric Key."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + PK11_FreeSymKey(pKey); + pKey = pNewKey; + + // Make SEC param + SECItem ivItem; + ivItem.type = siBuffer; + ivItem.data = iv; + ivItem.len = ivSize; + + // SEC Parameter + SECItem* SecParam; + if(NULL == (SecParam = PK11_ParamFromIV(cipherMech, &ivItem))){ + ERR_K2HPRN("could not get SEC Parameter."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // Context + PK11Context* Context; + if(NULL == (Context = PK11_CreateContextBySymKey(cipherMech, CKA_DECRYPT, pKey, SecParam))){ + ERR_K2HPRN("could not create Context by PK11_CreateContextBySymKey."); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // do decrypt + if(SECSuccess != PK11_CipherOp(Context, decryptdata, &decbodylen, encbodylen, encbodypos, encbodylen)){ + ERR_K2HPRN("Failed to AES256 CBC decrypt."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // last decrypt + if(SECSuccess != PK11_DigestFinal(Context, (decryptdata + decbodylen), &declastlen, (encbodylen - decbodylen))){ + ERR_K2HPRN("Failed to AES256 CBC decrypt finally."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + + // length + declen = static_cast(decbodylen + declastlen); + + return decryptdata; +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF2(PCKS#5 v2) +//--------------------------------------------------------- +// These provide encryption of AES256 CBC PADDING and the common +// key is created with PBKDF2(PCKS#5 v2). +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// IV value: initialize vector - IV(16byte) +// Iteration: iteration count value(16byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc_pbkdf2(const char* pass, int iter, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || iter < 1 || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + unsigned char iv[K2H_ENCRYPT_IV_LENGTH]; // always IV size is 16 == ivSize on this case, this likes EVP_MAX_IV_LENGTH in OpenSSL. + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* ivpos; + unsigned char* iterpos; + unsigned char* setdatapos; + int encbodylen = 0; + unsigned int enclastlen = 0; + + // allocated for encrypted data area + if(NULL == (encryptdata = reinterpret_cast(malloc(orglen + K2H_ENCRYPTED_DATA_EX2_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + return NULL; + } + ivpos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // make iv and copy it + if(!k2h_generate_iv(iv, K2H_ENCRYPT_IV_LENGTH)){ // always IV size is 16 == ivSize on this case, this likes EVP_MAX_IV_LENGTH in OpenSSL. + ERR_K2HPRN("Could not make iv."); + K2H_Free(encryptdata); + return NULL; + } + memcpy(ivpos, iv, K2H_ENCRYPT_IV_LENGTH); // must be K2H_ENCRYPT_IV_LENGTH = 16 + iterpos = &ivpos[K2H_ENCRYPT_IV_LENGTH]; + + // copy iter + if(!k2h_copy_iteration_count(iterpos, iter, K2H_ENCRYPT_ITER_LENGTH)){ + ERR_K2HPRN("Could not save iteration count."); + K2H_Free(encryptdata); + return NULL; + } + setdatapos = &iterpos[K2H_ENCRYPT_ITER_LENGTH]; + + // SEC OID Tag & Cipher + SECOidTag cipherOid = SEC_OID_AES_256_CBC; + CK_MECHANISM_TYPE cipherMech; + if(CKM_INVALID_MECHANISM == (cipherMech = PK11_AlgtagToMechanism(cipherOid))){ + ERR_K2HPRN("could not get SEC_OID_AES_256_CBC Cipher."); + K2H_Free(encryptdata); + return NULL; + }else{ + CK_MECHANISM_TYPE paddedMech = PK11_GetPadMechanism(cipherMech); + if(CKM_INVALID_MECHANISM == paddedMech || cipherMech == paddedMech){ + ERR_K2HPRN("could not get Padding SEC_OID_AES_256_CBC Cipher."); + K2H_Free(encryptdata); + return NULL; + } + cipherMech = paddedMech; + } + + // Passphrase(key) and Salt Items + SECItem keyItem; + keyItem.type = siBuffer; + keyItem.data = const_cast(reinterpret_cast(pass)); + keyItem.len = strlen(pass); + + SECItem saltItem; + saltItem.type = siBuffer; + saltItem.data = saltpos; + saltItem.len = K2H_ENCRYPT_SALT_LENGTH; + + // Algorithm ID + SECAlgorithmID* algid; + if(NULL == (algid = PK11_CreatePBEV2AlgorithmID(cipherOid, cipherOid, SEC_OID_HMAC_SHA512, 32, iter, &saltItem))){ + ERR_K2HPRN("could not get Algorithm ID."); + K2H_Free(encryptdata); + return NULL; + } + + // Slot + PK11SlotInfo* Slot; + if(NULL == (Slot = PK11_GetBestSlot(cipherMech, NULL))){ + ERR_K2HPRN("could not get PKCS#11 slot."); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // Symmetric key + PK11SymKey* pKey; + if(NULL == (pKey = PK11_PBEKeyGen(Slot, algid, &keyItem, PR_FALSE, NULL))){ + ERR_K2HPRN("could not get Symmetric Key."); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + // [NOTE] + // If you need to print raw key binary, you can print it here. + // + // ex: print following as (rawKey->data and rawKey->len) + // PK11_ExtractKeyValue(pKey); + // SECItem* rawKey = PK11_GetKeyData(pKey); + // + + // get IV size + int ivSize = PK11_GetIVLength(cipherMech); + if(ivSize <= 0 || K2H_ENCRYPT_IV_LENGTH < ivSize){ + ERR_K2HPRN("IV size is wrong(%d)", ivSize); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // Make SEC Parameter + SECItem ivItem; + ivItem.type = siBuffer; + ivItem.data = iv; + ivItem.len = ivSize; + + // SEC Parameter + SECItem* SecParam; + if(NULL == (SecParam = PK11_ParamFromIV(cipherMech, &ivItem))){ + ERR_K2HPRN("could not get SEC Parameter."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // Context + PK11Context* Context; + if(NULL == (Context = PK11_CreateContextBySymKey(cipherMech, CKA_ENCRYPT, pKey, SecParam))){ + ERR_K2HPRN("could not create Context by PK11_CreateContextBySymKey."); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // do encrypt + if(SECSuccess != PK11_CipherOp(Context, setdatapos, &encbodylen, orglen + K2H_ENCRYPT_MAX_PADDING_LENGTH, orgdata, orglen)){ + ERR_K2HPRN("Failed to AES256 CBC encrypt."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + // last encrypt + if(SECSuccess != PK11_DigestFinal(Context, (setdatapos + encbodylen), &enclastlen, ((orglen + K2H_ENCRYPT_MAX_PADDING_LENGTH) - encbodylen))){ + ERR_K2HPRN("Failed to AES256 CBC encrypt finally."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(encryptdata); + return NULL; + } + + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH + encbodylen + enclastlen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc_pbkdf2(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + unsigned char* decryptdata; + int iter; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* iv = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + const unsigned char* iterpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH)); + int decbodylen = 0; + unsigned int declastlen = 0; + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(encbodylen)))){ // decbodylen < encbodylen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // get iteration count + if(-1 == (iter = k2h_get_iteration_count(iterpos))){ + ERR_K2HPRN("Could not get iteration count."); + K2H_Free(decryptdata); + return NULL; + } + + // SEC OID Tag & Cipher + SECOidTag cipherOid = SEC_OID_AES_256_CBC; + CK_MECHANISM_TYPE cipherMech; + if(CKM_INVALID_MECHANISM == (cipherMech = PK11_AlgtagToMechanism(cipherOid))){ + ERR_K2HPRN("could not get SEC_OID_AES_256_CBC Cipher."); + K2H_Free(decryptdata); + return NULL; + }else{ + CK_MECHANISM_TYPE paddedMech = PK11_GetPadMechanism(cipherMech); + if(CKM_INVALID_MECHANISM == paddedMech || cipherMech == paddedMech){ + ERR_K2HPRN("could not get Padding SEC_OID_AES_256_CBC Cipher."); + K2H_Free(decryptdata); + return NULL; + } + cipherMech = paddedMech; + } + + // Passphrase(key) and Salt Items + SECItem keyItem; + keyItem.type = siBuffer; + keyItem.data = const_cast(reinterpret_cast(pass)); + keyItem.len = strlen(pass); + + SECItem saltItem; + saltItem.type = siBuffer; + saltItem.data = const_cast(saltpos); + saltItem.len = K2H_ENCRYPT_SALT_LENGTH; + + // Algorithm ID + SECAlgorithmID* algid; + if(NULL == (algid = PK11_CreatePBEV2AlgorithmID(cipherOid, cipherOid, SEC_OID_HMAC_SHA512, 32, iter, &saltItem))){ + ERR_K2HPRN("could not get Algorithm ID."); + K2H_Free(decryptdata); + return NULL; + } + + // Slot + PK11SlotInfo* Slot; + if(NULL == (Slot = PK11_GetBestSlot(cipherMech, NULL))){ + ERR_K2HPRN("could not get PKCS#11 slot."); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // Symmetric key + PK11SymKey* pKey; + if(NULL == (pKey = PK11_PBEKeyGen(Slot, algid, &keyItem, PR_FALSE, NULL))){ + ERR_K2HPRN("could not get Symmetric Key."); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + // [NOTE] + // If you need to print raw key binary, you can print it here. + // + // ex: print following as (rawKey->data and rawKey->len) + // PK11_ExtractKeyValue(pKey); + // SECItem* rawKey = PK11_GetKeyData(pKey); + // + + // IV size + int ivSize = PK11_GetIVLength(cipherMech); + if(ivSize <= 0 || K2H_ENCRYPT_IV_LENGTH < ivSize){ + ERR_K2HPRN("IV size is wrong(%d)", ivSize); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // Make SEC Parameter + SECItem ivItem; + ivItem.type = siBuffer; + ivItem.data = const_cast(iv); + ivItem.len = ivSize; + + // SEC Parameter + SECItem* SecParam; + if(NULL == (SecParam = PK11_ParamFromIV(cipherMech, &ivItem))){ + ERR_K2HPRN("could not get SEC Parameter."); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // Context + PK11Context* Context; + if(NULL == (Context = PK11_CreateContextBySymKey(cipherMech, CKA_DECRYPT, pKey, SecParam))){ + ERR_K2HPRN("could not create Context by PK11_CreateContextBySymKey."); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // do decrypt + if(SECSuccess != PK11_CipherOp(Context, decryptdata, &decbodylen, encbodylen, encbodypos, encbodylen)){ + ERR_K2HPRN("Failed to AES256 CBC decrypt."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + // last decrypt + if(SECSuccess != PK11_DigestFinal(Context, (decryptdata + decbodylen), &declastlen, (encbodylen - decbodylen))){ + ERR_K2HPRN("Failed to AES256 CBC decrypt finally."); + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + K2H_Free(decryptdata); + return NULL; + } + + PK11_DestroyContext(Context, PR_TRUE); + SECITEM_FreeItem(SecParam, PR_TRUE); + PK11_FreeSymKey(pKey); + PK11_FreeSlot(Slot); + SECOID_DestroyAlgorithmID(algid, PR_TRUE); + + // length + declen = static_cast(decbodylen + declastlen); + + return decryptdata; +} + +/* + * VIM modelines + * + * vim:set ts=4 fenc=utf-8: + */ diff --git a/lib/k2hcryptopenssl.cc b/lib/k2hcryptopenssl.cc new file mode 100644 index 0000000..a41f062 --- /dev/null +++ b/lib/k2hcryptopenssl.cc @@ -0,0 +1,527 @@ +/* + * K2HASH + * + * Copyright 2018 Yahoo Japan Corporation. + * + * K2HASH is key-valuew store base libraries. + * K2HASH is made for the purpose of the construction of + * original KVS system and the offer of the library. + * The characteristic is this KVS library which Key can + * layer. And can support multi-processing and multi-thread, + * and is provided safely as available KVS. + * + * For the full copyright and license information, please view + * the license file that was distributed with this source code. + * + * AUTHOR: Takeshi Nakatani + * CREATE: Mon May 7 2018 + * REVISION: + * + */ + +#include +#include +#include +#include + +#include "k2hcommon.h" +#include "k2hcryptcommon.h" +#include "k2hutil.h" +#include "k2hdbg.h" + +using namespace std; + +//--------------------------------------------------------- +// Get crypt library name +//--------------------------------------------------------- +const char* k2h_crypt_lib_name(void) +{ + static const char name[] = "OpenSSL"; + return name; +} + +//--------------------------------------------------------- +// Initialize/Terminate +//--------------------------------------------------------- +bool k2h_crypt_lib_initialize(void) +{ + // [NOTE] + // Now(after 1.1.0) it does not need to call OpenSSL_add_all_algorithms() etc. + // + return true; +} + +bool k2h_crypt_lib_terminate(void) +{ + return true; +} + +//--------------------------------------------------------- +// MD5 +//--------------------------------------------------------- +string to_md5_string(const char* str) +{ + MD5_CTX md5ctx; + unsigned char md5hex[MD5_DIGEST_LENGTH]; + + // md5 + MD5_Init(&md5ctx); + for(size_t length = strlen(str); !ISEMPTYSTR(str) && 0 < length; length -= min(length, K2H_CVT_MD_PARTSIZE), str = &str[min(length, K2H_CVT_MD_PARTSIZE)]){ + MD5_Update(&md5ctx, str, min(length, K2H_CVT_MD_PARTSIZE)); + } + MD5_Final(md5hex, &md5ctx); + + // base64 + return to_base64(md5hex, MD5_DIGEST_LENGTH); +} + +//--------------------------------------------------------- +// SHA256 +//--------------------------------------------------------- +string to_sha256_string(const unsigned char* bin, size_t length) +{ + EVP_MD_CTX* sha256ctx = EVP_MD_CTX_create(); + unsigned int digest_len = SHA256_DIGEST_LENGTH; + unsigned char sha256hex[SHA256_DIGEST_LENGTH]; + + if(!sha256ctx){ + ERR_K2HPRN("Could not create context for sha256."); + return string(""); + } + EVP_MD_CTX_set_flags(sha256ctx, EVP_MD_CTX_FLAG_ONESHOT); + + // sha256 + if(1 != EVP_DigestInit_ex(sha256ctx, EVP_sha256(), NULL)){ + ERR_K2HPRN("Could not initialize context for sha256 digest."); + EVP_MD_CTX_destroy(sha256ctx); + return string(""); + } + + for(size_t onelength = 0; 0 < length; length -= onelength, bin = &bin[onelength]){ + onelength = min(length, K2H_CVT_MD_PARTSIZE); + if(1 != EVP_DigestUpdate(sha256ctx, bin, onelength)){ + ERR_K2HPRN("Could not update context for sha256 digest."); + EVP_MD_CTX_destroy(sha256ctx); + return string(""); + } + } + if(1 != EVP_DigestFinal_ex(sha256ctx, sha256hex, &digest_len)){ + ERR_K2HPRN("Could not final context for sha256 digest."); + EVP_MD_CTX_destroy(sha256ctx); + return string(""); + } + EVP_MD_CTX_destroy(sha256ctx); + + // base64 + return to_base64(sha256hex, static_cast(digest_len)); +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF1(PCKS#5 v1.5) +//--------------------------------------------------------- +// These provide encryption of AES256 CBC PADDING. +// However, the common key is created with PBKDF1(PCKS#5 v1.5). +// These functions are the old type of encryption with K2HASH, +// and now the type of PBKDF2 is used. +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc(const char* pass, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + EVP_CIPHER_CTX* cictx; + const EVP_CIPHER* cipher = EVP_aes_256_cbc(); + unsigned char key[EVP_MAX_KEY_LENGTH]; + unsigned char iv[EVP_MAX_IV_LENGTH]; + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* setdatapos; + int encbodylen = 0; + int enclastlen = 0; + + // allocated for encrypted data area + if(NULL == (encryptdata = reinterpret_cast(malloc(orglen + K2H_ENCRYPTED_DATA_EX_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + return NULL; + } + setdatapos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // make common encryption key(iteration count = 1) + if(0 == EVP_BytesToKey(cipher, EVP_md5(), saltpos, reinterpret_cast(pass), strlen(pass), 1, key, iv)){ + ERR_K2HPRN("Failed to make AES256 key from pass."); + K2H_Free(encryptdata); + return NULL; + } + + // initialize + if(NULL == (cictx = EVP_CIPHER_CTX_new())){ + ERR_K2HPRN("Failed to make EVP cipher context."); + K2H_Free(encryptdata); + return NULL; + } + + // initialize context + if(1 != EVP_EncryptInit_ex(cictx, cipher, NULL, key, iv)){ // type is normally supplied by a function such as EVP_aes_256_cbc(). + ERR_K2HPRN("Could not initialize EVP context."); + K2H_Free(encryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // do encrypt + if(1 != EVP_EncryptUpdate(cictx, setdatapos, &encbodylen, orgdata, static_cast(orglen))){ + ERR_K2HPRN("Failed to AES256 CBC encrypt."); + K2H_Free(encryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // last encrypt + if(1 != EVP_EncryptFinal_ex(cictx, &setdatapos[encbodylen], &enclastlen)){ // padding is on as default + ERR_K2HPRN("Failed to AES256 CBC encrypt finally."); + K2H_Free(encryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // destroy context + EVP_CIPHER_CTX_free(cictx); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + encbodylen + enclastlen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + EVP_CIPHER_CTX* cictx; + const EVP_CIPHER* cipher = EVP_aes_256_cbc(); + unsigned char key[EVP_MAX_KEY_LENGTH]; + unsigned char iv[EVP_MAX_IV_LENGTH]; + unsigned char* decryptdata; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH)); + int decbodylen = 0; + int declastlen = 0; + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(enclen)))){ // declen < enclen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // make common encryption key(iteration count = 1) + if(0 == EVP_BytesToKey(cipher, EVP_md5(), saltpos, reinterpret_cast(pass), strlen(pass), 1, key, iv)){ + ERR_K2HPRN("Failed to make AES256 key from pass."); + K2H_Free(decryptdata); + return NULL; + } + + // initialize + if(NULL == (cictx = EVP_CIPHER_CTX_new())){ + ERR_K2HPRN("Failed to make EVP cipher context."); + K2H_Free(decryptdata); + return NULL; + } + + // initialize context + if(1 != EVP_DecryptInit_ex(cictx, cipher, NULL, key, iv)){ // type is normally supplied by a function such as EVP_aes_256_cbc(). + ERR_K2HPRN("Could not initialize EVP context."); + K2H_Free(decryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // do decrypt + if(1 != EVP_DecryptUpdate(cictx, decryptdata, &decbodylen, encbodypos, encbodylen)){ + ERR_K2HPRN("Failed to AES256 CBC decrypt."); + K2H_Free(decryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // last decrypt + if(1 != EVP_DecryptFinal_ex(cictx, &decryptdata[decbodylen], &declastlen)){ + ERR_K2HPRN("Failed to AES256 CBC decrypt finally."); + K2H_Free(decryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // destroy context + EVP_CIPHER_CTX_free(cictx); + + // length + declen = static_cast(decbodylen + declastlen); + + return decryptdata; +} + +//--------------------------------------------------------- +// AES256 CBC PADDING by PBKDF2(PCKS#5 v2) +//--------------------------------------------------------- +// These provide encryption of AES256 CBC PADDING and the common +// key is created with PBKDF2(PCKS#5 v2). +// +// [DATA FORMAT] +// Encrypted binary data is formatted following: +// "Salted__xxxxxxxxyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzEEEE....EEEEPPPP..." +// +// SALT Prefix: start with "Salted__" with salt(8byte) +// IV value: initialize vector - IV(16byte) +// Iteration: iteration count value(16byte) +// Encrypted data: EEEE....EEEE, it is same original data length. +// Padding: Encrypted data is 16byte block(AES256 CBC), then +// max 31 bytes for padding.(== 32 bytes) +// +unsigned char* k2h_encrypt_aes256_cbc_pbkdf2(const char* pass, int iter, const unsigned char* orgdata, size_t orglen, size_t& enclen) +{ + static const char salt_prefix[] = K2H_ENCRYPT_SALT_PREFIX; + + if(ISEMPTYSTR(pass) || iter < 1 || !orgdata || 0 == orglen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + EVP_CIPHER_CTX* cictx; + const EVP_CIPHER* cipher = EVP_aes_256_cbc(); + unsigned char* key; + unsigned char iv[EVP_MAX_IV_LENGTH]; + unsigned char* encryptdata; + unsigned char* saltpos; + unsigned char* ivpos; + unsigned char* iterpos; + unsigned char* setdatapos; + int encbodylen = 0; + int enclastlen = 0; + + // allocated for encrypted data area + if(NULL == (encryptdata = reinterpret_cast(malloc(orglen + K2H_ENCRYPTED_DATA_EX2_LENGTH)))){ + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // copy salt prefix + memcpy(encryptdata, salt_prefix, K2H_ENCRYPT_SALT_PREFIX_LENGTH); + saltpos = &encryptdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + + // make salt + if(!k2h_pkcs5_salt(saltpos, K2H_ENCRYPT_SALT_LENGTH)){ + ERR_K2HPRN("Could not make salt."); + K2H_Free(encryptdata); + return NULL; + } + ivpos = &saltpos[K2H_ENCRYPT_SALT_LENGTH]; + + // make iv and copy it + if(!k2h_generate_iv(iv, EVP_MAX_IV_LENGTH)){ // EVP_CIPHER_iv_length(cipher) = EVP_MAX_IV_LENGTH + ERR_K2HPRN("Could not make iv."); + K2H_Free(encryptdata); + return NULL; + } + memcpy(ivpos, iv, K2H_ENCRYPT_IV_LENGTH); // must be K2H_ENCRYPT_IV_LENGTH = EVP_MAX_IV_LENGTH = 16 + iterpos = &ivpos[K2H_ENCRYPT_IV_LENGTH]; + + // copy iter + if(!k2h_copy_iteration_count(iterpos, iter, K2H_ENCRYPT_ITER_LENGTH)){ + ERR_K2HPRN("Could not save iteration count."); + K2H_Free(encryptdata); + return NULL; + } + setdatapos = &iterpos[K2H_ENCRYPT_ITER_LENGTH]; + + // allocated for key + if(NULL == (key = reinterpret_cast(malloc(EVP_CIPHER_key_length(cipher))))){ // <= EVP_MAX_KEY_LENGTH + ERR_K2HPRN("Could not allcation memory."); + K2H_Free(encryptdata); + return NULL; + } + + // create key from pass/salt/iteration + // + // Key length is specified by EVP_CIPHER_key_length(cipher) + // Slat is MAX = 16, and this case is 16. + // + if(0 == PKCS5_PBKDF2_HMAC(pass, strlen(pass), saltpos, K2H_ENCRYPT_SALT_LENGTH, iter, EVP_sha512(), EVP_CIPHER_key_length(cipher), key)){ + ERR_K2HPRN("Failed to make PBKDF2 key from pass/salt/iteration."); + K2H_Free(key); + K2H_Free(encryptdata); + return NULL; + } + + // initialize + if(NULL == (cictx = EVP_CIPHER_CTX_new())){ + ERR_K2HPRN("Failed to make EVP cipher context."); + K2H_Free(key); + K2H_Free(encryptdata); + return NULL; + } + + // initialize context + if(1 != EVP_EncryptInit_ex(cictx, cipher, NULL, key, iv)){ // type is normally supplied by a function such as EVP_aes_256_cbc(). + ERR_K2HPRN("Could not initialize EVP context."); + K2H_Free(key); + K2H_Free(encryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // do encrypt + if(1 != EVP_EncryptUpdate(cictx, setdatapos, &encbodylen, orgdata, static_cast(orglen))){ + ERR_K2HPRN("Failed to AES256 CBC encrypt."); + K2H_Free(key); + K2H_Free(encryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + // last encrypt + if(1 != EVP_EncryptFinal_ex(cictx, &setdatapos[encbodylen], &enclastlen)){ // padding is on as default + ERR_K2HPRN("Failed to AES256 CBC encrypt finally."); + K2H_Free(key); + K2H_Free(encryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // free and destroy context + K2H_Free(key); + EVP_CIPHER_CTX_free(cictx); + + // length + enclen = static_cast(K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH + encbodylen + enclastlen); + + return encryptdata; +} + +unsigned char* k2h_decrypt_aes256_cbc_pbkdf2(const char* pass, const unsigned char* encdata, size_t enclen, size_t& declen) +{ + if(ISEMPTYSTR(pass) || !encdata || 0 == enclen){ + ERR_K2HPRN("parameters are wrong."); + return NULL; + } + + EVP_CIPHER_CTX* cictx; + const EVP_CIPHER* cipher = EVP_aes_256_cbc(); + unsigned char* key; + unsigned char* decryptdata; + int iter; + const unsigned char* saltpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH]; + const unsigned char* iv = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH]; + const unsigned char* iterpos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH]; + const unsigned char* encbodypos = &encdata[K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH]; + int encbodylen = static_cast(enclen - (K2H_ENCRYPT_SALT_PREFIX_LENGTH + K2H_ENCRYPT_SALT_LENGTH + K2H_ENCRYPT_IV_LENGTH + K2H_ENCRYPT_ITER_LENGTH)); + int decbodylen = 0; + int declastlen = 0; + + // allocated for decrypted data area + if(NULL == (decryptdata = reinterpret_cast(malloc(enclen)))){ // declen < enclen + ERR_K2HPRN("Could not allcation memory."); + return NULL; + } + + // get iteration count + if(-1 == (iter = k2h_get_iteration_count(iterpos))){ + ERR_K2HPRN("Could not get iteration count."); + K2H_Free(decryptdata); + return NULL; + } + + // allocated for key + if(NULL == (key = reinterpret_cast(malloc(EVP_CIPHER_key_length(cipher))))){ // <= EVP_MAX_KEY_LENGTH + ERR_K2HPRN("Could not allcation memory."); + K2H_Free(decryptdata); + return NULL; + } + + // create key from pass/salt/iteration + // + // Key length is specified by EVP_CIPHER_key_length(cipher) + // Slat is MAX = 16, and this case is 16. + // + if(0 == PKCS5_PBKDF2_HMAC(pass, strlen(pass), saltpos, K2H_ENCRYPT_SALT_LENGTH, iter, EVP_sha512(), EVP_CIPHER_key_length(cipher), key)){ + ERR_K2HPRN("Failed to make PBKDF2 key from pass/salt/iteration."); + K2H_Free(key); + K2H_Free(decryptdata); + return NULL; + } + + // initialize + if(NULL == (cictx = EVP_CIPHER_CTX_new())){ + ERR_K2HPRN("Failed to make EVP cipher context."); + K2H_Free(key); + K2H_Free(decryptdata); + return NULL; + } + + // initialize context + if(1 != EVP_DecryptInit_ex(cictx, cipher, NULL, key, iv)){ // type is normally supplied by a function such as EVP_aes_256_cbc(). + ERR_K2HPRN("Could not initialize EVP context."); + K2H_Free(key); + K2H_Free(decryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // do decrypt + if(1 != EVP_DecryptUpdate(cictx, decryptdata, &decbodylen, encbodypos, encbodylen)){ + ERR_K2HPRN("Failed to AES256 CBC decrypt."); + K2H_Free(key); + K2H_Free(decryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + // last decrypt + if(1 != EVP_DecryptFinal_ex(cictx, &decryptdata[decbodylen], &declastlen)){ + ERR_K2HPRN("Failed to AES256 CBC decrypt finally."); + K2H_Free(key); + K2H_Free(decryptdata); + EVP_CIPHER_CTX_free(cictx); + return NULL; + } + + // free and destroy context + K2H_Free(key); + EVP_CIPHER_CTX_free(cictx); + + // length + declen = static_cast(decbodylen + declastlen); + + return decryptdata; +} + +/* + * VIM modelines + * + * vim:set ts=4 fenc=utf-8: + */ diff --git a/lib/k2hdaccess.cc b/lib/k2hdaccess.cc index cfbd231..5b5ce72 100644 --- a/lib/k2hdaccess.cc +++ b/lib/k2hdaccess.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Mar 28 2014 diff --git a/lib/k2hdaccess.h b/lib/k2hdaccess.h index 331ea16..82a8cef 100644 --- a/lib/k2hdaccess.h +++ b/lib/k2hdaccess.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Mar 28 2014 diff --git a/lib/k2hdbg.cc b/lib/k2hdbg.cc index 5416187..57780e1 100644 --- a/lib/k2hdbg.cc +++ b/lib/k2hdbg.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 @@ -25,48 +25,100 @@ #include "k2hdbg.h" +//--------------------------------------------------------- +// Global variable +//--------------------------------------------------------- +K2hDbgMode debug_mode = K2HDBG_SILENT; +FILE* k2h_dbg_fp = NULL; + //--------------------------------------------------------- // Class K2HDbgControl //--------------------------------------------------------- +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// class K2HDbgControl { protected: - static const char* DBGENVNAME; - static const char* DBGENVFILE; - static K2HDbgControl singleton; - static bool isSetSignal; + static const char* DBGENVNAME; + static const char* DBGENVFILE; + static bool isSetSignal; - public: - static bool LoadEnv(void); - static bool LoadEnvName(void); - static bool LoadEnvFile(void); + K2hDbgMode* pdebug_mode; // pointer to global variable + FILE** pk2h_dbg_fp; // pointer to global variable + + protected: static void User1Handler(int Signal); - static bool SetUser1Handler(bool isEnable); - K2HDbgControl(); - virtual ~K2HDbgControl(); + K2HDbgControl() : pdebug_mode(&debug_mode), pk2h_dbg_fp(&k2h_dbg_fp) + { + *pdebug_mode = K2HDBG_SILENT; + *pk2h_dbg_fp = NULL; + LoadEnv(); + } + + virtual ~K2HDbgControl() + { + SetUser1Handler(false); + } + + bool LoadEnvName(void); + bool LoadEnvFile(void); + + public: + static K2HDbgControl& GetDbgCntl(void) + { + static K2HDbgControl singleton; // singleton + return singleton; + } + + bool SetUser1Handler(bool isEnable); + bool LoadEnv(void); + K2hDbgMode SetDbgMode(K2hDbgMode mode); + K2hDbgMode BumpupDbgMode(void); + K2hDbgMode GetDbgMode(void); + bool SetDbgFile(const char* filepath); + bool UnsetDbgFile(void); }; +// // Class valiables -const char* K2HDbgControl::DBGENVNAME = "K2HDBGMODE"; -const char* K2HDbgControl::DBGENVFILE = "K2HDBGFILE"; -K2HDbgControl K2HDbgControl::singleton; -bool K2HDbgControl::isSetSignal= false; +// +const char* K2HDbgControl::DBGENVNAME = "K2HDBGMODE"; +const char* K2HDbgControl::DBGENVFILE = "K2HDBGFILE"; +bool K2HDbgControl::isSetSignal= false; -// Constructor / Destructor -K2HDbgControl::K2HDbgControl() +// +// Methods +// +void K2HDbgControl::User1Handler(int Signal) { - K2HDbgControl::LoadEnv(); + MSG_K2HPRN("Caught signal SIGUSR1(%d), bumpup the logging level.", Signal); + K2HDbgControl::GetDbgCntl().BumpupDbgMode(); } -K2HDbgControl::~K2HDbgControl() + +bool K2HDbgControl::SetUser1Handler(bool isEnable) { - K2HDbgControl::SetUser1Handler(false); + if(isEnable != K2HDbgControl::isSetSignal){ + struct sigaction sa; + + sigemptyset(&sa.sa_mask); + sigaddset(&sa.sa_mask, SIGUSR1); + sa.sa_flags = isEnable ? 0 : SA_RESETHAND; + sa.sa_handler = isEnable ? K2HDbgControl::User1Handler : SIG_DFL; + + if(0 > sigaction(SIGUSR1, &sa, NULL)){ + WAN_K2HPRN("Could not %s signal USER1 handler. errno = %d", isEnable ? "set" : "unset", errno); + return false; + } + K2HDbgControl::isSetSignal = isEnable; + } + return true; } -// Class Methods bool K2HDbgControl::LoadEnv(void) { - if(!K2HDbgControl::LoadEnvName() || !K2HDbgControl::LoadEnvFile()){ + if(!LoadEnvName() || !LoadEnvFile()){ return false; } return true; @@ -80,13 +132,13 @@ bool K2HDbgControl::LoadEnvName(void) return true; } if(0 == strcasecmp(pEnvVal, "SILENT")){ - SetK2hDbgMode(K2HDBG_SILENT); + SetDbgMode(K2HDBG_SILENT); }else if(0 == strcasecmp(pEnvVal, "ERR")){ - SetK2hDbgMode(K2HDBG_ERR); + SetDbgMode(K2HDBG_ERR); }else if(0 == strcasecmp(pEnvVal, "WAN")){ - SetK2hDbgMode(K2HDBG_WARN); + SetDbgMode(K2HDBG_WARN); }else if(0 == strcasecmp(pEnvVal, "INFO")){ - SetK2hDbgMode(K2HDBG_MSG); + SetDbgMode(K2HDBG_MSG); }else{ MSG_K2HPRN("%s ENV is not unknown string(%s).", K2HDbgControl::DBGENVNAME, pEnvVal); return false; @@ -101,54 +153,23 @@ bool K2HDbgControl::LoadEnvFile(void) MSG_K2HPRN("%s ENV is not set.", K2HDbgControl::DBGENVFILE); return true; } - if(!SetK2hDbgFile(pEnvVal)){ + if(!SetDbgFile(pEnvVal)){ MSG_K2HPRN("%s ENV is unsafe string(%s).", K2HDbgControl::DBGENVFILE, pEnvVal); return false; } return true; } -void K2HDbgControl::User1Handler(int Signal) +K2hDbgMode K2HDbgControl::SetDbgMode(K2hDbgMode mode) { - MSG_K2HPRN("Caught signal SIGUSR1(%d), bumpup the logging level.", Signal); - BumpupK2hDbgMode(); -} - -bool K2HDbgControl::SetUser1Handler(bool isEnable) -{ - if(isEnable != K2HDbgControl::isSetSignal){ - struct sigaction sa; - - sigemptyset(&sa.sa_mask); - sigaddset(&sa.sa_mask, SIGUSR1); - sa.sa_flags = isEnable ? 0 : SA_RESETHAND; - sa.sa_handler = isEnable ? K2HDbgControl::User1Handler : SIG_DFL; - - if(0 > sigaction(SIGUSR1, &sa, NULL)){ - WAN_K2HPRN("Could not %s signal USER1 handler. errno = %d", isEnable ? "set" : "unset", errno); - return false; - } - K2HDbgControl::isSetSignal = isEnable; - } - return true; -} - -//--------------------------------------------------------- -// Global variable -//--------------------------------------------------------- -K2hDbgMode debug_mode = K2HDBG_SILENT; -FILE* k2h_dbg_fp = NULL; - -K2hDbgMode SetK2hDbgMode(K2hDbgMode mode) -{ - K2hDbgMode oldmode = debug_mode; - debug_mode = mode; + K2hDbgMode oldmode = *pdebug_mode; + *pdebug_mode = mode; return oldmode; } -K2hDbgMode BumpupK2hDbgMode(void) +K2hDbgMode K2HDbgControl::BumpupDbgMode(void) { - K2hDbgMode mode = GetK2hDbgMode(); + K2hDbgMode mode = GetDbgMode(); if(K2HDBG_SILENT == mode){ mode = K2HDBG_ERR; @@ -159,26 +180,21 @@ K2hDbgMode BumpupK2hDbgMode(void) }else{ // K2HDBG_MSG == mode mode = K2HDBG_SILENT; } - return ::SetK2hDbgMode(mode); -} - -K2hDbgMode GetK2hDbgMode(void) -{ - return debug_mode; + return SetDbgMode(mode); } -bool LoadK2hDbgEnv(void) +K2hDbgMode K2HDbgControl::GetDbgMode(void) { - return K2HDbgControl::LoadEnv(); + return *pdebug_mode; } -bool SetK2hDbgFile(const char* filepath) +bool K2HDbgControl::SetDbgFile(const char* filepath) { if(ISEMPTYSTR(filepath)){ ERR_K2HPRN("Parameter is wrong."); return false; } - if(!UnsetK2hDbgFile()){ + if(!UnsetDbgFile()){ return false; } FILE* newfp; @@ -186,26 +202,59 @@ bool SetK2hDbgFile(const char* filepath) ERR_K2HPRN("Could not open debug file(%s). errno = %d", filepath, errno); return false; } - k2h_dbg_fp = newfp; + *pk2h_dbg_fp = newfp; return true; } -bool UnsetK2hDbgFile(void) +bool K2HDbgControl::UnsetDbgFile(void) { - if(k2h_dbg_fp){ - if(0 != fclose(k2h_dbg_fp)){ + if(*pk2h_dbg_fp){ + if(0 != fclose(*pk2h_dbg_fp)){ ERR_K2HPRN("Could not close debug file. errno = %d", errno); - k2h_dbg_fp = NULL; // On this case, k2h_dbg_fp is not correct pointer after error... + *pk2h_dbg_fp = NULL; // On this case, k2h_dbg_fp is not correct pointer after error... return false; } - k2h_dbg_fp = NULL; + *pk2h_dbg_fp = NULL; } return true; } +//--------------------------------------------------------- +// Global Functions +//--------------------------------------------------------- +K2hDbgMode SetK2hDbgMode(K2hDbgMode mode) +{ + return K2HDbgControl::GetDbgCntl().SetDbgMode(mode); +} + +K2hDbgMode BumpupK2hDbgMode(void) +{ + return K2HDbgControl::GetDbgCntl().BumpupDbgMode(); +} + +K2hDbgMode GetK2hDbgMode(void) +{ + return K2HDbgControl::GetDbgCntl().GetDbgMode(); +} + +bool LoadK2hDbgEnv(void) +{ + return K2HDbgControl::GetDbgCntl().LoadEnv(); +} + +bool SetK2hDbgFile(const char* filepath) +{ + return K2HDbgControl::GetDbgCntl().SetDbgFile(filepath); +} + +bool UnsetK2hDbgFile(void) +{ + return K2HDbgControl::GetDbgCntl().UnsetDbgFile(); +} + bool SetSignalUser1(void) { - return K2HDbgControl::SetUser1Handler(true); + return K2HDbgControl::GetDbgCntl().SetUser1Handler(true); } /* diff --git a/lib/k2hdbg.h b/lib/k2hdbg.h index c5de3a1..1444408 100644 --- a/lib/k2hdbg.h +++ b/lib/k2hdbg.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hfilemonitor.cc b/lib/k2hfilemonitor.cc index d0beff7..35df0f8 100644 --- a/lib/k2hfilemonitor.cc +++ b/lib/k2hfilemonitor.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Wed Apr 16 2014 @@ -53,8 +53,8 @@ using namespace std; //--------------------------------------------------------- // Class valiable //--------------------------------------------------------- -const string K2HFileMonitor::base_prefix(MONITORFILE_BASE); -mode_t K2HFileMonitor::file_umask = 0; +const char K2HFileMonitor::base_prefix[] = MONITORFILE_BASE; +mode_t K2HFileMonitor::file_umask = 0; //--------------------------------------------------------- // Constructor / Destructor @@ -233,9 +233,16 @@ bool K2HFileMonitor::InitializeFileMonitor(PSFMONWRAP pfmonwrap, bool noupdate) // wait for initializing monitor file MSG_K2HPRN("Monitor File %s exists(errno=%d), so check file size and try to open it.", bup_monfile.c_str(), errno); + // [NOTE] + // It loops here assuming that Monitor file has been locked, but it could be a dead loop + // if the process that locking the file after checking the lock finished without initializing + // the file. + // Therefore, an upper limit(1000) is set for the number of loops. + // struct stat st; - bool is_retry_loop = false; - for(is_retry_loop = false; !is_retry_loop; ){ + bool is_retry_loop = false; + int check_loop_cnt = 0; + for(is_retry_loop = false, check_loop_cnt = 0; !is_retry_loop && check_loop_cnt < 1000; ++check_loop_cnt){ if(0 != stat(bup_monfile.c_str(), &st)){ if(ENOENT != errno){ ERR_K2HPRN("Could not get stat for file %s by errno(%d).", bup_monfile.c_str(), errno); @@ -253,7 +260,7 @@ bool K2HFileMonitor::InitializeFileMonitor(PSFMONWRAP pfmonwrap, bool noupdate) MSG_K2HPRN("file %s is not initialized yet, so continue to wait...", bup_monfile.c_str()); } } - if(is_retry_loop){ + if(is_retry_loop || 1000 <= check_loop_cnt){ // retry to create file. continue; } diff --git a/lib/k2hfilemonitor.h b/lib/k2hfilemonitor.h index a758321..b54bb3a 100644 --- a/lib/k2hfilemonitor.h +++ b/lib/k2hfilemonitor.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Wed Apr 16 2014 @@ -75,16 +75,16 @@ typedef union share_file_monitor_wrap{ class K2HFileMonitor { private: - static const std::string base_prefix; - static mode_t file_umask; // umask for monitor file + static const char base_prefix[]; + static mode_t file_umask; // umask for monitor file - std::string bup_shmfile; - std::string bup_monfile; - PSFMON psfmon; - int fmfd; - unsigned char bup_inode_cnt; - unsigned char bup_area_cnt; - ino_t bup_inode_val; + std::string bup_shmfile; + std::string bup_monfile; + PSFMON psfmon; + int fmfd; + unsigned char bup_inode_cnt; + unsigned char bup_area_cnt; + ino_t bup_inode_val; public: K2HFileMonitor(); diff --git a/lib/k2hfind.cc b/lib/k2hfind.cc index 809305a..bb2cd39 100644 --- a/lib/k2hfind.cc +++ b/lib/k2hfind.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hfind.h b/lib/k2hfind.h index 9e13787..b2cfcd7 100644 --- a/lib/k2hfind.h +++ b/lib/k2hfind.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hlock.cc b/lib/k2hlock.cc index 09753e1..1dceac8 100644 --- a/lib/k2hlock.cc +++ b/lib/k2hlock.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Feb 4 2014 @@ -32,18 +32,25 @@ using namespace std; const bool K2HLock::RDLOCK; const bool K2HLock::RWLOCK; -// [NOTE] About fdmodes -// We do not use lock for this mapping because this method is -// called only at initializing under another exclusive control. +// [NOTE] +// To avoid static object initialization order problem(SIOF) // -fdmodemap_t K2HLock::fdmodes; +fdmodemap_t& K2HLock::GetFdModes(void) +{ + // [NOTE] About fdmodes + // We do not use lock for this mapping because this method is + // called only at initializing under another exclusive control. + // + static fdmodemap_t fmmap; // singleton + return fmmap; +} bool K2HLock::AddReadModeFd(int fd) { if(FLCK_INVALID_HANDLE == fd){ return false; } - K2HLock::fdmodes[fd] = true; // value is dummy(always true) + K2HLock::GetFdModes()[fd] = true; // value is dummy(always true) return true; } @@ -53,10 +60,10 @@ bool K2HLock::RemoveReadModeFd(int fd) if(FLCK_INVALID_HANDLE == fd){ return false; } - if(K2HLock::fdmodes.end() == K2HLock::fdmodes.find(fd)){ + if(K2HLock::GetFdModes().end() == K2HLock::GetFdModes().find(fd)){ return false; } - K2HLock::fdmodes.erase(fd); + K2HLock::GetFdModes().erase(fd); return true; } @@ -95,7 +102,7 @@ bool K2HLock::Lock(bool IsRead) return true; } // if fd is read only mode, do nothing. - if(K2HLock::fdmodes.end() != K2HLock::fdmodes.find(lock_fd)){ + if(K2HLock::GetFdModes().end() != K2HLock::GetFdModes().find(lock_fd)){ return true; } return FLRwlRcsv::Lock(IsRead); @@ -115,7 +122,7 @@ bool K2HLock::Lock(int fd, off_t offset) return true; } // if fd is read only mode, do nothing. - if(K2HLock::fdmodes.end() != K2HLock::fdmodes.find(lock_fd)){ + if(K2HLock::GetFdModes().end() != K2HLock::GetFdModes().find(lock_fd)){ return true; } return FLRwlRcsv::Lock(fd, offset, 1L, (FLCK_READ_LOCK == lock_type)); @@ -131,7 +138,7 @@ bool K2HLock::Lock(int fd, off_t offset, bool IsRead) return true; } // if fd is read only mode, do nothing. - if(K2HLock::fdmodes.end() != K2HLock::fdmodes.find(lock_fd)){ + if(K2HLock::GetFdModes().end() != K2HLock::GetFdModes().find(lock_fd)){ return true; } return FLRwlRcsv::Lock(fd, offset, 1L, IsRead); @@ -148,7 +155,7 @@ bool K2HLock::Lock(void) return true; } // if fd is read only mode, do nothing. - if(K2HLock::fdmodes.end() != K2HLock::fdmodes.find(lock_fd)){ + if(K2HLock::GetFdModes().end() != K2HLock::GetFdModes().find(lock_fd)){ return true; } return FLRwlRcsv::Lock(); @@ -165,7 +172,7 @@ bool K2HLock::Unlock(void) return true; } // if fd is read only mode, do nothing. - if(K2HLock::fdmodes.end() != K2HLock::fdmodes.find(lock_fd)){ + if(K2HLock::GetFdModes().end() != K2HLock::GetFdModes().find(lock_fd)){ return true; } return FLRwlRcsv::Unlock(); @@ -198,7 +205,7 @@ bool K2HLock::Dup(const K2HLock& other) } if(is_mutex_locked){ - FLRwlRcsv::Stack.unlock(); + FLRwlRcsv::StackUnlock(); } return bresult; } diff --git a/lib/k2hlock.h b/lib/k2hlock.h index 5b108b4..45edcd9 100644 --- a/lib/k2hlock.h +++ b/lib/k2hlock.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Feb 4 2014 @@ -37,14 +37,13 @@ typedef std::map fdmodemap_t; // class K2HLock : public FLRwlRcsv { - protected: - static fdmodemap_t fdmodes; // read only mode fds - public: static const bool RDLOCK = true; static const bool RWLOCK = false; protected: + static fdmodemap_t& GetFdModes(void); + bool Dup(const K2HLock& other); public: diff --git a/lib/k2hmmapinfo.cc b/lib/k2hmmapinfo.cc index a5df8e0..1da79c7 100644 --- a/lib/k2hmmapinfo.cc +++ b/lib/k2hmmapinfo.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 @@ -405,9 +405,16 @@ bool K2HMmapMan::IsMmaped(const K2HShm* pk2hshm, const char* file, bool needlock } //--------------------------------------------------------- -// K2HMmapInfo: Class valiable +// K2HMmapInfo: Class Method //--------------------------------------------------------- -K2HMmapMan K2HMmapInfo::mmapman; +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +K2HMmapMan& K2HMmapInfo::GetMan(void) +{ + static K2HMmapMan mmapman; // singleton + return mmapman; +} //--------------------------------------------------------- // K2HMmapInfo: Constructor / Destructor @@ -416,12 +423,12 @@ K2HMmapInfo::K2HMmapInfo(K2HShm* pk2hshm) : pK2Hshm(pk2hshm), ppInfoTop(NULL) { assert(NULL != pK2Hshm); - ppInfoTop = K2HMmapInfo::mmapman.AddMapInfo(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm->GetRawK2hashFd(), pK2Hshm->GetRawK2hashReadMode(), true); + ppInfoTop = K2HMmapInfo::GetMan().AddMapInfo(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm->GetRawK2hashFd(), pK2Hshm->GetRawK2hashReadMode(), true); } K2HMmapInfo::~K2HMmapInfo() { - K2HMmapInfo::mmapman.RemoveMapInfo(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), true); + K2HMmapInfo::GetMan().RemoveMapInfo(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), true); } //--------------------------------------------------------- @@ -435,7 +442,7 @@ inline bool K2HMmapInfo::SetInternalMmapInfo(void) const if(ppInfoTop){ return true; } - if(NULL == ((const_cast(this))->ppInfoTop = K2HMmapInfo::mmapman.GetMmapInfo(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), false))){ + if(NULL == ((const_cast(this))->ppInfoTop = K2HMmapInfo::GetMan().GetMmapInfo(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), false))){ MSG_K2HPRN("There is no mapping info for \"%s\"(%p)", pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); return false; } @@ -447,13 +454,13 @@ inline bool K2HMmapInfo::SetInternalMmapInfo(void) const // bool K2HMmapInfo::GetFd(const char* file, int& fd) { - return K2HMmapInfo::mmapman.GetFd(file, &fd, true); + return K2HMmapInfo::GetMan().GetFd(file, &fd, true); } bool K2HMmapInfo::ReplaceMapInfo(const char* oldfile) { PK2HMMAPINFO* ppTmp; - if(NULL == (ppTmp = K2HMmapInfo::mmapman.ReplaceMapInfo(pK2Hshm, oldfile, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm->GetRawK2hashFd(), pK2Hshm->GetRawK2hashReadMode(), true))){ + if(NULL == (ppTmp = K2HMmapInfo::GetMan().ReplaceMapInfo(pK2Hshm, oldfile, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm->GetRawK2hashFd(), pK2Hshm->GetRawK2hashReadMode(), true))){ return false; } ppInfoTop = ppTmp; @@ -465,26 +472,26 @@ bool K2HMmapInfo::ReplaceMapInfo(const char* oldfile) // bool K2HMmapInfo::IsMmaped(void) const { - return K2HMmapInfo::mmapman.IsMmaped(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), true); + return K2HMmapInfo::GetMan().IsMmaped(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), true); } void K2HMmapInfo::UnmapAll(void) { - K2HMmapInfo::mmapman.UnmapAll(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), true); + K2HMmapInfo::GetMan().UnmapAll(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), true); ppInfoTop = NULL; } bool K2HMmapInfo::Unmap(long type, off_t file_offset, size_t length) { - return K2HMmapInfo::mmapman.Unmap(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), type, file_offset, length, true); + return K2HMmapInfo::GetMan().Unmap(pK2Hshm, pK2Hshm->GetRawK2hashFilePath(), type, file_offset, length, true); } bool K2HMmapInfo::Unmap(PK2HMMAPINFO pexistareatop) { - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return false; } @@ -509,16 +516,16 @@ bool K2HMmapInfo::Unmap(PK2HMMAPINFO pexistareatop) } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return true; } bool K2HMmapInfo::AddArea(long type, off_t file_offset, void* mmap_base, size_t length) { - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return false; } @@ -532,26 +539,26 @@ bool K2HMmapInfo::AddArea(long type, off_t file_offset, void* mmap_base, size_t MSG_K2HPRN("Added Area(type=%ld, file_offset=%jd, length=%zu, base=%p) for \"%s\"(%p)", type, static_cast(file_offset), length, mmap_base, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return true; } void* K2HMmapInfo::GetMmapAddrBase(off_t file_offset, bool is_update_check) const { - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return NULL; } for(PK2HMMAPINFO pinfo = *ppInfoTop; pinfo; pinfo = pinfo->next){ if(pinfo->file_offset <= file_offset && file_offset < static_cast(pinfo->file_offset + pinfo->length)){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return pinfo->mmap_base; } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); MSG_K2HPRN("Could not find base(file_offset=%jd, file=\"%s\", K2HShm=%p)", static_cast(file_offset), pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); @@ -567,20 +574,20 @@ void* K2HMmapInfo::GetMmapAddrBase(off_t file_offset, bool is_update_check) cons off_t K2HMmapInfo::GetMmapAddrOffset(off_t file_offset, bool is_update_check) const { - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return -1; } for(PK2HMMAPINFO pinfo = *ppInfoTop; pinfo; pinfo = pinfo->next){ if(pinfo->file_offset <= file_offset && file_offset < static_cast(pinfo->file_offset + pinfo->length)){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return reinterpret_cast(pinfo->mmap_base) - pinfo->file_offset; } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); MSG_K2HPRN("Could not find base(file_offset=%jd, file=\"%s\", K2HShm=%p)", static_cast(file_offset), pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); @@ -596,20 +603,20 @@ off_t K2HMmapInfo::GetMmapAddrOffset(off_t file_offset, bool is_update_check) co off_t K2HMmapInfo::GetFileOffsetBase(void* address, bool is_update_check) const { - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return 0; } for(PK2HMMAPINFO pinfo = *ppInfoTop; pinfo; pinfo = pinfo->next){ if(pinfo->mmap_base <= address && reinterpret_cast(address) < (reinterpret_cast(pinfo->mmap_base) + pinfo->length)){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return pinfo->file_offset; } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); MSG_K2HPRN("Could not find offset(address=%p, file=\"%s\", K2HShm=%p)", address, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); @@ -625,20 +632,20 @@ off_t K2HMmapInfo::GetFileOffsetBase(void* address, bool is_update_check) const off_t K2HMmapInfo::GetMmapAddressToFileOffset(void* address, bool is_update_check) const { - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return 0; } for(PK2HMMAPINFO pinfo = *ppInfoTop; pinfo; pinfo = pinfo->next){ if(pinfo->mmap_base <= address && reinterpret_cast(address) < (reinterpret_cast(pinfo->mmap_base) + pinfo->length)){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return pinfo->file_offset - reinterpret_cast(pinfo->mmap_base); } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); MSG_K2HPRN("Could not find offset(address=%p, file=\"%s\", K2HShm=%p)", address, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); @@ -674,20 +681,20 @@ off_t K2HMmapInfo::CvtRel(void* address, bool isAllowNull) const void* K2HMmapInfo::begin(int type, bool is_update_check) const { - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return NULL; } for(PK2HMMAPINFO pinfo = *ppInfoTop; pinfo; pinfo = pinfo->next){ if(type == pinfo->type){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return pinfo->mmap_base; } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); MSG_K2HPRN("Could not get begin(file=\"%s\", K2HShm=%p)", pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); @@ -705,10 +712,10 @@ void* K2HMmapInfo::next(void* address, size_t datasize, bool is_update_check) co { off_t file_offset = CvtRel(address); - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return NULL; } @@ -717,7 +724,7 @@ void* K2HMmapInfo::next(void* address, size_t datasize, bool is_update_check) co // found area if((file_offset + datasize) < (pinfo->file_offset + pinfo->length)){ // return address in same area - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return CvtAbs(file_offset + datasize); } // next address is over area => search next area @@ -725,14 +732,14 @@ void* K2HMmapInfo::next(void* address, size_t datasize, bool is_update_check) co for(pinfo = pinfo->next; pinfo; pinfo = pinfo->next){ if(Type == pinfo->type){ // found same area type => return head address of area - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return pinfo->mmap_base; } } break; } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); MSG_K2HPRN("Could not get next from (address=%p, length=%zd, file=\"%s\", K2HShm=%p)", address, datasize, pK2Hshm->GetRawK2hashFilePath(), pK2Hshm); @@ -754,10 +761,10 @@ bool K2HMmapInfo::AreaMsync(void* address) const file_offset = CvtRel(address); } - K2HMmapInfo::mmapman.Lock(); + K2HMmapInfo::GetMan().Lock(); if(!SetInternalMmapInfo()){ - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return NULL; } @@ -772,7 +779,7 @@ bool K2HMmapInfo::AreaMsync(void* address) const } } } - K2HMmapInfo::mmapman.Unlock(); + K2HMmapInfo::GetMan().Unlock(); return result; } diff --git a/lib/k2hmmapinfo.h b/lib/k2hmmapinfo.h index e020c49..4f13cc4 100644 --- a/lib/k2hmmapinfo.h +++ b/lib/k2hmmapinfo.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 @@ -177,15 +177,16 @@ class K2HMmapMan class K2HMmapInfo { private: - static K2HMmapMan mmapman; - K2HShm* pK2Hshm; - PK2HMMAPINFO* ppInfoTop; + K2HShm* pK2Hshm; + PK2HMMAPINFO* ppInfoTop; public: K2HMmapInfo(K2HShm* pk2hshm = NULL); virtual ~K2HMmapInfo(); private: + static K2HMmapMan& GetMan(void); + inline bool SetInternalMmapInfo(void) const; void* GetMmapAddrBase(off_t file_offset, bool is_update_check = true) const; diff --git a/lib/k2hpage.cc b/lib/k2hpage.cc index 85616f5..0f04a1a 100644 --- a/lib/k2hpage.cc +++ b/lib/k2hpage.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hpage.h b/lib/k2hpage.h index f8d22e6..e72467f 100644 --- a/lib/k2hpage.h +++ b/lib/k2hpage.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hpagefile.cc b/lib/k2hpagefile.cc index eac3fe5..19c3101 100644 --- a/lib/k2hpagefile.cc +++ b/lib/k2hpagefile.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hpagefile.h b/lib/k2hpagefile.h index 3af1ac9..bf475d7 100644 --- a/lib/k2hpagefile.h +++ b/lib/k2hpagefile.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hpagemem.cc b/lib/k2hpagemem.cc index f5aa504..fc4dec0 100644 --- a/lib/k2hpagemem.cc +++ b/lib/k2hpagemem.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hpagemem.h b/lib/k2hpagemem.h index 16b06c0..34f45ab 100644 --- a/lib/k2hpagemem.h +++ b/lib/k2hpagemem.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hqueue.cc b/lib/k2hqueue.cc index f967fcd..3884978 100644 --- a/lib/k2hqueue.cc +++ b/lib/k2hqueue.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013-2015 Yahoo! JAPAN corporation. + * Copyright 2013-2015 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Jan 30 2015 @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -56,13 +55,11 @@ using namespace std; class K2HUniqTimespec { protected: - static K2HUniqTimespec singleton; - - struct timespec uniq_count; // uses for building uniq key name - int LockVal; // like mutex for valiables + struct timespec uniq_count; // uses for building uniq key name + int LockVal; // like mutex for valiables public: - static K2HUniqTimespec* Get(void) { return &K2HUniqTimespec::singleton; } + static K2HUniqTimespec* Get(void); void GetUniqTimespec(struct timespec& ts); @@ -74,24 +71,26 @@ class K2HUniqTimespec //--------------------------------------------------------- // K2HUniqTimespec : Methods //--------------------------------------------------------- -K2HUniqTimespec K2HUniqTimespec::singleton; +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +K2HUniqTimespec* K2HUniqTimespec::Get(void) +{ + static K2HUniqTimespec uniqts; // singleton + return &uniqts; +} K2HUniqTimespec::K2HUniqTimespec() : LockVal(FLCK_NOSHARED_MUTEX_VAL_UNLOCKED) { - if(this == K2HUniqTimespec::Get()){ - if(-1 == clock_gettime(CLOCK_REALTIME, &uniq_count)){ - WAN_K2HPRN("Could not get timespec(errno=%d), but continue with other initial value...", errno); - uniq_count.tv_sec = time(NULL); - uniq_count.tv_nsec = 0L; - } - }else{ - assert(false); + if(-1 == clock_gettime(CLOCK_REALTIME, &uniq_count)){ + WAN_K2HPRN("Could not get timespec(errno=%d), but continue with other initial value...", errno); + uniq_count.tv_sec = time(NULL); + uniq_count.tv_nsec = 0L; } } K2HUniqTimespec::~K2HUniqTimespec() { - assert(this == K2HUniqTimespec::Get()); } void K2HUniqTimespec::GetUniqTimespec(struct timespec& ts) diff --git a/lib/k2hqueue.h b/lib/k2hqueue.h index 573910a..91600a5 100644 --- a/lib/k2hqueue.h +++ b/lib/k2hqueue.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Feb 3 2015 diff --git a/lib/k2hshm.cc b/lib/k2hshm.cc index 038a484..4cc786a 100644 --- a/lib/k2hshm.cc +++ b/lib/k2hshm.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hshm.h b/lib/k2hshm.h index e09398c..79072f7 100644 --- a/lib/k2hshm.h +++ b/lib/k2hshm.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hshmcomp.cc b/lib/k2hshmcomp.cc index 7c8d3b8..a430e65 100644 --- a/lib/k2hshmcomp.cc +++ b/lib/k2hshmcomp.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Apr 25 2014 diff --git a/lib/k2hshmdirect.cc b/lib/k2hshmdirect.cc index d71087f..6add312 100644 --- a/lib/k2hshmdirect.cc +++ b/lib/k2hshmdirect.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Jul 01 2016 diff --git a/lib/k2hshmdirect.h b/lib/k2hshmdirect.h index faf1946..08748f3 100644 --- a/lib/k2hshmdirect.h +++ b/lib/k2hshmdirect.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2016 Yahoo! JAPAN corporation. + * Copyright 2016 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Jul 01 2016 diff --git a/lib/k2hshmdump.cc b/lib/k2hshmdump.cc index 0317be8..b8b01bf 100644 --- a/lib/k2hshmdump.cc +++ b/lib/k2hshmdump.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hshminit.cc b/lib/k2hshminit.cc index 4a3c827..7bc3e96 100644 --- a/lib/k2hshminit.cc +++ b/lib/k2hshminit.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hshmque.cc b/lib/k2hshmque.cc index f104756..9a1756b 100644 --- a/lib/k2hshmque.cc +++ b/lib/k2hshmque.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013-2015 Yahoo! JAPAN corporation. + * Copyright 2013-2015 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 2 2015 diff --git a/lib/k2hshmupdater.cc b/lib/k2hshmupdater.cc index ba50a9a..2193a2d 100644 --- a/lib/k2hshmupdater.cc +++ b/lib/k2hshmupdater.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Apr 18 2014 diff --git a/lib/k2hshmupdater.h b/lib/k2hshmupdater.h index 82fb83f..2a0840c 100644 --- a/lib/k2hshmupdater.h +++ b/lib/k2hshmupdater.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Apr 18 2014 diff --git a/lib/k2hstream.h b/lib/k2hstream.h index e5c9239..5e1e774 100644 --- a/lib/k2hstream.h +++ b/lib/k2hstream.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Apr 1 2014 diff --git a/lib/k2hstructure.h b/lib/k2hstructure.h index c0f1fb5..e225b48 100644 --- a/lib/k2hstructure.h +++ b/lib/k2hstructure.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hsubkeys.cc b/lib/k2hsubkeys.cc index 360e5b1..1e49cf7 100644 --- a/lib/k2hsubkeys.cc +++ b/lib/k2hsubkeys.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hsubkeys.h b/lib/k2hsubkeys.h index 1cc74e3..2411161 100644 --- a/lib/k2hsubkeys.h +++ b/lib/k2hsubkeys.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2htrans.cc b/lib/k2htrans.cc index 55b895e..b7bedba 100644 --- a/lib/k2htrans.cc +++ b/lib/k2htrans.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,14 +11,14 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 10 2014 * REVISION: * */ -#include + #include #include @@ -120,11 +120,19 @@ const long K2HTransManager::MINIMUM_WAIT_SEEP; const long K2HTransManager::FINISH_WAIT_BLOCK; const unsigned char K2HTransManager::default_prefix[] = {'\0', 'K', '2', 'H', 'T', 'R', 'A', 'N', 'S', '_', 'P', 'R', 'E', 'F', 'I', 'X', '_'}; const time_t K2HTransManager::DEFAULT_INTERVAL; -K2HTransManager K2HTransManager::singleton; //--------------------------------------------------------- // K2HTransManager : Class Methods //--------------------------------------------------------- +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +K2HTransManager* K2HTransManager::Get(void) +{ + static K2HTransManager transman; // singleton + return &transman; +} + void* K2HTransManager::WorkerProc(void* param) { PTRTHPARAM ptrparam = reinterpret_cast(param); @@ -213,24 +221,16 @@ bool K2HTransManager::Do(k2h_h handle, PBCOM pBinCom) //--------------------------------------------------------- K2HTransManager::K2HTransManager() : LockVal(FLCK_NOSHARED_MUTEX_VAL_UNLOCKED), LockPool(FLCK_NOSHARED_MUTEX_VAL_UNLOCKED), threadcnt(0), interval(K2HTransManager::DEFAULT_INTERVAL) { - if(this == K2HTransManager::Get()){ - trfilemap.clear(); - trprefmap.clear(); - trpoolmap.clear(); - }else{ - assert(false); - } + trfilemap.clear(); + trprefmap.clear(); + trpoolmap.clear(); } K2HTransManager::~K2HTransManager() { - if(this == K2HTransManager::Get()){ - for(trprefmap_t::iterator iter = trprefmap.begin(); iter != trprefmap.end(); trprefmap.erase(iter++)){ - Stop(iter->first, false); - K2H_Delete(iter->second); - } - }else{ - assert(false); + for(trprefmap_t::iterator iter = trprefmap.begin(); iter != trprefmap.end(); trprefmap.erase(iter++)){ + Stop(iter->first, false); + K2H_Delete(iter->second); } } diff --git a/lib/k2htrans.h b/lib/k2htrans.h index 54da37d..8174182 100644 --- a/lib/k2htrans.h +++ b/lib/k2htrans.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 10 2014 @@ -150,7 +150,6 @@ class K2HTransManager protected: static const unsigned char default_prefix[]; // default transaction key prefix static const time_t DEFAULT_INTERVAL = 10; // 10s - static K2HTransManager singleton; volatile int LockVal; // like mutex for valiables(transaction file etc) volatile int LockPool; // like mutex for thread and thread pool @@ -161,10 +160,7 @@ class K2HTransManager time_t interval; // interval for checking transaction file public: - static K2HTransManager* Get(void) { return &K2HTransManager::singleton; } - - K2HTransManager(); - virtual ~K2HTransManager(); + static K2HTransManager* Get(void); bool CompareTransactionKeyPrefix(const K2HShm* pk2hshm, const unsigned char* pkey, size_t keylen); @@ -193,6 +189,9 @@ class K2HTransManager static void* WorkerProc(void* param); static bool Do(k2h_h handle, PBCOM pBinCom); + K2HTransManager(); + virtual ~K2HTransManager(); + bool SetTransactionKeyPrefix(const K2HShm* pk2hshm, const unsigned char* pprefix, size_t prefixlen, const time_t* expire = NULL); bool ResetTransactionKeyPrefix(const K2HShm* pk2hshm) { return SetTransactionKeyPrefix(pk2hshm, NULL, 0, NULL); } bool RemoveTransactionKeyPrefix(const K2HShm* pk2hshm); diff --git a/lib/k2htransfunc.cc b/lib/k2htransfunc.cc index c770fd8..87873f2 100644 --- a/lib/k2htransfunc.cc +++ b/lib/k2htransfunc.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 @@ -21,7 +21,6 @@ #include #include -#include #include #include "k2htransfunc.h" @@ -98,20 +97,21 @@ bool k2h_trans_cntl(k2h_h handle, PTRANSOPT pOpt) //--------------------------------------------------------- // K2HTransDynLib Class //--------------------------------------------------------- -K2HTransDynLib K2HTransDynLib::Singleton; +// [NOTE] +// To avoid static object initialization order problem(SIOF) +// +K2HTransDynLib* K2HTransDynLib::get(void) +{ + static K2HTransDynLib dynlib; // singleton + return &dynlib; +} K2HTransDynLib::K2HTransDynLib() : hDynLib(NULL), fp_k2h_trans(NULL), fp_k2h_trans_version(NULL), fp_k2h_trans_cntl(NULL) { - if(this != K2HTransDynLib::get()){ - assert(false); - } } K2HTransDynLib::~K2HTransDynLib() { - if(this != K2HTransDynLib::get()){ - assert(false); - } Unload(); } @@ -145,9 +145,9 @@ bool K2HTransDynLib::Load(const char* path) } // check symbol - if( NULL == (fp_k2h_trans = reinterpret_cast(dlsym(hDynLib, "k2h_trans"))) || - NULL == (fp_k2h_trans_version = reinterpret_cast(dlsym(hDynLib, "k2h_trans_version"))) || - NULL == (fp_k2h_trans_cntl = reinterpret_cast(dlsym(hDynLib, "k2h_trans_cntl"))) ) + if( NULL == (fp_k2h_trans = reinterpret_cast(dlsym(hDynLib, "k2h_trans"))) || + NULL == (fp_k2h_trans_version = reinterpret_cast(dlsym(hDynLib, "k2h_trans_version"))) || + NULL == (fp_k2h_trans_cntl = reinterpret_cast(dlsym(hDynLib, "k2h_trans_cntl"))) ) { const char* pError = dlerror(); ERR_K2HPRN("Failed to load library(%s), error = %s", path, pError ? pError : "unknown"); diff --git a/lib/k2htransfunc.h b/lib/k2htransfunc.h index 8d6062c..ec6dee5 100644 --- a/lib/k2htransfunc.h +++ b/lib/k2htransfunc.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Thu Feb 6 2014 @@ -140,21 +140,19 @@ DECL_EXTERN_C_END // extern "C" - end //--------------------------------------------------------- class K2HTransDynLib { - private: - static K2HTransDynLib Singleton; - private: void* hDynLib; Tfp_k2h_trans fp_k2h_trans; Tfp_k2h_trans_version fp_k2h_trans_version; Tfp_k2h_trans_cntl fp_k2h_trans_cntl; - public: - static K2HTransDynLib* get(void) { return &K2HTransDynLib::Singleton; } - + private: K2HTransDynLib(); virtual ~K2HTransDynLib(); + public: + static K2HTransDynLib* get(void); + bool Unload(void); bool Load(const char* path); diff --git a/lib/k2hutil.cc b/lib/k2hutil.cc index 9f03cb9..894d314 100644 --- a/lib/k2hutil.cc +++ b/lib/k2hutil.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/lib/k2hutil.h b/lib/k2hutil.h index b3bbbad..c652a0c 100644 --- a/lib/k2hutil.h +++ b/lib/k2hutil.h @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Dec 2 2013 diff --git a/make_release_version_file.sh b/make_release_version_file.sh deleted file mode 100755 index 97375e1..0000000 --- a/make_release_version_file.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/bin/sh -# -# Make RELEASE_VERSION file - make_release_version_file.sh -# -# Copyright 2016 Yahoo! JAPAN corporation. -# -# Templates for customizing screwdriver CPP and autotools. -# This template files are provided by yjcore team. -# -# This script makes RELEASE_VERSION from github release tag -# or ChangeLog file, and check version. -# -# For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. -# -# AUTHOR: Takeshi Nakatani -# CREATE: Thu, Jun 2 2016 -# REVISION: -# - -##################################################################### -# -# Usage: make_release_version_file.sh [-not_use_git] [-no_check_ver_diff] [-f changelog file path] -# -PROGRAM_NAME=`basename $0` -SRCTOP=`dirname $0` - -##################################################################### -# -# Parameter -# -NOGIT="no" -NOCHECKVERDIFF="no" -CHANGELOGFILE="ChangeLog" -while [ $# -ne 0 ]; do - if [ "X$1" = "X-not_use_git" ]; then - NOGIT="yes" - - elif [ "X$1" = "X-no_check_ver_diff" ]; then - NOCHECKVERDIFF="yes" - - elif [ "X$1" = "X-f" ]; then - shift - if [ $# -eq 0 ]; then - echo "ERROR: Must set changelog file name after -f option." - echo "Usage: ${PROGRAM_NAME} [-not_use_git] [-no_check_ver_diff] [-f changefile path]" - exit 1 - fi - if [ ! -f ${SRCTOP}/$1 ]; then - echo "ERROR: Not found changelog($1) file " - exit 1 - fi - CHANGELOGFILE=$1 - - elif [ "X$1" = "X-h" -o "X$1" = "X--help" ]; then - echo "Usage: ${PROGRAM_NAME} [-not_use_git] [-no_check_ver_diff] [-f changefile path]" - exit 1 - else - echo "ERROR: Unkown option $1" - echo "Usage: ${PROGRAM_NAME} [-not_use_git] [-no_check_ver_diff] [-f changefile path]" - exit 1 - fi - - shift -done - -##################################################################### -# -# Version number from Github -# -# get version number from git release tag formatted following: -# "v10", "v 10", "ver10", "ver-10", "version10", "version,10" -# "v10.0.0", "v 10.0", "ver 10.0.0a", "v10.0.0-1", etc -# -# and the last build number is cut.(ex, "v10.0.1-1" -> "10.0.1") -# -if [ "X${NOGIT}" = "Xno" ]; then - GIT_RELEASE_VERSION=`git tag | grep '^[v|V]\([e|E][r|R]\([s|S][i|I][o|O][n|N]\)\{0,1\}\)\{0,1\}' | sed 's/^[v|V]\([e|E][r|R]\([s|S][i|I][o|O][n|N]\)\{0,1\}\)\{0,1\}//' | grep -o '[0-9]\+\([\.]\([0-9]\)\+\)\+\(.\)*$' | sed 's/-\(.\)*$//' | sort -t . -n -k 1,1 -k 2,2 -k 3,3 -k 4,4 | uniq | tail -1 | tr -d '\n'` - - if [ "X${GIT_RELEASE_VERSION}" = "X" ]; then - echo "WARNING: Could not get latest release tag from github release tag" - GIT_RELEASE_VERSION= - fi -else - GIT_RELEASE_VERSION= -fi - -##################################################################### -# -# Version number from ChangeLog -# -# get version number from ChangeLog file formatted like debian. -# and the last build number is cut.(ex, "10.0.1-1" -> "10.0.1") -# -if [ -f ${SRCTOP}/${CHANGELOGFILE} ]; then - CH_RELEASE_VERSION=`grep -o '^.*[(].*[)].*[;].*$' ${SRCTOP}/${CHANGELOGFILE} | grep -o '[(].*[)]' | head -1 | sed 's/[(|)]//g'` - - if [ "X${CH_RELEASE_VERSION}" = "X" ]; then - echo "WARNING: Could not get latest release tag from ChangeLog file ( ${SRCTOP}/${CHANGELOGFILE} )" - CH_RELEASE_VERSION= - fi -else - echo "MESSAGE: not found ChangeLog file ( ${SRCTOP}/${CHANGELOGFILE} )" - CH_RELEASE_VERSION= -fi - -##################################################################### -# -# Check version number between github release tag and ChangeLog file -# -# If version number from Github is later than one from ChangeLog, -# this script puts error and exits. -# The other case, this script continue to work and puts version number -# to RELEASE_VERION file. -# If there are no version number from Github and ChangeLog, this script -# checks RELEASE_VERION file existing. -# -IS_PUT_RELEASE_VERSION_FILE=yes - -if [ "X${GIT_RELEASE_VERSION}" != "X" -a "X${CH_RELEASE_VERSION}" != "X" ]; then - if [ "X${NOCHECKVERDIFF}" = "Xno" ]; then - # - # Check latest version - # - GIT_VERS=`echo ${GIT_RELEASE_VERSION} | sed 's/\./ /g'` - CH_VERS=`echo ${CH_RELEASE_VERSION} | sed 's/\./ /g'` - - GIT_VER_PART_CNT=0 - LATEST_VER_TYPE= - for git_ver_part in ${GIT_VERS}; do - ch_ver_part= - - CH_VER_PART_CNT=0 - for ver_tmp in ${CH_VERS}; do - ch_ver_part=`echo ${ver_tmp}` - CH_VER_PART_CNT=`expr ${CH_VER_PART_CNT} + 1` - - if [ ${GIT_VER_PART_CNT} -lt ${CH_VER_PART_CNT} ]; then - break - fi - done - - if [ "X${ch_ver_part}" != "X" ]; then - if [ ${git_ver_part} -gt ${ch_ver_part} ]; then - LATEST_VER_TYPE=gitver - break - elif [ ${git_ver_part} -lt ${ch_ver_part} ]; then - LATEST_VER_TYPE=chver - break - fi - else - LATEST_VER_TYPE=gitver - break - fi - - GIT_VER_PART_CNT=`expr ${GIT_VER_PART_CNT} + 1` - done - - if [ "X${LATEST_VER_TYPE}" = "X" ]; then - GIT_VER_PART_CNT=0 - for git_ver_part in ${GIT_VERS}; do - GIT_VER_PART_CNT=`expr ${GIT_VER_PART_CNT} + 1` - done - - CH_VER_PART_CNT=0 - for ver_tmp in ${CH_VERS}; do - CH_VER_PART_CNT=`expr ${CH_VER_PART_CNT} + 1` - done - - if [ ${GIT_VER_PART_CNT} -lt ${CH_VER_PART_CNT} ]; then - LATEST_VER_TYPE=chver - fi - fi - - if [ "X${LATEST_VER_TYPE}" = "Xgitver" ]; then - echo "ERROR: Github release tag ( ${GIT_RELEASE_VERSION} ) is later than ChangeLog file ( ${CHANGELOGFILE} ) version ( ${CH_RELEASE_VERSION} )." - exit 1 - - elif [ "X${LATEST_VER_TYPE}" = "Xchver" ]; then - echo "WARNING: ChangeLog file ( ${CHANGELOGFILE} ) version ( ${CH_RELEASE_VERSION} ) is later than Github release tag ( ${GIT_RELEASE_VERSION} )." - echo " Then RELEASE_VERSION file is put Github release tag ( ${GIT_RELEASE_VERSION} )" - - RELEASE_VERSION=${GIT_RELEASE_VERSION} - - else - # LATEST_VER_TYPE is not set, this means same version. - - RELEASE_VERSION=${GIT_RELEASE_VERSION} - fi - - else - # - # Not check version number, so only use it from Github - # - RELEASE_VERSION=${GIT_RELEASE_VERSION} - fi - -elif [ "X${GIT_RELEASE_VERSION}" != "X" ]; then - RELEASE_VERSION=${GIT_RELEASE_VERSION} - -elif [ "X${CH_RELEASE_VERSION}" != "X" ]; then - RELEASE_VERSION=${CH_RELEASE_VERSION} - -elif [ -f ${SRCTOP}/RELEASE_VERSION ]; then - RELEASE_VERSION=`cat ${SRCTOP}/RELEASE_VERSION` - IS_PUT_RELEASE_VERSION_FILE=no - -else - echo "ERROR: There is no version number information." - echo " The version number must be given by Github release tag" - echo " or ChangeLog file or RELEASE_VERSION file." - exit 1 -fi - -##################################################################### -# -# Make RELEASE_VERSION file -# -if [ "X${IS_PUT_RELEASE_VERSION_FILE}" = "Xyes" ]; then - echo "MESSAGE: Put version number ${RELEASE_VERSION} to RELEASE_VERSION file" - - echo -n ${RELEASE_VERSION} > ${SRCTOP}/RELEASE_VERSION -fi - - -##################################################################### -# -# finish -# -echo "SUCCEED: ${PROGRAM_NAME}" -echo " Result - Version number is ${RELEASE_VERSION}" -echo "" - -exit 0 - -# -# VIM modelines -# -# vim:set ts=4 fenc=utf-8: -# diff --git a/make_rev.sh b/make_rev.sh deleted file mode 100755 index d1da479..0000000 --- a/make_rev.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/sh -# -# K2HASH -# -# Copyright 2013 Yahoo! JAPAN corporation. -# -# K2HASH is key-valuew store base libraries. -# K2HASH is made for the purpose of the construction of -# original KVS system and the offer of the library. -# The characteristic is this KVS library which Key can -# layer. And can support multi-processing and multi-thread, -# and is provided safely as available KVS. -# -# For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. -# -# AUTHOR: Takeshi Nakatani -# CREATE: Thu May 1 2014 -# REVISION: -# - -# -# This script puts git commit hash string to C header file. -# ex: static const char version[] = "...."; -# -# Usage: make_rev.sh -# - -PRGNAME=`basename $0` - -if [ $# -ne 2 ]; then - echo "${PRGNAME}: Error - parameter is not found." - exit 1 -fi -FILEPATH=$1 -VALUENAME=$2 - -REVISION=`git rev-parse --short HEAD` -if [ $? -ne 0 ]; then - echo "${PRGNAME}: Warning - git commit hash code is not found, so set to \"unknown\"." - REVISION="unknown" -fi -#echo ${REVISION} - -NEWCODES="char ${VALUENAME}[] = \"${REVISION}\";" -#echo ${NEWCODES} - -if [ -f ${FILEPATH} ]; then - FILECODES=`cat ${FILEPATH}` - #echo ${FILECODES} - - if [ "X${FILECODES}" = "X${NEWCODES}" ]; then - #echo "${PRGNAME}: ${FILEPATH} is not updated." - exit 0 - fi -fi - -echo ${NEWCODES} > ${FILEPATH} -echo "${PRGNAME}: ${FILEPATH} is updated." - -exit 0 - -# -# VIM modelines -# -# vim:set ts=4 fenc=utf-8: -# diff --git a/make_valiables.sh b/make_valiables.sh deleted file mode 100755 index 8f7a1bb..0000000 --- a/make_valiables.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# Puts project variables for building package - make_valiables.sh -# -# Copyright 2016 Yahoo! JAPAN corporation. -# -# Templates for customizing screwdriver CPP and autotools. -# This template files are provided by yjcore team. -# -# This script file puts some formatted variables for building -# packages. For building package, the project needs common -# valiables for example package name, version, etc. -# This script puts those variables from project files. -# -# For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. -# -# AUTHOR: Takeshi Nakatani -# CREATE: Thu, Jun 2 2016 -# REVISION: -# - -# -# Usage make_valiables.sh [-pkg_version | -lib_version_info | -major_number] -# -PROGRAM_NAME=`basename $0` -PROGRAM_DIR=`dirname $0` -TOP_DIR=`cd ${PROGRAM_DIR}; pwd` - -if [ $# -ne 1 ]; then - echo "ERROR: ${PROGRAM_NAME} needs parameter." - echo "Usage: ${PROGRAM_NAME} [-pkg_version | -lib_version_info | -lib_version_for_link | -major_number]" - exit 1 -fi - -# -# Make result -# -if [ "X$1" = "X-h" -o "X$1" = "X--help" ]; then - echo "Usage: ${PROGRAM_NAME} [-pkg_version | -lib_version_info | -major_number]" - exit 0 - -elif [ "X$1" = "X-pkg_version" ]; then - RESULT=`cat ${TOP_DIR}/RELEASE_VERSION` - -elif [ "X$1" = "X-lib_version_info" -o "X$1" = "X-lib_version_for_link" ]; then - MAJOR_VERSION=`cat ${TOP_DIR}/RELEASE_VERSION | sed 's/["|\.]/ /g' | awk '{print $1}'` - MID_VERSION=`cat ${TOP_DIR}/RELEASE_VERSION | sed 's/["|\.]/ /g' | awk '{print $2}'` - LAST_VERSION=`cat ${TOP_DIR}/RELEASE_VERSION | sed 's/["|\.]/ /g' | awk '{print $3}'` - - # check version number - expr "${MAJOR_VERSION}" + 1 >/dev/null 2>&1 - if [ $? -ge 2 ]; then - echo "ERROR: wrong version number in RELEASE_VERSION file" - exit 1 - fi - expr "${MID_VERSION}" + 1 >/dev/null 2>&1 - if [ $? -ge 2 ]; then - echo "ERROR: wrong version number in RELEASE_VERSION file" - exit 1 - fi - expr "${LAST_VERSION}" + 1 >/dev/null 2>&1 - if [ $? -ge 2 ]; then - echo "ERROR: wrong version number in RELEASE_VERSION file" - exit 1 - fi - - # make library revision number - if [ ${MID_VERSION} -gt 0 ]; then - REV_VERSION=`expr ${MID_VERSION} \* 1000` - REV_VERSION=`expr ${LAST_VERSION} + ${REV_VERSION}` - else - REV_VERSION=${LAST_VERSION} - fi - - if [ "X$1" = "X-lib_version_info" ]; then - RESULT="${MAJOR_VERSION}:${REV_VERSION}:0" - else - RESULT="${MAJOR_VERSION}.0.${REV_VERSION}" - fi - -elif [ "X$1" = "X-major_number" ]; then - RESULT=`cat ${TOP_DIR}/RELEASE_VERSION | sed 's/["|\.]/ /g' | awk '{print $1}'` - -else - echo "ERROR: unkown parameter $1" - echo "Usage: ${PROGRAM_NAME} [-pkg_version | -lib_version_info | -lib_version_for_link | -major_number]" - exit 1 -fi - -# -# Output result -# -echo -n $RESULT - -exit 0 - -# -# VIM modelines -# -# vim:set ts=4 fenc=utf-8: -# diff --git a/tests/Makefile.am b/tests/Makefile.am index fbd5a88..c66eebe 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -11,56 +11,54 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Fri Dec 2 2013 # REVISION: # -## TEST = @echo "NOTICE: this is test" bin_PROGRAMS = k2hlinetool k2hreplace k2hcompress k2htouch k2himport k2hbench noinst_PROGRAMS = k2hinittest k2hrwtest k2hmemtest k2hexttest k2hstreamtest k2hinittest_SOURCES = k2hinittest.cc -k2hinittest_LDADD = -L../lib/.libs -lk2hash +k2hinittest_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hrwtest_SOURCES = k2hrwtest.cc -k2hrwtest_LDADD = -L../lib/.libs -lk2hash +k2hrwtest_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hmemtest_SOURCES = k2hmemtest.cc -k2hmemtest_LDADD = -L../lib/.libs -lk2hash +k2hmemtest_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hlinetool_SOURCES = k2hlinetool.cc -k2hlinetool_LDADD = -L../lib/.libs -lk2hash +k2hlinetool_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hexttest_SOURCES = k2hexttest.cc -k2hexttest_LDADD = -L../lib/.libs -lk2hash +k2hexttest_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hstreamtest_SOURCES = k2hstreamtest.cc -k2hstreamtest_LDADD = -L../lib/.libs -lk2hash +k2hstreamtest_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hreplace_SOURCES = k2hreplace.cc -k2hreplace_LDADD = -L../lib/.libs -lk2hash +k2hreplace_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hcompress_SOURCES = k2hcompress.cc -k2hcompress_LDADD = -L../lib/.libs -lk2hash +k2hcompress_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2htouch_SOURCES = k2htouch.cc -k2htouch_LDADD = -L../lib/.libs -lk2hash +k2htouch_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2himport_SOURCES = k2himport.cc -k2himport_LDADD = -L../lib/.libs -lk2hash +k2himport_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash k2hbench_SOURCES = k2hbench.cc -k2hbench_LDADD = -L../lib/.libs -lk2hash -lpthread +k2hbench_LDADD = $(fullock_LIBS) -L../lib/.libs -lk2hash -lpthread # # [NOTE] # If you need *.so shared libraries, use lib_LTLIBRARIES macro. # -#lib_LTLIBRARIES = libk2htesthash.la libk2htesttrans.la libk2htestattr.la noinst_LTLIBRARIES = libk2htesthash.la libk2htesttrans.la libk2htestattr.la libk2htesthash_la_SOURCES = k2htesthashfunc.cc diff --git a/tests/k2hbench.cc b/tests/k2hbench.cc index ada9545..211d1f7 100644 --- a/tests/k2hbench.cc +++ b/tests/k2hbench.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2015 Yahoo! JAPAN corporation. + * Copyright 2015 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Apr 20 2015 diff --git a/tests/k2hcompress.cc b/tests/k2hcompress.cc index bacc622..b002324 100644 --- a/tests/k2hcompress.cc +++ b/tests/k2hcompress.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Apr 22 2014 diff --git a/tests/k2hedit b/tests/k2hedit index c0be2ec..db4fbcb 100755 --- a/tests/k2hedit +++ b/tests/k2hedit @@ -2,7 +2,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -12,7 +12,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Tetsuya Mochizuki # CREATE: Thu Jan 15 2015 diff --git a/tests/k2hexttest.cc b/tests/k2hexttest.cc index f2b727e..669f0b7 100644 --- a/tests/k2hexttest.cc +++ b/tests/k2hexttest.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 diff --git a/tests/k2himport.cc b/tests/k2himport.cc index 0699a1a..e1031e9 100644 --- a/tests/k2himport.cc +++ b/tests/k2himport.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2015 Yahoo! JAPAN corporation. + * Copyright 2015 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Tetsuya Mochizuki * CREATE: Thu Jan 26 2015 diff --git a/tests/k2hinittest.cc b/tests/k2hinittest.cc index a2dc5bf..6684e36 100644 --- a/tests/k2hinittest.cc +++ b/tests/k2hinittest.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 diff --git a/tests/k2hlinetool.cc b/tests/k2hlinetool.cc index 0656de3..b2c9241 100644 --- a/tests/k2hlinetool.cc +++ b/tests/k2hlinetool.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 diff --git a/tests/k2hmemtest.cc b/tests/k2hmemtest.cc index 144d291..d5c4267 100644 --- a/tests/k2hmemtest.cc +++ b/tests/k2hmemtest.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 diff --git a/tests/k2hreplace.cc b/tests/k2hreplace.cc index 560e05b..3bcb2b7 100644 --- a/tests/k2hreplace.cc +++ b/tests/k2hreplace.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Apr 22 2014 diff --git a/tests/k2hrwtest.cc b/tests/k2hrwtest.cc index fc5d372..19ff09a 100644 --- a/tests/k2hrwtest.cc +++ b/tests/k2hrwtest.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 diff --git a/tests/k2hstreamtest.cc b/tests/k2hstreamtest.cc index 7464659..27a79a7 100644 --- a/tests/k2hstreamtest.cc +++ b/tests/k2hstreamtest.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Fri Nor 31 2013 diff --git a/tests/k2htestattrfunc.cc b/tests/k2htestattrfunc.cc index 99f4b29..966d3ff 100644 --- a/tests/k2htestattrfunc.cc +++ b/tests/k2htestattrfunc.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Tue Dec 22 2015 diff --git a/tests/k2htesthashfunc.cc b/tests/k2htesthashfunc.cc index c46f9dc..aae4d59 100644 --- a/tests/k2htesthashfunc.cc +++ b/tests/k2htesthashfunc.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 17 2014 diff --git a/tests/k2htesttransfunc.cc b/tests/k2htesttransfunc.cc index fddeefa..22c085e 100644 --- a/tests/k2htesttransfunc.cc +++ b/tests/k2htesttransfunc.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2013 Yahoo! JAPAN corporation. + * Copyright 2013 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Takeshi Nakatani * CREATE: Mon Feb 17 2014 diff --git a/tests/k2htouch.cc b/tests/k2htouch.cc index 28c164a..3d3dbe9 100644 --- a/tests/k2htouch.cc +++ b/tests/k2htouch.cc @@ -1,7 +1,7 @@ /* * K2HASH * - * Copyright 2015 Yahoo! JAPAN corporation. + * Copyright 2015 Yahoo Japan Corporation. * * K2HASH is key-valuew store base libraries. * K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ * and is provided safely as available KVS. * * For the full copyright and license information, please view - * the LICENSE file that was distributed with this source code. + * the license file that was distributed with this source code. * * AUTHOR: Tetsuya Mochizuki * CREATE: Thu Jan 15 2015 diff --git a/tests/test.sh b/tests/test.sh index 51a3202..a7824ac 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -2,7 +2,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -12,7 +12,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Tue Nov 29 2013 diff --git a/tests/test_encrypt_keys b/tests/test_encrypt_keys index e5f6784..a5c7294 100644 --- a/tests/test_encrypt_keys +++ b/tests/test_encrypt_keys @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2016 Yahoo! JAPAN corporation. +# Copyright 2016 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Tetsuya Mochizuki # CREATE: Fri Feb 2 2015 diff --git a/tests/test_k2htouch.sh b/tests/test_k2htouch.sh index 876eeee..f357825 100755 --- a/tests/test_k2htouch.sh +++ b/tests/test_k2htouch.sh @@ -2,7 +2,7 @@ # # K2HASH # -# Copyright 2016 Yahoo! JAPAN corporation. +# Copyright 2016 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -12,7 +12,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Tetsuya Mochizuki # CREATE: Wed Jan 27 2016 diff --git a/tests/test_k2htouch_attr.sh b/tests/test_k2htouch_attr.sh index 876eeee..f357825 100755 --- a/tests/test_k2htouch_attr.sh +++ b/tests/test_k2htouch_attr.sh @@ -2,7 +2,7 @@ # # K2HASH # -# Copyright 2016 Yahoo! JAPAN corporation. +# Copyright 2016 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -12,7 +12,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Tetsuya Mochizuki # CREATE: Wed Jan 27 2016 diff --git a/tests/test_linetool.cmd b/tests/test_linetool.cmd index 1bb09f3..add8cb6 100644 --- a/tests/test_linetool.cmd +++ b/tests/test_linetool.cmd @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2016 Yahoo! JAPAN corporation. +# Copyright 2016 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Wed Jan 20 2016 diff --git a/tests/test_linetool.log b/tests/test_linetool.log index 5311301..d26a59b 100644 --- a/tests/test_linetool.log +++ b/tests/test_linetool.log @@ -12,9 +12,9 @@ Attached paraemters: Permanent file(/tmp/k2hash_test_linetool.k2h) is completely initialied. -K2HASH library Version 1.0.55 (commit: e8109f9) +K2HASH library Version 1.0.57 (commit: 36eb21e) with OpenSSL -Copyright 2013 Yahoo! JAPAN corporation. +Copyright(C) 2013 Yahoo Japan Corporation. K2HASH is key-valuew store base libraries. K2HASH is made for the purpose of the construction of original KVS system and the @@ -23,9 +23,9 @@ which Key can layer. And can support multi-processing and multi-thread, and is provided safely as available KVS. -FULLOCK Version 1.0.24 (commit: eea4e33) +FULLOCK Version 1.0.27 (commit: 7d68fdc) -Copyright 2015 Yahoo! JAPAN corporation. +Copyright(C) 2015 Yahoo Japan Corporation. FULLOCK is fast locking library on user level by Yahoo! JAPAN. FULLOCK is following specifications. diff --git a/tests/test_linetool.sh b/tests/test_linetool.sh index 362fab4..f39037c 100755 --- a/tests/test_linetool.sh +++ b/tests/test_linetool.sh @@ -2,7 +2,7 @@ # # K2HASH # -# Copyright 2016 Yahoo! JAPAN corporation. +# Copyright 2016 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -12,7 +12,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Wed Jan 20 2016 diff --git a/tests/test_linetool_dsave.cmd b/tests/test_linetool_dsave.cmd index 565fc53..d551a38 100644 --- a/tests/test_linetool_dsave.cmd +++ b/tests/test_linetool_dsave.cmd @@ -1,7 +1,7 @@ # # K2HASH # -# Copyright 2016 Yahoo! JAPAN corporation. +# Copyright 2016 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -11,7 +11,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Mon Jul 04 2016 diff --git a/tests/testk2htouch.sh b/tests/testk2htouch.sh index 58db3df..c845eee 100755 --- a/tests/testk2htouch.sh +++ b/tests/testk2htouch.sh @@ -2,7 +2,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -12,7 +12,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Tetsuya Mochizuki # CREATE: Fri Feb 2 2015 diff --git a/tests/testtool.sh b/tests/testtool.sh index 73b3a63..dd669ac 100755 --- a/tests/testtool.sh +++ b/tests/testtool.sh @@ -2,7 +2,7 @@ # # K2HASH # -# Copyright 2013 Yahoo! JAPAN corporation. +# Copyright 2013 Yahoo Japan Corporation. # # K2HASH is key-valuew store base libraries. # K2HASH is made for the purpose of the construction of @@ -12,7 +12,7 @@ # and is provided safely as available KVS. # # For the full copyright and license information, please view -# the LICENSE file that was distributed with this source code. +# the license file that was distributed with this source code. # # AUTHOR: Takeshi Nakatani # CREATE: Mon Feb 24 2014