diff --git a/Makefile.am b/Makefile.am index 360297020..22d4860d9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -41,6 +41,7 @@ win64: mv src/sha256deep.exe src/sha256deep64.exe mv src/tigerdeep.exe src/tigerdeep64.exe mv src/whirlpooldeep.exe src/whirlpooldeep64.exe + mv src/nulldeep.exe src/nulldeep64.exe mv src/hashdeep.exe src/hashdeep64.exe strip32: diff --git a/doc/hashdeep.html b/doc/hashdeep.html index 3f7d29e0a..e6987e70d 100644 --- a/doc/hashdeep.html +++ b/doc/hashdeep.html @@ -37,7 +37,7 @@

Description

-c <alg1>[,<alg2>...]
Computation mode. Compute hashes of FILES using the algorithms specified. Legal values are -md5, sha1, sha256, tiger, and whirlpool. +md5, sha1, sha256, tiger, whirlpool, and null.

diff --git a/man/Makefile.am b/man/Makefile.am index cef4f2bd0..e8e50dc38 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -1,6 +1,6 @@ -ALL_GOALS={md5,sha1,sha256,sha3,tiger,whirlpool,hashdeep} +ALL_GOALS={md5,sha1,sha256,sha3,tiger,whirlpool,null,hashdeep} -man_MANS=md5deep.1 sha1deep.1 sha256deep.1 tigerdeep.1 whirlpooldeep.1 hashdeep.1 #sha3deep.1 +man_MANS=md5deep.1 sha1deep.1 sha256deep.1 tigerdeep.1 whirlpooldeep.1 nulldeep.1 hashdeep.1 #sha3deep.1 # We erase the old man pages, just in case they might be symbolic links # (symbolic links were used in a previous release) @@ -14,6 +14,7 @@ update-manpages: md5deep.1 # $(INSTALL) $< sha3deep.1 $(INSTALL) $< tigerdeep.1 $(INSTALL) $< whirlpooldeep.1 + $(INSTALL) $< nulldeep.1 EXTRA_DIST=${man_MANS} diff --git a/man/hashdeep.1 b/man/hashdeep.1 index 169516679..4071bc7b7 100644 --- a/man/hashdeep.1 +++ b/man/hashdeep.1 @@ -31,8 +31,8 @@ reads from standard input. .TP \fB\-c [,...]\fR Computation mode. Compute hashes of FILES using the algorithms -specified. Legal values are md5, sha1, sha256, tiger, and -whirlpool. +specified. Legal values are md5, sha1, sha256, tiger, whirlpool, +and null. .TP diff --git a/man/nulldeep.1 b/man/nulldeep.1 new file mode 100644 index 000000000..adfcf86d0 --- /dev/null +++ b/man/nulldeep.1 @@ -0,0 +1,378 @@ +.TH MD5DEEP "1" "v4.4 \- 29 Jan 2014" "AFOSI" "United States Air Force" + +.SH NAME +md5deep \- Compute and compare MD5 message digests +.br +sha1deep \- Compute and compare SHA-1 message digests +.br +sha256deep \- Compute and compare SHA-256 message digests +.br +sha3deep \- Compute and compare SHA-3-256 message digests +.br +tigerdeep \- Compute and compare Tiger message digests +.br +whirlpooldeep \- Compute and compare Whirlpool message digests +.br +nulldeep \- Compute and compare null message digests + +.SH SYNOPSIS +.B md5deep +-v | -V | -h +.br +.B md5deep +[\-m|\-M|\-x|\-X ] [-a|-A ] [\-f ] +[\-p ] [\-i ] [\-tnwzresS0lbkqZud] [\-F ] +[\-o ] [\-j ] [[\fBFILES\fR] + +.SH DESCRIPTION +.PP +Computes the hashes, or message digest, +for any number of files while +optionally recursively digging through the directory structure. +Can also take a list of known hashes and display the filenames +of input files whose hashes either do or do not match any of the +known hashes. +Errors are reported to standard error. If no FILES are specified, +reads from standard input. + +.TP +\fB\-p \fR +Piecewise mode. Breaks files into chunks before hashing. +Chunks may be specified +using IEC multipliers b, k, m, g, t, p, or e. +(Never let it be +said that the author didn't plan ahead!) +This mode cannot be used with the \-z mode. + +.TP +\fB\-i|\-I \fR +Size threshold mode. Only hash files smaller than the given the +threshold. In \-i mode, simply omits those files larger than the +threshold. In \-I mode, displays all files, but uses asterisks +for the hashes of files larger than the threshold. +Sizes may be specified +using IEC multipliers b, k, m, g, t, p, or e. + +.TP +\fB\-r\fR +Enables recursive mode. All subdirectories are traversed. Please note +that recursive mode cannot be used to examine all files of a given +file extension. For example, calling md5deep -r *.txt will examine +all files in \fIdirectories\fR that end in .txt. + +.TP +\fB\-e\fR +Displays a progress indicator and estimate of time +remaining for each file being processed. Time estimates for files +larger than 4GB are not available on Windows. This mode may not be +used with th \-p mode. + +.TP +\fB\-m\fR +Enables matching mode. The file given should be a list of known hashes. The +input files are examined one at a time, and only those files that match +the list of known hashes are output. This flag may be used more than once +to add multiple sets of known hashes. Acceptable formats for lists of +known hashes are plain (such as those generated by md5deep or md5sum), +Hashkeeper files, iLook, and the National Software Reference Library +(NSRL) as produced by the National Institute for Standards in Technology. +.br +\fB\fR +If standard input is used with the -m flag, displays "stdin" +if the input matches one of the hashes in the list of known hashes. If the +hash does not match, the program displays no output. +.br +\fB\fR +This flag may not be used in conjunction with the \-x, \-X, or \-A flags. +See the section "UNICODE SUPPORT" below. + +.TP +\fB\-x\fR +Same as the \-m flag above, but does negative matching. That is, only +those files NOT in the list of known hashes are displayed. +.br +\fB\fR +This flag may not be used in conjunction with the \-m, \-M, or \-a flags. +See the section "UNICODE SUPPORT" below. +.TP +\fB\-M\fR and \fB-X\fR +Same as \-m and \-x above, but displays the hash for each file that +does (or does not) match the list of known hashes. + +.TP +\fB\-a\fR +Adds a single hash to the list of known hashes used for matching mode, +and if not already enabled, enables matching mode. Adding single +hashes cannot, by itself, be used to print the hashes of matching files +like the \-M flag does. When used in conjunction with the \-w flag, the +filename displayed is just the hash submitted on the command line. +.br +\fB\fR +This flag may not be used in conjunction with the \-x, \-X, or \-A flags. + +.TP +\fB\-A\fR +Same as \-a above, but does negative matching. +This flag may not be used in conjunction with the \-m, \-M, or \-A flags. + +.TP +\fB\-f\fR +Takes a list of files to be hashed from the specified file. Each +line is assumed to be a filename. This flag can only be used once +per invocation. If it's used a second time, the second instance will +clobber the first. +.br +Note that you can still use other flags, such as the \-m or \-x modes, +and submit additional FILES on the command line. + +.TP +\fB\-w\fR +During any of the matching modes (\-m,\-M,\-x,or \-X), displays the filename +of the known hash that matched the input file. +See the section "UNICODE SUPPORT" below. + +.TP +\fB\-t\fR +Display a timestamp in GMT with each result. On Windows this timestamp +will be the file's creation time. On all other systems it should be +the file's change time. + +.TP +\fB\-n\fR +During any of the matching modes (\-m,\-M,\-x,or \-X), displays only the +filenames of any known hashes that were not matched by any of the input files. + +.TP +\fB\-s\fR +Enables silent mode. All error messages are supressed. + +.TP +\fB\-S\fR +Like silent mode, but still displays warnings on improperly formatted +hashes in the list of known hashes. + +.TP +\fB\-z\fR +Enables file size mode. Prepends the hash with +a ten digit representation of the size of +each file processed. If the file size is greater than +9999999999 bytes (about 9.3GB) +the program displays 9999999999 for the size. + +.TP +\fB\-q\fR +Quiet mode. File names are omitted from the output. Each hash is still +followed by two spaces before the newline. + +.TP +\fB\-Z\fR +Produces output in Triage format. Each line contans +the file's size, a tab, a hash of the first 512 bytes, a tab, +the hash of the complete file, a tab, and the file name. +These values are intended in increasing order of specificity. That +is, two files with different sizes cannot possibly match. This is +a fast comparison and should be done first. Next, two files +with different partial hashes cannot possibly match. This is often +faster than hashing the whole file. Finally, if those two pieces +align, then it's worth reading and hashing the entire file. + +.TP +\fB\-0\fR +Uses a NULL character (/0) to terminate each line instead of a newline. +Useful for processing filenames with strange characters. + +.TP +\fB\-l\fR +Enables relative file paths. Instead of printing the absolute path for +each file, displays the relative file path as indicated on the command +line. This flag may not be used in conjunction with the \-b flag. + +.TP +\fB\-b\fR +Enables bare mode. Strips any leading directory information from +displayed filenames. +This flag may not be used in conjunction with the \-l flag. + +.TP +\fB\-k\fR +Enables asterisk mode. An asterisk is inserted in lieu of a second +space between the filename and the hash, just like md5sum in +its binary (\-b) mode. + +.TP +\fB\-c\fR +Enables comma separated values output, or CSV mode. This mode has the +side effect of removing the 10 digit size limitation from \-z mode. +Also note that asterisks from \-k mode are not displayed when in CSV mode. + +.TP +\fB\-o\fR +Enables expert mode. Allows the user specify which (and only which) types of +files are processed. Directory processing is still controlled with the +\-r flag. The expert mode options allowed are: +.br +f \- Regular files +.br +b \- Block Devices +.br +c \- Character Devices +.br +p \- Named Pipes +.br +l \- Symbolic Links +.br +s \- Sockets +.br +d \- Solaris Doors +.br +e \- Windows PE executables + +.TP +\fB-jnn\fR +Controls multi-threading. By default the program will create one +producer thread to scan the file system and one hashing thread per CPU +core. Multi-threading causes output filenames to be in +non-deterministic order, as files that take longer to hash will be +delayed while they are hashed. If a deterministic order is required, +specify \fB-j0\fR to disable multi-threading + +.TP +\fB-d\fR +Output in Digital Forensics XML (DFXML) format. + +.TP +\fB-u\fR +Quote Unicode output. For example, the snowman is shown as +\fBU+C426\fR. + +.TP +\fB-F\fR +Specifies the input mode that is used to read files. The default is +\fB-Fb\fR (buffered I/O) which reads files with fopen(). Specifying +\fB-Fu\fR will use unbuffered I/O and read the file with +open(). Specifying \fB-Fm\fR will use memory-mapped I/O which will be +faster on some platforms, but which (currently) will not work with +files that produce I/O errors. + +.TP +\fB\-h\fR +Show a help screen and exit. + +.TP +\fB\-v\fR +Show the version number and exit. + +.TP +\fB\-V\fR +Show copyright information and exit. + +.SH UNICODE SUPPORT +As of version 3.0 the program supports Unicode characters in filenames +on Microsoft Windows systems for filenames specified on the command +line with globbing (e.g. *), for files specified with the +\fB-f\fR of files to hash, and for files read from directories using +the \fB-r\fR option. + +By default all program input and output +should be in UTF-8. The program automatically converts this to UTF-16 +for opening files). + +On Unix/Linux/MacOS, you should use a terminal emulator that supports +UTF-8 and UTF-8 characters in filenames will be properly displayed. + +On Windows, the programs do not display Unicode characters on the console. +You must either redirect output to a file and open the +file with Wordpad (which can display Unicode), or you must specify the +\fB-u\fR option to quote Unicode using standard \fBU+XXXX\fR notation. + +Currently the file name of a file containing known hashes may not be +specified as a unicode filename, but you can specify the name using +tab completition or an asterisk (e.g. md5deep -m *.txt where there is +only one file with a .txt extension). + +.SH RETURN VALUE +Returns a bit-wise value based on the success of the operation and the +status of any matching operations. +.PP +.TP +0 +Success. Note that the program considers itself successful even when it +encounters read errors, permission denied errors, or finds directories +when not in recursive mode. +.TP +1 +Unused hashes. Under any of the matching modes, returns this +value if one or more of the +known hashes was not matched by any of the input files. +.TP +2 +Unmatched inputs. Under any of the matching modes, returns this value +if one or more of the input values did not match any of the known hashes. +.TP +64 +User error, such as trying to do both positive and negative matching at +the same time. +.TP +128 +Internal error, such as memory corruption or uncaught cycle. +All internal errors should +be reported to the developer! See the section "Reporting Bugs" below. + + +.SH AUTHOR +md5deep was written by Jesse Kornblum, research@jessekornblum.com +and Simson Garfinkel. + +.SH KNOWN ISSUES +Using the \-r flag cannot be used to recursively process all files +of a given extension in a directory. This is a feature, not a bug. +If you need to do this, use the \fBfind\fR(1) command. + +.SH REPORTING BUGS +We take all bug reports \fIvery\fR seriously. Any bug that jeopardizes the +forensic integrity of this program could have serious consequences on +people's lives. When submitting a bug report, please include a description +of the problem, how you found it, and your contact information. +.PP +Send bug reports to the author at the address above. + +.PP +.SH COPYRIGHT +This program is a work of the US Government. In accordance with 17 USC 105, +copyright protection is not available for any work of the US Government. +This program is PUBLIC DOMAIN. Portions of this program contain code +that is licensed under the terms of the General Public License (GPL). +Those portions retain their original copyright and license. See the file +COPYING for more details. +.PP +There is NO warranty for this program; +not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +.SH SEE ALSO +More information and installation instructions can be found in the README +file. Current versions of both documents can be found on the project homepage: +http://md5deep.sourceforge.net/ +.PP +The MD5 specification, RFC 1321, is available at +.br +http://www.ietf.org/rfc/rfc1321.txt +.PP +The SHA-1 specification, RFC 3174, is available at +.br +http://www.faqs.org/rfcs/rfc3174.html +.PP +The SHA-256 specification, FIPS 180-2, is available at +.br +http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf +.PP +The SHA-3-256 specification is available at +.br +http://keccak.noekeon.org/ +.PP +The Tiger specification is available at +.br +http://www.cs.technion.ac.il/~biham/Reports/Tiger/ +.PP +The Whirlpool specification is available at +.br +http://planeta.terra.com.br/informatica/paulobarreto/WhirlpoolPage.html diff --git a/src/Makefile.am b/src/Makefile.am index f1afdf208..6621e547e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,7 +8,7 @@ ALGS=md5.c md5.h sha1.c sha1.h sha256.c sha256.h whirlpool.c whirlpool.h tiger.c all_sources = $(ALGS) main.cpp hashlist.cpp multihash.cpp display.cpp \ hash.cpp dig.cpp helpers.cpp xml.cpp xml.h files.cpp common.h main.h \ utf8.h utf8/checked.h utf8/core.h utf8/unchecked.h \ - threadpool.h threadpool.cpp winpe.cpp winpe.h + threadpool.h threadpool.cpp winpe.cpp winpe.h hashdeep_SOURCES = $(all_sources) md5deep_SOURCES = $(all_sources) @@ -16,8 +16,9 @@ sha1deep_SOURCES = $(all_sources) sha256deep_SOURCES = $(all_sources) whirlpooldeep_SOURCES = $(all_sources) tigerdeep_SOURCES = $(all_sources) +nulldeep_SOURCES = $(all_sources) -bin_PROGRAMS = hashdeep md5deep sha1deep sha256deep whirlpooldeep tigerdeep +bin_PROGRAMS = hashdeep md5deep sha1deep sha256deep whirlpooldeep tigerdeep nulldeep # Yes, this is gross; it would be better to make them all with hard links. # But this works. That didn't. diff --git a/src/main.cpp b/src/main.cpp index b605d2b82..8f06b1cdf 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -422,7 +422,20 @@ void cc_sha256_final(void *ctx, unsigned char *digest) } #endif +/* + * Why aren't these gotten from null.c? + */ +void hash_init_null(void * ctx) +{ +} +void hash_update_null(void * ctx, const unsigned char *buf, size_t len) +{ +} + +void hash_final_null(void * ctx, unsigned char *digest) +{ +} /* * Load the hashing algorithms array. @@ -445,6 +458,7 @@ void algorithm_t::load_hashing_algorithms() #endif add_algorithm(alg_tiger, "tiger", 192, hash_init_tiger, hash_update_tiger, hash_final_tiger, DEFAULT_ENABLE_TIGER); add_algorithm(alg_whirlpool, "whirlpool", 512, hash_init_whirlpool, hash_update_whirlpool, hash_final_whirlpool, DEFAULT_ENABLE_WHIRLPOOL); + add_algorithm(alg_null, "null", 0, hash_init_null, hash_update_null, hash_final_null, DEFAULT_ENABLE_NULL); //add_algorithm(alg_sha3, // "sha3", diff --git a/src/main.h b/src/main.h index 99df49a6e..e17567dbb 100644 --- a/src/main.h +++ b/src/main.h @@ -62,6 +62,7 @@ typedef enum { alg_sha256, alg_tiger, alg_whirlpool, + alg_null, alg_sha3, // alg_unknown must always be last in this list. It's used @@ -78,6 +79,7 @@ inline std::ostream & operator << (std::ostream &os,const hashid_t &h) case alg_sha256: os << "alg_sha256" ; break ; case alg_tiger: os << "alg_tiger" ; break ; case alg_whirlpool: os << "alg_whirlpool" ; break ; + case alg_null: os << "alg_null" ; break ; case alg_sha3: os << "alg_sha3" ; break ; case alg_unknown: os << "alg_unknown" ; break ; } @@ -93,6 +95,7 @@ inline std::ostream & operator << (std::ostream &os,const hashid_t &h) #define DEFAULT_ENABLE_SHA256 TRUE #define DEFAULT_ENABLE_TIGER FALSE #define DEFAULT_ENABLE_WHIRLPOOL FALSE +#define DEFAULT_ENABLE_NULL FALSE #define DEFAULT_ENABLE_SHA3 FALSE class iomode { diff --git a/tests/tests.sh b/tests/tests.sh index 21320f019..8c76b8d74 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -211,36 +211,40 @@ do 30) cmd="$BASE/whirlpooldeep$EXE -r $HTMP " ;; 31) cmd="$BASE/whirlpooldeep$EXE -p512 -r $HTMP " ;; - 32) cmd="$BASE/hashdeep$EXE -r $HTMP " ;; - 33) cmd="$BASE/hashdeep$EXE -p512 -r $HTMP " ;; - - 34) cmd="$BASE/hashdeep$EXE -m -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; - 35) cmd="$BASE/hashdeep$EXE -M -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; - 36) cmd="$BASE/hashdeep$EXE -w -m -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; - 37) cmd="$BASE/hashdeep$EXE -x -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; - 38) cmd="$BASE/hashdeep$EXE -x -w -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; - 39) cmd="$BASE/hashdeep$EXE -r -a -k hashlist-hashdeep-full.txt $HTMP " ;; - 40) cmd="$BASE/hashdeep$EXE -v -r -a -k hashlist-hashdeep-full.txt $HTMP " ;; - 41) cmd="$BASE/hashdeep$EXE /dev/null ";; + 32) cmd="$BASE/nulldeep$EXE -r $HTMP " ;; + 33) cmd="$BASE/nulldeep$EXE -p512 -r $HTMP " ;; + + 34) cmd="$BASE/hashdeep$EXE -r $HTMP " ;; + 35) cmd="$BASE/hashdeep$EXE -p512 -r $HTMP " ;; + + 36) cmd="$BASE/hashdeep$EXE -m -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; + 37) cmd="$BASE/hashdeep$EXE -M -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; + 38) cmd="$BASE/hashdeep$EXE -w -m -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; + 39) cmd="$BASE/hashdeep$EXE -x -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; + 40) cmd="$BASE/hashdeep$EXE -x -w -k hashlist-hashdeep-partial.txt $HTMP/*.txt " ;; + 41) cmd="$BASE/hashdeep$EXE -r -a -k hashlist-hashdeep-full.txt $HTMP " ;; + 42) cmd="$BASE/hashdeep$EXE -v -r -a -k hashlist-hashdeep-full.txt $HTMP " ;; + 43) cmd="$BASE/hashdeep$EXE /dev/null ";; # The stdin tests - 42) cmd="echo README.txt | $BASE/hashdeep$EXE" ;; - 43) cmd="echo README.txt | $BASE/md5deep$EXE" ;; - 44) cmd="echo README.txt | $BASE/sha1deep$EXE" ;; - 45) cmd="echo README.txt | $BASE/sha256deep$EXE" ;; - 46) cmd="echo README.txt | $BASE/whirlpooldeep$EXE" ;; + 44) cmd="echo README.txt | $BASE/hashdeep$EXE" ;; + 45) cmd="echo README.txt | $BASE/md5deep$EXE" ;; + 46) cmd="echo README.txt | $BASE/sha1deep$EXE" ;; + 47) cmd="echo README.txt | $BASE/sha256deep$EXE" ;; + 48) cmd="echo README.txt | $BASE/whirlpooldeep$EXE" ;; + 49) cmd="echo README.txt | $BASE/nulldeep$EXE" ;; # Additional tests as errors are discovered - 47) cmd="$BASE/hashdeep$EXE -vvvbak known1 -k known2 foo bar moo cow" ;; + 50) cmd="$BASE/hashdeep$EXE -vvvbak known1 -k known2 foo bar moo cow" ;; # BSD style hashes, iLook hashes. # iLook has different behavior with the algorithms, so we test with all - 48) cmd="$BASE/md5deep$EXE -Sm $TESTFILES_DIR/bsd-hashes.txt -r $HTMP" ;; - 49) cmd="$BASE/md5deep$EXE -m $TESTFILES_DIR/ilookv4.hsh -r $HTMP" ;; - 50) cmd="$BASE/sha1deep$EXE -m $TESTFILES_DIR/ilookv4.hsh -r $HTMP" ;; - 51) cmd="$BASE/md5deep$EXE -m $TESTFILES_DIR/nsrlfile.txt -r $HTMP" ;; - 52) cmd="$BASE/sha1deep$EXE -m $TESTFILES_DIR/nsrlfile.txt -r $HTMP" ;; + 51) cmd="$BASE/md5deep$EXE -Sm $TESTFILES_DIR/bsd-hashes.txt -r $HTMP" ;; + 52) cmd="$BASE/md5deep$EXE -m $TESTFILES_DIR/ilookv4.hsh -r $HTMP" ;; + 53) cmd="$BASE/sha1deep$EXE -m $TESTFILES_DIR/ilookv4.hsh -r $HTMP" ;; + 54) cmd="$BASE/md5deep$EXE -m $TESTFILES_DIR/nsrlfile.txt -r $HTMP" ;; + 55) cmd="$BASE/sha1deep$EXE -m $TESTFILES_DIR/nsrlfile.txt -r $HTMP" ;; esac @@ -279,12 +283,12 @@ do tr -d \\r < test$i.out > ref/test$i.out tr -d \\r < test$i.err > ref/test$i.err - # extra addition for test 47 + # extra addition for test 50 if [ $i = 47 ]; then if grep 'Input files examined' ref/test$i.out ; then - echo Fixup for test 47 no longer required + echo Fixup for test 50 no longer required else - echo Applying fixup for test 47 + echo Applying fixup for test 50 echo " Input files examined: 0" >> ref/test$i.out echo " Known files expecting: 0" >> ref/test$i.out sort ref/test$i.out > ref/test$i.out2