Merge pull request #16 from glmcdona/glmcdona/msbuild-fix

Fix msbuild with newer compilers

setup.py

@@ -8,12 +8,11 @@
 from pybind11.setup_helpers import Pybind11Extension, build_ext
 from setuptools import setup, Extension, find_packages
 
-__version__ = "0.1.7"
+__version__ = "0.1.8"
 
 ext_modules = [
     Pybind11Extension("binary2strings",
         sorted(glob("src/*.cpp")),  # Sort source files for reproducibility
-        headers = sorted(glob("src/*.hpp")),
         define_macros = [('VERSION_INFO', __version__)],
         include_dirs = ["src"],
         ),

src/binary2strings.cpp

@@ -2,7 +2,7 @@
 
 using namespace std;
 
-size_t try_utf8_char_step(const unsigned char* buffer, size_t buffer_size, long offset)
+size_t try_utf8_char_step(const unsigned char* buffer, size_t buffer_size, size_t offset)
 {
 	// Returns 0 if it's not likely a valid utf8 character. For ascii range of characters it requires
 	// the character to be a displayable character.
@@ -99,14 +99,14 @@ int get_language_group(char16_t c)
 // Switch the definition based on platform:
 #if defined(_WIN32) || defined(_WIN64)
 // Note: Buffer overrun security checks disabled, since they added ~50% overhead.
-__declspec(safebuffers) extracted_string* try_extract_string(const unsigned char* buffer, size_t buffer_size, long offset, size_t min_chars)
+__declspec(safebuffers) extracted_string* try_extract_string(const unsigned char* buffer, size_t buffer_size, size_t offset, size_t min_chars)
 #else
-extracted_string* try_extract_string(const unsigned char* buffer, size_t buffer_size, long offset, size_t min_chars)
+extracted_string* try_extract_string(const unsigned char* buffer, size_t buffer_size, size_t offset, size_t min_chars)
 #endif
 {
 	// Try extracting the string as either utf8 or unicode wchar format. Returns None if it's not a valid string.
-	int i;
-	int char_count;
+	size_t i;
+	size_t char_count;
 
 	// Try to parse as utf8 first
 	size_t utf_char_len;
@@ -192,7 +192,7 @@ extracted_string* try_extract_string(const unsigned char* buffer, size_t buffer_
 }
 
 
-std::tuple<string, string, std::pair<int, int>, bool> try_extract_string_tuple(const unsigned char* buffer, size_t buffer_size, long offset, size_t min_chars, bool only_interesting)
+std::tuple<string, string, std::pair<size_t, size_t>, bool> try_extract_string_tuple(const unsigned char* buffer, size_t buffer_size, size_t offset, size_t min_chars, bool only_interesting)
 {
 	// Simple wrapper to return a tuple instead
 	extracted_string* s = try_extract_string(buffer, buffer_size, offset, min_chars);
@@ -206,7 +206,7 @@ std::tuple<string, string, std::pair<int, int>, bool> try_extract_string_tuple(c
 			auto result = std::make_tuple(
 				s->get_string(),
 				s->get_type_string(),
-				std::pair<int, int>(s->get_offset_start(), s->get_offset_end()),
+				std::pair<size_t, size_t>(s->get_offset_start(), s->get_offset_end()),
 				is_interesting
 			);
 
@@ -220,11 +220,11 @@ std::tuple<string, string, std::pair<int, int>, bool> try_extract_string_tuple(c
 }
 
 
-vector<std::tuple<string, string, std::pair<int, int>, bool>> extract_all_strings(const unsigned char buffer[], size_t buffer_size, size_t min_chars, bool only_interesting)
+vector<std::tuple<string, string, std::pair<size_t, size_t>, bool>> extract_all_strings(const unsigned char buffer[], size_t buffer_size, size_t min_chars, bool only_interesting)
 {
 	// Process the specified binary buffer and extract all strings
-	long offset = 0;
-	vector<std::tuple<string, string, std::pair<int, int>, bool>> r_vect;
+	size_t offset = 0;
+	vector<std::tuple<string, string, std::pair<size_t, size_t>, bool>> r_vect;
 	vector<float> proba_interesting_vect;
 	vector<float> proba_interesting_avg_vect;
 	extracted_string* s;
@@ -242,10 +242,10 @@ vector<std::tuple<string, string, std::pair<int, int>, bool>> extract_all_string
 
 			// Add the new string
 			r_vect.push_back(
-				tuple<string, string, std::pair<int, int>, bool>(
+				tuple<string, string, std::pair<size_t, size_t>, bool>(
 					s->get_string(),
 					s->get_type_string(),
-					std::pair<int, int>(s->get_offset_start(), s->get_offset_end()),
+					std::pair<size_t, size_t>(s->get_offset_start(), s->get_offset_end()),
 					proba_interesting > 0.5
 					)
 			);
@@ -268,7 +268,7 @@ vector<std::tuple<string, string, std::pair<int, int>, bool>> extract_all_string
 			proba_interesting_vect.push_back(proba_interesting);
 
 			// Advance by the byte-length of the string
-			offset += (long)s->get_size_in_bytes();
+			offset += s->get_size_in_bytes();
 
 			// Cleanup
 			delete s;
@@ -280,7 +280,7 @@ vector<std::tuple<string, string, std::pair<int, int>, bool>> extract_all_string
 	}
 
 	// Have a pass through the strings averaging the interestingness and filtering
-	vector<std::tuple<string, string, std::pair<int, int>, bool>> r_vect_filt;
+	vector<std::tuple<string, string, std::pair<size_t, size_t>, bool>> r_vect_filt;
 	for (int i = 0; i < r_vect.size(); i++)
 	{
 		// Get the interestingness
@@ -298,7 +298,7 @@ vector<std::tuple<string, string, std::pair<int, int>, bool>> extract_all_string
 		if (!only_interesting || proba_interesting_avg >= 0.2 || proba_interesting_vect[i] >= 0.5)
 		{
 			r_vect_filt.push_back(
-				tuple<string, string, std::pair<int, int>, bool>(
+				tuple<string, string, std::pair<size_t, size_t>, bool>(
 					std::get<0>(r_vect[i]),
 					std::get<1>(r_vect[i]),
 					std::get<2>(r_vect[i]),

src/binary2strings.hpp

@@ -63,12 +63,12 @@ static std::unordered_set<char16_t> is_seen_commoncrawl({
 	}
 );
 
-size_t try_utf8_char_step(const unsigned char* buffer, size_t buffer_size, long offset);
+size_t try_utf8_char_step(const unsigned char* buffer, size_t buffer_size, size_t offset);
 
 int get_language_group(char16_t c);
 
-extracted_string* try_extract_string(const unsigned char* buffer, size_t buffer_size, long offset, size_t min_chars);
+extracted_string* try_extract_string(const unsigned char* buffer, size_t buffer_size, size_t offset, size_t min_chars);
 
-std::tuple<string, string, std::pair<int, int>, bool> try_extract_string_tuple(const unsigned char* buffer, size_t buffer_size, long offset, size_t min_chars, bool only_interesting);
+std::tuple<string, string, std::pair<size_t, size_t>, bool> try_extract_string_tuple(const unsigned char* buffer, size_t buffer_size, size_t offset, size_t min_chars, bool only_interesting);
 
-vector<std::tuple<string, string, std::pair<int, int>, bool>> extract_all_strings(const unsigned char buffer[], size_t buffer_size, size_t min_chars, bool only_interesting);
+vector<std::tuple<string, string, std::pair<size_t, size_t>, bool>> extract_all_strings(const unsigned char buffer[], size_t buffer_size, size_t min_chars, bool only_interesting);

src/extracted_string.cpp

@@ -3,18 +3,19 @@
 
 using namespace std;
 
+#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING
 std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> _conv16;
 
 extracted_string::extracted_string()
 {
 	m_type = TYPE_UNDETERMINED;
-	m_string = (std::string)NULL;
+	m_string = "";
 	m_size_in_bytes = 0;
 	m_offset_start = 0;
 	m_offset_end = 0;
 }
 
-extracted_string::extracted_string(const char* string, size_t size_in_bytes, STRING_TYPE type, int offset_start, int offset_end)
+extracted_string::extracted_string(const char* string, size_t size_in_bytes, STRING_TYPE type, size_t offset_start, size_t offset_end)
 {
 	m_type = type;
 	m_string = std::string(string, size_in_bytes);
@@ -23,7 +24,7 @@ extracted_string::extracted_string(const char* string, size_t size_in_bytes, STR
 	m_offset_end = offset_end;
 }
 
-extracted_string::extracted_string(const char16_t* string, size_t size_in_bytes, STRING_TYPE type, int offset_start, int offset_end)
+extracted_string::extracted_string(const char16_t* string, size_t size_in_bytes, STRING_TYPE type, size_t offset_start, size_t offset_end)
 {
 	m_type = type;
 
@@ -126,12 +127,12 @@ string extracted_string::get_type_string()
 	}
 }
 
-int extracted_string::get_offset_start()
+size_t extracted_string::get_offset_start()
 {
 	return m_offset_start;
 }
 
-int extracted_string::get_offset_end()
+size_t extracted_string::get_offset_end()
 {
 	return m_offset_end;
 }

src/extracted_string.hpp

@@ -1,6 +1,7 @@
 // Class for extracted strings
 #pragma once
 #include <string>
+#define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS
 #include <codecvt>
 #include "string_model.hpp"
 #include <unordered_set>
@@ -24,21 +25,21 @@ class extracted_string
 	STRING_TYPE m_type;
 	std::string m_string; // Supports Utf8
 	size_t m_size_in_bytes;
-	int m_offset_start;
-	int m_offset_end;
+	size_t m_offset_start;
+	size_t m_offset_end;
 
 public:
 	extracted_string();
-	extracted_string(const char* string, size_t size_in_bytes, STRING_TYPE type, int offset_start, int offset_end);
-	extracted_string(const char16_t* string, size_t size_in_bytes, STRING_TYPE type, int offset_start, int offset_end);
+	extracted_string(const char* string, size_t size_in_bytes, STRING_TYPE type, size_t offset_start, size_t offset_end);
+	extracted_string(const char16_t* string, size_t size_in_bytes, STRING_TYPE type, size_t offset_start, size_t offset_end);
 
 	float get_proba_interesting();
 	size_t get_size_in_bytes();
 	string get_string();
 	STRING_TYPE get_type();
 	string get_type_string();
-	int get_offset_start();
-	int get_offset_end();
+	size_t get_offset_start();
+	size_t get_offset_end();
 
 	bool is_interesting();