Merge pull request #1 from blechschmidt/master

blechschmidt · Apr 29, 2020 · 351cce8 · 351cce8
2 parents 68482eb + adbe221
commit 351cce8
Show file tree

Hide file tree

Showing 24 changed files with 295 additions and 176 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -3,6 +3,7 @@ project(massdns)
 
 set(CMAKE_C_STANDARD 11)
 
-set(SOURCE_FILES main.c module.h list.h hashmap.h massdns.h security.h mixed_list.h net.h string.h buffers.h dns.h
-        timed_ring.h random.h cmd.h flow.h)
-add_executable(massdns ${SOURCE_FILES})
+set(SOURCE_FILES src/main.c src/module.h src/list.h src/hashmap.h src/massdns.h src/security.h src/net.h src/string.h src/buffers.h src/dns.h
+        src/timed_ring.h src/random.h src/cmd.h src/flow.h)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+add_executable(massdns ${SOURCE_FILES})
diff --git a/Makefile b/Makefile
@@ -2,13 +2,16 @@ PREFIX=/usr/local
 
 all:
 	mkdir -p bin
-	$(CC) $(CFLAGS) -O3 -std=c11 -DHAVE_EPOLL -DHAVE_SYSINFO -Wall -fstack-protector-strong main.c -o bin/massdns
+	$(CC) $(CFLAGS) -O3 -std=c11 -DHAVE_EPOLL -DHAVE_SYSINFO -Wall -fstack-protector-strong src/main.c -o bin/massdns
 debug:
 	mkdir -p bin
-	$(CC) $(CFLAGS) -O0 -std=c11 -DHAVE_EPOLL -DHAVE_SYSINFO -Wall -g -DDEBUG main.c -o bin/massdns
+	$(CC) $(CFLAGS) -O0 -std=c11 -DHAVE_EPOLL -DHAVE_SYSINFO -Wall -g -DDEBUG src/main.c -o bin/massdns
 nolinux:
 	mkdir -p bin
-	$(CC) $(CFLAGS) -O3 -std=c11 -Wall -fstack-protector-strong main.c -o bin/massdns
+	$(CC) $(CFLAGS) -O3 -std=c11 -Wall -fstack-protector-strong src/main.c -o bin/massdns
+debugnolinux:
+	mkdir -p bin
+	$(CC) $(CFLAGS) -O0 -std=c11 -Wall -fstack-protector-strong -g -DDEBUG src/main.c -o bin/massdns
 install:
 	test -d $(PREFIX) || mkdir $(PREFIX)
 	test -d $(PREFIX)/bin || mkdir $(PREFIX)/bin

diff --git a/README.md b/README.md
@@ -1,19 +1,12 @@
-# MassDNS 0.3 (experimental)
+# MassDNS 0.3
 ## A high-performance DNS stub resolver
 
-MassDNS is a simple high-performance DNS stub resolver targetting those who seek to resolve a massive amount of domain
+MassDNS is a simple high-performance DNS stub resolver targeting those who seek to resolve a massive amount of domain
 names in the order of millions or even billions. Without special configuration, MassDNS is capable of resolving over
 350,000 names per second using publicly available resolvers.
 
 ## Major changes
-This version of MassDNS is currently experimental. In order to speed up the resolving process, the `ldns` dependency has
-been replaced by a custom stack-based DNS implementation (which currently only supports the text representation of the
-most common DNS records). Furthermore, epoll has been introduced in order to lighten CPU usage when operating with a low
-concurrency which may have broken compatibility with some platforms. In case of bugs, please create an issue and
-[switch to the more mature version 0.2](https://github.com/blechschmidt/massdns/tree/v0.2).
-
-Also note that the command line interface has changed slightly due to criticism of the output complexity. Additionally,
-the default values of the `-s` and `-i` parameters have been changed. The repository structure has been changed as well.
+**The NDJSON output format has changed in order to provide more detailed information and allow better filtering.**
 
 ## Contributors
 * [Quirin Scheitle](https://github.com/quirins), [Technical University of Munich](https://www.net.in.tum.de/members/scheitle/)
@@ -46,6 +39,7 @@ Usage: ./bin/massdns [options] [domainlist]
       --root             Do not drop privileges when running as root. Not recommended.
   -s  --hashmap-size     Number of concurrent lookups. (Default: 10000)
       --sndbuf           Size of the send buffer in bytes.
+      --status-format    Format for real-time status updates, json or ansi (Default: ansi)
       --sticky           Do not switch the resolver when retrying.
       --socket-count     Socket count per process. (Default: 1)
   -t  --type             Record type to be resolved. (Default: A)
@@ -57,7 +51,20 @@ Output flags:
   F - full text output
   B - binary output
   J - ndjson output
+
+Advanced flags for the simple output mode:
+  d - Include records from the additional section.
+  i - Indent any reply record.
+  l - Separate replies using a line feed.
+  m - Only output reply records that match the question name.
+  n - Include records from the answer section.
+  q - Print the question.
+  r - Prepend resolver IP address, Unix timestamp and return code to the question line.
+  s - Separate packet sections using a line feed.
+  t - Include TTL and record class within the output.
+  u - Include records from the authority section.
 ```
+
 This overview may be incomplete. For more options, especially concerning output formatting, use `--help`.
 
 ### Example

diff --git a/lists/resolvers.txt b/lists/resolvers.txt
@@ -984,3 +984,10 @@
 95.158.129.2 
 95.173.193.3 
 95.85.9.86 
+80.67.188.188
+108.61.201.119
+159.69.198.101
+47.101.136.37
+118.24.208.197
+114.115.240.175
+119.29.107.85
diff --git a/mixed_list.h b/mixed_list.h
diff --git a/modules/binary_output/CMakeLists.txt b/modules/binary_output/CMakeLists.txt
diff --git a/modules/binary_output/Makefile b/modules/binary_output/Makefile
diff --git a/modules/binary_output/main.c b/modules/binary_output/main.c
diff --git a/scripts/ct.py b/scripts/ct.py
@@ -3,7 +3,7 @@
 import sys
 import urllib.request
 import urllib.parse
-import re
+import json
 
 
 if len(sys.argv) == 1:
@@ -12,10 +12,10 @@
 
 for i, arg in enumerate(sys.argv, 1):
 	domains = set()
-	with urllib.request.urlopen('https://crt.sh/?q=' + urllib.parse.quote('%.' + arg)) as f:
-		code = f.read().decode('utf-8')
-		for cert, domain in re.findall('<tr>(?:\s|\S)*?href="\?id=([0-9]+?)"(?:\s|\S)*?<td>([*_a-zA-Z0-9.-]+?\.' + re.escape(arg) + ')</td>(?:\s|\S)*?</tr>', code, re.IGNORECASE):
-			domain = domain.split('@')[-1]
-			if not domain in domains:
-				domains.add(domain)
-				print(domain)
+	with urllib.request.urlopen('https://crt.sh/?output=json&q=' + urllib.parse.quote('%.' + arg)) as f:
+		data = json.loads(f.read().decode('utf-8'))
+		for crt in data:
+			for domain in crt['name_value'].split('\n'):
+				if not domain in domains:
+					domains.add(domain)
+					print(domain)
diff --git a/buffers.h → src/buffers.h b/buffers.h → src/buffers.h
diff --git a/cmd.h → src/cmd.h b/cmd.h → src/cmd.h
diff --git a/dns.h → src/dns.h b/dns.h → src/dns.h
@@ -940,6 +940,7 @@ ssize_t dns_str2namebuf(const char *name, uint8_t *buffer)
             *lenptr = label_len;
             if (total_len == 1)
             {
+                total_len--;
                 break;
             }
             if (*name == 0)
@@ -1452,6 +1453,22 @@ char *dns_section2str(dns_section_t section)
     return "UNKNOWN";
 }
 
+char *dns_section2str_lower_plural(dns_section_t section)
+{
+    switch(section)
+    {
+        case DNS_SECTION_ANSWER:
+            return "answers";
+        case DNS_SECTION_ADDITIONAL:
+            return "additionals";
+        case DNS_SECTION_AUTHORITY:
+            return "authorities";
+        case DNS_SECTION_QUESTION:
+            return "questions";
+    }
+    return "unknowns";
+}
+
 bool dns_in_zone(dns_name_t *name, dns_name_t *zone)
 {
     return zone->length == 1 // Provided that the label is a FQDN, this is the root zone containing everything else

diff --git a/flow.h → src/flow.h b/flow.h → src/flow.h
diff --git a/hashmap.h → src/hashmap.h b/hashmap.h → src/hashmap.h
diff --git a/list.h → src/list.h b/list.h → src/list.h