NOnion, Tests: support for onion-style hs server

This commit also contains general refactoring of the
hidden service client code.

NOnion.Tests/HiddenServicesTests.cs

@@ -1,14 +1,14 @@
 using Microsoft.FSharp.Core;
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Security.Cryptography;
-using System.Text;
 using System.Threading.Tasks;
-using System.IO;
 
 using NUnit.Framework;
 using Org.BouncyCastle.Crypto;
+using Org.BouncyCastle.Crypto.Parameters;
+using Org.BouncyCastle.Crypto.Generators;
+using Org.BouncyCastle.Security;
 
 using NOnion.Network;
 using NOnion.Http;
@@ -92,13 +92,40 @@ private async Task<int> ReadExact(TorStream stream, byte[] buffer, int off, int
             return bytesRead + await ReadExact(stream, buffer, off + bytesRead, len);
         }
 
-        public async Task EstablishAndCommunicateOverHSConnectionNOnionStyle()
+        public async Task BrowseFacebookOverHS()
         {
             TorDirectory directory = await TorDirectory.BootstrapAsync(FallbackDirectorySelector.GetRandomFallbackDirectory());
 
-            var host = new TorServiceHost(directory, TestsRetryCount);
+            var client = await TorServiceClient.ConnectAsync(directory, "facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion");
+            var httpClient = new TorHttpClient(client.GetStream(), "facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion");
+            await httpClient.GetAsStringAsync("/", false);
+        }
+
+        [Test]
+        [Retry(TestsRetryCount)]
+        public void CanBrowseFacebookOverHS()
+        {
+            Assert.ThrowsAsync(typeof(UnsuccessfulHttpRequestException), BrowseFacebookOverHS);
+        }
+
+        public async Task EstablishAndCommunicateOverHSConnectionOnionStyle()
+        {
+            int descriptorUploadRetryLimit = 2;
+
+            TorDirectory directory = await TorDirectory.BootstrapAsync(FallbackDirectorySelector.GetRandomFallbackDirectory());
+
+            TorLogger.Log("Finished bootstraping");
+
+            SecureRandom random = new SecureRandom();
+            Ed25519KeyPairGenerator kpGen = new Ed25519KeyPairGenerator();
+            kpGen.Init(new Ed25519KeyGenerationParameters(random));
+            Ed25519PrivateKeyParameters masterPrivateKey = (Ed25519PrivateKeyParameters)kpGen.GenerateKeyPair().Private;
+
+            TorServiceHost host = new TorServiceHost(directory, descriptorUploadRetryLimit, TestsRetryCount, FSharpOption<Ed25519PrivateKeyParameters>.Some(masterPrivateKey));
             await host.StartAsync();
 
+            TorLogger.Log("Finished starting HS host");
+
             var dataToSendAndReceive = new byte[] { 1, 2, 3, 4 };
 
             var serverSide =
@@ -111,7 +138,7 @@ public async Task EstablishAndCommunicateOverHSConnectionNOnionStyle()
 
             var clientSide =
                 Task.Run(async () => {
-                    var client = await TorServiceClient.ConnectAsync(directory, TorServiceDescriptors.NewNOnion (host.Export()));
+                    var client = await TorServiceClient.ConnectAsync(directory, host.ExportUrl());
                     var stream = client.GetStream();
                     var lengthBytes = new byte[sizeof(int)];
                     await ReadExact(stream, lengthBytes, 0, lengthBytes.Length);
@@ -122,31 +149,14 @@ public async Task EstablishAndCommunicateOverHSConnectionNOnionStyle()
                     CollectionAssert.AreEqual(buffer, dataToSendAndReceive);
                 });
 
-
             await TaskUtils.WhenAllFailFast(serverSide, clientSide);
         }
-
-        [Test]
-        [Retry(TestsRetryCount)]
-        public void CanEstablishAndCommunicateOverHSConnectionNOnionStyle()
-        {
-            Assert.DoesNotThrowAsync(EstablishAndCommunicateOverHSConnectionNOnionStyle);
-        }
-
-        public async Task EstablishAndCommunicateOverHSConnectionOnionStyle()
-        {
-            TorDirectory directory = await TorDirectory.BootstrapAsync(FallbackDirectorySelector.GetRandomFallbackDirectory());
-
-            var client = await TorServiceClient.ConnectAsync(directory, TorServiceDescriptors.NewOnionURL("facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion"));
-            var httpClient = new TorHttpClient(client.GetStream(), "facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd.onion");
-            await httpClient.GetAsStringAsync("/", false);
-        }
-
+
         [Test]
         [Retry(TestsRetryCount)]
         public void CanEstablishAndCommunicateOverHSConnectionOnionStyle()
         {
-            Assert.ThrowsAsync(typeof(UnsuccessfulHttpRequestException), EstablishAndCommunicateOverHSConnectionOnionStyle);
+            Assert.DoesNotThrowAsync(EstablishAndCommunicateOverHSConnectionOnionStyle);
         }
     }
 }

NOnion.Tests/NOnion.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.1</TargetFramework>
@@ -10,6 +10,7 @@
     <PackageReference Include="NUnit" Version="3.12.0" />
     <PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.8.10" />
   </ItemGroup>
 
   <ItemGroup>

NOnion/Cells/Relay/CellPlainRelay.fs

@@ -81,6 +81,9 @@ type RelayData =
             RelayRendezvous.FromBytes reader |> RelayRendezvous2
         | RelayCommands.RelayIntroduceAck ->
             RelayIntroduceAck.FromBytes reader |> RelayIntroduceAck
+        | RelayCommands.RelaySendMe ->
+            //TODO: complete sendme implementation
+            RelaySendMe
         | _ -> failwith "Unsupported command"
 
     member self.GetCommand() : byte =
@@ -106,7 +109,7 @@ type RelayData =
         | RelayBegin relayBegin -> relayBegin.ToBytes()
         | RelayConnected data -> data
         | RelayData data -> data
-        | RelaySendMe _ -> Array.zeroCreate 3
+        | RelaySendMe -> Array.zeroCreate 3
         | RelayEnd reason -> reason |> byte |> Array.singleton
         | RelayExtend2 extend2 -> extend2.ToBytes()
         | RelayEstablishIntro establishIntro -> establishIntro.ToBytes true true

NOnion/Cells/Relay/RelayBegin.fs

@@ -25,7 +25,10 @@ type RelayBegin =
 
         {
             Address = readAddress String.Empty
-            Flags = BinaryIO.ReadBigEndianUInt32 reader
+            // This flag apparently doesn't exists in Tor's service stream begin
+            // calls, which caused deserialization excpetion, since we don't really
+            // use this flag, we ignore it when deserializing for now.
+            Flags = 0u
         }
 
     member self.ToBytes() =

NOnion/Constants.fs

@@ -90,6 +90,8 @@ module Constants =
 
     let internal HttpClientBufferSize = 1024
 
+    let internal DefaultHttpHost = "127.0.0.1"
+
     // NTor Handshake Constants
     let private nTorProtoIdStr = "ntor-curve25519-sha256-1"
     let internal NTorProtoId = nTorProtoIdStr |> Encoding.ASCII.GetBytes
@@ -128,37 +130,63 @@ module Constants =
 
     let internal RelayIntroduceKeyType = 1
 
-    // HS NTor Handshake Constants
-    module HiddenServiceNTor =
-        let private protoIdStr = "tor-hs-ntor-curve25519-sha3-256-1"
+    module internal HiddenServices =
+        let IntroductionPointCount = 3
+
+        let Version = 3
+
+        // HS NTor Handshake Constants
+        module NTorEncryption =
+            let private protoIdStr = "tor-hs-ntor-curve25519-sha3-256-1"
+
+            let ProtoId = protoIdStr |> Encoding.ASCII.GetBytes
+            let TMac = protoIdStr + ":hs_mac" |> Encoding.ASCII.GetBytes
+
+            let TKey = protoIdStr + ":key_extract" |> Encoding.ASCII.GetBytes
+
+            let TVerify = protoIdStr + ":hs_verify" |> Encoding.ASCII.GetBytes
 
-        let internal ProtoId = protoIdStr |> Encoding.ASCII.GetBytes
-        let internal TMac = protoIdStr + ":hs_mac" |> Encoding.ASCII.GetBytes
+            let MExpand =
+                protoIdStr + ":hs_key_expand" |> Encoding.ASCII.GetBytes
 
-        let internal TKey =
-            protoIdStr + ":key_extract" |> Encoding.ASCII.GetBytes
+            let TEncrypt =
+                protoIdStr + ":hs_key_extract" |> Encoding.ASCII.GetBytes
 
-        let internal TVerify =
-            protoIdStr + ":hs_verify" |> Encoding.ASCII.GetBytes
+            let AuthInputSuffix =
+                protoIdStr + "Server" |> Encoding.ASCII.GetBytes
 
-        let internal MExpand =
-            protoIdStr + ":hs_key_expand" |> Encoding.ASCII.GetBytes
+        module DirectoryEncryption =
+            let SuperEncrypted = "hsdir-superencrypted-data"
+            let Encrypted = "hsdir-encrypted-data"
 
-        let internal TEncrypt =
-            protoIdStr + ":hs_key_extract" |> Encoding.ASCII.GetBytes
+            let SaltLength = 16
+            let MacKeyLength = 32
 
-        let internal AuthInputSuffix =
-            protoIdStr + "Server" |> Encoding.ASCII.GetBytes
+        module Hashring =
+            let ReplicasNum = 2
+            let SpreadFetch = 3
+            let SpreadStore = 4
+
+        module Descriptor =
+            let CertificateLifetime = TimeSpan.FromHours 5.
+
+            let Lifetime = (TimeSpan.FromMinutes 3.).TotalMinutes |> int
+
+            let SigningPrefix = "Tor onion service descriptor sig v3"
+
+        module OnionUrl =
+            let ChecksumLength = 2
+            let ChecksumPrefix = ".onion checksum"
+            let PublicKeyLength = 32
 
-    module HSDirEncryption =
-        let SuperEncrypted = "hsdir-superencrypted-data"
-        let Encrypted = "hsdir-encrypted-data"
 
     let internal NewConnectionCheckDelay = TimeSpan.FromSeconds 1.
 
     // https://github.com/torproject/tor/blob/22552ad88e1e95ef9d2c6655c7602b7b25836075/src/feature/hs_common/shared_random_client.h#L33
     let internal SharedRandomNRounds = 12u
     let internal SharedRandomNPhases = 2u
 
-    let HsDirNReplicas = 2
-    let HsDirSpreadFetch = 3
+    let internal DirectoryBlockLineLength = 64
+
+    let internal CertificateCertifiedKeyLength = 32
+    let internal CertificateSignatureLength = 64