Skip to content

Commit fabf119

Browse files
Martin-MolineroMartin-Molinero
authored
Improve live trading streaming packets (#8584)
* Improve live trading streaming packets * Further improvements * Add test for default empty holding - Improve deserialization
1 parent 9be02d7 commit fabf119

File tree

8 files changed

+241
-44
lines changed

8 files changed

+241
-44
lines changed

‎Brokerages/Brokerage.cs

+5
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,11 @@ protected virtual List<Holding> GetAccountHoldings(Dictionary<string, string> br
334334

335335
if (brokerageData != null && brokerageData.Remove("live-holdings", out var value) && !string.IsNullOrEmpty(value))
336336
{
337+
if (Log.DebuggingEnabled)
338+
{
339+
Log.Debug($"Brokerage.GetAccountHoldings(): raw value: {value}");
340+
}
341+
337342
// remove the key, we really only want to return the cached value on the first request
338343
var result = JsonConvert.DeserializeObject<List<Holding>>(value);
339344
if (result == null)

‎Common/Extensions.cs

+25
Original file line numberDiff line numberDiff line change
@@ -1309,6 +1309,31 @@ public static decimal SmartRounding(this decimal input)
13091309
return input.RoundToSignificantDigits(7).Normalize();
13101310
}
13111311

1312+
/// <summary>
1313+
/// Provides global smart rounding to a shorter version
1314+
/// </summary>
1315+
public static decimal SmartRoundingShort(this decimal input)
1316+
{
1317+
input = Normalize(input);
1318+
if (input <= 1)
1319+
{
1320+
// 0.99 > input
1321+
return input;
1322+
}
1323+
else if (input <= 10)
1324+
{
1325+
// 1.01 to 9.99
1326+
return Math.Round(input, 2);
1327+
}
1328+
else if (input <= 100)
1329+
{
1330+
// 99.9 to 10.1
1331+
return Math.Round(input, 1);
1332+
}
1333+
// 100 to inf
1334+
return Math.Truncate(input);
1335+
}
1336+
13121337
/// <summary>
13131338
/// Casts the specified input value to a decimal while acknowledging the overflow conditions
13141339
/// </summary>

‎Common/Global.cs

+121-15
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
using System;
1717
using Newtonsoft.Json;
18+
using Newtonsoft.Json.Linq;
19+
using System.ComponentModel;
1820
using QuantConnect.Securities;
1921
using Newtonsoft.Json.Converters;
2022
using System.Runtime.Serialization;
@@ -60,48 +62,104 @@ public static class DateFormat
6062
/// <summary>
6163
/// Singular holding of assets from backend live nodes:
6264
/// </summary>
63-
[JsonObject]
65+
[JsonConverter(typeof(HoldingJsonConverter))]
6466
public class Holding
6567
{
68+
private decimal? _conversionRate;
69+
private decimal _marketValue;
70+
private decimal _unrealizedPnl;
71+
private decimal _unrealizedPnLPercent;
72+
6673
/// Symbol of the Holding:
67-
[JsonProperty(PropertyName = "symbol")]
74+
[JsonIgnore]
6875
public Symbol Symbol { get; set; } = Symbol.Empty;
6976

7077
/// Type of the security
71-
[JsonProperty(PropertyName = "type")]
78+
[JsonIgnore]
7279
public SecurityType Type => Symbol.SecurityType;
7380

7481
/// The currency symbol of the holding, such as $
75-
[JsonProperty(PropertyName = "currencySymbol")]
82+
[DefaultValue("$")]
83+
[JsonProperty(PropertyName = "c", DefaultValueHandling = DefaultValueHandling.Ignore)]
7684
public string CurrencySymbol { get; set; }
7785

7886
/// Average Price of our Holding in the currency the symbol is traded in
79-
[JsonProperty(PropertyName = "averagePrice", DefaultValueHandling = DefaultValueHandling.Ignore)]
87+
[JsonConverter(typeof(DecimalJsonConverter))]
88+
[JsonProperty(PropertyName = "a", DefaultValueHandling = DefaultValueHandling.Ignore)]
8089
public decimal AveragePrice { get; set; }
8190

8291
/// Quantity of Symbol We Hold.
83-
[JsonProperty(PropertyName = "quantity", DefaultValueHandling = DefaultValueHandling.Ignore)]
92+
[JsonConverter(typeof(DecimalJsonConverter))]
93+
[JsonProperty(PropertyName = "q", DefaultValueHandling = DefaultValueHandling.Ignore)]
8494
public decimal Quantity { get; set; }
8595

8696
/// Current Market Price of the Asset in the currency the symbol is traded in
87-
[JsonProperty(PropertyName = "marketPrice", DefaultValueHandling = DefaultValueHandling.Ignore)]
97+
[JsonConverter(typeof(DecimalJsonConverter))]
98+
[JsonProperty(PropertyName = "p", DefaultValueHandling = DefaultValueHandling.Ignore)]
8899
public decimal MarketPrice { get; set; }
89100

90101
/// Current market conversion rate into the account currency
91-
[JsonProperty(PropertyName = "conversionRate", DefaultValueHandling = DefaultValueHandling.Ignore)]
92-
public decimal? ConversionRate { get; set; }
102+
[JsonConverter(typeof(DecimalJsonConverter))]
103+
[JsonProperty(PropertyName = "r", DefaultValueHandling = DefaultValueHandling.Ignore)]
104+
public decimal? ConversionRate
105+
{
106+
get
107+
{
108+
return _conversionRate;
109+
}
110+
set
111+
{
112+
if (value != 1)
113+
{
114+
_conversionRate = value;
115+
}
116+
}
117+
}
93118

94119
/// Current market value of the holding
95-
[JsonProperty(PropertyName = "marketValue", DefaultValueHandling = DefaultValueHandling.Ignore)]
96-
public decimal MarketValue { get; set; }
120+
[JsonConverter(typeof(DecimalJsonConverter))]
121+
[JsonProperty(PropertyName = "v", DefaultValueHandling = DefaultValueHandling.Ignore)]
122+
public decimal MarketValue
123+
{
124+
get
125+
{
126+
return _marketValue;
127+
}
128+
set
129+
{
130+
_marketValue = value.SmartRoundingShort();
131+
}
132+
}
97133

98134
/// Current unrealized P/L of the holding
99-
[JsonProperty(PropertyName = "unrealizedPnl", DefaultValueHandling = DefaultValueHandling.Ignore)]
100-
public decimal UnrealizedPnL { get; set; }
135+
[JsonConverter(typeof(DecimalJsonConverter))]
136+
[JsonProperty(PropertyName = "u", DefaultValueHandling = DefaultValueHandling.Ignore)]
137+
public decimal UnrealizedPnL
138+
{
139+
get
140+
{
141+
return _unrealizedPnl;
142+
}
143+
set
144+
{
145+
_unrealizedPnl = value.SmartRoundingShort();
146+
}
147+
}
101148

102149
/// Current unrealized P/L % of the holding
103-
[JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)]
104-
public decimal UnrealizedPnLPercent { get; set; }
150+
[JsonConverter(typeof(DecimalJsonConverter))]
151+
[JsonProperty(PropertyName = "up", DefaultValueHandling = DefaultValueHandling.Ignore)]
152+
public decimal UnrealizedPnLPercent
153+
{
154+
get
155+
{
156+
return _unrealizedPnLPercent;
157+
}
158+
set
159+
{
160+
_unrealizedPnLPercent = value.SmartRoundingShort();
161+
}
162+
}
105163

106164
/// Create a new default holding:
107165
public Holding()
@@ -159,6 +217,54 @@ public override string ToString()
159217
{
160218
return Messages.Holding.ToString(this);
161219
}
220+
221+
private class DecimalJsonConverter : JsonConverter
222+
{
223+
public override bool CanRead => false;
224+
public override bool CanConvert(Type objectType) => typeof(decimal) == objectType;
225+
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
226+
{
227+
writer.WriteRawValue(((decimal)value).NormalizeToStr());
228+
}
229+
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
230+
{
231+
throw new NotImplementedException();
232+
}
233+
}
234+
private class HoldingJsonConverter : JsonConverter
235+
{
236+
public override bool CanWrite => false;
237+
public override bool CanConvert(Type objectType) => typeof(Holding) == objectType;
238+
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
239+
{
240+
var jObject = JObject.Load(reader);
241+
var result = new Holding
242+
{
243+
Symbol = jObject["symbol"]?.ToObject<Symbol>() ?? jObject["Symbol"]?.ToObject<Symbol>() ?? Symbol.Empty,
244+
CurrencySymbol = jObject["c"]?.Value<string>() ?? jObject["currencySymbol"]?.Value<string>() ?? jObject["CurrencySymbol"]?.Value<string>() ?? string.Empty,
245+
AveragePrice = jObject["a"]?.Value<decimal>() ?? jObject["averagePrice"]?.Value<decimal>() ?? jObject["AveragePrice"]?.Value<decimal>() ?? 0,
246+
Quantity = jObject["q"]?.Value<decimal>() ?? jObject["quantity"]?.Value<decimal>() ?? jObject["Quantity"]?.Value<decimal>() ?? 0,
247+
MarketPrice = jObject["p"]?.Value<decimal>() ?? jObject["marketPrice"]?.Value<decimal>() ?? jObject["MarketPrice"]?.Value<decimal>() ?? 0,
248+
ConversionRate = jObject["r"]?.Value<decimal>() ?? jObject["conversionRate"]?.Value<decimal>() ?? jObject["ConversionRate"]?.Value<decimal>() ?? null,
249+
MarketValue = jObject["v"]?.Value<decimal>() ?? jObject["marketValue"]?.Value<decimal>() ?? jObject["MarketValue"]?.Value<decimal>() ?? 0,
250+
UnrealizedPnL = jObject["u"]?.Value<decimal>() ?? jObject["unrealizedPnl"]?.Value<decimal>() ?? jObject["UnrealizedPnl"]?.Value<decimal>() ?? 0,
251+
UnrealizedPnLPercent = jObject["up"]?.Value<decimal>() ?? jObject["unrealizedPnLPercent"]?.Value<decimal>() ?? jObject["UnrealizedPnLPercent"]?.Value<decimal>() ?? 0,
252+
};
253+
if (!result.ConversionRate.HasValue)
254+
{
255+
result.ConversionRate = 1;
256+
}
257+
if (string.IsNullOrEmpty(result.CurrencySymbol))
258+
{
259+
result.CurrencySymbol = "$";
260+
}
261+
return result;
262+
}
263+
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
264+
{
265+
throw new NotImplementedException();
266+
}
267+
}
162268
}
163269

164270
/// <summary>

‎Common/Messages/Messages.QuantConnect.cs

+8-3
Original file line numberDiff line numberDiff line change
@@ -470,10 +470,15 @@ public static class Holding
470470
[MethodImpl(MethodImplOptions.AggressiveInlining)]
471471
public static string ToString(QuantConnect.Holding instance)
472472
{
473-
var value = Invariant($@"{instance.Symbol.Value}: {instance.Quantity} @ {
474-
instance.CurrencySymbol}{instance.AveragePrice} - Market: {instance.CurrencySymbol}{instance.MarketPrice}");
473+
var currencySymbol = instance.CurrencySymbol;
474+
if (string.IsNullOrEmpty(currencySymbol))
475+
{
476+
currencySymbol = "$";
477+
}
478+
var value = Invariant($@"{instance.Symbol?.Value}: {instance.Quantity} @ {
479+
currencySymbol}{instance.AveragePrice} - Market: {currencySymbol}{instance.MarketPrice}");
475480

476-
if (instance.ConversionRate != 1m)
481+
if (instance.ConversionRate.HasValue && instance.ConversionRate != 1m)
477482
{
478483
value += Invariant($" - Conversion: {instance.ConversionRate}");
479484
}

‎Common/Packets/LiveResultPacket.cs

-21
Original file line numberDiff line numberDiff line change
@@ -39,31 +39,16 @@ public class LiveResultPacket : Packet
3939
/// </summary>
4040
public int ProjectId { get; set; }
4141

42-
/// <summary>
43-
/// User session Id who issued the result packet
44-
/// </summary>
45-
public string SessionId { get; set; } = string.Empty;
46-
4742
/// <summary>
4843
/// Live Algorithm Id (DeployId) for this result packet
4944
/// </summary>
5045
public string DeployId { get; set; } = string.Empty;
5146

52-
/// <summary>
53-
/// Compile Id algorithm which generated this result packet
54-
/// </summary>
55-
public string CompileId { get; set; } = string.Empty;
56-
5747
/// <summary>
5848
/// Result data object for this result packet
5949
/// </summary>
6050
public LiveResult Results { get; set; } = new LiveResult();
6151

62-
/// <summary>
63-
/// Processing time / running time for the live algorithm.
64-
/// </summary>
65-
public double ProcessingTime { get; set; }
66-
6752
/// <summary>
6853
/// Default constructor for JSON Serialization
6954
/// </summary>
@@ -80,15 +65,12 @@ public LiveResultPacket(string json)
8065
try
8166
{
8267
var packet = JsonConvert.DeserializeObject<LiveResultPacket>(json);
83-
CompileId = packet.CompileId;
8468
Channel = packet.Channel;
85-
SessionId = packet.SessionId;
8669
DeployId = packet.DeployId;
8770
Type = packet.Type;
8871
UserId = packet.UserId;
8972
ProjectId = packet.ProjectId;
9073
Results = packet.Results;
91-
ProcessingTime = packet.ProcessingTime;
9274
}
9375
catch (Exception err)
9476
{
@@ -106,13 +88,10 @@ public LiveResultPacket(LiveNodePacket job, LiveResult results)
10688
{
10789
try
10890
{
109-
SessionId = job.SessionId;
110-
CompileId = job.CompileId;
11191
DeployId = job.DeployId;
11292
Results = results;
11393
UserId = job.UserId;
11494
ProjectId = job.ProjectId;
115-
SessionId = job.SessionId;
11695
Channel = job.Channel;
11796
}
11897
catch (Exception err) {

‎Engine/Results/LiveTradingResultHandler.cs

+2-5
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,6 @@ private void Update()
218218
Log.Debug("LiveTradingResultHandler.Update(): End build delta charts");
219219

220220
//Profit loss changes, get the banner statistics, summary information on the performance for the headers.
221-
var deltaStatistics = new Dictionary<string, string>();
222221
var serverStatistics = GetServerStatistics(utcNow);
223222
var holdings = GetHoldings(Algorithm.Securities.Values, Algorithm.SubscriptionManager.SubscriptionDataConfigService);
224223

@@ -232,7 +231,7 @@ private void Update()
232231

233232
// since we're sending multiple packets, let's do it async and forget about it
234233
// chart data can get big so let's break them up into groups
235-
var splitPackets = SplitPackets(deltaCharts, deltaOrders, holdings, Algorithm.Portfolio.CashBook, deltaStatistics, runtimeStatistics, serverStatistics, deltaOrderEvents);
234+
var splitPackets = SplitPackets(deltaCharts, deltaOrders, holdings, Algorithm.Portfolio.CashBook, runtimeStatistics, serverStatistics, deltaOrderEvents);
236235

237236
foreach (var liveResultPacket in splitPackets)
238237
{
@@ -256,6 +255,7 @@ private void Update()
256255

257256
var orderEvents = GetOrderEventsToStore();
258257

258+
var deltaStatistics = new Dictionary<string, string>();
259259
var orders = new Dictionary<int, Order>(TransactionHandler.Orders);
260260
var complete = new LiveResultPacket(_job, new LiveResult(new LiveResultParameters(chartComplete, orders, Algorithm.Transactions.TransactionRecord, holdings, Algorithm.Portfolio.CashBook, deltaStatistics, runtimeStatistics, orderEvents, serverStatistics, state: GetAlgorithmState())));
261261
StoreResult(complete);
@@ -469,7 +469,6 @@ private IEnumerable<LiveResultPacket> SplitPackets(Dictionary<string, Chart> del
469469
Dictionary<int, Order> deltaOrders,
470470
Dictionary<string, Holding> holdings,
471471
CashBook cashbook,
472-
Dictionary<string, string> deltaStatistics,
473472
SortedDictionary<string, string> runtimeStatistics,
474473
Dictionary<string, string> serverStatistics,
475474
List<OrderEvent> deltaOrderEvents)
@@ -514,7 +513,6 @@ private IEnumerable<LiveResultPacket> SplitPackets(Dictionary<string, Chart> del
514513
new LiveResultPacket(_job, new LiveResult { Holdings = holdings, CashBook = cashbook}),
515514
new LiveResultPacket(_job, new LiveResult
516515
{
517-
Statistics = deltaStatistics,
518516
RuntimeStatistics = runtimeStatistics,
519517
ServerStatistics = serverStatistics
520518
})
@@ -824,7 +822,6 @@ protected void SendFinalResult()
824822
result = LiveResultPacket.CreateEmpty(_job);
825823
result.Results.State = endState;
826824
}
827-
result.ProcessingTime = (endTime - StartTime).TotalSeconds;
828825

829826
StoreInsights();
830827

0 commit comments

Comments
 (0)