Skip to content

Commit 47b7d2c

Browse files
jskeetjskeet
committed
Add DiscardUnknownFields support for C#
By default, unknown fields are preserved when parsing. To discard them, use a parser configured to do so: var parser = MyMessage.Parser.WithDiscardUnknownFields(true);
1 parent 9f80df0 commit 47b7d2c

File tree

5 files changed

+173
-56
lines changed

5 files changed

+173
-56
lines changed

‎csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#endregion
3232

3333
using System;
34+
using System.IO;
3435
using Google.Protobuf.TestProtos;
3536
using NUnit.Framework;
3637

@@ -124,5 +125,52 @@ public void TestClone()
124125
Assert.AreEqual(message.CalculateSize(), otherEmptyMessage.CalculateSize());
125126
Assert.AreEqual(message.ToByteArray(), otherEmptyMessage.ToByteArray());
126127
}
128+
129+
[Test]
130+
public void TestDiscardUnknownFields()
131+
{
132+
var message = SampleMessages.CreateFullTestAllTypes();
133+
var goldenEmptyMessage = new TestEmptyMessage();
134+
byte[] data = message.ToByteArray();
135+
int fullSize = message.CalculateSize();
136+
137+
Action<IMessage> assertEmpty = msg =>
138+
{
139+
Assert.AreEqual(0, msg.CalculateSize());
140+
Assert.AreEqual(goldenEmptyMessage, msg);
141+
};
142+
143+
Action<IMessage> assertFull = msg => Assert.AreEqual(fullSize, msg.CalculateSize());
144+
145+
// Test the behavior of the parsers with and without discarding, both generic and non-generic.
146+
MessageParser<TestEmptyMessage> retainingParser1 = TestEmptyMessage.Parser;
147+
MessageParser retainingParser2 = retainingParser1;
148+
MessageParser<TestEmptyMessage> discardingParser1 = retainingParser1.WithDiscardUnknownFields(true);
149+
MessageParser discardingParser2 = retainingParser2.WithDiscardUnknownFields(true);
150+
151+
// Test parse from byte[]
152+
assertFull(retainingParser1.ParseFrom(data));
153+
assertFull(retainingParser2.ParseFrom(data));
154+
assertEmpty(discardingParser1.ParseFrom(data));
155+
assertEmpty(discardingParser2.ParseFrom(data));
156+
157+
// Test parse from byte[] with offset
158+
assertFull(retainingParser1.ParseFrom(data, 0, data.Length));
159+
assertFull(retainingParser2.ParseFrom(data, 0, data.Length));
160+
assertEmpty(discardingParser1.ParseFrom(data, 0, data.Length));
161+
assertEmpty(discardingParser2.ParseFrom(data, 0, data.Length));
162+
163+
// Test parse from CodedInputStream
164+
assertFull(retainingParser1.ParseFrom(new CodedInputStream(data)));
165+
assertFull(retainingParser2.ParseFrom(new CodedInputStream(data)));
166+
assertEmpty(discardingParser1.ParseFrom(new CodedInputStream(data)));
167+
assertEmpty(discardingParser2.ParseFrom(new CodedInputStream(data)));
168+
169+
// Test parse from Stream
170+
assertFull(retainingParser1.ParseFrom(new MemoryStream(data)));
171+
assertFull(retainingParser2.ParseFrom(new MemoryStream(data)));
172+
assertEmpty(discardingParser1.ParseFrom(new MemoryStream(data)));
173+
assertEmpty(discardingParser2.ParseFrom(new MemoryStream(data)));
174+
}
127175
}
128176
}

‎csharp/src/Google.Protobuf/CodedInputStream.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,11 @@ public long Position
267267
/// </value>
268268
public int RecursionLimit { get { return recursionLimit; } }
269269

270+
/// <summary>
271+
/// Internal-only property; when set to true, unknown fields will be discarded while parsing.
272+
/// </summary>
273+
internal bool DiscardUnknownFields { get; set; }
274+
270275
/// <summary>
271276
/// Disposes of this instance, potentially closing any underlying stream.
272277
/// </summary>

‎csharp/src/Google.Protobuf/MessageExtensions.cs

Lines changed: 61 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,8 @@ public static class MessageExtensions
4444
/// </summary>
4545
/// <param name="message">The message to merge the data into.</param>
4646
/// <param name="data">The data to merge, which must be protobuf-encoded binary data.</param>
47-
public static void MergeFrom(this IMessage message, byte[] data)
48-
{
49-
ProtoPreconditions.CheckNotNull(message, "message");
50-
ProtoPreconditions.CheckNotNull(data, "data");
51-
CodedInputStream input = new CodedInputStream(data);
52-
message.MergeFrom(input);
53-
input.CheckReadEndOfStreamTag();
54-
}
47+
public static void MergeFrom(this IMessage message, byte[] data) =>
48+
MergeFrom(message, data, false);
5549

5650
/// <summary>
5751
/// Merges data from the given byte array slice into an existing message.
@@ -60,42 +54,24 @@ public static void MergeFrom(this IMessage message, byte[] data)
6054
/// <param name="data">The data containing the slice to merge, which must be protobuf-encoded binary data.</param>
6155
/// <param name="offset">The offset of the slice to merge.</param>
6256
/// <param name="length">The length of the slice to merge.</param>
63-
public static void MergeFrom(this IMessage message, byte[] data, int offset, int length)
64-
{
65-
ProtoPreconditions.CheckNotNull(message, "message");
66-
ProtoPreconditions.CheckNotNull(data, "data");
67-
CodedInputStream input = new CodedInputStream(data, offset, length);
68-
message.MergeFrom(input);
69-
input.CheckReadEndOfStreamTag();
70-
}
57+
public static void MergeFrom(this IMessage message, byte[] data, int offset, int length) =>
58+
MergeFrom(message, data, offset, length, false);
7159

7260
/// <summary>
7361
/// Merges data from the given byte string into an existing message.
7462
/// </summary>
7563
/// <param name="message">The message to merge the data into.</param>
7664
/// <param name="data">The data to merge, which must be protobuf-encoded binary data.</param>
77-
public static void MergeFrom(this IMessage message, ByteString data)
78-
{
79-
ProtoPreconditions.CheckNotNull(message, "message");
80-
ProtoPreconditions.CheckNotNull(data, "data");
81-
CodedInputStream input = data.CreateCodedInput();
82-
message.MergeFrom(input);
83-
input.CheckReadEndOfStreamTag();
84-
}
65+
public static void MergeFrom(this IMessage message, ByteString data) =>
66+
MergeFrom(message, data, false);
8567

8668
/// <summary>
8769
/// Merges data from the given stream into an existing message.
8870
/// </summary>
8971
/// <param name="message">The message to merge the data into.</param>
9072
/// <param name="input">Stream containing the data to merge, which must be protobuf-encoded binary data.</param>
91-
public static void MergeFrom(this IMessage message, Stream input)
92-
{
93-
ProtoPreconditions.CheckNotNull(message, "message");
94-
ProtoPreconditions.CheckNotNull(input, "input");
95-
CodedInputStream codedInput = new CodedInputStream(input);
96-
message.MergeFrom(codedInput);
97-
codedInput.CheckReadEndOfStreamTag();
98-
}
73+
public static void MergeFrom(this IMessage message, Stream input) =>
74+
MergeFrom(message, input, false);
9975

10076
/// <summary>
10177
/// Merges length-delimited data from the given stream into an existing message.
@@ -106,14 +82,8 @@ public static void MergeFrom(this IMessage message, Stream input)
10682
/// </remarks>
10783
/// <param name="message">The message to merge the data into.</param>
10884
/// <param name="input">Stream containing the data to merge, which must be protobuf-encoded binary data.</param>
109-
public static void MergeDelimitedFrom(this IMessage message, Stream input)
110-
{
111-
ProtoPreconditions.CheckNotNull(message, "message");
112-
ProtoPreconditions.CheckNotNull(input, "input");
113-
int size = (int) CodedInputStream.ReadRawVarint32(input);
114-
Stream limitedStream = new LimitedInputStream(input, size);
115-
message.MergeFrom(limitedStream);
116-
}
85+
public static void MergeDelimitedFrom(this IMessage message, Stream input) =>
86+
MergeDelimitedFrom(message, input, false);
11787

11888
/// <summary>
11989
/// Converts the given message into a byte array in protobuf encoding.
@@ -168,6 +138,56 @@ public static ByteString ToByteString(this IMessage message)
168138
{
169139
ProtoPreconditions.CheckNotNull(message, "message");
170140
return ByteString.AttachBytes(message.ToByteArray());
171-
}
141+
}
142+
143+
// Implementations allowing unknown fields to be discarded.
144+
internal static void MergeFrom(this IMessage message, byte[] data, bool discardUnknownFields)
145+
{
146+
ProtoPreconditions.CheckNotNull(message, "message");
147+
ProtoPreconditions.CheckNotNull(data, "data");
148+
CodedInputStream input = new CodedInputStream(data);
149+
input.DiscardUnknownFields = discardUnknownFields;
150+
message.MergeFrom(input);
151+
input.CheckReadEndOfStreamTag();
152+
}
153+
154+
internal static void MergeFrom(this IMessage message, byte[] data, int offset, int length, bool discardUnknownFields)
155+
{
156+
ProtoPreconditions.CheckNotNull(message, "message");
157+
ProtoPreconditions.CheckNotNull(data, "data");
158+
CodedInputStream input = new CodedInputStream(data, offset, length);
159+
input.DiscardUnknownFields = discardUnknownFields;
160+
message.MergeFrom(input);
161+
input.CheckReadEndOfStreamTag();
162+
}
163+
164+
internal static void MergeFrom(this IMessage message, ByteString data, bool discardUnknownFields)
165+
{
166+
ProtoPreconditions.CheckNotNull(message, "message");
167+
ProtoPreconditions.CheckNotNull(data, "data");
168+
CodedInputStream input = data.CreateCodedInput();
169+
input.DiscardUnknownFields = discardUnknownFields;
170+
message.MergeFrom(input);
171+
input.CheckReadEndOfStreamTag();
172+
}
173+
174+
internal static void MergeFrom(this IMessage message, Stream input, bool discardUnknownFields)
175+
{
176+
ProtoPreconditions.CheckNotNull(message, "message");
177+
ProtoPreconditions.CheckNotNull(input, "input");
178+
CodedInputStream codedInput = new CodedInputStream(input);
179+
codedInput.DiscardUnknownFields = discardUnknownFields;
180+
message.MergeFrom(codedInput);
181+
codedInput.CheckReadEndOfStreamTag();
182+
}
183+
184+
internal static void MergeDelimitedFrom(this IMessage message, Stream input, bool discardUnknownFields)
185+
{
186+
ProtoPreconditions.CheckNotNull(message, "message");
187+
ProtoPreconditions.CheckNotNull(input, "input");
188+
int size = (int) CodedInputStream.ReadRawVarint32(input);
189+
Stream limitedStream = new LimitedInputStream(input, size);
190+
MergeFrom(message, limitedStream, discardUnknownFields);
191+
}
172192
}
173193
}

‎csharp/src/Google.Protobuf/MessageParser.cs

Lines changed: 52 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,13 @@ namespace Google.Protobuf
4242
public class MessageParser
4343
{
4444
private Func<IMessage> factory;
45+
// TODO: When we use a C# 7.1 compiler, make this private protected.
46+
internal bool DiscardUnknownFields { get; }
4547

46-
internal MessageParser(Func<IMessage> factory)
48+
internal MessageParser(Func<IMessage> factory, bool discardUnknownFields)
4749
{
4850
this.factory = factory;
51+
DiscardUnknownFields = discardUnknownFields;
4952
}
5053

5154
/// <summary>
@@ -65,7 +68,7 @@ internal IMessage CreateTemplate()
6568
public IMessage ParseFrom(byte[] data)
6669
{
6770
IMessage message = factory();
68-
message.MergeFrom(data);
71+
message.MergeFrom(data, DiscardUnknownFields);
6972
return message;
7073
}
7174

@@ -79,7 +82,7 @@ public IMessage ParseFrom(byte[] data)
7982
public IMessage ParseFrom(byte[] data, int offset, int length)
8083
{
8184
IMessage message = factory();
82-
message.MergeFrom(data, offset, length);
85+
message.MergeFrom(data, offset, length, DiscardUnknownFields);
8386
return message;
8487
}
8588

@@ -91,7 +94,7 @@ public IMessage ParseFrom(byte[] data, int offset, int length)
9194
public IMessage ParseFrom(ByteString data)
9295
{
9396
IMessage message = factory();
94-
message.MergeFrom(data);
97+
message.MergeFrom(data, DiscardUnknownFields);
9598
return message;
9699
}
97100

@@ -103,7 +106,7 @@ public IMessage ParseFrom(ByteString data)
103106
public IMessage ParseFrom(Stream input)
104107
{
105108
IMessage message = factory();
106-
message.MergeFrom(input);
109+
message.MergeFrom(input, DiscardUnknownFields);
107110
return message;
108111
}
109112

@@ -119,7 +122,7 @@ public IMessage ParseFrom(Stream input)
119122
public IMessage ParseDelimitedFrom(Stream input)
120123
{
121124
IMessage message = factory();
122-
message.MergeDelimitedFrom(input);
125+
message.MergeDelimitedFrom(input, DiscardUnknownFields);
123126
return message;
124127
}
125128

@@ -131,7 +134,7 @@ public IMessage ParseDelimitedFrom(Stream input)
131134
public IMessage ParseFrom(CodedInputStream input)
132135
{
133136
IMessage message = factory();
134-
message.MergeFrom(input);
137+
MergeFrom(message, input);
135138
return message;
136139
}
137140

@@ -148,6 +151,29 @@ public IMessage ParseJson(string json)
148151
JsonParser.Default.Merge(message, json);
149152
return message;
150153
}
154+
155+
// TODO: When we're using a C# 7.1 compiler, make this private protected.
156+
internal void MergeFrom(IMessage message, CodedInputStream codedInput)
157+
{
158+
bool originalDiscard = codedInput.DiscardUnknownFields;
159+
try
160+
{
161+
codedInput.DiscardUnknownFields = DiscardUnknownFields;
162+
message.MergeFrom(codedInput);
163+
}
164+
finally
165+
{
166+
codedInput.DiscardUnknownFields = originalDiscard;
167+
}
168+
}
169+
170+
/// <summary>
171+
/// Creates a new message parser which optionally discards unknown fields when parsing.
172+
/// </summary>
173+
/// <param name="discardUnknownFields">Whether or not to discard unknown fields when parsing.</param>
174+
/// <returns>A newly configured message parser.</returns>
175+
public MessageParser WithDiscardUnknownFields(bool discardUnknownFields) =>
176+
new MessageParser(factory, discardUnknownFields);
151177
}
152178

153179
/// <summary>
@@ -182,7 +208,11 @@ public sealed class MessageParser<T> : MessageParser where T : IMessage<T>
182208
/// to require a parameterless constructor: delegates are significantly faster to execute.
183209
/// </remarks>
184210
/// <param name="factory">Function to invoke when a new, empty message is required.</param>
185-
public MessageParser(Func<T> factory) : base(() => factory())
211+
public MessageParser(Func<T> factory) : this(factory, false)
212+
{
213+
}
214+
215+
internal MessageParser(Func<T> factory, bool discardUnknownFields) : base(() => factory(), discardUnknownFields)
186216
{
187217
this.factory = factory;
188218
}
@@ -204,7 +234,7 @@ public MessageParser(Func<T> factory) : base(() => factory())
204234
public new T ParseFrom(byte[] data)
205235
{
206236
T message = factory();
207-
message.MergeFrom(data);
237+
message.MergeFrom(data, DiscardUnknownFields);
208238
return message;
209239
}
210240

@@ -218,7 +248,7 @@ public MessageParser(Func<T> factory) : base(() => factory())
218248
public new T ParseFrom(byte[] data, int offset, int length)
219249
{
220250
T message = factory();
221-
message.MergeFrom(data, offset, length);
251+
message.MergeFrom(data, offset, length, DiscardUnknownFields);
222252
return message;
223253
}
224254

@@ -230,7 +260,7 @@ public MessageParser(Func<T> factory) : base(() => factory())
230260
public new T ParseFrom(ByteString data)
231261
{
232262
T message = factory();
233-
message.MergeFrom(data);
263+
message.MergeFrom(data, DiscardUnknownFields);
234264
return message;
235265
}
236266

@@ -242,7 +272,7 @@ public MessageParser(Func<T> factory) : base(() => factory())
242272
public new T ParseFrom(Stream input)
243273
{
244274
T message = factory();
245-
message.MergeFrom(input);
275+
message.MergeFrom(input, DiscardUnknownFields);
246276
return message;
247277
}
248278

@@ -258,7 +288,7 @@ public MessageParser(Func<T> factory) : base(() => factory())
258288
public new T ParseDelimitedFrom(Stream input)
259289
{
260290
T message = factory();
261-
message.MergeDelimitedFrom(input);
291+
message.MergeDelimitedFrom(input, DiscardUnknownFields);
262292
return message;
263293
}
264294

@@ -270,7 +300,7 @@ public MessageParser(Func<T> factory) : base(() => factory())
270300
public new T ParseFrom(CodedInputStream input)
271301
{
272302
T message = factory();
273-
message.MergeFrom(input);
303+
MergeFrom(message, input);
274304
return message;
275305
}
276306

@@ -287,5 +317,13 @@ public MessageParser(Func<T> factory) : base(() => factory())
287317
JsonParser.Default.Merge(message, json);
288318
return message;
289319
}
320+
321+
/// <summary>
322+
/// Creates a new message parser which optionally discards unknown fields when parsing.
323+
/// </summary>
324+
/// <param name="discardUnknownFields">Whether or not to discard unknown fields when parsing.</param>
325+
/// <returns>A newly configured message parser.</returns>
326+
public new MessageParser<T> WithDiscardUnknownFields(bool discardUnknownFields) =>
327+
new MessageParser<T>(factory, discardUnknownFields);
290328
}
291329
}

0 commit comments

Comments
 (0)