Fix failing test cases for proto1 -> proto2 migration

api/src/main/java/com/google/appengine/api/datastore/AdminDatastoreService.java

@@ -260,10 +260,10 @@ public Index compositeIndexForQuery(Query query) {
   }
 
   public Set<Index> compositeIndexesForQuery(Query query) {
-    List<DatastoreV3Pb.Query> pbQueries =
+    List<DatastoreV3Pb.Query.Builder> pbQueries =
         convertQueryToPbs(query, FetchOptions.Builder.withDefaults());
     Set<Index> resultSet = new HashSet<Index>();
-    for (DatastoreV3Pb.Query queryProto : pbQueries) {
+    for (DatastoreV3Pb.Query.Builder queryProto : pbQueries) {
       IndexComponentsOnlyQuery indexQuery = new IndexComponentsOnlyQuery(queryProto);
 
       OnestoreEntity.Index.Builder index =
@@ -281,7 +281,7 @@ public Index minimumCompositeIndexForQuery(Query query, Collection<Index> indexe
   }
 
   public Set<Index> minimumCompositeIndexesForQuery(Query query, Collection<Index> indexes) {
-    List<DatastoreV3Pb.Query> pbQueries =
+    List<DatastoreV3Pb.Query.Builder> pbQueries =
         convertQueryToPbs(query, FetchOptions.Builder.withDefaults());
 
     List<OnestoreEntity.Index> indexPbs = Lists.newArrayListWithCapacity(indexes.size());
@@ -290,7 +290,7 @@ public Set<Index> minimumCompositeIndexesForQuery(Query query, Collection<Index>
     }
 
     Set<Index> resultSet = new HashSet<Index>();
-    for (DatastoreV3Pb.Query queryProto : pbQueries) {
+    for (DatastoreV3Pb.Query.Builder queryProto : pbQueries) {
       IndexComponentsOnlyQuery indexQuery = new IndexComponentsOnlyQuery(queryProto);
 
       OnestoreEntity.Index.Builder index =
@@ -304,19 +304,19 @@ public Set<Index> minimumCompositeIndexesForQuery(Query query, Collection<Index>
 
   /** Convert a query to a list of ProtocolBuffer Queries. */
   @SuppressWarnings("deprecation")
-  private static List<DatastoreV3Pb.Query> convertQueryToPbs(
+  private static List<DatastoreV3Pb.Query.Builder> convertQueryToPbs(
       Query query, FetchOptions fetchOptions) {
     List<MultiQueryBuilder> queriesToRun = QuerySplitHelper.splitQuery(query);
     // All Filters should be in queriesToRun
     query.setFilter(null);
     query.getFilterPredicates().clear();
-    List<DatastoreV3Pb.Query> resultQueries = new ArrayList<DatastoreV3Pb.Query>();
+    List<DatastoreV3Pb.Query.Builder> resultQueries = new ArrayList<DatastoreV3Pb.Query.Builder>();
     for (MultiQueryBuilder multiQuery : queriesToRun) {
       for (List<List<FilterPredicate>> parallelQueries : multiQuery) {
         for (List<FilterPredicate> singleQuery : parallelQueries) {
           Query newQuery = new Query(query);
           newQuery.getFilterPredicates().addAll(singleQuery);
-          DatastoreV3Pb.Query queryProto = QueryTranslator.convertToPb(newQuery, fetchOptions);
+          DatastoreV3Pb.Query.Builder queryProto = QueryTranslator.convertToPb(newQuery, fetchOptions);
           resultQueries.add(queryProto);
         }
       }

api/src/main/java/com/google/appengine/api/datastore/AsyncDatastoreServiceImpl.java

@@ -123,7 +123,7 @@ int getMaxCount() {
         @Override
         protected Future<DeleteResponse> makeCall(DeleteRequest batch) {
           return makeAsyncCall(
-              apiConfig, DatastoreService_3.Method.Delete, batch, DeleteResponse.newBuilder().build());
+              apiConfig, DatastoreService_3.Method.Delete, batch, DeleteResponse.getDefaultInstance());
         }
       };
 
@@ -141,7 +141,7 @@ int getMaxCount() {
 
         @Override
         protected Future<GetResponse> makeCall(GetRequest batch) {
-          return makeAsyncCall(apiConfig, DatastoreService_3.Method.Get, batch, GetResponse.newBuilder().build());
+          return makeAsyncCall(apiConfig, DatastoreService_3.Method.Get, batch, GetResponse.getDefaultInstance());
         }
       };
 
@@ -169,7 +169,7 @@ int getMaxCount() {
 
         @Override
         protected Future<GetResponse> makeCall(GetRequest batch) {
-          return makeAsyncCall(apiConfig, DatastoreService_3.Method.Get, batch, GetResponse.newBuilder().build());
+          return makeAsyncCall(apiConfig, DatastoreService_3.Method.Get, batch, GetResponse.getDefaultInstance());
         }
       };
 
@@ -192,7 +192,7 @@ int getMaxCount() {
 
         @Override
         protected Future<PutResponse> makeCall(PutRequest batch) {
-          return makeAsyncCall(apiConfig, DatastoreService_3.Method.Put, batch, PutResponse.newBuilder().build());
+          return makeAsyncCall(apiConfig, DatastoreService_3.Method.Put, batch, PutResponse.getDefaultInstance());
         }
 
         @Override
@@ -247,7 +247,7 @@ protected TransactionImpl.InternalTransaction doBeginTransaction(TransactionOpti
 
     Future<DatastoreV3Pb.Transaction> future =
         DatastoreApiHelper.makeAsyncCall(
-            apiConfig, DatastoreService_3.Method.BeginTransaction, request.build(), remoteTxn.build());
+            apiConfig, DatastoreService_3.Method.BeginTransaction, request.build(), remoteTxn.buildPartial());
 
     return new InternalTransactionV3(apiConfig, request.getApp(), future);
   }
@@ -527,7 +527,7 @@ public Future<KeyRange> allocateIds(final Key parent, final String kind, long nu
     req.setModelKey(allocateIdsRef);
     AllocateIdsResponse.Builder resp = AllocateIdsResponse.newBuilder();
     Future<AllocateIdsResponse> future =
-        makeAsyncCall(apiConfig, DatastoreService_3.Method.AllocateIds, req.build(), resp.build());
+        makeAsyncCall(apiConfig, DatastoreService_3.Method.AllocateIds, req.build(), resp.buildPartial());
     return new FutureWrapper<AllocateIdsResponse, KeyRange>(future) {
       @Override
       protected KeyRange wrap(AllocateIdsResponse resp) throws Exception {
@@ -596,7 +596,7 @@ public Future<Map<Index, IndexState>> getIndexes() {
             .build();
     return new FutureWrapper<CompositeIndices, Map<Index, IndexState>>(
         makeAsyncCall(
-            apiConfig, DatastoreService_3.Method.GetIndices, req, CompositeIndices.newBuilder().build())) {
+            apiConfig, DatastoreService_3.Method.GetIndices, req, CompositeIndices.getDefaultInstance())) {
       @Override
       protected Map<Index, IndexState> wrap(CompositeIndices indices) throws Exception {
         Map<Index, IndexState> answer = new LinkedHashMap<Index, IndexState>();

api/src/main/java/com/google/appengine/api/datastore/CompositeIndexManager.java

@@ -362,7 +362,7 @@ class EqPropsAndAncestorConstraint {
    */
   protected static class IndexComponentsOnlyQuery
       extends com.google.appengine.api.datastore.IndexComponentsOnlyQuery {
-    public IndexComponentsOnlyQuery(DatastoreV3Pb.Query query) {
+    public IndexComponentsOnlyQuery(DatastoreV3Pb.Query.Builder query) {
       super(query);
     }
   }
@@ -372,7 +372,7 @@ public IndexComponentsOnlyQuery(DatastoreV3Pb.Query query) {
    * publicly exposing it in the api.
    */
   protected static class ValidatedQuery extends com.google.appengine.api.datastore.ValidatedQuery {
-    public ValidatedQuery(DatastoreV3Pb.Query query) {
+    public ValidatedQuery(DatastoreV3Pb.Query.Builder query) {
       super(query);
     }
   }

api/src/main/java/com/google/appengine/api/datastore/DataTypeTranslator.java

@@ -209,7 +209,7 @@ public final class DataTypeTranslator {
    * @param map A not {@code null} map of all the properties which will be set on {@code proto}
    * @param proto A not {@code null} protocol buffer
    */
-  public static void addPropertiesToPb(Map<String, ?> map, EntityProto proto) {
+  public static void addPropertiesToPb(Map<String, ?> map, EntityProto.Builder proto) {
     for (Map.Entry<String, ?> entry : map.entrySet()) {
       String name = entry.getKey();
 
@@ -233,15 +233,15 @@ public static void addPropertiesToPb(Map<String, ?> map, EntityProto proto) {
     }
   }
 
-  /** @see #addPropertiesToPb(Map, EntityProto) */
+  /** @see #addPropertiesToPb(Map, EntityProto.Builder) */
   static void addPropertiesToPb(Map<String, ?> map, com.google.datastore.v1.Entity.Builder proto) {
     for (Map.Entry<String, ?> entry : map.entrySet()) {
       proto.putProperties(entry.getKey(), toV1Value(entry.getValue()).build());
     }
   }
 
   private static void addListPropertyToPb(
-      EntityProto proto,
+      EntityProto.Builder proto,
       String name,
       boolean indexed,
       Collection<?> values,
@@ -258,11 +258,12 @@ private static void addListPropertyToPb(
         // If the value is indexed it appears in queries, but distinction between
         // null and empty list is lost.
       }
+      property.setValue(PropertyValue.getDefaultInstance());
       property.getValue(); // Indicate to the proto that we have set this field
       if (indexed) {
-        proto = proto.toBuilder().addProperty(property.build()).build();
+        proto.addProperty(property);
       } else {
-        proto = proto.toBuilder().addRawProperty(property.build()).build();
+        proto.addRawProperty(property);
       }
     } else {
       // Write every element to the PB
@@ -291,7 +292,7 @@ private static void addPropertyToPb(
       boolean indexed,
       boolean forceIndexedEmbeddedEntity,
       boolean multiple,
-      EntityProto entity) {
+      EntityProto.Builder entity) {
     Property.Builder property = Property.newBuilder();
     property.setName(name);
     property.setMultiple(multiple);
@@ -317,9 +318,9 @@ private static void addPropertyToPb(
       }
     }
     if (indexed) {
-      entity = entity.toBuilder().addProperty(property).build();
+      entity.addProperty(property);
     } else {
-      entity = entity.toBuilder().addRawProperty(property).build();
+      entity.addRawProperty(property);
     }
   }
 
@@ -395,6 +396,7 @@ private static Property buildImplicitKeyProperty(EntityProto proto) {
     PropertyValue.Builder propVal = PropertyValue.newBuilder();
     propVal.setReferenceValue(KeyType.toReferenceValue(proto.getKey()));
     keyProp.setValue(propVal.build());
+    keyProp.setMultiple(false);
     return keyProp.build();
   }
 
@@ -1137,6 +1139,8 @@ public RawValue getValue(Value propertyValue) {
       // All possible values except byte[] are already comparable.
       if (value2 instanceof byte[]) {
         return new ComparableByteArray((byte[]) value2);
+      } else if (value2 instanceof ByteString) {
+        return new ComparableByteArray(((ByteString) value2).toByteArray());
       }
       return (Comparable<?>) value2;
     }
@@ -1683,7 +1687,7 @@ public void toV3Value(Object value, PropertyValue.Builder propertyValue) {
       if (structProp.getKey() != null) {
         proto.setKey(KeyTranslator.convertToPb(structProp.getKey()));
       }
-      addPropertiesToPb(structProp.getPropertyMap(), proto.build());
+      addPropertiesToPb(structProp.getPropertyMap(), proto);
       // TODO: Figure out how to do partial serialization.
       propertyValue.setStringValueBytes(proto.build().toByteString()).build();
     }

api/src/main/java/com/google/appengine/api/datastore/DatastoreApiHelper.java

@@ -130,23 +130,25 @@ static <T extends Message> Future<T> makeAsyncCall(
     return new FutureWrapper<byte[], T>(response) {
       @Override
       protected T wrap(byte[] responseBytes) throws InvalidProtocolBufferException {
+        T updatedResponseProto = null;
         // This null check is mainly for the benefit of unit tests
         // (specifically ones using EasyMock, where the default behavior
         // is to return null).
         if (responseBytes != null && responseProto != null) {
+
           try {
-            responseProto.getParserForType().parseFrom(responseBytes);
+            updatedResponseProto = (T) responseProto.getParserForType().parseFrom(responseBytes);
           }
           catch(InvalidProtocolBufferException e) {
             throw new InvalidProtocolBufferException(
                 String.format("Invalid %s.%s response", DATASTORE_V3_PACKAGE, method.name()));
           }
-          List<String> initializationErrors = responseProto.findInitializationErrors();
+          List<String> initializationErrors = updatedResponseProto.findInitializationErrors();
           if (initializationErrors != null && !initializationErrors.isEmpty()) {
             throw new InvalidProtocolBufferException(initializationErrors.toString());
           }
         }
-        return responseProto;
+        return updatedResponseProto;
       }
 
       @Override

api/src/main/java/com/google/appengine/api/datastore/EntityTranslator.java

@@ -80,13 +80,12 @@ public static EntityProto convertToPb(Entity entity) {
 
     // If we've already been stored, make sure the entity group is set
     // to match our key.
-    Path.Builder entityGroup = proto.getEntityGroup().toBuilder();
+    Path.Builder entityGroup = proto.getEntityGroupBuilder();
     Key key = entity.getKey();
     if (key.isComplete()) {
       entityGroup.addElement(reference.getPath().getElement(0));
     }
-
-    DataTypeTranslator.addPropertiesToPb(entity.getPropertyMap(), proto.build());
+    DataTypeTranslator.addPropertiesToPb(entity.getPropertyMap(), proto);
     return proto.build();
   }
 

api/src/main/java/com/google/appengine/api/datastore/IndexComponentsOnlyQuery.java

@@ -54,7 +54,7 @@ class IndexComponentsOnlyQuery extends ValidatedQuery {
 
   private boolean hasKeyProperty = false;
 
-  public IndexComponentsOnlyQuery(DatastoreV3Pb.Query query) {
+  public IndexComponentsOnlyQuery(DatastoreV3Pb.Query.Builder query) {
     super(query);
     removeNativelySupportedComponents();
     categorizeQuery();
@@ -75,7 +75,7 @@ private void removeNativelySupportedComponents() {
     // Pulling out __key__ asc orders since is supported natively for perfect plans
     boolean hasKeyDescOrder = false;
     if (query.getOrderCount() > 0) {
-      Order lastOrder = query.getOrder(query.getCount() - 1);
+      Order lastOrder = query.getOrder(query.getOrderCount() - 1);
       if (lastOrder.getProperty().equals(Entity.KEY_RESERVED_PROPERTY)) {
         if (lastOrder.getDirection() == Order.Direction.ASCENDING) {
           query.removeOrder(query.getOrderCount() - 1).build();

api/src/main/java/com/google/appengine/api/datastore/IndexTranslator.java

@@ -18,6 +18,7 @@
 
 import com.google.apphosting.api.AppEngineInternal;
 import com.google.storage.onestore.v3.proto2api.OnestoreEntity;
+import com.google.storage.onestore.v3.proto2api.OnestoreEntity.CompositeIndex.State;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -73,6 +74,7 @@ public static Index.Property convertFromPb(OnestoreEntity.Index.Property propert
   }
 
   public static Index convertFromPb(OnestoreEntity.Index index) {
-    return convertFromPb(OnestoreEntity.CompositeIndex.newBuilder().setId(0).setDefinition(index).build());
+    return convertFromPb(OnestoreEntity.CompositeIndex.newBuilder().setId(0).setDefinition(index).setAppId("").setState(
+            State.WRITE_ONLY).build());
   }
 }

api/src/main/java/com/google/appengine/api/datastore/InternalTransactionV3.java

@@ -84,7 +84,7 @@ private <T extends Message> Future<Void> makeAsyncTxnCall(
 
   @Override
   public Future<Void> doCommitAsync() {
-    return makeAsyncTxnCall(DatastoreService_3.Method.Commit, CommitResponse.newBuilder().build());
+    return makeAsyncTxnCall(DatastoreService_3.Method.Commit, CommitResponse.getDefaultInstance());
   }
 
   @Override

api/src/main/java/com/google/appengine/api/datastore/KeyFactory.java

@@ -203,9 +203,9 @@ public static Key stringToKey(String encoded) {
       throw new IllegalArgumentException("Cannot parse: " + encoded, ex);
     }
 
-    Reference reference = Reference.newBuilder().build();
+    Reference reference = Reference.newBuilder().buildPartial();
     try{
-      reference.getParserForType().parseFrom(decodedBytes);
+      reference = reference.getParserForType().parseFrom(decodedBytes);
     } catch (InvalidProtocolBufferException e){
       throw new IllegalArgumentException("Could not parse Reference");
     }

api/src/main/java/com/google/appengine/api/datastore/KeyTranslator.java

@@ -19,6 +19,7 @@
 import com.google.storage.onestore.v3.proto2api.OnestoreEntity.Path;
 import com.google.storage.onestore.v3.proto2api.OnestoreEntity.Path.Element;
 import com.google.storage.onestore.v3.proto2api.OnestoreEntity.Reference;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -71,7 +72,7 @@ public static Reference convertToPb(Key key) {
       reference.setNameSpace(nameSpace);
     }
 
-    Path.Builder path = reference.build().getPath().toBuilder();
+    Path.Builder path = reference.getPathBuilder();
     while (key != null) {
       Element.Builder pathElement = Element.newBuilder();
       pathElement.setType(key.getKind());
@@ -83,7 +84,13 @@ public static Reference convertToPb(Key key) {
       path.addElement(pathElement.build());
       key = key.getParent();
     }
-    Collections.reverse(path.build().getElementList());
+    List<Element> elements = new ArrayList<>(path.getElementList());
+    Collections.reverse(elements);
+    Path.Builder reversedPath = Path.newBuilder();
+    for(Element element : elements){
+      reversedPath.addElement(element);
+    }
+    reference.setPath(reversedPath.build());
     return reference.build();
   }
 

api/src/main/java/com/google/appengine/api/datastore/NormalizedQuery.java

@@ -37,8 +37,8 @@ class NormalizedQuery {
 
   protected final Query.Builder query;
 
-  public NormalizedQuery(Query query) {
-    this.query = query.toBuilder().clone();
+  public NormalizedQuery(Query.Builder query) {
+    this.query = query.clone();
     normalizeQuery();
   }
 

api/src/main/java/com/google/appengine/api/datastore/QueryResultsSourceV3.java

@@ -61,7 +61,7 @@ public NextRequest buildNextCallPrototype(QueryResult initialResult) {
       req.setCompile(true);
     }
     // This used to call .freeze() but that method has been deleted, see go/javaproto1freezeremoval
-    return req.build();
+    return req.buildPartial();
   }
 
   @Override
@@ -77,7 +77,7 @@ public Future<QueryResult> makeNextCall(
     if (offsetOrNull != null) {
       req.setOffset(offsetOrNull);
     }
-    return makeAsyncCall(apiConfig, Method.Next, req.build(), DatastoreV3Pb.QueryResult.newBuilder().build());
+    return makeAsyncCall(apiConfig, Method.Next, req.build(), DatastoreV3Pb.QueryResult.getDefaultInstance());
   }
 
   @Override