Adds Async entity From Android

src/main/java/org/commcare/cases/entity/AsyncEntity.java

@@ -0,0 +1,236 @@
+package org.commcare.cases.entity;
+
+
+import org.commcare.cases.util.StringUtils;
+import org.commcare.suite.model.DetailField;
+import org.commcare.suite.model.Text;
+import org.javarosa.core.model.condition.EvaluationContext;
+import org.javarosa.core.model.instance.TreeReference;
+import org.javarosa.core.services.Logger;
+import org.javarosa.xpath.XPathException;
+import org.javarosa.xpath.expr.FunctionUtils;
+import org.javarosa.xpath.expr.XPathExpression;
+import org.javarosa.xpath.parser.XPathSyntaxException;
+
+import java.util.Enumeration;
+import java.util.Hashtable;
+
+/**
+ * An AsyncEntity is an entity reference which is capable of building its
+ * values (evaluating all Text elements/background data elements) lazily
+ * rather than upfront when the entity is constructed.
+ *
+ * It is threadsafe.
+ *
+ * It will attempt to Cache its values persistently by a derived entity key rather
+ * than evaluating them each time when possible. This can be slow to perform across
+ * all entities internally due to the overhead of establishing the db connection, it
+ * is recommended that the entities be primed externally with a bulk query.
+ *
+ * @author ctsims
+ */
+public class AsyncEntity extends Entity<TreeReference> {
+
+    private final DetailField[] fields;
+    private final Object[] data;
+    private final String[] sortData;
+    private final boolean[] relevancyData;
+    private final String[][] sortDataPieces;
+    private final EvaluationContext context;
+    private final Hashtable<String, XPathExpression> mVariableDeclarations;
+    private boolean mVariableContextLoaded = false;
+    private final String mCacheIndex;
+    private final String mDetailId;
+
+    private final EntityStorageCache mEntityStorageCache;
+
+    /*
+     * the Object's lock. NOTE: _DO NOT LOCK ANY CODE WHICH READS/WRITES THE CACHE
+     * UNTIL YOU HAVE A LOCK FOR THE DB!
+     * 
+     * The lock is for the integrity of this object, not the larger environment, 
+     * and any DB access has its own implict lock between threads, so it's easy
+     * to accidentally deadlock if you don't already have the db lock
+     * 
+     * Basically you should never be calling mEntityStorageCache from inside of
+     * a lock that 
+     */
+    private final Object mAsyncLock = new Object();
+
+    public AsyncEntity(DetailField[] fields, EvaluationContext ec,
+                       TreeReference t, Hashtable<String, XPathExpression> variables,
+                       EntityStorageCache cache, String cacheIndex, String detailId,
+                       String extraKey) {
+        super(t, extraKey);
+
+        this.fields = fields;
+        this.data = new Object[fields.length];
+        this.sortData = new String[fields.length];
+        this.sortDataPieces = new String[fields.length][];
+        this.relevancyData = new boolean[fields.length];
+        this.context = ec;
+        this.mVariableDeclarations = variables;
+        this.mEntityStorageCache = cache;
+
+        //TODO: It's weird that we pass this in, kind of, but the thing is that we don't want to figure out
+        //if this ref is _cachable_ every time, since it's a pretty big lift
+        this.mCacheIndex = cacheIndex;
+
+        this.mDetailId = detailId;
+    }
+
+    private void loadVariableContext() {
+        synchronized (mAsyncLock) {
+            if (!mVariableContextLoaded) {
+                //These are actually in an ordered hashtable, so we can't just get the keyset, since it's
+                //in a 1.3 hashtable equivalent
+                for (Enumeration<String> en = mVariableDeclarations.keys(); en.hasMoreElements(); ) {
+                    String key = en.nextElement();
+                    context.setVariable(key, FunctionUtils.unpack(mVariableDeclarations.get(key).eval(context)));
+                }
+                mVariableContextLoaded = true;
+            }
+        }
+    }
+
+    @Override
+    public Object getField(int i) {
+        synchronized (mAsyncLock) {
+            loadVariableContext();
+            if (data[i] == null) {
+                try {
+                    data[i] = fields[i].getTemplate().evaluate(context);
+                } catch (XPathException xpe) {
+                    Logger.exception("Error while evaluating field for case list ", xpe);
+                    xpe.printStackTrace();
+                    data[i] = "<invalid xpath: " + xpe.getMessage() + ">";
+                }
+            }
+            return data[i];
+        }
+    }
+
+    @Override
+    public String getNormalizedField(int i) {
+        String normalized = this.getSortField(i);
+        if (normalized == null) {
+            return "";
+        }
+        return normalized;
+    }
+
+    @Override
+    public String getSortField(int i) {
+        if (mEntityStorageCache.lockCache()) {
+            //get our second lock.
+            synchronized (mAsyncLock) {
+                if (sortData[i] == null) {
+                    // sort data not in search field cache; load and store it
+                    Text sortText = fields[i].getSort();
+                    if (sortText == null) {
+                        mEntityStorageCache.releaseCache();
+                        return null;
+                    }
+
+                    String cacheKey = mEntityStorageCache.getCacheKey(mDetailId, String.valueOf(i));
+
+                    if (mCacheIndex != null) {
+                        //Check the cache!
+                        String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey);
+                        if (value != null) {
+                            this.setSortData(i, value);
+                            mEntityStorageCache.releaseCache();
+                            return sortData[i];
+                        }
+                    }
+
+                    loadVariableContext();
+                    try {
+                        sortText = fields[i].getSort();
+                        if (sortText == null) {
+                            this.setSortData(i, getFieldString(i));
+                        } else {
+                            this.setSortData(i, StringUtils.normalize(sortText.evaluate(context)));
+                        }
+
+                        mEntityStorageCache.cache(mCacheIndex, cacheKey, sortData[i]);
+                    } catch (XPathException xpe) {
+                        Logger.exception("Error while evaluating sort field", xpe);
+                        xpe.printStackTrace();
+                        sortData[i] = "<invalid xpath: " + xpe.getMessage() + ">";
+                    }
+                }
+                mEntityStorageCache.releaseCache();
+                return sortData[i];
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public int getNumFields() {
+        return fields.length;
+    }
+
+    @Override
+    public boolean isValidField(int fieldIndex) {
+        //NOTE: This totally jacks the asynchronicity. It's only used in
+        //detail fields for now, so not super important, but worth bearing
+        //in mind
+        synchronized (mAsyncLock) {
+            loadVariableContext();
+            if (getField(fieldIndex).equals("")) {
+                return false;
+            }
+
+            try {
+                this.relevancyData[fieldIndex] = this.fields[fieldIndex].isRelevant(this.context);
+            } catch (XPathSyntaxException e) {
+                final String msg = "Invalid relevant condition for field : " + fields[fieldIndex].getHeader().toString();
+                Logger.exception(msg, e);
+                throw new RuntimeException(msg);
+            }
+            return this.relevancyData[fieldIndex];
+        }
+    }
+
+    @Override
+    public Object[] getData() {
+        for (int i = 0; i < this.getNumFields(); ++i) {
+            this.getField(i);
+        }
+        return data;
+    }
+
+    @Override
+    public String[] getSortFieldPieces(int i) {
+        if (getSortField(i) == null) {
+            return new String[0];
+        }
+        return sortDataPieces[i];
+    }
+
+    private void setSortData(int i, String val) {
+        synchronized (mAsyncLock) {
+            this.sortData[i] = val;
+            this.sortDataPieces[i] = breakUpField(val);
+        }
+    }
+
+    public void setSortData(String cacheKey, String val) {
+        int sortIndex = mEntityStorageCache.getSortFieldIdFromCacheKey(mDetailId, cacheKey);
+        if (sortIndex != -1) {
+            setSortData(sortIndex, val);
+        }
+    }
+
+    private static String[] breakUpField(String input) {
+        if (input == null) {
+            return new String[0];
+        } else {
+            //We always fuzzy match on the sort field and only if it is available
+            //(as a way to restrict possible matching)
+            return input.split("\\s+");
+        }
+    }
+}

src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java

@@ -0,0 +1,128 @@
+package org.commcare.cases.entity;
+
+
+import org.commcare.suite.model.Detail;
+import org.javarosa.core.model.condition.EvaluationContext;
+import org.javarosa.core.model.instance.TreeReference;
+import org.javarosa.core.model.utils.CacheHost;
+import org.javarosa.core.util.OrderedHashtable;
+import org.javarosa.xpath.expr.XPathExpression;
+
+import java.util.Hashtable;
+import java.util.List;
+
+/**
+ * @author ctsims
+ */
+public class AsyncNodeEntityFactory extends NodeEntityFactory {
+    private static final String TAG = AsyncNodeEntityFactory.class.getSimpleName();
+    private final OrderedHashtable<String, XPathExpression> mVariableDeclarations;
+
+    private final Hashtable<String, AsyncEntity> mEntitySet = new Hashtable<>();
+    private final EntityStorageCache mEntityCache;
+
+    private CacheHost mCacheHost = null;
+    private Boolean mTemplateIsCachable = null;
+    private static final Object mAsyncLock = new Object();
+    private Thread mAsyncPrimingThread;
+
+    // Don't show entity list until we primeCache and caches all fields
+    private final boolean isBlockingAsyncMode;
+
+    public AsyncNodeEntityFactory(Detail d, EvaluationContext ec, EntityStorageCache entityStorageCache) {
+        super(d, ec);
+
+        mVariableDeclarations = detail.getVariableDeclarations();
+        mEntityCache = entityStorageCache;
+        isBlockingAsyncMode = detail.hasSortField();
+    }
+
+    @Override
+    public Entity<TreeReference> getEntity(TreeReference data) {
+        EvaluationContext nodeContext = new EvaluationContext(ec, data);
+
+        mCacheHost = nodeContext.getCacheHost(data);
+
+        String mCacheIndex = null;
+        if (mTemplateIsCachable == null) {
+            mTemplateIsCachable = mCacheHost != null && mCacheHost.isReferencePatternCachable(data);
+        }
+        if (mTemplateIsCachable && mCacheHost != null) {
+            mCacheIndex = mCacheHost.getCacheIndex(data);
+        }
+
+        String entityKey = loadCalloutDataMapKey(nodeContext);
+        AsyncEntity entity =
+                new AsyncEntity(detail.getFields(), nodeContext, data, mVariableDeclarations,
+                        mEntityCache, mCacheIndex, detail.getId(), entityKey);
+
+        if (mCacheIndex != null) {
+            mEntitySet.put(mCacheIndex, entity);
+        }
+        return entity;
+    }
+
+    @Override
+    protected void setEvaluationContextDefaultQuerySet(EvaluationContext ec,
+                                                       List<TreeReference> result) {
+
+        //Don't do anything for asynchronous lists. In theory the query set could help expand the
+        //first cache more quickly, but otherwise it's just keeping around tons of cases in memory
+        //that don't even need to be loaded.
+    }
+
+
+    /**
+     * Bulk loads search field cache from db.
+     * Note that the cache is lazily built upon first case list search.
+     */
+    private void primeCache() {
+        if (mTemplateIsCachable == null || !mTemplateIsCachable || mCacheHost == null) {
+            return;
+        }
+
+        String[][] cachePrimeKeys = mCacheHost.getCachePrimeGuess();
+        if (cachePrimeKeys == null) {
+            return;
+        }
+        mEntityCache.primeCache(mEntitySet,cachePrimeKeys, detail);
+    }
+
+    @Override
+    protected void prepareEntitiesInternal(List<Entity<TreeReference>> entities) {
+        // if blocking mode load cache on the same thread and set any data thats not cached
+        if (isBlockingAsyncMode) {
+            primeCache();
+            setUnCachedData(entities);
+        } else {
+            // otherwise we want to show the entity list asap and hence want to offload the loading cache part to a separate
+            // thread while caching any uncached data later on UI thread during Adapter's getView
+            synchronized (mAsyncLock) {
+                if (mAsyncPrimingThread == null) {
+                    mAsyncPrimingThread = new Thread(this::primeCache);
+                    mAsyncPrimingThread.start();
+                }
+            }
+        }
+    }
+
+    private void setUnCachedData(List<Entity<TreeReference>> entities) {
+        for (int i = 0; i < entities.size(); i++) {
+            AsyncEntity e = (AsyncEntity)entities.get(i);
+            for (int col = 0; col < e.getNumFields(); ++col) {
+                e.getSortField(col);
+            }
+        }
+    }
+
+    @Override
+    protected boolean isEntitySetReadyInternal() {
+        synchronized (mAsyncLock) {
+            return mAsyncPrimingThread == null || !mAsyncPrimingThread.isAlive();
+        }
+    }
+
+    public boolean isBlockingAsyncMode() {
+        return isBlockingAsyncMode;
+    }
+}

src/main/java/org/commcare/cases/entity/EntityStorageCache.java

@@ -0,0 +1,24 @@
+package org.commcare.cases.entity;
+
+import org.commcare.suite.model.Detail;
+
+import java.util.Hashtable;
+
+/**
+ * Interface for evaluated entity fields cache
+ */
+public interface EntityStorageCache {
+    boolean lockCache();
+
+    void releaseCache();
+
+    String getCacheKey(String detailId, String detailFieldIndex);
+
+    String retrieveCacheValue(String cacheIndex, String cacheKey);
+
+    void cache(String cacheIndex, String cacheKey, String data);
+
+    int getSortFieldIdFromCacheKey(String detailId, String cacheKey);
+
+    void primeCache(Hashtable<String, AsyncEntity> entitySet, String[][] cachePrimeKeys, Detail detail);
+}