diff --git a/src/main/java/org/commcare/cases/entity/AsyncEntity.java b/src/main/java/org/commcare/cases/entity/AsyncEntity.java new file mode 100755 index 000000000..fbe5b43e1 --- /dev/null +++ b/src/main/java/org/commcare/cases/entity/AsyncEntity.java @@ -0,0 +1,236 @@ +package org.commcare.cases.entity; + + +import org.commcare.cases.util.StringUtils; +import org.commcare.suite.model.DetailField; +import org.commcare.suite.model.Text; +import org.javarosa.core.model.condition.EvaluationContext; +import org.javarosa.core.model.instance.TreeReference; +import org.javarosa.core.services.Logger; +import org.javarosa.xpath.XPathException; +import org.javarosa.xpath.expr.FunctionUtils; +import org.javarosa.xpath.expr.XPathExpression; +import org.javarosa.xpath.parser.XPathSyntaxException; + +import java.util.Enumeration; +import java.util.Hashtable; + +/** + * An AsyncEntity is an entity reference which is capable of building its + * values (evaluating all Text elements/background data elements) lazily + * rather than upfront when the entity is constructed. + * + * It is threadsafe. + * + * It will attempt to Cache its values persistently by a derived entity key rather + * than evaluating them each time when possible. This can be slow to perform across + * all entities internally due to the overhead of establishing the db connection, it + * is recommended that the entities be primed externally with a bulk query. + * + * @author ctsims + */ +public class AsyncEntity extends Entity { + + private final DetailField[] fields; + private final Object[] data; + private final String[] sortData; + private final boolean[] relevancyData; + private final String[][] sortDataPieces; + private final EvaluationContext context; + private final Hashtable mVariableDeclarations; + private boolean mVariableContextLoaded = false; + private final String mCacheIndex; + private final String mDetailId; + + private final EntityStorageCache mEntityStorageCache; + + /* + * the Object's lock. NOTE: _DO NOT LOCK ANY CODE WHICH READS/WRITES THE CACHE + * UNTIL YOU HAVE A LOCK FOR THE DB! + * + * The lock is for the integrity of this object, not the larger environment, + * and any DB access has its own implict lock between threads, so it's easy + * to accidentally deadlock if you don't already have the db lock + * + * Basically you should never be calling mEntityStorageCache from inside of + * a lock that + */ + private final Object mAsyncLock = new Object(); + + public AsyncEntity(DetailField[] fields, EvaluationContext ec, + TreeReference t, Hashtable variables, + EntityStorageCache cache, String cacheIndex, String detailId, + String extraKey) { + super(t, extraKey); + + this.fields = fields; + this.data = new Object[fields.length]; + this.sortData = new String[fields.length]; + this.sortDataPieces = new String[fields.length][]; + this.relevancyData = new boolean[fields.length]; + this.context = ec; + this.mVariableDeclarations = variables; + this.mEntityStorageCache = cache; + + //TODO: It's weird that we pass this in, kind of, but the thing is that we don't want to figure out + //if this ref is _cachable_ every time, since it's a pretty big lift + this.mCacheIndex = cacheIndex; + + this.mDetailId = detailId; + } + + private void loadVariableContext() { + synchronized (mAsyncLock) { + if (!mVariableContextLoaded) { + //These are actually in an ordered hashtable, so we can't just get the keyset, since it's + //in a 1.3 hashtable equivalent + for (Enumeration en = mVariableDeclarations.keys(); en.hasMoreElements(); ) { + String key = en.nextElement(); + context.setVariable(key, FunctionUtils.unpack(mVariableDeclarations.get(key).eval(context))); + } + mVariableContextLoaded = true; + } + } + } + + @Override + public Object getField(int i) { + synchronized (mAsyncLock) { + loadVariableContext(); + if (data[i] == null) { + try { + data[i] = fields[i].getTemplate().evaluate(context); + } catch (XPathException xpe) { + Logger.exception("Error while evaluating field for case list ", xpe); + xpe.printStackTrace(); + data[i] = ""; + } + } + return data[i]; + } + } + + @Override + public String getNormalizedField(int i) { + String normalized = this.getSortField(i); + if (normalized == null) { + return ""; + } + return normalized; + } + + @Override + public String getSortField(int i) { + if (mEntityStorageCache.lockCache()) { + //get our second lock. + synchronized (mAsyncLock) { + if (sortData[i] == null) { + // sort data not in search field cache; load and store it + Text sortText = fields[i].getSort(); + if (sortText == null) { + mEntityStorageCache.releaseCache(); + return null; + } + + String cacheKey = mEntityStorageCache.getCacheKey(mDetailId, String.valueOf(i)); + + if (mCacheIndex != null) { + //Check the cache! + String value = mEntityStorageCache.retrieveCacheValue(mCacheIndex, cacheKey); + if (value != null) { + this.setSortData(i, value); + mEntityStorageCache.releaseCache(); + return sortData[i]; + } + } + + loadVariableContext(); + try { + sortText = fields[i].getSort(); + if (sortText == null) { + this.setSortData(i, getFieldString(i)); + } else { + this.setSortData(i, StringUtils.normalize(sortText.evaluate(context))); + } + + mEntityStorageCache.cache(mCacheIndex, cacheKey, sortData[i]); + } catch (XPathException xpe) { + Logger.exception("Error while evaluating sort field", xpe); + xpe.printStackTrace(); + sortData[i] = ""; + } + } + mEntityStorageCache.releaseCache(); + return sortData[i]; + } + } + return null; + } + + @Override + public int getNumFields() { + return fields.length; + } + + @Override + public boolean isValidField(int fieldIndex) { + //NOTE: This totally jacks the asynchronicity. It's only used in + //detail fields for now, so not super important, but worth bearing + //in mind + synchronized (mAsyncLock) { + loadVariableContext(); + if (getField(fieldIndex).equals("")) { + return false; + } + + try { + this.relevancyData[fieldIndex] = this.fields[fieldIndex].isRelevant(this.context); + } catch (XPathSyntaxException e) { + final String msg = "Invalid relevant condition for field : " + fields[fieldIndex].getHeader().toString(); + Logger.exception(msg, e); + throw new RuntimeException(msg); + } + return this.relevancyData[fieldIndex]; + } + } + + @Override + public Object[] getData() { + for (int i = 0; i < this.getNumFields(); ++i) { + this.getField(i); + } + return data; + } + + @Override + public String[] getSortFieldPieces(int i) { + if (getSortField(i) == null) { + return new String[0]; + } + return sortDataPieces[i]; + } + + private void setSortData(int i, String val) { + synchronized (mAsyncLock) { + this.sortData[i] = val; + this.sortDataPieces[i] = breakUpField(val); + } + } + + public void setSortData(String cacheKey, String val) { + int sortIndex = mEntityStorageCache.getSortFieldIdFromCacheKey(mDetailId, cacheKey); + if (sortIndex != -1) { + setSortData(sortIndex, val); + } + } + + private static String[] breakUpField(String input) { + if (input == null) { + return new String[0]; + } else { + //We always fuzzy match on the sort field and only if it is available + //(as a way to restrict possible matching) + return input.split("\\s+"); + } + } +} diff --git a/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java new file mode 100755 index 000000000..e766b0619 --- /dev/null +++ b/src/main/java/org/commcare/cases/entity/AsyncNodeEntityFactory.java @@ -0,0 +1,128 @@ +package org.commcare.cases.entity; + + +import org.commcare.suite.model.Detail; +import org.javarosa.core.model.condition.EvaluationContext; +import org.javarosa.core.model.instance.TreeReference; +import org.javarosa.core.model.utils.CacheHost; +import org.javarosa.core.util.OrderedHashtable; +import org.javarosa.xpath.expr.XPathExpression; + +import java.util.Hashtable; +import java.util.List; + +/** + * @author ctsims + */ +public class AsyncNodeEntityFactory extends NodeEntityFactory { + private static final String TAG = AsyncNodeEntityFactory.class.getSimpleName(); + private final OrderedHashtable mVariableDeclarations; + + private final Hashtable mEntitySet = new Hashtable<>(); + private final EntityStorageCache mEntityCache; + + private CacheHost mCacheHost = null; + private Boolean mTemplateIsCachable = null; + private static final Object mAsyncLock = new Object(); + private Thread mAsyncPrimingThread; + + // Don't show entity list until we primeCache and caches all fields + private final boolean isBlockingAsyncMode; + + public AsyncNodeEntityFactory(Detail d, EvaluationContext ec, EntityStorageCache entityStorageCache) { + super(d, ec); + + mVariableDeclarations = detail.getVariableDeclarations(); + mEntityCache = entityStorageCache; + isBlockingAsyncMode = detail.hasSortField(); + } + + @Override + public Entity getEntity(TreeReference data) { + EvaluationContext nodeContext = new EvaluationContext(ec, data); + + mCacheHost = nodeContext.getCacheHost(data); + + String mCacheIndex = null; + if (mTemplateIsCachable == null) { + mTemplateIsCachable = mCacheHost != null && mCacheHost.isReferencePatternCachable(data); + } + if (mTemplateIsCachable && mCacheHost != null) { + mCacheIndex = mCacheHost.getCacheIndex(data); + } + + String entityKey = loadCalloutDataMapKey(nodeContext); + AsyncEntity entity = + new AsyncEntity(detail.getFields(), nodeContext, data, mVariableDeclarations, + mEntityCache, mCacheIndex, detail.getId(), entityKey); + + if (mCacheIndex != null) { + mEntitySet.put(mCacheIndex, entity); + } + return entity; + } + + @Override + protected void setEvaluationContextDefaultQuerySet(EvaluationContext ec, + List result) { + + //Don't do anything for asynchronous lists. In theory the query set could help expand the + //first cache more quickly, but otherwise it's just keeping around tons of cases in memory + //that don't even need to be loaded. + } + + + /** + * Bulk loads search field cache from db. + * Note that the cache is lazily built upon first case list search. + */ + private void primeCache() { + if (mTemplateIsCachable == null || !mTemplateIsCachable || mCacheHost == null) { + return; + } + + String[][] cachePrimeKeys = mCacheHost.getCachePrimeGuess(); + if (cachePrimeKeys == null) { + return; + } + mEntityCache.primeCache(mEntitySet,cachePrimeKeys, detail); + } + + @Override + protected void prepareEntitiesInternal(List> entities) { + // if blocking mode load cache on the same thread and set any data thats not cached + if (isBlockingAsyncMode) { + primeCache(); + setUnCachedData(entities); + } else { + // otherwise we want to show the entity list asap and hence want to offload the loading cache part to a separate + // thread while caching any uncached data later on UI thread during Adapter's getView + synchronized (mAsyncLock) { + if (mAsyncPrimingThread == null) { + mAsyncPrimingThread = new Thread(this::primeCache); + mAsyncPrimingThread.start(); + } + } + } + } + + private void setUnCachedData(List> entities) { + for (int i = 0; i < entities.size(); i++) { + AsyncEntity e = (AsyncEntity)entities.get(i); + for (int col = 0; col < e.getNumFields(); ++col) { + e.getSortField(col); + } + } + } + + @Override + protected boolean isEntitySetReadyInternal() { + synchronized (mAsyncLock) { + return mAsyncPrimingThread == null || !mAsyncPrimingThread.isAlive(); + } + } + + public boolean isBlockingAsyncMode() { + return isBlockingAsyncMode; + } +} diff --git a/src/main/java/org/commcare/cases/entity/EntityStorageCache.java b/src/main/java/org/commcare/cases/entity/EntityStorageCache.java new file mode 100644 index 000000000..ff771a5e3 --- /dev/null +++ b/src/main/java/org/commcare/cases/entity/EntityStorageCache.java @@ -0,0 +1,24 @@ +package org.commcare.cases.entity; + +import org.commcare.suite.model.Detail; + +import java.util.Hashtable; + +/** + * Interface for evaluated entity fields cache + */ +public interface EntityStorageCache { + boolean lockCache(); + + void releaseCache(); + + String getCacheKey(String detailId, String detailFieldIndex); + + String retrieveCacheValue(String cacheIndex, String cacheKey); + + void cache(String cacheIndex, String cacheKey, String data); + + int getSortFieldIdFromCacheKey(String detailId, String cacheKey); + + void primeCache(Hashtable entitySet, String[][] cachePrimeKeys, Detail detail); +}