parser: Rework parsing of attribute and entity values

Don't use a separate function to handle "complex" attributes. Validate
UTF-8 byte sequences without decoding. This should improve performance
considerably when parsing multi-byte UTF-8 sequences.

Use a string buffer to avoid unnecessary allocations and copying when
expanding entities.

Normalize attribute values in a single pass while expanding entities.

Be more lenient in recovery mode.

If no entity substitution was requested, validate entities without
expanding. Fixes #596.

Also fixes #655.

Author: nwellnhof

SAX2.c

@@ -970,10 +970,8 @@ xmlSAX2AttributeInternal(void *ctx, const xmlChar *fullname,
         (void) nsret;
 
         if (!ctxt->replaceEntities) {
-	    ctxt->depth++;
-	    val = xmlStringDecodeEntities(ctxt, value, XML_SUBSTITUTE_REF,
-		                          0,0,0);
-	    ctxt->depth--;
+            /* TODO: normalize if needed */
+	    val = xmlExpandEntitiesInAttValue(ctxt, value, /* normalize */ 0);
 	    if (val == NULL) {
 	        xmlSAX2ErrMemory(ctxt);
 		if (name != NULL)
@@ -1038,10 +1036,8 @@ xmlSAX2AttributeInternal(void *ctx, const xmlChar *fullname,
         (void) nsret;
 
         if (!ctxt->replaceEntities) {
-	    ctxt->depth++;
-	    val = xmlStringDecodeEntities(ctxt, value, XML_SUBSTITUTE_REF,
-		                          0,0,0);
-	    ctxt->depth--;
+            /* TODO: normalize if needed */
+	    val = xmlExpandEntitiesInAttValue(ctxt, value, /* normalize */ 0);
 	    if (val == NULL) {
 	        xmlSAX2ErrMemory(ctxt);
 	        xmlFree(ns);
@@ -1179,10 +1175,8 @@ xmlSAX2AttributeInternal(void *ctx, const xmlChar *fullname,
         if (!ctxt->replaceEntities) {
 	    xmlChar *val;
 
-	    ctxt->depth++;
-	    val = xmlStringDecodeEntities(ctxt, value, XML_SUBSTITUTE_REF,
-		                          0,0,0);
-	    ctxt->depth--;
+            /* TODO: normalize if needed */
+	    val = xmlExpandEntitiesInAttValue(ctxt, value, /* normalize */ 0);
 
 	    if (val == NULL)
 		ctxt->valid &= xmlValidateOneAttribute(&ctxt->vctxt,
@@ -1736,19 +1730,19 @@ static xmlChar *
 xmlSAX2DecodeAttrEntities(xmlParserCtxtPtr ctxt, const xmlChar *str,
                           const xmlChar *end) {
     const xmlChar *in;
-    xmlChar *ret;
 
     in = str;
     while (in < end)
         if (*in++ == '&')
 	    goto decode;
     return(NULL);
 decode:
-    ctxt->depth++;
-    ret = xmlStringLenDecodeEntities(ctxt, str, end - str,
-				     XML_SUBSTITUTE_REF, 0,0,0);
-    ctxt->depth--;
-    return(ret);
+    /*
+     * If the value contains '&', we can be sure it was allocated and is
+     * zero-terminated.
+     */
+    /* TODO: normalize if needed */
+    return(xmlExpandEntitiesInAttValue(ctxt, str, /* normalize */ 0));
 }
 #endif /* LIBXML_VALID_ENABLED */
 

include/private/entities.h

@@ -9,13 +9,17 @@
  *
  * XML_ENT_PARSED: The entity was parsed and `children` points to the
  * content.
- * XML_ENT_CHECKED: The entity was checked for loops.
+ *
+ * XML_ENT_CHECKED: The entity was checked for loops and amplification.
+ * expandedSize was set.
+ *
+ * XML_ENT_VALIDATED: The entity contains a valid attribute value.
+ * Only used when entities aren't substituted.
  */
-#define XML_ENT_PARSED      (1<<0)
-#define XML_ENT_CHECKED     (1<<1)
-#define XML_ENT_EXPANDING   (1<<2)
-#define XML_ENT_CHECKED_LT  (1<<3)
-#define XML_ENT_CONTAINS_LT (1<<4)
+#define XML_ENT_PARSED      (1u << 0)
+#define XML_ENT_CHECKED     (1u << 1)
+#define XML_ENT_VALIDATED   (1u << 2)
+#define XML_ENT_EXPANDING   (1u << 3)
 
 XML_HIDDEN xmlChar *
 xmlEncodeAttributeEntities(xmlDocPtr doc, const xmlChar *input);

include/private/parser.h

@@ -87,4 +87,8 @@ XML_HIDDEN xmlParserInputPtr
 xmlNewInputPush(xmlParserCtxtPtr ctxt, const char *url,
                 const char *chunk, int size, const char *encoding);
 
+XML_HIDDEN xmlChar *
+xmlExpandEntitiesInAttValue(xmlParserCtxtPtr ctxt, const xmlChar *str,
+                            int normalize);
+
 #endif /* XML_PARSER_H_PRIVATE__ */