Made field existence optional, fixed message with missing inequality constraints, refactor of spec. parsing

Author: dangunter

.gitignore

@@ -38,4 +38,7 @@ nosetests.xml
 # Mr Developer
 .mr.developer.cfg
 .project
-.pydevproject
+.pydevproject
+
+# Docs
+_build/

matgendb/vv/validate.py

@@ -261,7 +261,7 @@ def compare(self, lhs_value, rhs_value):
             raise RuntimeError('unexpected size operator: {}'.format(self._op))
         if self.is_inequality():
             if not isinstance(lhs_value, Number):
-                raise ValueError('Number required for inequality')
+                return False
             py_op = self.PY_INEQ.get(self._op, self._op)
             return eval('{} {} {}'.format(lhs_value, py_op, rhs_value))
         if self.is_type():
@@ -774,6 +774,65 @@ def update(self, *args):
         self._count = 0
 
 
+class ConstraintSpec:
+    """Specification of a set of constraints for a collection.
+    """
+    FILTER_SECT = 'filter'
+    CONSTRAINT_SECT = 'constraints'
+
+    def __init__(self, spec):
+        """Create specification from a configuration.
+
+        :param spec: Configuration for a single collection
+        :type spec: dict
+        :raise: ValueError if specification is wrong
+        """
+        self._sections = {}
+        for item in spec:
+            if isinstance(item, dict):
+                self._add_filtered_section(item)
+            else:
+                self._add_simple_section(item)
+
+    def _add_filtered_section(self, item):
+        """Add a section that has a filter and set of constraints
+
+        :raise: ValueError if filter or constraints is missing
+        """
+        # extract filter and constraints
+        cond_raw, constraints = None, None
+        try:
+            cond_raw = item[self.FILTER_SECT]
+            constraints = item[self.CONSTRAINT_SECT]
+        except KeyError:
+            if cond_raw is None:
+                raise ValueError("configuration is missing '{}'".format(self.FILTER_SECT))
+            else:
+                raise ValueError("configuration is missing '{}'".format(self.CONSTRAINT_SECT))
+
+        # make condition(s) into a tuple
+        if isinstance(cond_raw, basestring):
+            cond = (cond_raw,)
+        elif cond_raw is None:
+            cond = None
+        else:
+            cond = tuple(cond_raw)   # tuples can be used as keys
+            # add
+        if cond in self._sections:
+            self._sections[cond].extend(constraints)
+        else:
+            self._sections[cond] = constraints
+
+    def _add_simple_section(self, item):
+        self._sections[None] = [item]
+
+    def __iter__(self):
+        """When invoked as an iterator, return the key, value
+        pairs of the filter and constraints.
+        """
+        return self._sections.iteritems()
+
+
 class Validator(DoesLogging):
     """Validate a collection.
     """
@@ -792,7 +851,7 @@ class Validator(DoesLogging):
         )
         \s*''', re.VERBOSE)
 
-    def __init__(self, max_violations=50, max_dberrors=10, aliases=None):
+    def __init__(self, max_violations=50, max_dberrors=10, aliases=None, add_exists=False):
         DoesLogging.__init__(self, name='mg.validator')
         self.set_progress(0)
         self._aliases = aliases if aliases else {}
@@ -803,6 +862,7 @@ def __init__(self, max_violations=50, max_dberrors=10, aliases=None):
             self._find_kw = {}
         self._max_dberr = max_dberrors
         self._base_report_fields = {'_id': 1, 'task_id': 1}
+        self._add_exists = add_exists
 
     def set_aliases(self, a):
         """Set aliases.
@@ -824,15 +884,21 @@ def num_violations(self):
             return 0
         return self._progress._count
 
-    def validate(self, coll, constraint_sections, subject='collection'):
+    def validate(self, coll, constraint_spec, subject='collection'):
         """Validation of  a collection.
         This is a generator that yields ConstraintViolationGroups.
 
+        :param coll: Mongo collection
+        :type coll: pymongo.Collection
+        :param constraint_spec: Constraint specification
+        :type constraint_spec: ConstraintSpec
+        :param subject: Name of the thing being validated
+        :type subject: str
         :return: Sets of constraint violation, one for each constraint_section
         :rtype: ConstraintViolationGroup
         """
         self._progress.set_subject(subject)
-        self._build(constraint_sections)
+        self._build(constraint_spec)
         for cond, body in self._sections:
             cvg = self._validate_section(subject, coll, cond, body)
             if cvg is not None:
@@ -919,16 +985,19 @@ def _get_violations(self, query, record):
                 reasons.append(ConstraintViolation(clause.constraint, fval, expected))
         return reasons
 
-    def _build(self, constraint_sections):
+    def _build(self, constraint_spec):
         """Generate queries to execute.
 
         Sets instance variables so that Mongo query strings, etc. can now
         be extracted from the object.
+
+        :param constraint_spec: Constraint specification
+        :type constraint_spec: ConstraintSpec
         """
         self._sections = []
         self._report_fields = self._base_report_fields
         # loopover each condition on the records
-        for cond_expr_list, expr_list in constraint_sections.iteritems():
+        for cond_expr_list, expr_list in constraint_spec:
             #print("@@ CONDS = {}".format(cond_expr_list))
             #print("@@ MAIN = {}".format(expr_list))
             groups = self._process_constraint_expressions(expr_list)
@@ -938,8 +1007,9 @@ def _build(self, constraint_sections):
                 for c in cg:
                     projection.add(c.field, c.op, c.value)
                     query.add_clause(MongoClause(c))
-                for c in cg.existence_constraints:
-                    query.add_clause(MongoClause(c, exists_main=True))
+                if self._add_exists:
+                    for c in cg.existence_constraints:
+                        query.add_clause(MongoClause(c, exists_main=True))
             self._report_fields.update(projection.to_mongo())
             cond_query = MongoQuery()
             if cond_expr_list is not None:

scripts/mgvv

@@ -25,7 +25,7 @@ import urlparse
 import yaml
 # local module
 from matgendb.util import get_settings
-from matgendb.vv.validate import Validator, DBError
+from matgendb.vv.validate import ConstraintSpec, Validator, DBError
 from matgendb.vv import report
 
 # Initialize module logging
@@ -165,6 +165,9 @@ def parse_mongo_url(url):
 
 def get_constraint_sections(spec):
     """Parse out sections within a collection.
+
+    :return: Dictionary keyed by the 'filter' of the 'constraints'
+    :rtype: dict
     """
     sections = {}
     for item in spec:
@@ -218,6 +221,8 @@ def main(cmdline=None):
                              'The "from" and "to" are required; default server is localhost. '
                              'This information can also be in the main config file under the "_email" key, '
                              'which should be a mapping with keys "from", "to", and "server".')
+    parser.add_argument('--exonly', dest='must_exist', action='store_true', default=False,
+                        help='Only show results where all fields in the constraints are also present in the record')
     parser.add_argument('--file', '-f', dest='constfile', metavar='FILE', default=None,
                         help='Main configuration file. Has constraints, and optionally email info.')
     parser.add_argument('--format', '-F', dest='report_format', metavar='FORMAT', default='html',
@@ -267,6 +272,11 @@ def main(cmdline=None):
             parser.error('Cannot authenticate to database as user {}'.format(user))
     else:
         _log.info('Connecting to DB "{}" without authentication'.format(config[db_key]))
+    # get collection name if given
+    if 'collection' in config:
+        db_coll_name = config['collection']
+    else:
+        db_coll_name = None
 
     # Init constraints from file or command-line
     constraints_from_file = args.constfile is not None
@@ -275,14 +285,18 @@ def main(cmdline=None):
         with Timing('load.yaml', file=f.name):
             constraints = yaml.safe_load(f)
     else:
-        if args.coll is None:
-            parser.error('collection name required for command-line constraints')
+        coll_name = args.coll
+        if coll_name is None:
+            if db_coll_name is None:
+                parser.error('collection name not found')
+            else:
+                coll_name = db_coll_name
         if not args.constraint:
             parser.error('at least one constraint is required')
         # re-split command-line arguments on commas (not whitespace)
         plist = ' '.join(args.constraint).split(',')
         # make dict(key=collection) of list of the constraints, just like YAML config file
-        constraints = {args.coll: plist}
+        constraints = {coll_name: plist}
 
     # Init aliases from file or command-line
     if constraints_from_file:
@@ -321,19 +335,24 @@ def main(cmdline=None):
                     .format(fmt, textlist(formatters.keys())))
 
     # Run validation for each collection
-    validator = Validator(aliases=aliases, max_violations=args.limit, max_dberrors=10)
+    validator = Validator(aliases=aliases, max_violations=args.limit, max_dberrors=10, add_exists=args.must_exist)
     if args.progress > 0:
         validator.set_progress(args.progress)
     with Timing("validate"):
-        for coll_name, constraint_spec in constraints.iteritems():
+        for coll_name, constraint_spec_cfg in constraints.iteritems():
             sect_hdr = report.SectionHeader(title="Collection {}".format(coll_name))
             rpt_sect = report.ReportSection(sect_hdr)
             if coll_name.startswith(PATTERN_KEY_PREFIX_IGNORE):
                 continue
             coll = db[coll_name]
             try:
-                sect = get_constraint_sections(constraint_spec)
-                for vnum, vgroup in enumerate(validator.validate(coll, sect, subject=coll_name)):
+                try:
+                    cspec = ConstraintSpec(constraint_spec_cfg)
+                except ValueError, err:
+                    _log.error('processing constraints for {}: {}'.format(coll_name, err))
+                    break
+                #sect = get_constraint_sections(constraint_spec)
+                for vnum, vgroup in enumerate(validator.validate(coll, cspec, subject=coll_name)):
                     sect_hdr = report.SectionHeader(title='Violations {:d}'.format(vnum + 1))
                     sect_hdr.add('Condition', str(vgroup.condition))
                     table = report.Table(colnames=('Id', 'TaskId', 'Field', 'Constraint', 'Value'))