Control list filters

app/control_lists/views.py

@@ -7,7 +7,7 @@
 
 
 from app.main.views.utils import render_template_with_nav_info
-from app.models import ControlLists, AllowedLemma, WordToken, User, PublicationStatus, CorpusCustomDictionary
+from app.models import ControlLists, ControlListsUser, AllowedLemma, WordToken, User, PublicationStatus, CorpusCustomDictionary
 from app import db, email
 from ..utils import PyrrhaError
 from ..utils.forms import strip_or_none
@@ -374,3 +374,40 @@ def information_edit(control_list_id, control_list):
 def information_read(control_list_id):
     control_list, is_owner = ControlLists.get_linked_or_404(control_list_id=control_list_id, user=current_user)
     return render_template_with_nav_info('control_lists/information_read.html', control_list=control_list)
+
+
+@control_lists_bp.route("/controls/<int:control_list_id>/ignore_terms", methods=["POST", "GET"])
+@login_required
+def ignore_terms_filter(control_list_id):
+    current_controlListUser = ControlListsUser.retrieve(
+        user_id=current_user.id,
+        control_list_id=control_list_id
+    ).first_or_404()
+
+    list_filter = []
+    if request.method == "POST":
+        list_filter.append(request.form.get("punct"))
+        list_filter.append(request.form.get("numeral"))
+        list_filter.append(request.form.get('ignore'))
+        list_filter.append(request.form.get('metadata'))
+        filtered_filter = []
+        for el in list_filter:
+            if el is not None:
+                filtered_filter.append(el)
+
+        current_controlListUser.filter_punct = 'punct' in filtered_filter
+        current_controlListUser.filter_metadata = 'metadata' in filtered_filter
+        current_controlListUser.filter_numeral = 'numeral' in filtered_filter
+        current_controlListUser.filter_ignore = 'ignore' in filtered_filter
+        db.session.add(current_controlListUser)
+        db.session.commit()
+
+        flash('The filters have been updated.', 'success')
+        current_controlListUser = ControlListsUser.retrieve(
+            user_id=current_user.id,
+            control_list_id=control_list_id
+        ).first()
+        return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id,
+                                             current_control_list=current_controlListUser)
+
+    return render_template_with_nav_info('control_lists/ignore_filter.html', control_list_id=control_list_id, current_control_list=current_controlListUser)

app/main/views/corpus.py

@@ -3,6 +3,7 @@
 import sqlalchemy.exc
 from sqlalchemy import func, distinct, text
 from typing import List
+import logging
 
 
 from app import db
@@ -36,12 +37,14 @@ def _get_available():
         lists[cl.str_public].append(cl)
     return lists
 
-
 @main.route('/corpus/new', methods=["POST", "GET"])
 @login_required
 def corpus_new():
     """ Register a new corpus
     """
+    logging.basicConfig(filename='./pyrrha_corpus_creation.log', level=logging.DEBUG,
+                        format='%(asctime)s %(levelname)s %(name)s %(message)s')
+    logger = logging.getLogger(__name__)
     lemmatizers = current_app.config.get("LEMMATIZERS", [])
 
     def normal_view():
@@ -93,8 +96,9 @@ def error():
                 tokens = read_input_tokens(request.form.get("tsv"))
                 try:
                     control_list = ControlLists.query.get_or_404(request.form.get("control_list_select"))
-                except Exception:
+                except Exception as e:
                     flash("This control list does not exist", category="error")
+                    logger.error(e)
                     return error()
                 form_kwargs.update({"word_tokens_dict": tokens,
                                     "control_list": control_list})
@@ -110,17 +114,33 @@ def error():
                 form_kwargs.update({"word_tokens_dict": tokens, "allowed_lemma": allowed_lemma,
                                     "allowed_POS": allowed_POS, "allowed_morph": allowed_morph})
 
+            list_filter = []
+            list_filter.append(request.form.get("punct"))
+            list_filter.append(request.form.get("numeral"))
+            list_filter.append(request.form.get("ignore"))
+            list_filter.append(request.form.get("metadata"))
+            list_filter = [flt for flt in list_filter if flt]
+
             try:
                 corpus: Corpus = Corpus.create(**form_kwargs)
                 db.session.add(CorpusUser(corpus=corpus, user=current_user, is_owner=True))
                 # Add a link to the control list
                 ControlLists.link(corpus.control_lists_id, current_user.id, is_owner=cl_owner)
                 db.session.commit()
+                current_controlListUser = ControlListsUser.query.filter_by(
+                    **{"control_lists_id": corpus.control_lists_id, "user_id": current_user.id}).first_or_404()
+                current_controlListUser.filter_punct = 'punct' in list_filter
+                current_controlListUser.filter_metadata = 'metadata' in list_filter
+                current_controlListUser.filter_numeral = 'numeral' in list_filter
+                current_controlListUser.filter_ignore = 'ignore' in list_filter
+                db.session.commit()
                 flash("New corpus registered", category="success")
             except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
+                print(e)
                 db.session.rollback()
                 flash("The corpus cannot be registered. Check your data", category="error")
                 flash(str(e.orig).lower())
+                logger.error(e)
                 if db.session.get_bind().dialect.name == "postgresql":
                     unique_constraint = 'duplicate key value violates unique constraint "corpus_name_key"'
                 else:
@@ -133,6 +153,7 @@ def error():
                 db.session.rollback()
                 flash("At least one line of your corpus is missing a token/form. Check line %s " % exc.line,
                       category="error")
+                logger.error(exc)
                 return error()
             except NoTokensInput:
                 db.session.rollback()
@@ -143,8 +164,10 @@ def error():
                 flash(exception, category="error")
                 return error()
             except Exception as e:
+                print(e)
                 db.session.rollback()
                 flash("The corpus cannot be registered. Check your data", category="error")
+                logger.error(e)
                 return error()
             return redirect(url_for(".corpus_get", corpus_id=corpus.id))
 

app/main/views/tokens.py

@@ -52,8 +52,9 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     :param allowed_type: Type of allowed value to check agains (lemma, POS, morph)
     """
     corpus = Corpus.query.filter_by(**{"id": corpus_id}).first()
+    user_id = current_user.id
     tokens = corpus\
-        .get_unallowed(allowed_type)\
+        .get_unallowed(user_id, corpus_id, allowed_type)\
         .paginate(
             page=int_or(request.args.get("page"), 1),
             per_page=int_or(request.args.get("limit"), current_app.config["PAGINATION_DEFAULT_TOKENS"])
@@ -67,6 +68,7 @@ def tokens_correct_unallowed(corpus_id, allowed_type):
     )
 
 
+
 @main.route('/corpus/<int:corpus_id>/tokens/changes/similar/<int:record_id>')
 @login_required
 @requires_corpus_access("corpus_id")
@@ -133,7 +135,7 @@ def tokens_correct_single(corpus_id, token_id):
             token_id=token_id, corpus_id=corpus_id,
             lemma=string_to_none(request.form.get("lemma")),
             POS=string_to_none(request.form.get("POS")),
-            morph=string_to_none(request.form.get("morph"))
+            morph=string_to_none(request.form.get("morph")),
         )
         if "similar" in corpus.displayed_columns_by_name:
             similar = {

app/models/control_lists.py

@@ -8,9 +8,11 @@
 from collections import Counter
 # PIP Packages
 import unidecode
+import regex as re
 import yaml
+from flask_sqlalchemy.query import Query as FlaskQuery
 from sqlalchemy.ext.associationproxy import association_proxy
-from sqlalchemy.orm import backref
+from sqlalchemy.orm import backref, Query
 from sqlalchemy import literal, case
 from werkzeug.exceptions import BadRequest
 # APP Logic
@@ -240,6 +242,7 @@ def has_list(self, allowed_type):
             ).exists()
         ).scalar()
 
+
     @staticmethod
     def add_default_lists(path=None):
         """ Loads the default lists from the config folder
@@ -255,7 +258,7 @@ def add_default_lists(path=None):
             print("[ControlLists] Adding %s " % data["name"])
             cl = ControlLists(**data, public=PublicationStatus.public)
             db.session.add(cl)
-            db.session.flush()  # Get the AutoIncrement ID
+            db.session.flush()  # Get the AutoIncrement ID/home/jjanes
             configs = [
                 ("lemma.txt", AllowedLemma, read_input_lemma),
                 ("POS.txt", AllowedPOS, read_input_POS),
@@ -281,9 +284,25 @@ class ControlListsUser(db.Model):
     user_id = db.Column(db.Integer, db.ForeignKey(User.id), primary_key=True)
     is_owner = db.Column(db.Boolean, default=False)
 
+    filter_punct = db.Column(db.Boolean, unique=False, default=False)
+    filter_numeral = db.Column(db.Boolean, unique=False, default=False)
+    filter_metadata = db.Column(db.Boolean, unique=False, default=False)
+    filter_ignore = db.Column(db.Boolean, unique=False, default=False)
+
     control = db.relationship("ControlLists", backref=backref("control_lists_user", cascade="all, delete-orphan"))
     user = db.relationship(User, backref=backref("control_lists_user", cascade="all, delete-orphan"))
 
+    re_filter_metadata = r'(\[[^\]]+:[^\]]*\]$)'
+    re_filter_ignore = r'(^\[IGNORE\])'
+    re_filter_punct = "(^[^\w\s]$)"
+    re_filter_numeral = r'(^\d+$)'
+
+    @classmethod
+    def retrieve(cls, user_id: int, control_list_id: int) -> FlaskQuery:
+        return cls.query.filter(db.and_(
+            cls.user_id == user_id,
+            cls.control_lists_id == control_list_id
+        ))
 
 class AllowedLemma(db.Model):
     """ An allowed lemma is a lemma that is accepted

app/models/corpus.py

@@ -7,6 +7,7 @@
 # PIP Packages
 import unidecode
 import sqlalchemy.exc
+import regex as re
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import backref
 from sqlalchemy import func, literal, not_, or_, and_
@@ -23,7 +24,7 @@
 
 # Models
 from .user import User
-from .control_lists import ControlLists, AllowedPOS, AllowedMorph, AllowedLemma, PublicationStatus
+from .control_lists import ControlLists, ControlListsUser, AllowedPOS, AllowedMorph, AllowedLemma, PublicationStatus
 
 
 from collections import namedtuple
@@ -297,7 +298,7 @@ def get_allowed_values(self, allowed_type="lemma", label=None, order_by="label",
             ).order_by(order_by)
         return db.session.query(cls).filter(cls.control_list == self.control_lists_id).order_by(order_by)
 
-    def get_unallowed(self, allowed_type="lemma"):
+    def get_unallowed(self, user_id, corpus_id, allowed_type="lemma"):
         """ Search for WordToken that would not comply with Allowed Values (in AllowedLemma,
         AllowedPOS, AllowedMorph) nor with a corpus custom dictionary
 
@@ -326,12 +327,33 @@ def get_unallowed(self, allowed_type="lemma"):
             CorpusCustomDictionary.category == allowed_type,
             CorpusCustomDictionary.label == prop
         )
+
+        list_darguments = [
+            WordToken.corpus == self.id,
+            not_(allowed.exists()),
+            not_(custom_dict.exists())
+        ]
+
+        current_corpus = Corpus.query.filter_by(**{"id":corpus_id}).first_or_404()
+        current_controlListUser = ControlListsUser.query.filter_by(
+            **{"control_lists_id":current_corpus.control_lists_id, "user_id": user_id}).first_or_404()
+
+        regex_liste = []
+        if current_controlListUser:
+            if current_controlListUser.filter_metadata:
+                regex_liste.append(ControlListsUser.re_filter_metadata)
+            if current_controlListUser.filter_ignore:
+                regex_liste.append(ControlListsUser.re_filter_ignore)
+            if current_controlListUser.filter_punct:
+                regex_liste.append(ControlListsUser.re_filter_punct)
+            if current_controlListUser.filter_numeral:
+                regex_liste.append(ControlListsUser.re_filter_numeral)
+
+        if regex_liste:
+            list_darguments.append(WordToken.form.op('~')("".join(regex_liste)))
+
         return db.session.query(WordToken).filter(
-            db.and_(
-                WordToken.corpus == self.id,
-                not_(allowed.exists()),
-                not_(custom_dict.exists())
-            )
+            db.and_(*list_darguments)
         ).order_by(WordToken.order_id)
 
     @property
@@ -437,6 +459,8 @@ def create(
             )
             db.session.add(c)
             db.session.commit()
+
+
         except (sqlalchemy.exc.StatementError, sqlalchemy.exc.IntegrityError) as e:
             db.session.rollback()
             raise e
@@ -1072,7 +1096,7 @@ def get_like(filter_id, form, group_by, type_like="lemma", allowed_list=False):
         return query
 
     @staticmethod
-    def is_valid(lemma, POS, morph, corpus):
+    def is_valid(lemma, POS, morph, corpus, user_id):
         """ Check if a token is valid for a given corpus
 
         :param lemma: Lemma value of the token to validate
@@ -1097,13 +1121,33 @@ def is_valid(lemma, POS, morph, corpus):
         }
 
         allowed_column = corpus.displayed_columns_by_name
-
-        if lemma is not None \
-                and "lemma" in allowed_column \
-                and allowed_lemma.count() > 0 \
-                and corpus.get_allowed_values("lemma", label=lemma).count() == 0:
-            if not corpus.has_custom_dictionary_value("lemma", lemma):
-                statuses["lemma"] = False
+        if lemma and "lemma" in allowed_column and allowed_lemma.count():
+            current_controlListUser = ControlListsUser.retrieve(
+                user_id=user_id, control_list_id=corpus.control_lists_id
+            ).first()
+            regex_liste = []
+            if current_controlListUser:
+                if current_controlListUser.filter_metadata:
+                    regex_liste.append(ControlListsUser.re_filter_metadata)
+                if current_controlListUser.filter_ignore:
+                    regex_liste.append(ControlListsUser.re_filter_ignore)
+                if current_controlListUser.filter_punct:
+                    regex_liste.append(ControlListsUser.re_filter_punct)
+                if current_controlListUser.filter_numeral:
+                    regex_liste.append(ControlListsUser.re_filter_numeral)
+
+            ignored_by_regex = False
+
+            for regex in regex_liste:
+                if re.match(regex, lemma) is not None:
+                    ignored_by_regex = True
+
+            if (
+                    not ignored_by_regex and
+                    corpus.has_custom_dictionary_value("lemma", lemma) is False and
+                    corpus.get_allowed_values("lemma", label=lemma).count() == 0
+                ):
+                    statuses["lemma"] = False
 
         if POS is not None \
                 and "POS" in allowed_column \
@@ -1118,6 +1162,7 @@ def is_valid(lemma, POS, morph, corpus):
                 and corpus.get_allowed_values("morph", label=morph).count() == 0:
             if not corpus.has_custom_dictionary_value("morph", morph):
                 statuses["morph"] = False
+
         return statuses
 
     @staticmethod
@@ -1305,9 +1350,10 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
             error = WordToken.NothingChangedError("No value where changed")
             error.msg = "No value where changed"
             raise error
-
+        print(token)
         # Check if values are correct regarding allowed values
-        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus)
+        validity = WordToken.is_valid(lemma=lemma, POS=POS, morph=morph, corpus=corpus, user_id=user_id)
+        print(token.POS, validity)
         if False in list(validity.values()):
             error_msg = "Invalid value in {}".format(
                 ", ".join([key for key in validity.keys() if validity[key] is False])
@@ -1317,7 +1363,6 @@ def update(user_id, corpus_id, token_id, lemma=None, POS=None, morph=None):
             error.statuses = validity
             error.invalid_columns = [key for key in validity.keys() if validity[key] is False]
             raise error
-
         # Updating
         if not lemma:
             lemma = token.lemma
@@ -1442,7 +1487,7 @@ def get_nearly_similar_to(token, mode):
                     WordToken.id != token.id,
                     *filtering
                 )
-            )
+        )
 
 
 class TokenHistory(db.Model):