Setup new code quality tools and drop pylint

Makefile

@@ -83,14 +83,21 @@ clean-services: stop-services ## Delete containers and attached volumes
 # Code quality
 # ------------
 
-# Run pylint on the whole project.
-pylint-all:
-	pylint --rcfile=.pylintrc --reports=no --output-format=colorized $(PACKAGE_DIR) || true
-
-# Run pylint on a specific file, e.g.:
-# make pylint FILE=labonneboite/web/app.py
-pylint:
-	pylint --rcfile=.pylintrc --reports=no --output-format=colorized $(FILE) || true
+quality: black isort flake8
+
+black:
+	black --line-length 119 .
+
+isort:
+	isort --recursive .
+
+flake8:
+	flake8 .
+
+setup_git_pre_commit_hook:
+	touch .git/hooks/pre-commit
+	chmod +x .git/hooks/pre-commit
+	echo "make black && make isort" > .git/hooks/pre-commit
 
 
 # Local dev

README.md

@@ -60,7 +60,7 @@ Table of contents:
 - [Elastic Search](#elasticsearch)
 - [DB content in the development environment](#db-content-in-the-development-environment)
 - [Running scripts](#running-scripts)
-- [Running Pylint](#running-pylint)
+- [Code quality](#code-quality)
 - [Debugging safely in development, staging or production](#debugging-safely-in-a-development-staging-or-production-environment)
 - [Importer](#importer)
 - [Single ROME versus multi-ROME search](#single-rome-vs-multi-rome-search)
@@ -234,25 +234,17 @@ For example `create_index`:
 
     $ python labonneboite/scripts/create_index.py
 
-## Running pylint
+## Code quality
 
-You can run [pylint](https://www.pylint.org) on the whole project:
+We use three code quality tools: black, isort and flake8.
 
-    $ make pylint-all
+Run `make quality` for run all three of them. Or `make black`, `make isort` and `make flake8` separately.
 
-Or on a specific python file:
+Black and isort will automatically apply changes for you, however flake8 requires you to manually fix every issue.
 
-    $ make pylint FILE=labonneboite/web/app.py
-
-We recommend you use a pylint git pre-commit hook:
-
-    $ pip install git-pylint-commit-hook
-    $ vim .git/hooks/pre-commit
-    #!/bin/bash
-    # (...) previous content which was already present (e.g. nosetests)
-    # add the following line at the end of your pre-commit hook file
-    git-pylint-commit-hook
+We recommend you run `make setup_git_pre_commit_hook` to setup a git pre-commit hook which will run both black and isort automatically before every commit.
 
+You will still have to run flake 8 manually though.
 
 ## Debugging safely in a development, staging or production environment
 

ROME_NAF/analyses/print_array.py

@@ -1,14 +1,22 @@
 import pandas as pd
 
-# Affichage des codes ROME les plus fréquents
 
-ROME_df = pd.read_csv('../referentiels/referentiel_ROME/20150921_arboprincipale28427_ROME.csv', index_col=0, sep='|', dtype=str)
-OGR_df = pd.read_csv('../referentiels/referentiel_OGR/20150921_arboprincipale28427_OGR.csv', sep='|', dtype=str).set_index('OGR')
-NAF_df = pd.read_csv('../referentiels/referentiel_NAF/naf2008_liste_n5_nouveau_header.csv', sep='|', encoding="utf-8").set_index(['NAF'])
+# Affichage des codes ROME les plus fréquents
 
-stats_lbb = pd.read_csv('../stats_lbb/stats_lbb.csv')
+ROME_df = pd.read_csv(
+    "../referentiels/referentiel_ROME/20150921_arboprincipale28427_ROME.csv", index_col=0, sep="|", dtype=str
+)
+OGR_df = pd.read_csv(
+    "../referentiels/referentiel_OGR/20150921_arboprincipale28427_OGR.csv", sep="|", dtype=str
+).set_index("OGR")
+NAF_df = pd.read_csv(
+    "../referentiels/referentiel_NAF/naf2008_liste_n5_nouveau_header.csv", sep="|", encoding="utf-8"
+).set_index(["NAF"])
+
+stats_lbb = pd.read_csv("../stats_lbb/stats_lbb.csv")
 top_slugs = stats_lbb[:34]
 
+
 def pprint_ROME(array):
     for i, line in enumerate(top_slugs.iterrows()):
         slugs, visite, uniq = line[1]
@@ -22,10 +30,10 @@ def pprint_ROME(array):
         poids_NAF = array[:, ROME_index, :].sum(1)
         top_NAF_indexes = (-poids_NAF).argsort()[:20]
 
-        print('{}. {} ({})'.format(i, ROME_label, ROME_code))
+        print("{}. {} ({})".format(i, ROME_label, ROME_code))
         for NAF_index in top_NAF_indexes:
             NAF_code = NAF_df.iloc[NAF_index].name
             NAF_label = NAF_df.loc[NAF_code].label
 
-            print('{}, {} : {}'.format(NAF_code, NAF_label, poids_NAF[NAF_index], ))
+            print("{}, {} : {}".format(NAF_code, NAF_label, poids_NAF[NAF_index]))
         print()

ROME_NAF/referentiels/referentiels.py

@@ -3,8 +3,11 @@
 
 import pandas as pd
 
-ROME_df = pd.read_csv('referentiel_ROME/20150921_arboprincipale28427_ROME.csv', index_col=0, sep='|', dtype=str)
 
-OGR_df = pd.read_csv('referentiel_OGR/20150921_arboprincipale28427_OGR.csv', sep='|', dtype=str).set_index('OGR')
+ROME_df = pd.read_csv("referentiel_ROME/20150921_arboprincipale28427_ROME.csv", index_col=0, sep="|", dtype=str)
 
-NAF_df = pd.read_csv('referentiel_NAF/naf2008_liste_n5_nouveau_header.csv', sep='|', encoding="utf-8").set_index(['NAF'])
+OGR_df = pd.read_csv("referentiel_OGR/20150921_arboprincipale28427_OGR.csv", sep="|", dtype=str).set_index("OGR")
+
+NAF_df = pd.read_csv("referentiel_NAF/naf2008_liste_n5_nouveau_header.csv", sep="|", encoding="utf-8").set_index(
+    ["NAF"]
+)

labonneboite/alembic/env.py

@@ -1,4 +1,3 @@
-
 from logging.config import fileConfig
 
 from alembic import context
@@ -44,10 +43,7 @@ def run_migrations_online():
     connectable = create_engine(get_db_string())
 
     with connectable.connect() as connection:
-        context.configure(
-            connection=connection,
-            target_metadata=target_metadata
-        )
+        context.configure(connection=connection, target_metadata=target_metadata)
 
         with context.begin_transaction():
             context.run_migrations()

labonneboite/alembic/versions/013a2cb893fc_add_flag_pmsmp_and_flag_afpr_poe.py

@@ -5,70 +5,68 @@
 Revises: c5cd5037cb31
 Create Date: 2019-03-01 11:39:05.272684
 """
+import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import mysql
-import sqlalchemy as sa
 
 
 # Revision identifiers, used by Alembic.
-revision = '013a2cb893fc'
-down_revision = 'c5cd5037cb31'
+revision = "013a2cb893fc"
+down_revision = "c5cd5037cb31"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
     op.add_column(
-        'etablissements_raw',
-        sa.Column('flag_poe_afpr', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_raw",
+        sa.Column("flag_poe_afpr", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements_raw',
-        sa.Column('flag_pmsmp', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_raw",
+        sa.Column("flag_pmsmp", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements',
-        sa.Column('flag_poe_afpr', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements",
+        sa.Column("flag_poe_afpr", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements',
-        sa.Column('flag_pmsmp', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements", sa.Column("flag_pmsmp", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
     )
     op.add_column(
-        'etablissements_admin_add',
-        sa.Column('flag_poe_afpr', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_admin_add",
+        sa.Column("flag_poe_afpr", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements_admin_add',
-        sa.Column('flag_pmsmp', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_admin_add",
+        sa.Column("flag_pmsmp", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements_exportable',
-        sa.Column('flag_poe_afpr', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_exportable",
+        sa.Column("flag_poe_afpr", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements_exportable',
-        sa.Column('flag_pmsmp', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_exportable",
+        sa.Column("flag_pmsmp", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements_backoffice',
-        sa.Column('flag_poe_afpr', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_backoffice",
+        sa.Column("flag_poe_afpr", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
     op.add_column(
-        'etablissements_backoffice',
-        sa.Column('flag_pmsmp', mysql.TINYINT(display_width=1), autoincrement=False, nullable=False)
+        "etablissements_backoffice",
+        sa.Column("flag_pmsmp", mysql.TINYINT(display_width=1), autoincrement=False, nullable=False),
     )
 
 
-
 def downgrade():
-    op.drop_column('etablissements_raw', 'flag_poe_afpr')
-    op.drop_column('etablissements_raw', 'flag_pmsmp')
-    op.drop_column('etablissements', 'flag_poe_afpr')
-    op.drop_column('etablissements', 'flag_pmsmp')
-    op.drop_column('etablissements_admin_add', 'flag_poe_afpr')
-    op.drop_column('etablissements_admin_add', 'flag_pmsmp')
-    op.drop_column('etablissements_exportable', 'flag_poe_afpr')
-    op.drop_column('etablissements_exportable', 'flag_pmsmp')
-    op.drop_column('etablissements_backoffice', 'flag_poe_afpr')
-    op.drop_column('etablissements_backoffice', 'flag_pmsmp')
+    op.drop_column("etablissements_raw", "flag_poe_afpr")
+    op.drop_column("etablissements_raw", "flag_pmsmp")
+    op.drop_column("etablissements", "flag_poe_afpr")
+    op.drop_column("etablissements", "flag_pmsmp")
+    op.drop_column("etablissements_admin_add", "flag_poe_afpr")
+    op.drop_column("etablissements_admin_add", "flag_pmsmp")
+    op.drop_column("etablissements_exportable", "flag_poe_afpr")
+    op.drop_column("etablissements_exportable", "flag_pmsmp")
+    op.drop_column("etablissements_backoffice", "flag_poe_afpr")
+    op.drop_column("etablissements_backoffice", "flag_pmsmp")

labonneboite/alembic/versions/025bcb5f723e_update_index_for_office_offer_matching.py

@@ -5,35 +5,38 @@
 Revises: 8f1ef5533cc7
 Create Date: 2018-12-13 12:18:58.492180
 """
-from alembic import op
-
 import sqlalchemy as sa
+from alembic import op
 
 
 # Revision identifiers, used by Alembic.
-revision = '025bcb5f723e'
-down_revision = '8f1ef5533cc7'
+revision = "025bcb5f723e"
+down_revision = "8f1ef5533cc7"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
-    op.drop_constraint('_raisonsociale_codecommune', 'etablissements', type_='unique')
-    op.drop_constraint('_raisonsociale_codecommune', 'etablissements_exportable', type_='unique')
-    op.drop_constraint('_enseigne_codecommune', 'etablissements', type_='unique')
-    op.drop_constraint('_enseigne_codecommune', 'etablissements_exportable', type_='unique')
-    op.create_index('_raisonsociale_departement', 'etablissements', ['raisonsociale', 'departement'], unique=False)
-    op.create_index('_raisonsociale_departement', 'etablissements_exportable', ['raisonsociale', 'departement'], unique=False)
-    op.create_index('_enseigne_departement', 'etablissements', ['enseigne', 'departement'], unique=False)
-    op.create_index('_enseigne_departement', 'etablissements_exportable', ['enseigne', 'departement'], unique=False)
+    op.drop_constraint("_raisonsociale_codecommune", "etablissements", type_="unique")
+    op.drop_constraint("_raisonsociale_codecommune", "etablissements_exportable", type_="unique")
+    op.drop_constraint("_enseigne_codecommune", "etablissements", type_="unique")
+    op.drop_constraint("_enseigne_codecommune", "etablissements_exportable", type_="unique")
+    op.create_index("_raisonsociale_departement", "etablissements", ["raisonsociale", "departement"], unique=False)
+    op.create_index(
+        "_raisonsociale_departement", "etablissements_exportable", ["raisonsociale", "departement"], unique=False
+    )
+    op.create_index("_enseigne_departement", "etablissements", ["enseigne", "departement"], unique=False)
+    op.create_index("_enseigne_departement", "etablissements_exportable", ["enseigne", "departement"], unique=False)
 
 
 def downgrade():
-    op.drop_constraint('_raisonsociale_departement', 'etablissements', type_='unique')
-    op.drop_constraint('_raisonsociale_departement', 'etablissements_exportable', type_='unique')
-    op.drop_constraint('_enseigne_departement', 'etablissements', type_='unique')
-    op.drop_constraint('_enseigne_departement', 'etablissements_exportable', type_='unique')
-    op.create_index('_raisonsociale_codecommune', 'etablissements', ['raisonsociale', 'codecommune'], unique=False)
-    op.create_index('_raisonsociale_codecommune', 'etablissements_exportable', ['raisonsociale', 'codecommune'], unique=False)
-    op.create_index('_enseigne_codecommune', 'etablissements', ['enseigne', 'codecommune'], unique=False)
-    op.create_index('_enseigne_codecommune', 'etablissements_exportable', ['enseigne', 'codecommune'], unique=False)
+    op.drop_constraint("_raisonsociale_departement", "etablissements", type_="unique")
+    op.drop_constraint("_raisonsociale_departement", "etablissements_exportable", type_="unique")
+    op.drop_constraint("_enseigne_departement", "etablissements", type_="unique")
+    op.drop_constraint("_enseigne_departement", "etablissements_exportable", type_="unique")
+    op.create_index("_raisonsociale_codecommune", "etablissements", ["raisonsociale", "codecommune"], unique=False)
+    op.create_index(
+        "_raisonsociale_codecommune", "etablissements_exportable", ["raisonsociale", "codecommune"], unique=False
+    )
+    op.create_index("_enseigne_codecommune", "etablissements", ["enseigne", "codecommune"], unique=False)
+    op.create_index("_enseigne_codecommune", "etablissements_exportable", ["enseigne", "codecommune"], unique=False)