diff --git a/.gitignore b/.gitignore index b8e138405..28a922101 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,6 @@ galaxy.min.css .DS_Store ansible-deployment/ yarn-error.log -galaxy/importer/galaxy-lint-rules/ # NOTE(cutwater): Temporary adding developer environment related files # to .gitignore, however these files should not be included in project diff --git a/galaxy/api/serializers/content.py b/galaxy/api/serializers/content.py index bc1727653..3e9dc42ea 100644 --- a/galaxy/api/serializers/content.py +++ b/galaxy/api/serializers/content.py @@ -19,6 +19,7 @@ from django.core import urlresolvers as urls from rest_framework import serializers +from collections import OrderedDict from galaxy.main import models @@ -96,7 +97,11 @@ class Meta: 'content_type', 'imported', 'download_count', - 'role_type' + 'role_type', + 'quality_score', + 'content_score', + 'metadata_score', + 'compatibility_score', ) def get_platforms(self, instance): @@ -124,6 +129,12 @@ def get_related(self, instance): } def get_summary_fields(self, instance): + # Support ansible-galaxy <= 2.6 by excluding unsupported messges + supported_types = ('INFO', 'WARNING', 'ERROR', 'SUCCESS', 'FAILED') + latest_task = models.ImportTask.objects.filter( + repository_id=instance.repository.id + ).latest('id') + return { 'namespace': _NamespaceSerializer().to_representation( instance.repository.provider_namespace.namespace), @@ -132,7 +143,25 @@ def get_summary_fields(self, instance): 'content_type': _ContentTypeSerializer().to_representation( instance.content_type), - 'dependencies': [str(g) for g in instance.dependencies.all()] + 'dependencies': [str(g) for g in instance.dependencies.all()], + 'task_messages': [ + OrderedDict([ + ('id', m.id), + ('message_type', m.message_type), + ('message_text', m.message_text), + ('content_id', m.content_id), + ('is_linter_rule_violation', m.is_linter_rule_violation), + ('linter_type', m.linter_type), + ('linter_rule_id', m.linter_rule_id), + ('rule_desc', m.rule_desc), + ('rule_severity', m.rule_severity), + ]) for m in models.ImportTaskMessage.objects.filter( + task_id=latest_task.id, + content_id=instance.id, + message_type__in=supported_types, + is_linter_rule_violation=True, + ) + ], } diff --git a/galaxy/api/serializers/repository.py b/galaxy/api/serializers/repository.py index 7b20139c0..742147963 100644 --- a/galaxy/api/serializers/repository.py +++ b/galaxy/api/serializers/repository.py @@ -60,7 +60,8 @@ class Meta: 'readme_html', 'download_url', 'deprecated', - 'community_score' + 'community_score', + 'quality_score', ) def get_related(self, instance): @@ -130,6 +131,7 @@ def get_summary_fields(self, instance): 'name': c.name, 'content_type': c.content_type.name, 'description': c.description, + 'quality_score': c.quality_score, } for c in instance.content_objects.all() ] diff --git a/galaxy/api/serializers/roles.py b/galaxy/api/serializers/roles.py index dfa39fecc..d30ee7c64 100644 --- a/galaxy/api/serializers/roles.py +++ b/galaxy/api/serializers/roles.py @@ -134,6 +134,7 @@ def get_summary_fields(self, obj): travis_build_url=obj.repository.travis_build_url, format=obj.repository.format, deprecated=obj.repository.deprecated, + quality_score=obj.repository.quality_score, ) d['tags'] = [g.name for g in obj.tags.all()] d['versions'] = [ diff --git a/galaxy/api/serializers/serializers.py b/galaxy/api/serializers/serializers.py index 0c16123f7..4805baf59 100644 --- a/galaxy/api/serializers/serializers.py +++ b/galaxy/api/serializers/serializers.py @@ -386,7 +386,11 @@ def get_summary_fields(self, obj): summary['task_messages'] = [OrderedDict([ ('id', g.id), ('message_type', g.message_type), - ('message_text', g.message_text) + ('message_text', g.message_text), + ('content_id', g.content_id), + ('is_linter_rule_violation', g.is_linter_rule_violation), + ('linter_type', g.linter_type), + ('linter_rule_id', g.linter_rule_id) ]) for g in obj.messages.filter( message_type__in=supported_types).order_by('id')] summary['role'] = { diff --git a/galaxy/common/logutils.py b/galaxy/common/logutils.py index f6cc048df..2d2e04284 100644 --- a/galaxy/common/logutils.py +++ b/galaxy/common/logutils.py @@ -26,6 +26,14 @@ class ImportTaskAdapter(logging.LoggerAdapter): def __init__(self, logger, task_id): super(ImportTaskAdapter, self).__init__(logger, {'task_id': task_id}) + def process(self, msg, kwargs): + if self.extra: + if 'extra' not in kwargs: + kwargs.update({'extra': {}}) + for key, value in self.extra.items(): + kwargs['extra'][key] = value + return msg, kwargs + class ContentTypeAdapter(logging.LoggerAdapter): def __init__(self, logger, content_type, content_name=None): @@ -35,14 +43,11 @@ def __init__(self, logger, content_type, content_name=None): }) def process(self, msg, kwargs): - if self.extra['content_name']: - prefix = '{}: {}'.format( - self.extra['content_type'].name, - self.extra['content_name']) - else: - prefix = self.extra['content_type'] - - msg = '[{}] {}'.format(prefix, msg) + if self.extra: + if 'extra' not in kwargs: + kwargs.update({'extra': {}}) + for key, value in self.extra.items(): + kwargs['extra'][key] = value return msg, kwargs @@ -50,10 +55,29 @@ class ImportTaskHandler(logging.Handler): def emit(self, record): # type: (logging.LogRecord) -> None from galaxy.main import models - msg = self.format(record) + + lint = { + 'is_linter_rule_violation': False, + 'linter_type': None, + 'linter_rule_id': None, + 'rule_desc': None, + 'content_name': '', + } + if set(lint.keys()).issubset(vars(record).keys()): + lint['is_linter_rule_violation'] = record.is_linter_rule_violation + lint['linter_type'] = record.linter_type + lint['linter_rule_id'] = record.linter_rule_id + lint['rule_desc'] = record.rule_desc + lint['content_name'] = record.content_name + models.ImportTaskMessage.objects.using('logging').create( task_id=record.task_id, message_type=constants.ImportTaskMessageType.from_logging_level( record.levelno).value, - message_text=msg, + message_text=record.msg, + is_linter_rule_violation=lint['is_linter_rule_violation'], + linter_type=lint['linter_type'], + linter_rule_id=lint['linter_rule_id'], + rule_desc=lint['rule_desc'], + content_name=lint['content_name'], ) diff --git a/galaxy/constants.py b/galaxy/constants.py index 6d1530091..b700acec4 100644 --- a/galaxy/constants.py +++ b/galaxy/constants.py @@ -129,6 +129,20 @@ class ImportTaskState(Enum): SUCCESS = 'SUCCESS' +class LinterType(Enum): + FLAKE8 = 'flake8' + YAMLLINT = 'yamllint' + ANSIBLELINT = 'ansible-lint' + + @classmethod + def choices(cls): + return [ + (cls.FLAKE8.value, 'flake8'), + (cls.YAMLLINT.value, 'yamllint'), + (cls.ANSIBLELINT.value, 'ansible-lint') + ] + + class RepositoryFormat(enum.Enum): ROLE = 'role' APB = 'apb' diff --git a/galaxy/importer/finders.py b/galaxy/importer/finders.py index e50ed9455..c47cd679d 100644 --- a/galaxy/importer/finders.py +++ b/galaxy/importer/finders.py @@ -66,7 +66,7 @@ class ApbFinder(BaseFinder): repository_format = constants.RepositoryFormat.APB def find_contents(self): - self.log.info('Content search - Looking for file "apb.yml"') + self.log.debug('Content search - Looking for file "apb.yml"') meta_path = _find_metadata(self.path, self.META_FILES) if meta_path: meta_path = os.path.join(self.path, meta_path) @@ -81,7 +81,7 @@ class RoleFinder(BaseFinder): repository_format = constants.RepositoryFormat.ROLE def find_contents(self): - self.log.info( + self.log.debug( 'Content search - Looking for top level role metadata file') meta_path = _find_metadata(self.path, ROLE_META_FILES) if meta_path: @@ -97,7 +97,7 @@ class FileSystemFinder(BaseFinder): repository_format = constants.RepositoryFormat.MULTI def find_contents(self): - self.log.info('Content search - Analyzing repository structure') + self.log.debug('Content search - Analyzing repository structure') content = self._find_content() # Check if finder has found at least one content try: diff --git a/galaxy/importer/linters/__init__.py b/galaxy/importer/linters/__init__.py index 21674b460..5e0719b36 100644 --- a/galaxy/importer/linters/__init__.py +++ b/galaxy/importer/linters/__init__.py @@ -46,6 +46,7 @@ def _check_files(self, paths): class Flake8Linter(BaseLinter): + name = 'flake8' cmd = 'flake8' def _check_files(self, paths): @@ -59,9 +60,28 @@ def _check_files(self, paths): yield line.strip() proc.wait() + def parse_id_and_desc(self, message): + try: + msg_parts = message.split(' ') + rule_desc = ' '.join(msg_parts[2:]) + + error_id = msg_parts[1] + if error_id[0] not in ['E', 'W']: + error_id = None + + except IndexError: + error_id = None + + if not error_id: + logger.error('No error_id found in {} message'.format(self.cmd)) + return (None, None) + + return (error_id, rule_desc) + class YamlLinter(BaseLinter): + name = 'yamllint' cmd = 'yamllint' config = os.path.join(LINTERS_DIR, 'yamllint.yaml') @@ -73,9 +93,28 @@ def _check_files(self, paths): yield line.strip() proc.wait() + def parse_id_and_desc(self, message): + try: + msg_parts = message.split(' ') + rule_desc = ' '.join(msg_parts[2:]) + + error_id = msg_parts[1][1:-1] + if error_id not in ['error', 'warning']: + error_id = None + + except IndexError: + error_id = None + + if not error_id: + logger.error('No error_id found in {} message'.format(self.cmd)) + return (None, None) + + return (error_id, rule_desc) + class AnsibleLinter(BaseLinter): + name = 'ansible-lint' cmd = 'ansible-lint' def _check_files(self, paths): @@ -102,3 +141,21 @@ def _check_files(self, paths): # returncode 1 is app exception, 0 is no linter err, 2 is linter err if proc.wait() not in (0, 2): yield 'Exception running ansible-lint, could not complete linting' + + def parse_id_and_desc(self, message): + try: + msg_parts = message.split(' ') + rule_desc = ' '.join(msg_parts[2:]) + + error_id = msg_parts[1][1:-1] + if error_id[0] not in ['E']: + error_id = None + + except IndexError: + error_id = None + + if not error_id: + logger.error('No error_id found in {} message'.format(self.cmd)) + return (None, None) + + return (error_id, rule_desc) diff --git a/galaxy/importer/loaders/base.py b/galaxy/importer/loaders/base.py index 377fd42cd..2507f5bbb 100644 --- a/galaxy/importer/loaders/base.py +++ b/galaxy/importer/loaders/base.py @@ -73,22 +73,35 @@ def load(self): def lint(self): if not self.linters: return - self.log.info('Linting...') linters = self.linters if not isinstance(linters, (list, tuple)): linters = [linters] - ok = True + all_linters_ok = True for linter_cls in linters: - for message in linter_cls(self.root).check_files(self.rel_path): - message = '[%s] %s' % (linter_cls.cmd, message) - self.log.error(message) - ok = False - - if ok: - self.log.info('Linting OK.') - return ok + linter_ok = True + linter_obj = linter_cls(self.root) + for message in linter_obj.check_files(self.rel_path): + if linter_ok: + self.log.info('{} Warnings:'.format(linter_obj.name)) + linter_ok = False + error_id, rule_desc = linter_obj.parse_id_and_desc(message) + if error_id: + extra = { + 'is_linter_rule_violation': True, + 'linter_type': linter_cls.cmd, + 'linter_rule_id': error_id, + 'rule_desc': rule_desc + } + self.log.warning(message, extra=extra) + else: + self.log.warning(message) + all_linters_ok = False + if linter_ok: + self.log.info('{} OK.'.format(linter_obj.name)) + + return all_linters_ok # FIXME(cutwater): Due to current object model current object limitation # this leads to copying README file over multiple roles. diff --git a/galaxy/importer/repository.py b/galaxy/importer/repository.py index 29154a518..5f2919e4b 100644 --- a/galaxy/importer/repository.py +++ b/galaxy/importer/repository.py @@ -78,10 +78,6 @@ def load(self): result = list(self._load_contents(contents)) readme = self._get_readme(finder.repository_format) - if not all(v[1] for v in result): - self.log.error('Lint failed') - # raise exc.ContentLoadError('Lint failed') - name = None if (finder.repository_format in (constants.RepositoryFormat.ROLE, constants.RepositoryFormat.APB) @@ -113,7 +109,10 @@ def _load_contents(self, contents): loader = loader_cls(content_type, rel_path, self.path, logger=self.log, **extra) content = loader.load() + self.log.info('===== LINTING {}: {} ====='.format( + content_type.name, content.name)) lint_result = loader.lint() + self.log.info(' ') yield content, lint_result def _get_readme(self, repository_format): diff --git a/galaxy/main/admin.py b/galaxy/main/admin.py index aa4f85611..0796f8714 100644 --- a/galaxy/main/admin.py +++ b/galaxy/main/admin.py @@ -39,8 +39,13 @@ class ContentBlockAdmin(admin.ModelAdmin): pass +class ContentRuleAdmin(admin.ModelAdmin): + pass + + admin.site.register(models.Platform, PlatformAdmin) admin.site.register(models.CloudPlatform, CloudPlatformAdmin) admin.site.register(models.Content, RoleAdmin) admin.site.register(models.RepositoryVersion, RepositoryVersionAdmin) admin.site.register(models.ContentBlock, ContentBlockAdmin) +admin.site.register(models.ContentRule, ContentRuleAdmin) diff --git a/galaxy/main/migrations/0115_content_scoring_20180918_1506.py b/galaxy/main/migrations/0115_content_scoring_20180918_1506.py new file mode 100644 index 000000000..d56e92749 --- /dev/null +++ b/galaxy/main/migrations/0115_content_scoring_20180918_1506.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.15 on 2018-09-18 19:06 +from __future__ import unicode_literals + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import galaxy.main.mixins + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0114_content_type'), + ] + + operations = [ + migrations.CreateModel( + name='ContentRule', + fields=[ + ('id', models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('rule_id', models.CharField(max_length=25)), + ('linter_id', models.CharField( + choices=[ + ('flake8', 'flake8'), + ('yamllint', 'yamllint'), + ('ansible-lint', 'ansible-lint')], + max_length=25)), + ('severity', models.IntegerField( + default=0, + validators=[ + django.core.validators.MinValueValidator(0), + django.core.validators.MaxValueValidator(5)])), + ], + bases=(models.Model, galaxy.main.mixins.DirtyMixin), + ), + migrations.AddField( + model_name='content', + name='compatibility_score', + field=models.FloatField( + null=True, + validators=[ + django.core.validators.MinValueValidator(0.0), + django.core.validators.MaxValueValidator(5.0)]), + ), + migrations.AddField( + model_name='content', + name='content_score', + field=models.FloatField( + null=True, + validators=[ + django.core.validators.MinValueValidator(0.0), + django.core.validators.MaxValueValidator(5.0)]), + ), + migrations.AddField( + model_name='content', + name='metadata_score', + field=models.FloatField( + null=True, + validators=[ + django.core.validators.MinValueValidator(0.0), + django.core.validators.MaxValueValidator(5.0)]), + ), + migrations.AddField( + model_name='content', + name='quality_score', + field=models.FloatField( + null=True, + validators=[ + django.core.validators.MinValueValidator(0.0), + django.core.validators.MaxValueValidator(5.0)]), + ), + migrations.AddField( + model_name='importtaskmessage', + name='content', + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='messages', + to='main.Content'), + ), + migrations.AddField( + model_name='importtaskmessage', + name='content_name', + field=models.CharField(max_length=60, null=True), + ), + migrations.AddField( + model_name='importtaskmessage', + name='is_linter_rule_violation', + field=models.NullBooleanField(default=False), + ), + migrations.AddField( + model_name='importtaskmessage', + name='linter_rule_id', + field=models.CharField(max_length=25, null=True), + ), + migrations.AddField( + model_name='importtaskmessage', + name='linter_type', + field=models.CharField(max_length=25, null=True), + ), + migrations.AddField( + model_name='importtaskmessage', + name='rule_desc', + field=models.CharField(max_length=256, null=True), + ), + migrations.AddField( + model_name='importtaskmessage', + name='rule_severity', + field=models.IntegerField( + null=True, + validators=[ + django.core.validators.MinValueValidator(0), + django.core.validators.MaxValueValidator(5)]), + ), + migrations.AddField( + model_name='repository', + name='quality_score', + field=models.FloatField( + null=True, + validators=[ + django.core.validators.MinValueValidator(0.0), + django.core.validators.MaxValueValidator(5.0)]), + ), + migrations.AlterField( + model_name='contenttype', + name='name', + field=models.CharField( + choices=[ + ('apb', 'Ansible Playbook Bundle'), + ('role', 'Role'), + ('module', 'Module'), + ('module_utils', 'Module Utils'), + ('action', 'Action Plugin'), + ('cache', 'Cache Plugin'), + ('callback', 'Callback Plugin'), + ('cliconf', 'CLI Conf Plugin'), + ('connection', 'Connection Plugin'), + ('filter', 'Filter Plugin'), + ('inventory', 'Inventory Plugin'), + ('lookup', 'Lookup Plugin'), + ('netconf', 'Netconf Plugin'), + ('shell', 'Shell Plugin'), + ('strategy', 'Strategy Plugin'), + ('terminal', 'Terminal Plugin'), + ('test', 'Test Plugin')], + db_index=True, + max_length=512, + unique=True), + ), + migrations.AlterUniqueTogether( + name='contentrule', + unique_together=set([('rule_id', 'linter_id')]), + ), + ] diff --git a/galaxy/main/migrations/0116_set_contentrule_values_20180918_1509.py b/galaxy/main/migrations/0116_set_contentrule_values_20180918_1509.py new file mode 100644 index 000000000..1c69d1b88 --- /dev/null +++ b/galaxy/main/migrations/0116_set_contentrule_values_20180918_1509.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.15 on 2018-09-17 22:01 +from __future__ import unicode_literals + +from django.db import migrations + + +def set_initial_rule_severity(apps, schema_editor): + LINTER_RULE_SEVERITY = [ + ('ansible-lint', 'E101', 3), + ('ansible-lint', 'E102', 4), + ('ansible-lint', 'E103', 5), + ('ansible-lint', 'E104', 5), + ('ansible-lint', 'E105GAL', 0), + ('ansible-lint', 'E106GAL', 0), + ('ansible-lint', 'E201', 0), + ('ansible-lint', 'E202', 5), + ('ansible-lint', 'E203GAL', 2), + ('ansible-lint', 'E204GAL', 1), + ('ansible-lint', 'E205GAL', 1), + ('ansible-lint', 'E206GAL', 3), + ('ansible-lint', 'E207GAL', 3), + ('ansible-lint', 'E208GAL', 2), + ('ansible-lint', 'E301', 4), + ('ansible-lint', 'E302', 5), + ('ansible-lint', 'E303', 4), + ('ansible-lint', 'E304', 5), + ('ansible-lint', 'E305', 4), + ('ansible-lint', 'E306GAL', 0), + ('ansible-lint', 'E401', 3), + ('ansible-lint', 'E402', 3), + ('ansible-lint', 'E403', 1), + ('ansible-lint', 'E404GAL', 4), + ('ansible-lint', 'E405GAL', 3), + ('ansible-lint', 'E406GAL', 0), + ('ansible-lint', 'E501', 5), + ('ansible-lint', 'E502', 3), + ('ansible-lint', 'E503', 3), + ('ansible-lint', 'E504GAL', 3), + ('ansible-lint', 'E601GAL', 4), + ('ansible-lint', 'E602GAL', 4), + ('yamllint', 'error', 0), + ('yamllint', 'warning', 0), + ] + ContentRule = apps.get_model('main', 'ContentRule') + for linter, rule, severity in LINTER_RULE_SEVERITY: + c = ContentRule( + linter_id=linter, + rule_id=rule, + severity=severity, + ) + c.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0115_content_scoring_20180918_1506'), + ] + + operations = [ + migrations.RunPython(set_initial_rule_severity), + ] diff --git a/galaxy/main/models.py b/galaxy/main/models.py index a57e3abdb..732e79833 100644 --- a/galaxy/main/models.py +++ b/galaxy/main/models.py @@ -41,7 +41,7 @@ 'Content', 'ImportTask', 'ImportTaskMessage', 'RepositoryVersion', 'UserAlias', 'NotificationSecret', 'Notification', 'Repository', 'Subscription', 'Stargazer', 'Namespace', 'Provider', 'ProviderNamespace', - 'ContentBlock', 'ContentType' + 'ContentBlock', 'ContentType', 'ContentRule' ] # TODO(cutwater): Split models.py into multiple modules @@ -389,6 +389,22 @@ class Meta: null=True, verbose_name="Last Import" ) + quality_score = models.FloatField( + null=True, + validators=[MinValueValidator(0.0), MaxValueValidator(5.0)], + ) + content_score = models.FloatField( + null=True, + validators=[MinValueValidator(0.0), MaxValueValidator(5.0)], + ) + metadata_score = models.FloatField( + null=True, + validators=[MinValueValidator(0.0), MaxValueValidator(5.0)], + ) + compatibility_score = models.FloatField( + null=True, + validators=[MinValueValidator(0.0), MaxValueValidator(5.0)], + ) search_vector = psql_search.SearchVectorField() @@ -684,6 +700,11 @@ class ImportTaskMessage(PrimordialModel): 'ImportTask', related_name='messages', ) + content = models.ForeignKey( + 'Content', + related_name='messages', + null=True, + ) message_type = models.CharField( max_length=10, choices=constants.ImportTaskMessageType.choices(), @@ -691,6 +712,29 @@ class ImportTaskMessage(PrimordialModel): message_text = models.CharField( max_length=256, ) + is_linter_rule_violation = models.NullBooleanField( + default=False, + ) + linter_type = models.CharField( + max_length=25, + null=True, + ) + linter_rule_id = models.CharField( + max_length=25, + null=True, + ) + content_name = models.CharField( + max_length=60, + null=True, + ) + rule_desc = models.CharField( + max_length=256, + null=True, + ) + rule_severity = models.IntegerField( + validators=[MinValueValidator(0), MaxValueValidator(5)], + null=True, + ) def __str__(self): return "{}-{}-{}".format( @@ -953,14 +997,16 @@ class Meta: download_count = models.IntegerField( default=0 ) - deprecated = models.BooleanField( default=False, ) - community_score = models.FloatField( null=True ) + quality_score = models.FloatField( + null=True, + validators=[MinValueValidator(0.0), MaxValueValidator(5.0)], + ) @property def clone_url(self): @@ -1153,3 +1199,24 @@ class Meta: null=True, validators=[MinValueValidator(0), MaxValueValidator(5)] ) + + +class ContentRule(BaseModel): + class Meta: + unique_together = ('rule_id', 'linter_id') + + rule_id = models.CharField( + max_length=25, + ) + linter_id = models.CharField( + max_length=25, + choices=constants.LinterType.choices(), + ) + severity = models.IntegerField( + default=0, + validators=[MinValueValidator(0), MaxValueValidator(5)] + ) + + def __str__(self): + return '{} {} severity={}'.format( + self.linter_id, self.rule_id, self.severity) diff --git a/galaxy/worker/importers/base.py b/galaxy/worker/importers/base.py index 1da00d0a9..fda580b0e 100644 --- a/galaxy/worker/importers/base.py +++ b/galaxy/worker/importers/base.py @@ -45,6 +45,7 @@ def do_import(self): content.clean() content.save() + self.log.info(' ') return content def make_content(self): @@ -111,9 +112,5 @@ def _update_readme(self, content): content.save() def _log_create_content(self, content_id, is_created): - action = 'Created new' if is_created else 'Found' - self.log.info( - '{action} Content instance: id={id}, content_type="{content_type}"' - ', name="{name}"'.format( - action=action, id=content_id, - content_type=self.data.content_type, name=self.data.name)) + self.log.info('===== IMPORTING {}: {} ====='.format( + self.data.content_type.name, self.data.name)) diff --git a/galaxy/worker/tasks.py b/galaxy/worker/tasks.py index 37eb9e0df..d6f55ba96 100644 --- a/galaxy/worker/tasks.py +++ b/galaxy/worker/tasks.py @@ -22,6 +22,7 @@ import celery import github import pytz +from numbers import Number from django.conf import settings from django.db import transaction @@ -74,6 +75,7 @@ def _import_repository(import_task, logger): + "/" + repository.original_name) logger.info(u'Starting import: task_id={}, repository={}' .format(import_task.id, repo_full_name)) + logger.info(' ') token = _get_social_token(import_task) gh_api = github.Github(token) @@ -133,7 +135,112 @@ def _import_repository(import_task, logger): _update_repo_info(repository, gh_repo, repo_info.commit) repository.save() - import_task.finish_success(u'Import completed') + _update_task_msg_content_id(import_task) + _update_task_msg_severity(import_task) + _update_quality_score(import_task) + + warnings = import_task.messages.filter( + message_type=models.ImportTaskMessage.TYPE_WARNING).count() + errors = import_task.messages.filter( + message_type=models.ImportTaskMessage.TYPE_ERROR).count() + import_task.finish_success( + 'Import completed with {0} warnings and {1} ' + 'errors'.format(warnings, errors)) + + +def _update_task_msg_content_id(import_task): + repo_id = import_task.repository.id + contents = models.Content.objects.filter(repository_id=repo_id) + name_to_id = {c.original_name: c.id for c in contents} + + # single role repo content_name is None in ImportTaskMessage + if len(contents) == 1: + name_to_id[None] = name_to_id[name_to_id.keys()[0]] + + import_messages = models.ImportTaskMessage.objects.filter( + task_id=import_task.id, + is_linter_rule_violation=True + ) + + for msg in import_messages: + if msg.content_name not in name_to_id: + LOG.error(u'Importer could not associate content id to rule msg') + msg.is_linter_rule_violation = False + msg.save() + continue + msg.content_id = name_to_id[msg.content_name] + msg.save() + + +def _update_task_msg_severity(import_task): + rule_to_severity = {'{}_{}'.format(r.linter_id, r.rule_id).lower(): + r.severity + for r in models.ContentRule.objects.all()} + + import_messages = models.ImportTaskMessage.objects.filter( + task_id=import_task.id, + is_linter_rule_violation=True + ) + + for msg in import_messages: + rule_code = '{}_{}'.format(msg.linter_type, + msg.linter_rule_id).lower() + if rule_code not in rule_to_severity: + LOG.warning(u'Rule not found in database: {}'.format(rule_code)) + msg.is_linter_rule_violation = False + msg.save() + continue + msg.rule_severity = rule_to_severity[rule_code] + msg.save() + + +def _update_quality_score(import_task): + BASE_SCORE = 50 + SEVERITY_TO_WEIGHT = { + 0: 0, + 1: 0.75, + 2: 1.25, + 3: 2.5, + 4: 5, + 5: 10, + } + + rule_to_severity = {'{}_{}'.format(r.linter_id, r.rule_id).lower(): + r.severity + for r in models.ContentRule.objects.all()} + repository = import_task.repository + contents = models.Content.objects.filter(repository_id=repository.id) + repo_points = 0.0 + + for content in contents: + messages = models.ImportTaskMessage.objects.filter( + task_id=import_task.id, + content_id=content.id, + is_linter_rule_violation=True, + ) + + rule_vios = ['{}_{}'.format(m.linter_type, m.linter_rule_id).lower() + for m in messages] + severitys = [rule_to_severity[rule_vio] for rule_vio in rule_vios + if rule_vio in rule_to_severity] + weights = [SEVERITY_TO_WEIGHT[s] for s in severitys + if isinstance(SEVERITY_TO_WEIGHT[s], Number)] + score = max(0.0, (BASE_SCORE - sum(weights)) / 10) + + content.content_score = score + content.quality_score = score + content.save() + repo_points += score + + score_calc_log = ('score calc for content {} - ' + 'rule_vios, severitys, weights, score: ' + '{}, {}, {}, {}') + LOG.debug(score_calc_log.format(content.original_name, + rule_vios, severitys, weights, content.quality_score)) + + repository.quality_score = repo_points / contents.count() + repository.save() + LOG.debug(u'repo quality score: {}'.format(repository.quality_score)) def _update_namespace(repository): diff --git a/galaxyui/src/app/my-imports/import-detail/import-detail.component.html b/galaxyui/src/app/my-imports/import-detail/import-detail.component.html index b122a4a03..0166983c9 100644 --- a/galaxyui/src/app/my-imports/import-detail/import-detail.component.html +++ b/galaxyui/src/app/my-imports/import-detail/import-detail.component.html @@ -1,39 +1,39 @@
- {{ importTask.summary_fields.namespace.name + '.' + importTask.summary_fields.repository.name }} -
+ {{ importTask.summary_fields.namespace.name + '.' + importTask.summary_fields.repository.name }} +
+ +
- +
- +
-
+
{{ importTask.import_branch }} {{ importTask.commit_message }}
-
{{ importTask.last_run }}
+
{{ importTask.last_run }}
- Commit {{ importTask.commit.substring(0,7) }} + + Commit {{ importTask.commit.substring(0,7) }}
@@ -46,11 +46,16 @@
-
+
- {{ message.message_text }} + +
+ + {{ message.message_text }}