Skip to content

Commit

Permalink
Merge pull request #111 from AI4Bharat/multiple_annotators_change
Browse files Browse the repository at this point in the history 
added final changes
  • Loading branch information
@ishvindersethi22
ishvindersethi22 authored Oct 30, 2024
2 parents d8c2f45 + 3c6f941 commit c19840d
Show file tree
Hide file tree
Showing 8 changed files with 422 additions and 34 deletions.
140 changes: 140 additions & 0 deletions backend/dataset/migrations/0055_auto_20241029_0902.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
# Generated by Django 3.2.14 on 2024-10-29 09:02

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("tasks", "0050_alter_annotation_unique_together"),
("dataset", "0054_auto_20241016_0456"),
]

operations = [
migrations.RemoveField(
model_name="conversation",
name="datasetbase_ptr",
),
migrations.RemoveField(
model_name="ocrdocument",
name="datasetbase_ptr",
),
migrations.RemoveField(
model_name="sentencetext",
name="datasetbase_ptr",
),
migrations.RemoveField(
model_name="speechconversation",
name="datasetbase_ptr",
),
migrations.RemoveField(
model_name="translationpair",
name="datasetbase_ptr",
),
migrations.AlterField(
model_name="datasetinstance",
name="dataset_type",
field=models.CharField(
choices=[
("PromptBase", "PromptBase"),
("PromptAnswer", "PromptAnswer"),
("PromptAnswerEvaluation", "PromptAnswerEvaluation"),
("Interaction", "Interaction"),
("Instruction", "Instruction"),
("MultiModelInteraction", "MultiModelInteraction"),
],
help_text="Dataset Type which is specific for each annotation task",
max_length=100,
verbose_name="dataset_type",
),
),
migrations.AlterField(
model_name="promptanswer",
name="language",
field=models.CharField(
choices=[
("English", "English"),
("Assamese", "Assamese"),
("Bengali", "Bengali"),
("Bodo", "Bodo"),
("Dogri", "Dogri"),
("Filipino", "Filipino"),
("Gujarati", "Gujarati"),
("Hindi", "Hindi"),
("Kannada", "Kannada"),
("Kashmiri", "Kashmiri"),
("Konkani", "Konkani"),
("Maithili", "Maithili"),
("Malayalam", "Malayalam"),
("Manipuri", "Manipuri"),
("Marathi", "Marathi"),
("Nepali", "Nepali"),
("Odia", "Odia"),
("Punjabi", "Punjabi"),
("Sanskrit", "Sanskrit"),
("Santali", "Santali"),
("Sindhi", "Sindhi"),
("Sinhala", "Sinhala"),
("Tamil", "Tamil"),
("Telugu", "Telugu"),
("Thai", "Thai"),
("Urdu", "Urdu"),
],
max_length=15,
verbose_name="language",
),
),
migrations.AlterField(
model_name="promptbase",
name="language",
field=models.CharField(
choices=[
("English", "English"),
("Assamese", "Assamese"),
("Bengali", "Bengali"),
("Bodo", "Bodo"),
("Dogri", "Dogri"),
("Filipino", "Filipino"),
("Gujarati", "Gujarati"),
("Hindi", "Hindi"),
("Kannada", "Kannada"),
("Kashmiri", "Kashmiri"),
("Konkani", "Konkani"),
("Maithili", "Maithili"),
("Malayalam", "Malayalam"),
("Manipuri", "Manipuri"),
("Marathi", "Marathi"),
("Nepali", "Nepali"),
("Odia", "Odia"),
("Punjabi", "Punjabi"),
("Sanskrit", "Sanskrit"),
("Santali", "Santali"),
("Sindhi", "Sindhi"),
("Sinhala", "Sinhala"),
("Tamil", "Tamil"),
("Telugu", "Telugu"),
("Thai", "Thai"),
("Urdu", "Urdu"),
],
max_length=15,
verbose_name="language",
),
),
migrations.DeleteModel(
name="BlockText",
),
migrations.DeleteModel(
name="Conversation",
),
migrations.DeleteModel(
name="OCRDocument",
),
migrations.DeleteModel(
name="SentenceText",
),
migrations.DeleteModel(
name="SpeechConversation",
),
migrations.DeleteModel(
name="TranslationPair",
),
]
90 changes: 90 additions & 0 deletions backend/projects/migrations/0060_auto_20241029_0902.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Generated by Django 3.2.14 on 2024-10-29 09:02

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("projects", "0059_alter_project_project_type"),
]

operations = [
migrations.AlterField(
model_name="project",
name="src_language",
field=models.CharField(
blank=True,
choices=[
("English", "English"),
("Assamese", "Assamese"),
("Bengali", "Bengali"),
("Bodo", "Bodo"),
("Dogri", "Dogri"),
("Filipino", "Filipino"),
("Gujarati", "Gujarati"),
("Hindi", "Hindi"),
("Kannada", "Kannada"),
("Kashmiri", "Kashmiri"),
("Konkani", "Konkani"),
("Maithili", "Maithili"),
("Malayalam", "Malayalam"),
("Manipuri", "Manipuri"),
("Marathi", "Marathi"),
("Nepali", "Nepali"),
("Odia", "Odia"),
("Punjabi", "Punjabi"),
("Sanskrit", "Sanskrit"),
("Santali", "Santali"),
("Sindhi", "Sindhi"),
("Sinhala", "Sinhala"),
("Tamil", "Tamil"),
("Telugu", "Telugu"),
("Thai", "Thai"),
("Urdu", "Urdu"),
],
help_text="Source language of the project",
max_length=50,
null=True,
verbose_name="Source Language",
),
),
migrations.AlterField(
model_name="project",
name="tgt_language",
field=models.CharField(
blank=True,
choices=[
("English", "English"),
("Assamese", "Assamese"),
("Bengali", "Bengali"),
("Bodo", "Bodo"),
("Dogri", "Dogri"),
("Filipino", "Filipino"),
("Gujarati", "Gujarati"),
("Hindi", "Hindi"),
("Kannada", "Kannada"),
("Kashmiri", "Kashmiri"),
("Konkani", "Konkani"),
("Maithili", "Maithili"),
("Malayalam", "Malayalam"),
("Manipuri", "Manipuri"),
("Marathi", "Marathi"),
("Nepali", "Nepali"),
("Odia", "Odia"),
("Punjabi", "Punjabi"),
("Sanskrit", "Sanskrit"),
("Santali", "Santali"),
("Sindhi", "Sindhi"),
("Sinhala", "Sinhala"),
("Tamil", "Tamil"),
("Telugu", "Telugu"),
("Thai", "Thai"),
("Urdu", "Urdu"),
],
help_text="Target language of the project",
max_length=50,
null=True,
verbose_name="Target Language",
),
),
]
39 changes: 34 additions & 5 deletions backend/projects/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ def create_parameters_for_task_creation(
create_tasks_from_dataitems(sampled_items, project)


# @shared_task
@shared_task
def export_project_in_place(
annotation_fields, project_id, project_type, get_request_data
) -> None:
Expand Down Expand Up @@ -424,12 +424,19 @@ def export_project_in_place(
# List for storing the annotated tasks that have been accepted as correct annotation
annotated_tasks = []
export_excluded_task_ids = []
required_annotators_per_task = project.required_annotators_per_task
for task in tasks:
task_dict = model_to_dict(task)
# Rename keys to match label studio converter
# task_dict['id'] = task_dict['task_id']
# del task_dict['task_id']
if task.correct_annotation is not None:
ann_list = []
if required_annotators_per_task >= 2:
all_ann = Annotation.objects.filter(task=task)
for a in all_ann:
ann_list.append(a)
task_dict["annotations"] = ann_list
elif task.correct_annotation is not None:
annotated_tasks.append(task)
annotation_dict = model_to_dict(task.correct_annotation)
# annotation_dict['result'] = annotation_dict['result_json']
Expand All @@ -442,14 +449,36 @@ def export_project_in_place(
task.output_data = task.input_data
task.save()
data_item = dataset_model.objects.get(id__exact=task_dict["input_data"])
res = task_dict["annotations"][0]["result"]
complete_result = []
for i in range(len(task_dict["annotations"])):
a = task_dict["annotations"][i]
annotation_result = a.result
annotation_result = (
json.loads(annotation_result)
if isinstance(annotation_result, str)
else annotation_result
)
uid = a.completed_by.email
try:
p_ann = a.parent_annotation.id
except Exception as e:
p_ann = None
single_dict = {
"user_id": uid,
"annotation_id": a.id,
"annotation_result": annotation_result,
"annotation_type": a.annotation_type,
"annotation_status": a.annotation_status,
"parent_annotation": a.parent_annotation,
"parent_annotation_id": p_ann,
}
complete_result.append(single_dict)
try:
for field in annotation_fields:
setattr(data_item, field, res)
setattr(data_item, field, complete_result)
except Exception as e:
export_excluded_task_ids.append(task.id)
data_items.append(data_item)
# Write json to dataset columns
dataset_model.objects.bulk_update(data_items, annotation_fields)

tasks = tasks.exclude(id__in=export_excluded_task_ids)
Expand Down
Loading

0 comments on commit c19840d

Please sign in to comment.