diff --git a/application/onlinecourse/admin.py b/application/onlinecourse/admin.py index 00e5febb8..c1efdc34d 100644 --- a/application/onlinecourse/admin.py +++ b/application/onlinecourse/admin.py @@ -1,7 +1,5 @@ from django.contrib import admin -from .models import Institution, Platform, OnlineCourse +from .models import OnlineCourse -admin.site.register(Institution) -admin.site.register(Platform) admin.site.register(OnlineCourse) diff --git a/application/onlinecourse/models.py b/application/onlinecourse/models.py index 2d2f28cd3..204a3a147 100644 --- a/application/onlinecourse/models.py +++ b/application/onlinecourse/models.py @@ -1,82 +1,45 @@ from django.db import models from dataprocessing.models import Items -class Institution(models.Model): - """ - Модель для правообладателей онлайн курсов - """ - title = models.CharField(max_length=1024, verbose_name='Название', blank=False, null=False) - id_from_roo = models.CharField(max_length=1024, verbose_name='ID, который приходит от РОО', blank=True, null=True) - - class Meta: - verbose_name = 'Правообладатель' - verbose_name_plural = 'Правообладатели' - - def __str__(self): - return self.title - - -class Platform(models.Model): - """ - Модель для платформ, на которых размещены онлайн курсы - """ - title = models.CharField(max_length=1024, verbose_name='Название', blank=False, null=False) - id_from_roo = models.CharField(max_length=1024, verbose_name='ID, который приходит от РОО', blank=True, null=True) - - class Meta: - verbose_name = 'Платформа' - verbose_name_plural = 'Платформы' - - def __str__(self): - return self.title - class OnlineCourse(models.Model): """ Модель онлайн курса """ - title = models.CharField(max_length=1024, verbose_name='Название', blank=False, null=False) - id_from_roo = models.CharField(max_length=1024, verbose_name='ID, который приходит от РОО', blank=True, null=True) + title = models.CharField(max_length=4000, verbose_name='Название', blank=False, null=False) + id_from_roo = models.CharField(max_length=4000, verbose_name='ID, который приходит от РОО', blank=True, null=True) description = models.TextField(verbose_name='Описание', blank=False, null=False) - institution = models.ForeignKey('Institution', on_delete=models.CASCADE, verbose_name="Правообладатель", blank=False, - null=False) - platform = models.ForeignKey('Platform', on_delete=models.CASCADE, verbose_name="Платформа", blank=False, - null=False) + institution = models.CharField(max_length=4000, verbose_name='Правообладатель', blank=False, null=False) + platform = models.CharField(max_length=4000, verbose_name='Платформа', blank=False, null=False) LanguageChoices = [ ('ru', 'Русский'), ('en', 'Английский'), ('ru/en', 'Русский/Английский'), ] language = models.CharField( - max_length=5, + max_length=30, choices=LanguageChoices, verbose_name='Язык онлайн курса', blank=False, null=False ) - started_at = models.DateField(blank=True, null=True, verbose_name='Дата начала курса') - created_at = models.DateField(blank=True, null=True, verbose_name='Дата создания курса') - record_end_at = models.DateField(blank=True, null=True, verbose_name='Дата окончания записи на курс') - finished_at = models.DateField(blank=True, null=True, verbose_name='Дата окончания курса') + started_at = models.CharField(max_length=4000, blank=True, null=True, verbose_name='Дата начала курса') + record_end_at = models.DateField(null=True, verbose_name='Дата окончания записи на курс') + finished_at = models.DateField(null=True, verbose_name='Дата окончания курса') rating = models.FloatField(blank=True, null=True, verbose_name='Рейтинг пользователей') - experts_rating = models.FloatField(blank=True, null=True, verbose_name='Рейтинг экспертов') visitors_number = models.IntegerField(blank=True, null=True, verbose_name='Количество записавшихся на текущую сессию') - total_visitors_number = models.IntegerField(blank=True, null=True, - verbose_name='Количество записавшихся на все сессии онлайн курса') duration = models.IntegerField(blank=True, null=True, verbose_name='Длительность онлайн курса, недель') - volume = models.IntegerField(blank=True, null=True, verbose_name='Объем онлайн курса, часов') - intensity_per_week = models.IntegerField(blank=True, null=True, - verbose_name='Требуемое время для изучения онлайн-курса, часов в неделю') - content = models.TextField(blank=True, null=True, verbose_name='Содержание онлайн курса') + content = models.CharField(max_length=40000, blank=True, null=True, verbose_name='Содержание онлайн курса') lectures_number = models.IntegerField(blank=True, null=True, verbose_name='Количество лекций') - external_url = models.URLField(blank=True, null=True, verbose_name='Ссылка на онлайн курс') - has_certificate = models.BooleanField(blank=True, null=True, verbose_name='Возможность получить сертификат') + external_url = models.CharField(max_length=4000, blank=True, null=True, verbose_name='Ссылка на онлайн курс') + roc_url = models.CharField(max_length=4000, blank=True, null=True, verbose_name='Ссылка на онлайн курс online.edu.ru') + has_certificate = models.CharField(max_length=40, blank=True, null=True, verbose_name='Возможность получить сертификат') credits = models.FloatField(blank=True, null=True, verbose_name='Трудоемкость курса в з.е.') - requirements = models.TextField(verbose_name='Требования', blank=True, null=True) - competences = models.TextField(verbose_name='Компетенции', blank=True, null=True) - learning_outcome = models.TextField(verbose_name='Результаты', blank=True, null=True) + requirements = models.CharField(max_length=40000, verbose_name='Требования', blank=True, null=True) + competences = models.CharField(max_length=40000, verbose_name='Компетенции', blank=True, null=True) + learning_outcome = models.CharField(max_length=40000, verbose_name='Результаты', blank=True, null=True) learning_outcome_list = models.ManyToManyField(Items) - + actual = models.BooleanField(blank=False, default=True, verbose_name='Актуальный курс') class Meta: verbose_name = 'Онлайн курс' @@ -84,6 +47,3 @@ class Meta: def __str__(self): return self.title - - - diff --git a/application/onlinecourse/onlinecourse_parser.py b/application/onlinecourse/onlinecourse_parser.py new file mode 100644 index 000000000..d208fb3be --- /dev/null +++ b/application/onlinecourse/onlinecourse_parser.py @@ -0,0 +1,191 @@ +from bs4 import BeautifulSoup as bs +import requests +import re +import pandas as pd +import time +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry +from datetime import datetime + + +online_edu_url = 'https://online.edu.ru' + + +def get_course_links(url): + """ + По ссылке на платформу online.edu.ru собирает ссылки на страницу каждого курса в список + """ + course_links = [] + for i in range(0, 10**9): + time.sleep(0.5) + main_url = url + '/public/courses.xhtml?page=' + str(i) + '&name=asc' + print(main_url) + session = requests.Session() + retry = Retry(connect=3, backoff_factor=0.5) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + html_text = session.get(main_url) + soup = bs(html_text.text, "html.parser") + print(i) + if soup.body.find_all(class_="ui-outputpanel ui-widget course-name"): + course_widgets = soup.body.find_all(class_="ui-outputpanel ui-widget course-name") + for k in course_widgets: + course_links.append(k.find('a').get('href')) + else: + break + return course_links + + +def get_course_info(course_url): + """ + По ссылке на конкретный курс собирается вся информация о курсе в датафрейм + """ + time.sleep(5) + session = requests.Session() + retry = Retry(connect=3, backoff_factor=0.5) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + html_text = session.get(course_url) + soup = bs(html_text.text, "html.parser") + archived = soup.find(class_="archived-label") + title = '' + id_from_roo = '' + description = '' + institution = '' + platform = '' + language = '' + started_at = None + record_end_at = None + finished_at = None + rating = 0 + visitors_number = 0 + duration = 0 + content = '' + lectures_number = 0 + external_url = '' + roc_url = course_url + has_certificate = '' + credits = 0 + requirements = '' + competences = '' + learning_outcome = '' + actual = True + try: + if archived is None: + course_info = soup.find(class_="course-info") + title = re.search(r'.*', str(course_info.find(class_="course-name").text)).group(0) + course_info_table = course_info.find_all(class_="ui-panelgrid-cell") + course_info_table_list = [] + for i in course_info_table: + course_info_table_list.append(i.text) + started_at = course_info_table_list[3] + platform = re.sub(r' *', '', re.sub(r'\n', '', course_info_table_list[5])) + button_link = course_info.find( + class_="ui-button ui-widget ui-state-default ui-corner-all ui-button-text-only rc-action-button") + external_url = re.search(r"'.*'", button_link.get('onclick')).group(0) + course_main = soup.find(class_="course-view-main content") + institution = course_main.find(id="j_idt260:j_idt290").find('span').text + if course_main.find(id="j_idt260:j_idt264"): + rating = float(re.sub(',', '.', re.search(r'\d.*', course_main.find(id="j_idt260:j_idt264").text).group(0))) + description = re.sub(r'\nО курсе', '', course_main.find(class_="ui-outputpanel ui-widget").text) + if course_main.find(id="j_idt123:0:j_idt124"): + requirements = course_main.find(id="j_idt123:0:j_idt124").text + if course_main.find(id="j_idt130"): + content = course_main.find(id="j_idt130").text + if course_main.find(id="course-view-competences"): + competences = course_main.find(id="course-view-competences") + list_comp = competences.find_all('li') + text_list_comp = [] + for i in list_comp: + text_list_comp.append(i.text) + competences = '\n '.join(text_list_comp) + if course_main.find(id="course-view-results"): + results = course_main.find(id="course-view-results") + list_results = results.find_all('li') + text_list_results = [] + for i in list_results: + text_list_results.append(i.text) + learning_outcome = '\n '.join(text_list_results) + course_add_info_table = course_main.find_all(class_="ui-panelgrid-cell") + for i in range(0, len(course_add_info_table)): + if course_add_info_table[i].text == 'Количество лекций': + if str(course_add_info_table[i + 1].text) != '': + lectures_number = int(course_add_info_table[i + 1].text) + elif course_add_info_table[i].text == 'ID курса': + id_from_roo = course_add_info_table[i + 1].text + elif course_add_info_table[i].text == 'Дата ближайшего старта': + started_at = course_add_info_table[i + 1].text + elif course_add_info_table[i].text == 'Дата окончания': + finished_at = datetime.strptime(course_add_info_table[i + 1].text, '%d.%m.%Y').date() + elif course_add_info_table[i].text == 'К-во обучающихся на версии курса': + if str(course_add_info_table[i + 1].text) != '': + visitors_number = int(course_add_info_table[i + 1].text) + elif course_add_info_table[i].text == 'Язык': + language = course_add_info_table[i + 1].text + elif course_add_info_table[i].text == 'Длительность': + if re.search('\d*', course_add_info_table[i + 1].text)[0] != '': + duration = int(re.search('\d*', course_add_info_table[i + 1].text)[0]) + elif course_add_info_table[i].text == 'Сертификат': + has_certificate = course_add_info_table[i + 1].text + elif course_add_info_table[i].text == 'Дата окончания записи': + record_end_at = datetime.strptime(course_add_info_table[i + 1].text, '%d.%m.%Y') + elif course_add_info_table[i].text == 'Трудоёмкость в з.е.': + credits = float(course_add_info_table[i + 1].text) + df = pd.DataFrame([[id_from_roo, + title, + description, + institution, + platform, + language, + started_at, + record_end_at, + finished_at, + rating, + visitors_number, + duration, + content, + lectures_number, + external_url, + roc_url, + has_certificate, + credits, + requirements, + learning_outcome, + competences, + actual]], columns=['id_from_roo', 'title', 'description', 'institution', + 'platform', 'language', 'started_at', 'record_end_at', + 'finished_at', 'rating', 'visitors_number', 'duration', + 'content', 'lectures_number', 'external_url', 'roc_url', + 'has_certificate', 'credits', 'requirements', 'learning_outcome', + 'competences', 'actual']) + return df + else: + pass + except Exception as e: + print("Во время парсинга курса", title, 'произошла ошибка') + print(e) + + +def get_all_data(url): + """ + Функция сбора данных об онлайн курсах в датафрейм + """ + print('Собираем ссылки на курсы') + course_links = get_course_links(url) + print('Количество онлайн курсов', len(course_links)) + print("Собираем сами курсы") + df = pd.DataFrame( + columns=['id_from_roo', 'title', 'description', 'institution', 'platform', 'language', + 'started_at', 'record_end_at', 'finished_at', 'rating', + 'visitors_number', 'duration', + 'content', 'lectures_number', 'external_url', 'roc_url', + 'has_certificate', 'credits', 'requirements', 'learning_outcome', + 'competences', 'actual']) + for i in course_links: + print(url + i) + df = pd.concat([df, get_course_info(url + i)], ignore_index=True) + print(df.shape) + #df.to_csv('df_1954.csv') + return df diff --git a/application/onlinecourse/serializers.py b/application/onlinecourse/serializers.py index 3a6f3f4c9..dced2e30c 100644 --- a/application/onlinecourse/serializers.py +++ b/application/onlinecourse/serializers.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from .models import Institution, Platform, OnlineCourse +from .models import OnlineCourse from workprogramsapp.models import CourseCredit, CourseFieldOfStudy, FieldOfStudy, Topic, DisciplineSection, WorkProgram from dataprocessing.models import Items @@ -13,24 +13,9 @@ class Meta: fields = "__all__" -class InstitutionSerializer(serializers.ModelSerializer): - """Сериализатор Правообладателей""" - class Meta: - model = Institution - fields = '__all__' - - -class PlatformSerializer(serializers.ModelSerializer): - """Сериализатор Платформ""" - class Meta: - model = Platform - fields = '__all__' - - class CourseCreditSerializer(serializers.ModelSerializer): """Сериализатор Перезачетов""" course = serializers.SlugRelatedField(slug_field="title", read_only=True) - institution = InstitutionSerializer() field_of_study = FieldOfStudySerializer(many=False) class Meta: @@ -85,8 +70,6 @@ class OnlineCourseSerializer(serializers.ModelSerializer): """Сериализатор Онлайн курса""" course_field_of_study = CourseFieldOfStudySerializer(many=True) course_credit = CourseCreditSerializer(many=True) - institution = InstitutionSerializer(many=False) - platform = PlatformSerializer(many=False) learning_outcome_list = ItemsForOnlineCourseSerializer(many=True) topic_with_online_course = OnlineCourseInTopics(many=True) diff --git a/application/onlinecourse/thread.py b/application/onlinecourse/thread.py new file mode 100644 index 000000000..b69362543 --- /dev/null +++ b/application/onlinecourse/thread.py @@ -0,0 +1,115 @@ +import threading, time +from .models import OnlineCourse +from .onlinecourse_parser import get_all_data, online_edu_url +from rest_framework.response import Response +import requests +from bs4 import BeautifulSoup as bs + + +def upload_data_to_db(): + """ + Adding data to OnlineCourse + """ + #data_OnlineCourse = pd.read_csv('/Users/valeriaartamonova/Desktop/Work/analytics_backend/application/df3003_0900.csv') + data_OnlineCourse = get_all_data(online_edu_url) + for i in range(len(data_OnlineCourse)): + if OnlineCourse.objects.filter(id_from_roo=data_OnlineCourse.id_from_roo[i]).exists(): + continue + else: + onlinecourse = OnlineCourse(id_from_roo=data_OnlineCourse.id_from_roo[i], + title=data_OnlineCourse.title[i], + description=data_OnlineCourse.description[i], + institution=data_OnlineCourse.institution[i], + platform=data_OnlineCourse.platform[i], + language=data_OnlineCourse.language[i], + external_url=data_OnlineCourse.external_url[i], + roc_url=data_OnlineCourse.roc_url[i], + has_certificate=data_OnlineCourse.has_certificate[i], + rating=data_OnlineCourse.rating[i], + started_at=data_OnlineCourse.started_at[i], + ) + onlinecourse.save() + if len(str(data_OnlineCourse.record_end_at[i])) > 3: + onlinecourse.record_end_at = data_OnlineCourse.record_end_at[i] + if len(str(data_OnlineCourse.finished_at[i])) > 3: + onlinecourse.finished_at = data_OnlineCourse.finished_at[i] + try: + onlinecourse.duration = int(data_OnlineCourse.duration[i]) + except: + continue + if len(str(data_OnlineCourse.requirements[i])) > 3: + onlinecourse.requirements = data_OnlineCourse.requirements[i] + if len(str(data_OnlineCourse.competences[i])) > 3: + onlinecourse.competences = data_OnlineCourse.competences[i] + if len(str(data_OnlineCourse.learning_outcome[i])) > 3: + onlinecourse.learning_outcome = data_OnlineCourse.learning_outcome[i] + if len(str(data_OnlineCourse.content[i])) > 3: + onlinecourse.content = data_OnlineCourse.content[i] + + if data_OnlineCourse.lectures_number[i] > 0: + onlinecourse.lectures_number = data_OnlineCourse.lectures_number[i] + if data_OnlineCourse.visitors_number[i] > 0: + onlinecourse.visitors_number = data_OnlineCourse.visitors_number[i] + if data_OnlineCourse.credits[i] > 0: + onlinecourse.credits = data_OnlineCourse.credits[i] + + onlinecourse.save() + print("добавлен онлайн курс -", i) + return Response(status=200) + + +class OnlineCourseThread(threading.Thread): + """ + Класс потока для онлайн курсов + """ + def __init__(self, event): + threading.Thread.__init__(self) + self.stopped = event + + def run(self): + """ + Метод потока для парсинга онлайн курсов и их загрузки в БД + """ + while not self.stopped.wait(10): + print('Старт потока') + upload_data_to_db() + print('Завершение работы потока') + time.sleep(60 * 60 * 24 * 7) + + +class OnlineCourseActualThread(threading.Thread): + """ + Класс потока для проверки актуальности онлайн курсов + """ + def __init__(self, event): + threading.Thread.__init__(self) + self.stopped = event + + def run(self): + """ + Метод потока для проверки актуальности курсов + """ + while not self.stopped.wait(10): + print('Запущен поток для проверки актуальности курсов') + online_courses = OnlineCourse.objects.all() + for i in online_courses: + #print(i) + html_text = requests.get(i.roc_url, timeout=15) + soup = bs(html_text.text, "html.parser") + archived = soup.find(class_="archived-label") + if archived is None: + continue + else: + print('Найден устаревший курс', i.roc_url, i.title) + i.actual = False + i.save() + time.sleep(5) + print('Поток для проверки актуальности курсов завершен') + time.sleep(60*60*24*7) + + +stopFlag = threading.Event() +course_thread = OnlineCourseThread(stopFlag) + +stopFlag2 = threading.Event() +course_actual_thread = OnlineCourseActualThread(stopFlag) diff --git a/application/onlinecourse/urls.py b/application/onlinecourse/urls.py index da86e58ee..c8f89aaea 100644 --- a/application/onlinecourse/urls.py +++ b/application/onlinecourse/urls.py @@ -1,13 +1,9 @@ from django.urls import path -from .views import InstitutionViewSet, PlatformViewSet, OnlineCourseViewSet, CourseCreditViewSet, \ - CourseFieldOfStudyViewSet, CourseDataAPIView +from .views import OnlineCourseViewSet, CourseCreditViewSet, CourseFieldOfStudyViewSet urlpatterns = [ - path('api/course/institution/', InstitutionViewSet.as_view({'get': 'list'}), name='Institution-list'), - path('api/course/platform/', PlatformViewSet.as_view({'get': 'list'}), name='Platform-list'), path('api/course/onlinecourse/', OnlineCourseViewSet.as_view({'get': 'list'}), name='OnlineCourse-list'), path('api/course/onlinecourse//', OnlineCourseViewSet.as_view({'get': 'retrieve'}), name='OnlineCourse-item'), path('api/course/coursecredit/', CourseCreditViewSet.as_view({'get': 'list'}), name='CourseCredit-list'), path('api/course/coursefieldofstudy/', CourseFieldOfStudyViewSet.as_view({'get': 'list'}), name='CourseFieldOfStudy-list'), - path('api/course/upload_data/', CourseDataAPIView.as_view(), name='CourseDataUploader') ] diff --git a/application/onlinecourse/views.py b/application/onlinecourse/views.py index f02cbfb3d..d9ba6e6cf 100644 --- a/application/onlinecourse/views.py +++ b/application/onlinecourse/views.py @@ -1,33 +1,18 @@ +import threading, time from django_filters.rest_framework import DjangoFilterBackend from rest_framework import filters from rest_framework import viewsets from rest_framework.views import APIView from rest_framework.response import Response from rest_framework.permissions import IsAuthenticated - -from .models import Institution, Platform, OnlineCourse +from .models import OnlineCourse from workprogramsapp.models import FieldOfStudy, CourseCredit, CourseFieldOfStudy - -from .serializers import InstitutionSerializer, PlatformSerializer, OnlineCourseSerializer, \ - CourseCreditSerializer, CourseFieldOfStudySerializer - +from .serializers import OnlineCourseSerializer, CourseCreditSerializer, CourseFieldOfStudySerializer from .data_onlinecourse import get_data +import pandas as pd +import re - -class InstitutionViewSet(viewsets.ModelViewSet): - """Контроллер для модели Правообладатель""" - queryset = Institution.objects.all() - serializer_class = InstitutionSerializer - filter_backends = [filters.SearchFilter, filters.OrderingFilter] - search_fields = ['title'] - - -class PlatformViewSet(viewsets.ModelViewSet): - """Контроллер для модели Платформа""" - queryset = Platform.objects.all() - serializer_class = PlatformSerializer - filter_backends = [filters.SearchFilter, filters.OrderingFilter] - search_fields = ['title'] +from . import thread class OnlineCourseViewSet(viewsets.ModelViewSet): @@ -36,9 +21,9 @@ class OnlineCourseViewSet(viewsets.ModelViewSet): serializer_class = OnlineCourseSerializer permission_classes = [IsAuthenticated] filter_backends = [filters.SearchFilter, filters.OrderingFilter, DjangoFilterBackend] - search_fields = ['title', 'platform__title', 'institution__title', + search_fields = ['title', 'platform', 'institution', 'course_field_of_study__field_of_study__number', 'course_field_of_study__field_of_study__title'] - ordering_fields = ['title', 'platform__title', 'institution__title', 'language', 'started_at', 'rating'] + ordering_fields = ['title', 'platform', 'institution', 'language', 'started_at', 'rating'] filterset_fields = ['platform__title', 'institution__title', 'language', 'course_field_of_study__field_of_study__number', 'course_field_of_study__field_of_study__title'] @@ -55,194 +40,5 @@ class CourseFieldOfStudyViewSet(viewsets.ModelViewSet): serializer_class = CourseFieldOfStudySerializer -class CourseDataAPIView(APIView): - """ - Контроллер для загрузки данных из реестра онлайн курсов - """ - - def post(self, request): - print('Data parsing started') - data_Platform, data_Rigthholder, data_OnlineCourse, data_CourseFieldOfStudy, data_CourseCredit = get_data() - print('Data parsing ended') - - """ - Adding data to Platform - """ - for i in range(len(data_Platform)): - if Platform.objects.filter(id=data_Platform.index[i]).exists(): - continue - else: - platform = Platform.objects.create(id=data_Platform.index[i], - id_from_roo=data_Platform.platform_id[i], - title=data_Platform.title[i], ) - platform.save() - - """ - Adding data to Institution - """ - for i in range(len(data_Rigthholder)): - if Institution.objects.filter(id=data_Rigthholder.index[i]).exists(): - continue - else: - institution = Institution.objects.create(id=data_Rigthholder.index[i], - id_from_roo=data_Rigthholder.institution_id[i], - title=data_Rigthholder.title[i], ) - institution.save() - - """ - Adding data to OnlineCourse - """ - for i in range(len(data_OnlineCourse)): - if OnlineCourse.objects.filter(id=data_OnlineCourse.index[i]).exists(): - continue - else: - onlinecourse = OnlineCourse.objects.create(id=data_OnlineCourse.index[i], - id_from_roo=data_OnlineCourse.index[i], - title=data_OnlineCourse.title_x[i], - description=data_OnlineCourse.description[i], - institution=Institution.objects.get( - id=data_OnlineCourse.id_institution[i]), - platform=Platform.objects.get( - id=data_OnlineCourse.id_platform[i]), - language=data_OnlineCourse.language[i], - ) - onlinecourse.save() - if data_OnlineCourse.started_at[i] != 'null': - onlinecourse.started_at = data_OnlineCourse.started_at[i] - onlinecourse.save() - if data_OnlineCourse.created_at[i] != 'null': - onlinecourse.created_at = data_OnlineCourse.created_at[i] - onlinecourse.save() - if data_OnlineCourse.record_end_at[i] != 'null': - onlinecourse.record_end_at = data_OnlineCourse.record_end_at[i] - onlinecourse.save() - if data_OnlineCourse.finished_at[i] != 'null': - onlinecourse.finished_at = data_OnlineCourse.finished_at[i] - onlinecourse.save() - if data_OnlineCourse.rating[i] != 'null': - onlinecourse.rating = data_OnlineCourse.rating[i] - onlinecourse.save() - if data_OnlineCourse.experts_rating[i] != 'null': - try: - onlinecourse.experts_rating = float(data_OnlineCourse.experts_rating[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.visitors_number[i] != 'null': - try: - onlinecourse.visitors_number = int(data_OnlineCourse.visitors_number[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.total_visitors_number[i] != 'null': - try: - onlinecourse.total_visitors_number = int(data_OnlineCourse.total_visitors_number[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.duration[i] != 'null': - try: - onlinecourse.duration = int(data_OnlineCourse.duration[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.volume[i] != 'null': - try: - onlinecourse.volume = int(data_OnlineCourse.volume[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.intensity_per_week[i] != 'null': - try: - onlinecourse.intensity_per_week = int(data_OnlineCourse.intensity_per_week[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.content[i] != 'null': - try: - onlinecourse.content = str(data_OnlineCourse.content[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.lectures_number[i] != 'null': - try: - onlinecourse.lectures_number = int(data_OnlineCourse.lectures_number[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.external_url[i] != 'null': - try: - onlinecourse.external_url = str(data_OnlineCourse.external_url[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.has_certificate[i] != 'null': - try: - onlinecourse.has_certificate = bool(data_OnlineCourse.has_certificate[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.credits[i] != 'null': - try: - onlinecourse.credits = float(data_OnlineCourse.credits[i]) - onlinecourse.save() - except: - continue - if data_OnlineCourse.requirements[i] != 'null': - try: - onlinecourse.requirements = data_OnlineCourse.requirements[i] - onlinecourse.save() - except: - continue - if data_OnlineCourse.competences[i] != 'null': - try: - onlinecourse.competences = data_OnlineCourse.competences[i] - onlinecourse.save() - except: - continue - if data_OnlineCourse.learning_outcomes[i] != 'null': - try: - onlinecourse.learning_outcomes = data_OnlineCourse.learning_outcomes[i] - onlinecourse.save() - except: - continue - """ - Adding data to CourseFieldOfStudy - """ - for i in range(len(data_CourseFieldOfStudy)): - if CourseFieldOfStudy.objects.filter(id=data_CourseFieldOfStudy.index[i]).exists(): - continue - else: - if FieldOfStudy.objects.filter(number=data_CourseFieldOfStudy.field_of_study[i]).exists(): - onlinecourse_field_of_study = CourseFieldOfStudy.objects.create(id=data_CourseFieldOfStudy.index[i], - course=OnlineCourse.objects.get( - id= - data_CourseFieldOfStudy.id_course[ - i]), - field_of_study=FieldOfStudy.objects.get( - number= - data_CourseFieldOfStudy.field_of_study[ - i] - ), ) - onlinecourse_field_of_study.save() - - """ - Adding data to CourseCredit - """ - - for i in range(len(data_CourseCredit)): - if CourseCredit.objects.filter(id=data_CourseCredit.index[i]).exists(): - continue - else: - if FieldOfStudy.objects.filter(number=data_CourseCredit.field_of_study[i]).exists(): - onlinecourse_credit = CourseCredit.objects.create(id=data_CourseCredit.index[i], - course=OnlineCourse.objects.get( - id=data_CourseCredit.id_course[i]), - institution=Institution.objects.get( - id=data_CourseCredit.id_institution[i] - ), - field_of_study=FieldOfStudy.objects.get( - number=data_CourseCredit.field_of_study[i])) - onlinecourse_credit.save() - - return Response(status=200) +thread.course_thread.start() +thread.course_actual_thread.start()