From 72937725a323b0cc1ccec150c81b041abeacb575 Mon Sep 17 00:00:00 2001 From: Brian Rodriguez Date: Fri, 6 Aug 2021 21:16:01 -0400 Subject: [PATCH] Fix #11465: Migrates the codebase from Python 2 to Python 3 (#13395) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Migrates the codebase from Python 2 to Python 3 * fix lint * fix tests * fix blog tests * update pylintrc * run mypy checks * run mypy checks * revert changes to CODEOWNERS * Test backend error * Test backend error * Test backend error * Test backend error * Improve coverage * Improve coverage * Improve coverage * fix lint * fix frontend coverage * Fix backend tests * fix mypy checks * Improve coverage * Improve coverage * Fix tests and improve coverage * fix broken test * fix lint * Improve coverage * Improve coverage * Improve coverage * fix learner_progress_services * Improve coverage and fix types * Fix mypy * Fix tests and improve coverage * Increase coverage * inline mock_datetime_utcnow * fix lint * Split e2e suite to improve performance * Split e2e suite to improve performance * Split e2e suite to improve performance * Split e2e suite to improve performance * fix datetime tests * fix tests * disable incorrect pylint warning * Improve coverage * Improve coverage * Remove elastic search from lighthouse tests * Fix e2e tests * simplify metaclass * Address comments * try to store on disk * try to store on disk * update pylintrc * Address some comments * Address some comments * Address some comments * Address some comments * Address more comments * Fix backend tests * Fix backend tests * Address comments * Address comments * Merge related edits * Address more comments. * Address comments * Address comments * Fix backend tests * Fix backend tests * Fix backend tests * Debug portserver * Address comments * Address comments * Address comments * Fix prereqs * Add tests * Address comments * Fix tests * Fix tests * Address comments * Fix lighthouse tests * Fix tests and increase coverage * Fix backend tests * Fix backend tests * Small improvements * Fix lighthouse * [skip ci] Lighthouse changes * Fix encode/decode * Fix backend tests * Fix ighthouse tests * Fix lighthouse * Fix tests * Fix redis shutdown * Add debug lines * Add FE debugging * Fix e2e and backend tests * Fix backend tests * Fix portserver printing * Fix lighthouse, debug e2e * Fix portserver hangs * Fix portserver and add debugging * Fix debugging * Fix release * Debug and fix tests * Fix backend tests * Fix frontend * Fix e2e tests * Fix FE tests * Remove debugging statements * Fix e2e tests * Fix frontend tests * Fix tests and address comments * Fix frontend and e2e tests * Debug e2e tests * Fix e2e tests * Fix e2e and frontend tests * Change policies * Remove debugging statements * Fix e2e errors * Fix e2e tests * Reset flake policy for extensions * Partial mypy * Fix backend tests * Fix mypy * Fix backend tests and linting * Fix backend tests Co-authored-by: Vojtěch Jelínek Co-authored-by: Vojtěch Jelínek --- .circleci/config.yml | 2 +- .gcloudignore | 167 +++ .github/CODEOWNERS | 10 +- .github/workflows/backend_tests.yml | 13 +- .../e2e_additional_editor_and_player.yml | 9 +- ...ator_learner_dashboard_and_editor_tabs.yml | 4 +- .../e2e_email_dashboard_features.yml | 48 - ...history_statistics_tabs_and_extensions.yml | 4 +- ...earner_flow_skill_editor_and_embedding.yml | 4 +- .github/workflows/e2e_miscellaneous_tests.yml | 4 +- .github/workflows/e2e_other_tests.yml | 4 +- .github/workflows/e2e_topic_tests.yml | 12 +- ...ranslation_classroom_and_core_features.yml | 4 +- .../e2e_user_features_and_library.yml | 12 +- .github/workflows/e2e_user_profile.yml | 4 +- .github/workflows/eslint_tests.yml | 5 +- .github/workflows/frontend_tests.yml | 4 +- .../workflows/lighthouse_accessibility.yml | 10 +- .github/workflows/lighthouse_performance.yml | 10 +- .github/workflows/lint.yml | 5 +- .github/workflows/python_type_checks.yml | 5 +- .lighthouserc-1.js | 123 ++ .lighthouserc-2.js | 111 ++ .lighthouserc-accessibility-1.js | 100 ++ .lighthouserc-accessibility-2.js | 108 ++ .lighthouserc-accessibility.js | 254 ---- .lighthouserc-base.js | 105 ++ .lighthouserc.js | 429 ------ .pylintrc | 49 +- android_validation_constants_test.py | 6 +- app_dev.yaml | 222 +--- appengine_config.py | 130 -- appengine_config_test.py | 85 -- constants_test.py | 2 +- core/controllers/acl_decorators.py | 49 +- core/controllers/acl_decorators_test.py | 58 + core/controllers/admin.py | 25 +- core/controllers/admin_test.py | 69 +- core/controllers/base.py | 92 +- core/controllers/base_test.py | 243 +--- core/controllers/blog_admin_test.py | 4 +- core/controllers/blog_dashboard_test.py | 6 +- core/controllers/classifier_test.py | 4 +- core/controllers/collection_editor.py | 7 +- core/controllers/collection_editor_test.py | 8 +- core/controllers/contributor_dashboard.py | 23 +- .../contributor_dashboard_admin_test.py | 12 +- .../controllers/contributor_dashboard_test.py | 19 +- core/controllers/creator_dashboard.py | 2 +- core/controllers/creator_dashboard_test.py | 471 ++----- core/controllers/cron.py | 192 +-- core/controllers/cron_test.py | 373 +----- core/controllers/editor.py | 2 +- core/controllers/editor_test.py | 25 +- core/controllers/email_dashboard.py | 16 +- core/controllers/email_dashboard_test.py | 803 ++++-------- core/controllers/feedback_test.py | 75 +- core/controllers/learner_dashboard_test.py | 2 +- core/controllers/library.py | 4 +- core/controllers/pages.py | 10 +- core/controllers/payload_validator_test.py | 4 +- core/controllers/platform_feature_test.py | 5 +- core/controllers/profile.py | 9 +- core/controllers/profile_test.py | 107 +- core/controllers/question_editor_test.py | 3 +- core/controllers/reader.py | 3 +- core/controllers/reader_test.py | 20 +- core/controllers/recent_commits.py | 2 +- core/controllers/recent_commits_test.py | 3 +- core/controllers/release_coordinator.py | 85 +- core/controllers/release_coordinator_test.py | 172 --- core/controllers/resources.py | 6 +- core/controllers/resources_test.py | 82 +- core/controllers/story_viewer_test.py | 9 +- core/controllers/suggestion.py | 4 +- core/controllers/suggestion_test.py | 13 +- core/controllers/tasks.py | 14 +- core/controllers/tasks_test.py | 16 +- core/controllers/topic_editor_test.py | 21 +- core/controllers/topic_viewer_test.py | 4 +- .../topics_and_skills_dashboard.py | 4 +- .../topics_and_skills_dashboard_test.py | 16 +- core/controllers/voice_artist.py | 7 +- core/domain/activity_jobs_one_off.py | 50 - core/domain/activity_jobs_one_off_test.py | 104 -- core/domain/caching_services.py | 25 +- core/domain/caching_services_test.py | 41 +- core/domain/classifier_services.py | 13 +- core/domain/collection_domain.py | 14 +- core/domain/collection_domain_test.py | 29 + core/domain/collection_jobs_one_off.py | 97 -- core/domain/collection_jobs_one_off_test.py | 345 ----- core/domain/collection_services.py | 5 +- core/domain/collection_services_test.py | 72 +- core/domain/cron_services.py | 136 -- core/domain/customization_args_util_test.py | 2 +- core/domain/draft_upgrade_services.py | 2 +- core/domain/draft_upgrade_services_test.py | 13 +- core/domain/email_manager.py | 2 +- core/domain/email_manager_test.py | 489 +++---- core/domain/email_services.py | 9 +- core/domain/email_services_test.py | 12 +- core/domain/event_services.py | 2 +- core/domain/event_services_test.py | 87 +- core/domain/exp_domain.py | 30 +- core/domain/exp_domain_test.py | 112 +- core/domain/exp_fetchers.py | 6 +- core/domain/exp_fetchers_test.py | 37 +- core/domain/exp_jobs_one_off.py | 178 --- core/domain/exp_jobs_one_off_test.py | 585 --------- core/domain/exp_services.py | 35 +- core/domain/exp_services_test.py | 69 +- core/domain/expression_parser.py | 18 +- core/domain/expression_parser_test.py | 48 +- core/domain/feedback_services_test.py | 65 +- core/domain/fs_domain.py | 92 +- core/domain/fs_domain_test.py | 9 +- core/domain/fs_services.py | 18 +- core/domain/fs_services_test.py | 20 +- core/domain/html_cleaner.py | 14 +- core/domain/html_validation_service.py | 89 +- core/domain/html_validation_service_test.py | 223 +--- core/domain/image_services_test.py | 5 +- core/domain/image_validation_services_test.py | 2 +- core/domain/improvements_services.py | 6 +- core/domain/moderator_services_test.py | 14 +- core/domain/opportunity_domain.py | 2 +- core/domain/opportunity_domain_test.py | 31 +- core/domain/opportunity_services.py | 15 +- core/domain/opportunity_services_test.py | 5 +- core/domain/param_domain.py | 4 +- core/domain/platform_feature_services.py | 2 +- core/domain/platform_parameter_domain.py | 29 +- core/domain/platform_parameter_domain_test.py | 9 +- core/domain/platform_parameter_registry.py | 5 +- core/domain/question_domain_test.py | 9 +- core/domain/question_jobs_one_off.py | 235 ---- core/domain/question_jobs_one_off_test.py | 492 ------- core/domain/question_services.py | 6 +- core/domain/question_services_test.py | 3 +- core/domain/recommendations_jobs_one_off.py | 100 -- .../recommendations_jobs_one_off_test.py | 129 -- core/domain/rte_component_registry_test.py | 15 +- core/domain/skill_domain.py | 12 +- core/domain/skill_domain_test.py | 20 + core/domain/skill_fetchers.py | 2 +- core/domain/skill_jobs_one_off.py | 163 --- core/domain/skill_jobs_one_off_test.py | 422 ------ core/domain/skill_services.py | 10 +- core/domain/skill_services_test.py | 29 +- core/domain/state_domain.py | 15 +- core/domain/state_domain_test.py | 22 +- core/domain/stats_domain_test.py | 2 +- core/domain/stats_services.py | 2 +- core/domain/stats_services_test.py | 27 + core/domain/story_domain.py | 14 +- core/domain/story_fetchers.py | 2 +- core/domain/story_jobs_one_off.py | 179 --- core/domain/story_jobs_one_off_test.py | 507 ------- core/domain/story_services.py | 3 +- core/domain/story_services_test.py | 33 + core/domain/subtopic_page_domain.py | 6 +- core/domain/suggestion_jobs_one_off.py | 365 ------ core/domain/suggestion_jobs_one_off_test.py | 934 ------------- core/domain/suggestion_registry_test.py | 4 +- core/domain/suggestion_services_test.py | 15 +- core/domain/summary_services.py | 5 +- core/domain/taskqueue_services_test.py | 12 +- core/domain/topic_domain.py | 10 +- core/domain/topic_fetchers.py | 2 +- core/domain/topic_jobs_one_off.py | 179 --- core/domain/topic_jobs_one_off_test.py | 413 ------ core/domain/topic_services.py | 3 +- core/domain/topic_services_test.py | 26 +- core/domain/translation_services.py | 4 +- core/domain/translation_services_test.py | 7 +- core/domain/user_domain.py | 12 +- core/domain/user_domain_test.py | 33 +- core/domain/user_jobs_one_off.py | 40 - core/domain/user_jobs_one_off_test.py | 279 ---- core/domain/user_query_domain_test.py | 2 +- core/domain/user_query_jobs_one_off.py | 237 ---- core/domain/user_query_jobs_one_off_test.py | 506 ------- core/domain/user_query_services_test.py | 52 +- core/domain/user_services.py | 34 +- core/domain/user_services_test.py | 23 +- core/domain/value_generators_domain_test.py | 23 +- core/domain/visualization_registry_test.py | 23 +- core/domain/voiceover_services_test.py | 7 +- core/domain/wipeout_service.py | 10 +- core/domain/wipeout_service_test.py | 14 +- core/jobs.py | 1160 ----------------- core/jobs_registry.py | 61 - core/jobs_test.py | 518 -------- .../app_identity/gae_app_identity_services.py | 23 +- .../gae_app_identity_services_test.py | 35 +- .../auth/firebase_auth_services_test.py | 52 +- .../mailchimp_bulk_email_services.py | 7 +- .../mailchimp_bulk_email_services_test.py | 4 +- core/platform/cache/redis_cache_services.py | 32 +- .../cache/redis_cache_services_test.py | 40 +- ...ervices.py => cloud_datastore_services.py} | 170 +-- .../datastore/gae_datastore_services_test.py | 179 --- .../platform/email/dev_mode_email_services.py | 7 +- core/platform/email/mailgun_email_services.py | 18 +- .../email/mailgun_email_services_test.py | 143 +- core/platform/models.py | 100 +- core/platform/models_test.py | 63 +- core/platform/search/gae_search_services.py | 313 ----- .../search/gae_search_services_test.py | 491 ------- .../{cloud_translate => storage}/__init__.py | 0 .../storage/cloud_storage_emulator.py | 265 ++++ .../storage/cloud_storage_emulator_test.py | 186 +++ .../storage/cloud_storage_services.py | 145 +++ .../storage/cloud_storage_services_test.py | 287 ++++ .../storage/dev_mode_storage_services.py | 119 ++ .../storage/dev_mode_storage_services_test.py | 108 ++ .../taskqueue/cloud_taskqueue_services.py | 9 +- .../taskqueue/cloud_tasks_emulator_test.py | 12 +- .../taskqueue/dev_mode_taskqueue_services.py | 4 +- ...vices.py => cloud_transaction_services.py} | 29 +- .../platform/translate}/__init__.py | 0 .../cloud_translate_emulator.py | 0 .../cloud_translate_emulator_test.py | 2 +- .../cloud_translate_services.py | 8 +- .../cloud_translate_services_test.py | 2 +- .../dev_mode_translate_services.py} | 4 +- .../dev_mode_translate_services_test.py} | 14 +- core/storage/activity/gae_models_test.py | 3 +- .../storage/app_feedback_report/gae_models.py | 7 +- .../app_feedback_report/gae_models_test.py | 4 +- core/storage/audit/gae_models.py | 3 +- core/storage/audit/gae_models_test.py | 3 +- core/storage/auth/gae_models.py | 5 +- core/storage/auth/gae_models_test.py | 5 +- core/storage/base_model/gae_models.py | 38 +- core/storage/base_model/gae_models_test.py | 22 +- core/storage/blog/gae_models_test.py | 27 +- core/storage/classifier/gae_models.py | 27 +- core/storage/classifier/gae_models_test.py | 31 +- core/storage/collection/gae_models_test.py | 13 +- core/storage/config/gae_models_test.py | 3 +- core/storage/email/gae_models_test.py | 3 +- core/storage/exploration/gae_models_test.py | 6 +- core/storage/feedback/gae_models.py | 11 +- core/storage/job/gae_models.py | 39 - core/storage/opportunity/gae_models.py | 62 +- core/storage/opportunity/gae_models_test.py | 11 +- core/storage/skill/gae_models.py | 17 +- core/storage/skill/gae_models_test.py | 3 + core/storage/suggestion/gae_models.py | 1 - core/storage/suggestion/gae_models_test.py | 2 +- core/storage/user/gae_models.py | 20 +- .../exploration-creation.service.ts | 5 +- ...editor-navbar-breadcrumb.component.spec.ts | 5 +- .../editor-navbar-breadcrumb.component.ts | 19 +- .../preview-tab/preview-tab.component.spec.ts | 83 +- .../preview-tab/preview-tab.component.ts | 46 +- .../services/exploration-data.service.ts | 1 + .../services/exploration-property.service.ts | 2 +- .../services/parameter-metadata.service.ts | 1 - .../conversation-skin.directive.html | 4 +- .../conversation-skin.directive.ts | 6 +- core/tests/data/compressed_image.jpg | Bin 190429 -> 189969 bytes core/tests/data/failing_tests.py | 2 + core/tests/gae_suite.py | 30 +- core/tests/gae_suite_test.py | 2 +- core/tests/protractor.conf.js | 9 +- .../additionalEditorFeatures.js | 339 ----- .../additionalEditorFeaturesModals.js | 386 ++++++ .../protractor_desktop/emailDashboard.js | 77 -- core/tests/protractor_utils/general.js | 3 +- core/tests/puppeteer/lighthouse_setup.js | 5 +- core/tests/test_utils.py | 390 +++--- core/tests/test_utils_test.py | 83 +- extensions/interactions/base_test.py | 7 +- extensions/objects/models/objects_test.py | 358 +++-- extensions/rich_text_components/components.py | 12 +- .../rich_text_components/components_test.py | 65 +- extensions/visualizations/models.py | 4 +- feconf.py | 19 +- jobs/base_jobs.py | 27 +- jobs/base_jobs_test.py | 4 +- jobs/batch_jobs/validation_jobs.py | 29 +- jobs/blog_validation_errors_test.py | 8 +- jobs/blog_validation_jobs.py | 14 +- jobs/blog_validation_jobs_test.py | 5 +- jobs/decorators/validation_decorators_test.py | 28 +- jobs/io/job_io.py | 38 +- jobs/io/job_io_test.py | 14 +- jobs/io/ndb_io.py | 78 +- jobs/io/ndb_io_test.py | 45 +- jobs/io/stub_io.py | 326 ----- jobs/io/stub_io_test.py | 193 --- jobs/job_options.py | 25 +- jobs/job_options_test.py | 27 +- jobs/job_test_utils.py | 18 +- jobs/job_test_utils_test.py | 7 - jobs/job_utils.py | 201 +-- jobs/job_utils_test.py | 231 +--- jobs/transforms/base_validation.py | 132 +- jobs/transforms/blog_validation.py | 40 +- jobs/transforms/blog_validation_test.py | 77 +- jobs/transforms/skill_validation_test.py | 3 +- jobs/transforms/story_validation_test.py | 25 +- jobs/transforms/subtopic_validation_test.py | 3 +- jobs/types/base_validation_errors_test.py | 2 +- jobs/types/job_run_result_test.py | 8 +- main.py | 164 ++- main_cron.py | 66 - main_taskqueue.py | 56 - main_test.py | 88 ++ manifest.json | 2 +- mypy.ini | 2 +- mypy_imports.py | 37 +- mypy_requirements.txt | 1 + puppeteer-login-script.js | 14 +- python_utils.py | 93 +- python_utils_test.py | 59 +- requirements.in | 28 +- requirements.txt | 423 ++++-- schema_utils.py | 4 +- schema_utils_test.py | 321 +++-- scripts/backend_test_shards.json | 58 +- scripts/build.py | 34 +- scripts/build_test.py | 85 +- scripts/check_e2e_tests_are_captured_in_ci.py | 2 +- ...check_e2e_tests_are_captured_in_ci_test.py | 12 +- scripts/check_frontend_test_coverage.py | 7 +- scripts/check_frontend_test_coverage_test.py | 184 +-- scripts/common.py | 75 +- scripts/common_test.py | 101 +- scripts/concurrent_task_utils.py | 3 +- scripts/concurrent_task_utils_test.py | 16 +- ...e_topological_sort_of_all_services_test.py | 8 +- scripts/flake_checker.py | 18 +- scripts/install_backend_python_libs.py | 26 +- scripts/install_backend_python_libs_test.py | 31 +- scripts/install_chrome_for_ci.py | 3 +- scripts/install_prerequisites.sh | 13 +- scripts/install_third_party.py | 2 +- scripts/install_third_party_libs.py | 66 +- scripts/install_third_party_libs_test.py | 53 +- scripts/install_third_party_test.py | 15 +- scripts/linters/css_linter.py | 6 +- scripts/linters/css_linter_test.py | 5 +- scripts/linters/general_purpose_linter.py | 20 +- scripts/linters/html_linter.py | 4 +- scripts/linters/js_ts_linter.py | 11 +- scripts/linters/js_ts_linter_test.py | 6 +- scripts/linters/linter_utils_test.py | 2 +- scripts/linters/pre_commit_linter.py | 139 +- scripts/linters/pre_commit_linter_test.py | 11 +- scripts/linters/pylint_extensions.py | 117 +- scripts/linters/pylint_extensions_test.py | 93 +- scripts/linters/python_linter.py | 16 +- scripts/linters/python_linter_test.py | 13 +- .../linters/test_files/invalid_docstring.py | 4 +- ..._duplicate_prod_validation_jobs_one_off.py | 57 + .../test_files/invalid_import_order.py | 5 +- scripts/linters/test_files/valid.py | 4 +- scripts/pre_commit_hook.py | 34 +- scripts/pre_commit_hook_test.py | 35 +- scripts/pre_push_hook.py | 22 +- scripts/pre_push_hook_test.py | 89 +- scripts/regenerate_requirements.py | 3 +- .../update_changelog_and_credits.py | 5 +- .../update_changelog_and_credits_test.py | 40 +- scripts/release_scripts/update_configs.py | 64 +- .../release_scripts/update_configs_test.py | 26 +- scripts/run_backend_tests.py | 160 +-- scripts/run_custom_eslint_tests.py | 6 +- scripts/run_e2e_tests.py | 21 +- scripts/run_e2e_tests_test.py | 20 +- scripts/run_frontend_tests.py | 9 +- scripts/run_lighthouse_tests.py | 51 +- scripts/run_mypy_checks.py | 35 +- scripts/run_mypy_checks_test.py | 35 +- scripts/run_portserver.py | 19 +- scripts/script_import_test.py | 16 +- scripts/scripts_test_utils.py | 24 +- scripts/scripts_test_utils_test.py | 28 +- scripts/servers.py | 66 +- scripts/servers_test.py | 125 +- scripts/setup.py | 4 +- scripts/setup_gae.py | 5 +- scripts/setup_test.py | 2 +- scripts/start.py | 8 +- scripts/third_party_size_check.py | 38 +- scripts/typescript_checks.py | 4 +- .../{appengine/ext => cloud}/__init__.pyi | 0 .../{appengine/ext => cloud}/ndb/__init__.pyi | 6 +- .../ext => cloud}/ndb/google_imports.pyi | 0 stubs/google/cloud/storage/__init__.pyi | 1 + stubs/google/cloud/storage/blob.pyi | 9 + stubs/google/cloud/storage/bucket.pyi | 11 + stubs/google/cloud/storage/client.pyi | 11 + .../{appengine/ext => cloud}/vendor.pyi | 0 utils.py | 61 +- utils_test.py | 27 +- 400 files changed, 8106 insertions(+), 18570 deletions(-) create mode 100644 .gcloudignore delete mode 100644 .github/workflows/e2e_email_dashboard_features.yml create mode 100644 .lighthouserc-1.js create mode 100644 .lighthouserc-2.js create mode 100644 .lighthouserc-accessibility-1.js create mode 100644 .lighthouserc-accessibility-2.js delete mode 100644 .lighthouserc-accessibility.js create mode 100644 .lighthouserc-base.js delete mode 100644 .lighthouserc.js delete mode 100644 appengine_config.py delete mode 100644 appengine_config_test.py delete mode 100644 core/domain/activity_jobs_one_off.py delete mode 100644 core/domain/activity_jobs_one_off_test.py delete mode 100644 core/domain/collection_jobs_one_off.py delete mode 100644 core/domain/collection_jobs_one_off_test.py delete mode 100644 core/domain/exp_jobs_one_off.py delete mode 100644 core/domain/exp_jobs_one_off_test.py delete mode 100644 core/domain/question_jobs_one_off.py delete mode 100644 core/domain/question_jobs_one_off_test.py delete mode 100644 core/domain/recommendations_jobs_one_off.py delete mode 100644 core/domain/recommendations_jobs_one_off_test.py delete mode 100644 core/domain/skill_jobs_one_off.py delete mode 100644 core/domain/skill_jobs_one_off_test.py delete mode 100644 core/domain/story_jobs_one_off.py delete mode 100644 core/domain/story_jobs_one_off_test.py delete mode 100644 core/domain/suggestion_jobs_one_off.py delete mode 100644 core/domain/suggestion_jobs_one_off_test.py delete mode 100644 core/domain/topic_jobs_one_off.py delete mode 100644 core/domain/topic_jobs_one_off_test.py delete mode 100644 core/domain/user_jobs_one_off.py delete mode 100644 core/domain/user_jobs_one_off_test.py delete mode 100644 core/domain/user_query_jobs_one_off.py delete mode 100644 core/domain/user_query_jobs_one_off_test.py delete mode 100644 core/jobs.py delete mode 100644 core/jobs_registry.py delete mode 100644 core/jobs_test.py rename core/platform/datastore/{gae_datastore_services.py => cloud_datastore_services.py} (53%) delete mode 100644 core/platform/datastore/gae_datastore_services_test.py delete mode 100644 core/platform/search/gae_search_services.py delete mode 100644 core/platform/search/gae_search_services_test.py rename core/platform/{cloud_translate => storage}/__init__.py (100%) create mode 100644 core/platform/storage/cloud_storage_emulator.py create mode 100644 core/platform/storage/cloud_storage_emulator_test.py create mode 100644 core/platform/storage/cloud_storage_services.py create mode 100644 core/platform/storage/cloud_storage_services_test.py create mode 100644 core/platform/storage/dev_mode_storage_services.py create mode 100644 core/platform/storage/dev_mode_storage_services_test.py rename core/platform/transactions/{gae_transaction_services.py => cloud_transaction_services.py} (68%) rename {proto_files => core/platform/translate}/__init__.py (100%) rename core/platform/{cloud_translate => translate}/cloud_translate_emulator.py (100%) rename core/platform/{cloud_translate => translate}/cloud_translate_emulator_test.py (97%) rename core/platform/{cloud_translate => translate}/cloud_translate_services.py (91%) rename core/platform/{cloud_translate => translate}/cloud_translate_services_test.py (97%) rename core/platform/{cloud_translate/dev_mode_cloud_translate_services.py => translate/dev_mode_translate_services.py} (93%) rename core/platform/{cloud_translate/dev_mode_cloud_translate_services_test.py => translate/dev_mode_translate_services_test.py} (81%) create mode 100644 core/tests/protractor_desktop/additionalEditorFeaturesModals.js delete mode 100644 core/tests/protractor_desktop/emailDashboard.js delete mode 100644 jobs/io/stub_io.py delete mode 100644 jobs/io/stub_io_test.py delete mode 100644 main_cron.py delete mode 100644 main_taskqueue.py create mode 100644 main_test.py create mode 100644 scripts/linters/test_files/invalid_duplicate_prod_validation_jobs_one_off.py rename stubs/google/{appengine/ext => cloud}/__init__.pyi (100%) rename stubs/google/{appengine/ext => cloud}/ndb/__init__.pyi (97%) rename stubs/google/{appengine/ext => cloud}/ndb/google_imports.pyi (100%) create mode 100644 stubs/google/cloud/storage/__init__.pyi create mode 100644 stubs/google/cloud/storage/blob.pyi create mode 100644 stubs/google/cloud/storage/bucket.pyi create mode 100644 stubs/google/cloud/storage/client.pyi rename stubs/google/{appengine/ext => cloud}/vendor.pyi (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb0ee2b9dad1..19dd4d9889c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ workflows: jobs: - setup_and_typescript_tests -var_for_docker_image: &docker_image circleci/python:2.7.17-browsers +var_for_docker_image: &docker_image circleci/python:3.7.10-browsers anchor_for_job_defaults: &job_defaults working_directory: /home/circleci/oppia diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 000000000000..3440e65e4cdb --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,167 @@ +# Various compiled, temporary, and hidden files +*~ +*.pyc +*.pyo +*.swp +*.swo +*.bak +.* +# Typescript files in core +core/**.ts +# Typescript output log file +tsc_output_log.txt +# Python test files +*_test.py +# Other folders to ignore +core/tests/ +node_modules/ +scripts/ +third_party/python3_libs/ +# Some third_party static scripts are directly imported, namely: jquery, +# jqueryui, angularjs, jqueryui-touch-punch, MathJax, code-mirror, +# ui-codemirror, d3js, midi-js, ui-map, guppy, skulpt, math-expressions. +# We exclude some of the scripts that are not directly imported in order to +# reduce deployed file count. +# TODO(sll): Find a more structured way of doing this. +# +# Please do not remove line below this, as it is used for testing purpose. +# Third party files: +third_party/static/angular-audio-1.7.4/ +third_party/static/angular-toastr-1.7.0/ +third_party/static/bootstrap-4.3.1/ +third_party/static/bower-angular-translate-2.18.1/ +third_party/static/bower-angular-translate-interpolation-messageformat-2.18.1/ +third_party/static/bower-angular-translate-loader-partial-2.18.1/ +third_party/static/bower-angular-translate-loader-static-files-2.18.1/ +third_party/static/bower-angular-translate-storage-cookie-2.18.1/ +third_party/static/bower-material-1.1.19/ +third_party/python_libs/google/appengine/ +third_party/python_libs/google/net/ +third_party/python_libs/google/pyglib/ +third_party/python_libs/grpc/ +# CKEditor-4.12.1 plugins in the download from the CKEditor website include +# only a11yhelp, about, clipboard, colordialog, copyformatting, dialog, div, +# find, flash, forms, iframe, image, link, liststyle, magicline, pagebreak, +# pastefromword, preview, scayt, showblocks, smiley, specialchar, table, +# tableselection, tabletools, templates, widget and wsc. Our code is also using +# the sharedspace plugin. So, for now, we exclude all others, as well as flash, +# a11yhelp, about, colordialog, iframe, and anything related to tables, which +# we definitely don't use. +third_party/static/ckeditor-4.12.1/plugins/a11yhelp/ +third_party/static/ckeditor-4.12.1/plugins/about/ +third_party/static/ckeditor-4.12.1/plugins/adobeair/ +third_party/static/ckeditor-4.12.1/plugins/ajax/ +third_party/static/ckeditor-4.12.1/plugins/autocomplete/ +third_party/static/ckeditor-4.12.1/plugins/autoembed/ +third_party/static/ckeditor-4.12.1/plugins/autogrow/ +third_party/static/ckeditor-4.12.1/plugins/autolink/ +third_party/static/ckeditor-4.12.1/plugins/balloonpanel/ +third_party/static/ckeditor-4.12.1/plugins/balloontoolbar/ +third_party/static/ckeditor-4.12.1/plugins/bbcode/ +third_party/static/ckeditor-4.12.1/plugins/bidi/ +third_party/static/ckeditor-4.12.1/plugins/cloudservices/ +third_party/static/ckeditor-4.12.1/plugins/codesnippet/ +third_party/static/ckeditor-4.12.1/plugins/codesnippetgeshi/ +third_party/static/ckeditor-4.12.1/plugins/colorbutton/ +third_party/static/ckeditor-4.12.1/plugins/colordialog/ +third_party/static/ckeditor-4.12.1/plugins/devtools/ +third_party/static/ckeditor-4.12.1/plugins/dialogadvtab/ +third_party/static/ckeditor-4.12.1/plugins/divarea/ +third_party/static/ckeditor-4.12.1/plugins/docprops/ +third_party/static/ckeditor-4.12.1/plugins/easyimage/ +third_party/static/ckeditor-4.12.1/plugins/embed/ +third_party/static/ckeditor-4.12.1/plugins/embedbase/ +third_party/static/ckeditor-4.12.1/plugins/embedsemantic/ +third_party/static/ckeditor-4.12.1/plugins/emoji/ +third_party/static/ckeditor-4.12.1/plugins/flash/ +third_party/static/ckeditor-4.12.1/plugins/font/ +third_party/static/ckeditor-4.12.1/plugins/iframe/ +third_party/static/ckeditor-4.12.1/plugins/iframedialog/ +third_party/static/ckeditor-4.12.1/plugins/image2/ +third_party/static/ckeditor-4.12.1/plugins/imagebase/ +third_party/static/ckeditor-4.12.1/plugins/indentblock/ +third_party/static/ckeditor-4.12.1/plugins/justify/ +third_party/static/ckeditor-4.12.1/plugins/language/ +third_party/static/ckeditor-4.12.1/plugins/mathjax/ +third_party/static/ckeditor-4.12.1/plugins/mentions/ +third_party/static/ckeditor-4.12.1/plugins/newpage/ +third_party/static/ckeditor-4.12.1/plugins/panelbutton/ +third_party/static/ckeditor-4.12.1/plugins/placeholder/ +third_party/static/ckeditor-4.12.1/plugins/print/ +third_party/static/ckeditor-4.12.1/plugins/save/ +third_party/static/ckeditor-4.12.1/plugins/selectall/ +third_party/static/ckeditor-4.12.1/plugins/sourcedialog/ +third_party/static/ckeditor-4.12.1/plugins/stylesheetparser/ +third_party/static/ckeditor-4.12.1/plugins/table/ +third_party/static/ckeditor-4.12.1/plugins/tableresize/ +third_party/static/ckeditor-4.12.1/plugins/tabletools/ +third_party/static/ckeditor-4.12.1/plugins/textmatch/ +third_party/static/ckeditor-4.12.1/plugins/textwatcher/ +third_party/static/ckeditor-4.12.1/plugins/uicolor/ +third_party/static/ckeditor-4.12.1/plugins/uploadfile/ +third_party/static/ckeditor-4.12.1/plugins/xml/ +third_party/static/ckeditor-4.12.1/samples/ +third_party/static/ckeditor-4.12.1/skins/kama/ +third_party/static/ckeditor-4.12.1/skins/moono/ +third_party/static/ckeditor-bootstrapck-1.0.0/core/ +third_party/static/ckeditor-bootstrapck-1.0.0/lang/ +third_party/static/ckeditor-bootstrapck-1.0.0/plugins/ +third_party/static/ckeditor-bootstrapck-1.0.0/skins/bootstrapck-dev/ +third_party/static/ckeditor-bootstrapck-1.0.0/skins/moono/ +third_party/static/ckeditor-bootstrapck-1.0.0/skins/ckbuilder.jar/ +third_party/static/ckeditor-bootstrapck-1.0.0/skins/bootstrapck/sample/ +third_party/static/ckeditor-bootstrapck-1.0.0/skins/bootstrapck/scss/ +third_party/static/fontawesome-free-5.9.0-web/ +third_party/static/guppy-7509f3/site/ +third_party/static/guppy-7509f3/test/ +third_party/static/MathJax-2.7.5/docs/ +third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Gyre-Pagella/ +third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Gyre-Termes/ +third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Latin-Modern/ +third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Neo-Euler/ +third_party/static/MathJax-2.7.5/fonts/HTML-CSS/TeX/png/ +third_party/static/MathJax-2.7.5/localization/ast/ +third_party/static/MathJax-2.7.5/localization/bcc/ +third_party/static/MathJax-2.7.5/localization/bg/ +third_party/static/MathJax-2.7.5/localization/br/ +third_party/static/MathJax-2.7.5/localization/ca/ +third_party/static/MathJax-2.7.5/localization/cdo/ +third_party/static/MathJax-2.7.5/localization/cs/ +third_party/static/MathJax-2.7.5/localization/cy/ +third_party/static/MathJax-2.7.5/localization/da/ +third_party/static/MathJax-2.7.5/localization/de/ +third_party/static/MathJax-2.7.5/localization/eo/ +third_party/static/MathJax-2.7.5/localization/es/ +third_party/static/MathJax-2.7.5/localization/fa/ +third_party/static/MathJax-2.7.5/localization/fi/ +third_party/static/MathJax-2.7.5/localization/fr/ +third_party/static/MathJax-2.7.5/localization/gl/ +third_party/static/MathJax-2.7.5/localization/he/ +third_party/static/MathJax-2.7.5/localization/ia/ +third_party/static/MathJax-2.7.5/localization/it/ +third_party/static/MathJax-2.7.5/localization/ja/ +third_party/static/MathJax-2.7.5/localization/kn/ +third_party/static/MathJax-2.7.5/localization/ko/ +third_party/static/MathJax-2.7.5/localization/lb/ +third_party/static/MathJax-2.7.5/localization/lki/ +third_party/static/MathJax-2.7.5/localization/lt/ +third_party/static/MathJax-2.7.5/localization/mk/ +third_party/static/MathJax-2.7.5/localization/nl/ +third_party/static/MathJax-2.7.5/localization/oc/ +third_party/static/MathJax-2.7.5/localization/pl/ +third_party/static/MathJax-2.7.5/localization/pt/ +third_party/static/MathJax-2.7.5/localization/pt-br/ +third_party/static/MathJax-2.7.5/localization/qqq/ +third_party/static/MathJax-2.7.5/localization/ru/ +third_party/static/MathJax-2.7.5/localization/scn/ +third_party/static/MathJax-2.7.5/localization/sco/ +third_party/static/MathJax-2.7.5/localization/sl/ +third_party/static/MathJax-2.7.5/localization/sv/ +third_party/static/MathJax-2.7.5/localization/tr/ +third_party/static/MathJax-2.7.5/localization/uk/ +third_party/static/MathJax-2.7.5/localization/vi/ +third_party/static/MathJax-2.7.5/localization/zh-hans/ +third_party/static/MathJax-2.7.5/test/ +third_party/static/MathJax-2.7.5/unpacked/ +third_party/static/messageformat-2.0.5/ +third_party/static/popperJs-1.15.0/ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 752a0cc429ec..eec313e0faa5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -237,11 +237,9 @@ # Google-app-engine configurations -/appengine_config*.py @nithusha21 /cron.yaml @nithusha21 /index.yaml @vojtechjelinek /queue.yaml @nithusha21 -/main_taskqueue.py @nithusha21 # Global components and filters. @@ -460,8 +458,7 @@ /core/templates/domain/jobs/ @brianrodri /core/controllers/cron*.py @nithusha21 /core/domain/cron_services*.py @nithusha21 -/main_cron.py @nithusha21 -/main.py @DubeySandeep @nithusha21 +/main*.py @DubeySandeep @nithusha21 /feconf.py @DubeySandeep @nithusha21 @seanlip /constants*.py @nithusha21 /assets/constants.ts @nithusha21 @@ -469,7 +466,6 @@ /core/domain/beam_job*.py @brianrodri /core/domain/email*.py @aks681 /core/domain/image_service*.py @DubeySandeep -/core/jobs*.py @brianrodri /core/platform/ @brianrodri /core/templates/App*.ts @srijanreddy98 /core/templates/app.constants.ts @srijanreddy98 @@ -526,6 +522,7 @@ # Release team. +/.gcloudignore @nithusha21 /core/controllers/release_coordinator*.py @nithusha21 @vojtechjelinek /core/templates/pages/release-coordinator-page/ @nithusha21 @vojtechjelinek /scripts/release_scripts/ @nithusha21 @vojtechjelinek @@ -602,8 +599,7 @@ /core/templates/pages/header_js_libs.html @vojtechjelinek /core/templates/services/csrf-token.service*.ts @vojtechjelinek /core/templates/third-party-imports/ @nithusha21 @vojtechjelinek -/.lighthouserc.js @jimbyo @vojtechjelinek -/.lighthouserc-accessibility.js @jimbyo @vojtechjelinek +/.lighthouserc*.js @jimbyo @vojtechjelinek /puppeteer-login-script.js @jimbyo @vojtechjelinek /scripts/run_lighthouse_tests.py @vojtechjelinek /webpack.*.ts @nithusha21 @vojtechjelinek diff --git a/.github/workflows/backend_tests.yml b/.github/workflows/backend_tests.yml index 04c8219b383f..458a8ad36fe5 100644 --- a/.github/workflows/backend_tests.yml +++ b/.github/workflows/backend_tests.yml @@ -21,17 +21,11 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: pip install wheel==0.35.0 - name: Install third party if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: python -m scripts.install_third_party_libs - - name: Install packages for optimized coverage - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: sudo apt-get install -y python-dev gcc - name: Install coverage and configparser if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: pip install coverage configparser @@ -52,11 +46,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: pip install wheel==0.35.0 # We need this so that we can check coverage of autogenerated code - name: Install third party if: startsWith(github.head_ref, 'update-changelog-for-release') == false diff --git a/.github/workflows/e2e_additional_editor_and_player.yml b/.github/workflows/e2e_additional_editor_and_player.yml index 43974b2f085f..2335b6a77404 100644 --- a/.github/workflows/e2e_additional_editor_and_player.yml +++ b/.github/workflows/e2e_additional_editor_and_player.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs @@ -40,6 +38,11 @@ jobs: run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --suite="additionalEditorFeatures" --prod_env env: VIDEO_RECORDING_IS_ENABLED: 0 + - name: Run Additional Editor Modals E2E Test + if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} + run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-build --skip-install --suite="additionalEditorFeaturesModals" --prod_env + env: + VIDEO_RECORDING_IS_ENABLED: 0 - name: Run Additional Player E2E Test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-build --skip-install --suite="additionalPlayerFeatures" --prod_env diff --git a/.github/workflows/e2e_creator_learner_dashboard_and_editor_tabs.yml b/.github/workflows/e2e_creator_learner_dashboard_and_editor_tabs.yml index db582a99ab03..c94c8def4ab6 100644 --- a/.github/workflows/e2e_creator_learner_dashboard_and_editor_tabs.yml +++ b/.github/workflows/e2e_creator_learner_dashboard_and_editor_tabs.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/e2e_email_dashboard_features.yml b/.github/workflows/e2e_email_dashboard_features.yml deleted file mode 100644 index c75413f12f12..000000000000 --- a/.github/workflows/e2e_email_dashboard_features.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: End-to-End tests -on: - push: - branches: - - develop - - release-* - pull_request: - branches: - - develop - - release-* - -jobs: - e2e_email_dashboard_features: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-18.04] - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '2.x' - architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - - uses: ./.github/actions/merge - - name: Install third party - run: python -m scripts.install_third_party_libs - - id: check-risk - name: Check if PR is low-risk - uses: ./.github/actions/check-if-pr-is-low-risk - - name: Install chrome - if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} - run: python -m scripts.install_chrome_for_ci - - name: Install ffmpeg - if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} - run: sudo apt install ffmpeg - - name: Run Email Dashboard E2E Test - if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} - run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --suite="emailDashboard" --prod_env - env: - VIDEO_RECORDING_IS_ENABLED: 0 - - name: Uploading protractor-video as Artifacts - if: ${{ steps.check-risk.outputs.is-low-risk != 0 && always() }} - uses: actions/upload-artifact@v2 - with: - name: protractor-video - path: /home/runner/work/oppia/protractor-video diff --git a/.github/workflows/e2e_history_statistics_tabs_and_extensions.yml b/.github/workflows/e2e_history_statistics_tabs_and_extensions.yml index fc1115ba2ea9..f44a18e21cd3 100644 --- a/.github/workflows/e2e_history_statistics_tabs_and_extensions.yml +++ b/.github/workflows/e2e_history_statistics_tabs_and_extensions.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/e2e_learner_flow_skill_editor_and_embedding.yml b/.github/workflows/e2e_learner_flow_skill_editor_and_embedding.yml index d4a83c8b4231..4a4d9d7d3ae6 100644 --- a/.github/workflows/e2e_learner_flow_skill_editor_and_embedding.yml +++ b/.github/workflows/e2e_learner_flow_skill_editor_and_embedding.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/e2e_miscellaneous_tests.yml b/.github/workflows/e2e_miscellaneous_tests.yml index 867f191a7697..55e805524dfb 100644 --- a/.github/workflows/e2e_miscellaneous_tests.yml +++ b/.github/workflows/e2e_miscellaneous_tests.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/e2e_other_tests.yml b/.github/workflows/e2e_other_tests.yml index f227e599b80c..464a5de07b25 100644 --- a/.github/workflows/e2e_other_tests.yml +++ b/.github/workflows/e2e_other_tests.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/e2e_topic_tests.yml b/.github/workflows/e2e_topic_tests.yml index b34521b83c4a..bb7001262f76 100644 --- a/.github/workflows/e2e_topic_tests.yml +++ b/.github/workflows/e2e_topic_tests.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - name: Install third party run: python -m scripts.install_third_party_libs - id: check-risk @@ -37,22 +35,22 @@ jobs: - name: Run Topics and Skills Dashboard E2E Test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --suite="topicsAndSkillsDashboard" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Run Topic and Story Editor E2E Test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-build --skip-install --suite="topicAndStoryEditor" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Run Topic and Story Editor File Upload E2E Test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-build --skip-install --suite="topicAndStoryEditorFileUploadFeatures" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Run Topic and Story Viewer E2E Test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-build --skip-install --suite="topicAndStoryViewer" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Uploading protractor-video as Artifacts if: ${{ steps.check-risk.outputs.is-low-risk != 0 && always() }} diff --git a/.github/workflows/e2e_translation_classroom_and_core_features.yml b/.github/workflows/e2e_translation_classroom_and_core_features.yml index 2474dc0c9dfa..c9694f69de7b 100644 --- a/.github/workflows/e2e_translation_classroom_and_core_features.yml +++ b/.github/workflows/e2e_translation_classroom_and_core_features.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/e2e_user_features_and_library.yml b/.github/workflows/e2e_user_features_and_library.yml index c8a5d0912de3..359fea568ebb 100644 --- a/.github/workflows/e2e_user_features_and_library.yml +++ b/.github/workflows/e2e_user_features_and_library.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs @@ -38,22 +36,22 @@ jobs: - name: Run Library E2E Test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --suite="library" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Run e2e preferences test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-install --skip-build --suite="preferences" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Run e2e subscriptions test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-install --skip-build --suite="subscriptions" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Run e2e wipeout test if: ${{ steps.check-risk.outputs.is-low-risk != 0 }} run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" python -m scripts.run_e2e_tests --skip-install --skip-build --suite="wipeout" --prod_env - env: + env: VIDEO_RECORDING_IS_ENABLED: 0 - name: Uploading protractor-video as Artifacts if: ${{ steps.check-risk.outputs.is-low-risk != 0 && always() }} diff --git a/.github/workflows/e2e_user_profile.yml b/.github/workflows/e2e_user_profile.yml index 455fd33289c7..2095c87aaba1 100644 --- a/.github/workflows/e2e_user_profile.yml +++ b/.github/workflows/e2e_user_profile.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install third party run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/eslint_tests.yml b/.github/workflows/eslint_tests.yml index 6970c6528500..a4c8f5d4b93c 100644 --- a/.github/workflows/eslint_tests.yml +++ b/.github/workflows/eslint_tests.yml @@ -20,11 +20,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: pip install wheel==0.35.0 - name: Install Third Party Dependencies if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index d30674f56034..9f8e57d6a269 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -19,10 +19,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - run: pip install wheel==0.35.0 - name: Install third party run: python -m scripts.install_third_party_libs - name: Suppress ENOSPC error from chokidar file watcher. See https://stackoverflow.com/a/32600959. diff --git a/.github/workflows/lighthouse_accessibility.yml b/.github/workflows/lighthouse_accessibility.yml index f9baef935e75..0cd217018e14 100644 --- a/.github/workflows/lighthouse_accessibility.yml +++ b/.github/workflows/lighthouse_accessibility.yml @@ -15,21 +15,19 @@ jobs: strategy: matrix: os: [ubuntu-18.04] + shard: [1, 2] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: pip install wheel==0.35.0 - name: Install Third Party Dependencies if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: python -m scripts.install_third_party_libs - name: Install chrome if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: python -m scripts.install_chrome_for_ci - - name: Run lighthouse checks + - name: Run Lighthouse accessibility checks shard if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: python -m scripts.run_lighthouse_tests --mode accessibility + run: python -m scripts.run_lighthouse_tests --mode accessibility --shard ${{ matrix.shard }} diff --git a/.github/workflows/lighthouse_performance.yml b/.github/workflows/lighthouse_performance.yml index 38bf5ed9482b..79b711a43952 100644 --- a/.github/workflows/lighthouse_performance.yml +++ b/.github/workflows/lighthouse_performance.yml @@ -15,21 +15,19 @@ jobs: strategy: matrix: os: [ubuntu-18.04] + shard: [1, 2] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v1 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: pip install wheel==0.35.0 - name: Install Third Party Dependencies if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: python -m scripts.install_third_party_libs - name: Install chrome if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: python -m scripts.install_chrome_for_ci - - name: Run lighthouse checks + - name: Run Lighthouse performance checks shard if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: python -m scripts.run_lighthouse_tests --mode performance + run: python -m scripts.run_lighthouse_tests --mode performance --shard ${{ matrix.shard }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1ed20228c66e..91cb28f4d126 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,11 +20,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: pip install wheel==0.35.0 - name: Install Third Party Dependencies if: startsWith(github.head_ref, 'update-changelog-for-release') == false run: python -m scripts.install_third_party_libs diff --git a/.github/workflows/python_type_checks.yml b/.github/workflows/python_type_checks.yml index fd8fe6a24b7a..be3d0e702ad0 100644 --- a/.github/workflows/python_type_checks.yml +++ b/.github/workflows/python_type_checks.yml @@ -20,11 +20,8 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '2.x' + python-version: '3.7' architecture: 'x64' - - name: Setup python by installing wheel - if: startsWith(github.head_ref, 'update-changelog-for-release') == false - run: pip install wheel==0.35.0 - uses: ./.github/actions/merge - name: Install Third Party Dependencies if: startsWith(github.head_ref, 'update-changelog-for-release') == false diff --git a/.lighthouserc-1.js b/.lighthouserc-1.js new file mode 100644 index 000000000000..fd2f83102f63 --- /dev/null +++ b/.lighthouserc-1.js @@ -0,0 +1,123 @@ +// Copyright 2020 The Oppia Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS-IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @fileoverview Configuration for lighthouse-ci. + */ + +const baseConfig = require('./.lighthouserc-base.js') + +module.exports = { + 'ci': { + 'collect': { + 'numberOfRuns': baseConfig['numberOfRuns'], + 'puppeteerScript': baseConfig['puppeteerScript'], + 'url': baseConfig['urlShards'][1] + }, + 'assert': { + 'assertMatrix': [ + baseConfig['basePerformanceAssertMatrix'], + { + 'matchingUrlPattern': 'http://[^/]+/$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/admin$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/about$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/community-library$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/contact$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/contributor-dashboard$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/creator-dashboard$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/creator-guidelines$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/delete-account$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/donate$', + 'assertions': { + // The YouTube embed on donate page loads images in jpg format, thus + // we need to allow one image. + 'uses-webp-images': [ + 'error', {'maxLength': 1, 'strategy': 'pessimistic'} + ], + // The YouTube embed on donate page uses passive listeners. + 'uses-passive-event-listeners': ['error', {'minScore': 0}], + 'deprecations': ['error', {'minScore': 1}] + } + }, + { + 'matchingUrlPattern': 'http://[^/]+/emaildashboard$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/get-started$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/learner-dashboard$', + 'assertions': { + 'uses-webp-images': [ + 'error', {'maxLength': 0, 'strategy': 'pessimistic'} + ], + // We need to use passive event listeners on this page so that + // the page works correctly. + 'uses-passive-event-listeners': ['error', {'minScore': 0}], + 'deprecations': ['error', {'minScore': 1}], + 'redirects': ['error', {'minScore': 0.5}] + } + }, + { + 'matchingUrlPattern': 'http://[^/]+/license$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/nonprofits$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/moderator$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/parents$', + 'assertions': baseConfig['basePerformanceAssertions'] + } + ] + }, + 'upload': { + 'target': 'temporary-public-storage' + } + } +}; diff --git a/.lighthouserc-2.js b/.lighthouserc-2.js new file mode 100644 index 000000000000..199332397f18 --- /dev/null +++ b/.lighthouserc-2.js @@ -0,0 +1,111 @@ +// Copyright 2020 The Oppia Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS-IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @fileoverview Configuration for lighthouse-ci. + */ + +const baseConfig = require('./.lighthouserc-base.js') + +module.exports = { + 'ci': { + 'collect': { + 'numberOfRuns': baseConfig['numberOfRuns'], + 'puppeteerScript': baseConfig['puppeteerScript'], + 'url': baseConfig['urlShards'][2] + }, + 'assert': { + 'assertMatrix': [ + baseConfig['basePerformanceAssertMatrix'], + { + 'matchingUrlPattern': 'http://[^/]+/partners$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/preferences$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/privacy-policy$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/profile/username1$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/signup?return_url=%2F$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/teach$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/teachers$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/terms$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/thanks$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/volunteers$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/topics-and-skills-dashboard$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/create/.*$', + 'assertions': { + // TODO(#13465): Change this maxLength to 0 once images are migrated. + 'uses-webp-images': [ + 'error', {'maxLength': 3, 'strategy': 'pessimistic'} + ], + // We need to use passive event listeners on this page so that + // the page works correctly. + 'uses-passive-event-listeners': ['error', {'minScore': 0}], + // MIDI library uses some deprecated API. + 'deprecations': ['error', {'minScore': 0}] + } + }, + { + 'matchingUrlPattern': 'http://[^/]+/collection_editor/create/.*$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/topic_editor/.*$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': 'http://[^/]+/skill_editor/.*$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + { + 'matchingUrlPattern': '^http://[^/]+/story_editor/.*$', + 'assertions': baseConfig['basePerformanceAssertions'] + }, + ] + }, + 'upload': { + 'target': 'temporary-public-storage' + } + } +}; diff --git a/.lighthouserc-accessibility-1.js b/.lighthouserc-accessibility-1.js new file mode 100644 index 000000000000..664c98ae43cd --- /dev/null +++ b/.lighthouserc-accessibility-1.js @@ -0,0 +1,100 @@ +// Copyright 2020 The Oppia Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS-IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @fileoverview Configuration for accessibility. + */ + +const baseConfig = require('./.lighthouserc-base.js') + +module.exports = { + 'ci': { + 'collect': { + 'numberOfRuns': baseConfig['numberOfRuns'], + 'puppeteerScript': baseConfig['puppeteerScript'], + 'url': baseConfig['urlShards'][1] + }, + 'assert': { + 'assertMatrix': [ + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/about$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/admin$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/community-library$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/contact$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/contributor-dashboard$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/creator-dashboard$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/delete-account$', + 'assertions': { + 'categories:accessibility': ['error', {'minScore': 0.99}] + } + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/donate$', + 'assertions': { + 'categories:accessibility': ['error', {'minScore': 0.98}] + } + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/emailDashboard$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/get-started$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/learner-dashboard$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/nonprofits$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/moderator$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/parents$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + } + ] + }, + 'upload': { + 'target': 'temporary-public-storage' + } + } +}; diff --git a/.lighthouserc-accessibility-2.js b/.lighthouserc-accessibility-2.js new file mode 100644 index 000000000000..4fd70ad41850 --- /dev/null +++ b/.lighthouserc-accessibility-2.js @@ -0,0 +1,108 @@ +// Copyright 2020 The Oppia Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS-IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @fileoverview Configuration for accessibility. + */ + +const baseConfig = require('./.lighthouserc-base.js') + +module.exports = { + 'ci': { + 'collect': { + 'numberOfRuns': baseConfig['numberOfRuns'], + 'puppeteerScript': baseConfig['puppeteerScript'], + 'url': baseConfig['urlShards'][2] + }, + 'assert': { + 'assertMatrix': [ + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/partners$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/preferences$', + 'assertions': { + 'categories:accessibility': ['error', {'minScore': 0.85}] + } + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/privacy-policy$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/profile/username1$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/signup?return_url=%2F$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/teach$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/teachers$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/topics-and-skills-dashboard$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/terms$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/thanks$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/volunteers$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/create/.*$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/collection_editor/create/.*$', + 'assertions': { + 'categories:accessibility': ['error', {'minScore': 0.86}] + } + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/topic_editor/.*$', + 'assertions': { + 'categories:accessibility': ['error', {'minScore': 0.86}] + } + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/skill_editor/.*$', + 'assertions': baseConfig['baseAccessibilityAssertions'] + }, + { + 'matchingUrlPattern': '^http://127.0.0.1:8181/story_editor/.*$', + 'assertions': { + 'categories:accessibility': ['error', {'minScore': 0.86}] + } + }, + ] + }, + 'upload': { + 'target': 'temporary-public-storage' + } + } +}; diff --git a/.lighthouserc-accessibility.js b/.lighthouserc-accessibility.js deleted file mode 100644 index bdb6d59e48c2..000000000000 --- a/.lighthouserc-accessibility.js +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright 2020 The Oppia Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS-IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * @fileoverview Configuration for accessibility. - */ - -module.exports = { - 'ci': { - 'collect': { - 'numberOfRuns': 3, - 'puppeteerScript': 'puppeteer-login-script.js', - 'url': [ - 'http://127.0.0.1:8181/', - 'http://127.0.0.1:8181/about', - 'http://127.0.0.1:8181/admin', - 'http://127.0.0.1:8181/community-library', - 'http://127.0.0.1:8181/contact', - 'http://127.0.0.1:8181/contributor-dashboard', - 'http://127.0.0.1:8181/creator-dashboard', - 'http://127.0.0.1:8181/creator-guidelines', - 'http://127.0.0.1:8181/delete-account', - 'http://127.0.0.1:8181/donate', - "http://127.0.0.1:8181/emaildashboard", - 'http://127.0.0.1:8181/get-started', - 'http://127.0.0.1:8181/learner-dashboard', - 'http://127.0.0.1:8181/license', - 'http://127.0.0.1:8181/nonprofits', - "http://127.0.0.1:8181/moderator", - 'http://127.0.0.1:8181/parents', - 'http://127.0.0.1:8181/partners', - 'http://127.0.0.1:8181/preferences', - 'http://127.0.0.1:8181/privacy-policy', - 'http://127.0.0.1:8181/profile/username1', - 'http://127.0.0.1:8181/signup?return_url=%2F', - 'http://127.0.0.1:8181/teach', - 'http://127.0.0.1:8181/teachers', - 'http://127.0.0.1:8181/topics-and-skills-dashboard', - 'http://127.0.0.1:8181/terms', - 'http://127.0.0.1:8181/thanks', - 'http://127.0.0.1:8181/volunteers', - `http://127.0.0.1:8181/create/${process.env.exploration_id}`, - `http://127.0.0.1:8181/collection_editor/create/${process.env.collection_id}`, - `http://127.0.0.1:8181/topic_editor/${process.env.topic_id}`, - `http://127.0.0.1:8181/skill_editor/${process.env.skill_id}`, - `http://127.0.0.1:8181/story_editor/${process.env.story_id}`, - ] - }, - 'assert': { - 'assertMatrix': [ - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/about$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/admin$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/community-library$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/contact$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/contributor-dashboard$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/creator-dashboard$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/delete-account$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 0.99}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/donate$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 0.98}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/emailDashboard$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/get-started$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/learner-dashboard$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/nonprofits$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/moderator$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/parents$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/partners$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/preferences$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 0.85}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/privacy-policy$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/profile/username1$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/signup?return_url=%2F$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/teach$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/teachers$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/topics-and-skills-dashboard$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/terms$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/thanks$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/volunteers$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/create/.*$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/collection_editor/create/.*$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 0.86}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/topic_editor/.*$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 0.86}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/skill_editor/.*$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': '^http://127.0.0.1:8181/story_editor/.*$', - 'assertions': { - 'categories:accessibility': ['error', {'minScore': 0.86}] - } - }, - ] - }, - 'upload': { - 'target': 'temporary-public-storage' - } - } -}; diff --git a/.lighthouserc-base.js b/.lighthouserc-base.js new file mode 100644 index 000000000000..6ec388ae78c9 --- /dev/null +++ b/.lighthouserc-base.js @@ -0,0 +1,105 @@ +// Copyright 2020 The Oppia Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS-IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @fileoverview Configuration for lighthouse-ci. + */ + +module.exports = { + numberOfRuns: 3, + puppeteerScript: 'puppeteer-login-script.js', + urlShards: { + 1: [ + 'http://127.0.0.1:8181/', + 'http://127.0.0.1:8181/about', + 'http://127.0.0.1:8181/admin', + 'http://127.0.0.1:8181/community-library', + 'http://127.0.0.1:8181/contact', + 'http://127.0.0.1:8181/contributor-dashboard', + 'http://127.0.0.1:8181/creator-dashboard', + 'http://127.0.0.1:8181/creator-guidelines', + 'http://127.0.0.1:8181/delete-account', + 'http://127.0.0.1:8181/donate', + 'http://127.0.0.1:8181/emaildashboard', + 'http://127.0.0.1:8181/get-started', + 'http://127.0.0.1:8181/learner-dashboard', + 'http://127.0.0.1:8181/license', + 'http://127.0.0.1:8181/nonprofits', + 'http://127.0.0.1:8181/moderator', + 'http://127.0.0.1:8181/parents', + ], + 2: [ + 'http://127.0.0.1:8181/partners', + 'http://127.0.0.1:8181/preferences', + 'http://127.0.0.1:8181/privacy-policy', + 'http://127.0.0.1:8181/profile/username1', + 'http://127.0.0.1:8181/signup?return_url=%2F', + 'http://127.0.0.1:8181/teach', + 'http://127.0.0.1:8181/teachers', + 'http://127.0.0.1:8181/topics-and-skills-dashboard', + 'http://127.0.0.1:8181/terms', + 'http://127.0.0.1:8181/thanks', + 'http://127.0.0.1:8181/volunteers', + `http://127.0.0.1:8181/create/${process.env.exploration_id}`, + `http://127.0.0.1:8181/collection_editor/create/${process.env.collection_id}`, + `http://127.0.0.1:8181/topic_editor/${process.env.topic_id}`, + `http://127.0.0.1:8181/skill_editor/${process.env.skill_id}`, + `http://127.0.0.1:8181/story_editor/${process.env.story_id}` + ] + }, + basePerformanceAssertMatrix: { + 'matchingUrlPattern': '.*', + 'assertions': { + // Performance category. + 'first-contentful-paint': [ 'warn', {'maxNumericValue': 1230000}], + 'first-meaningful-paint': ['warn', {'maxNumericValue': 1280000}], + 'first-cpu-idle': ['warn', {'maxNumericValue': 1460000}], + 'speed-index': ['warn', {'maxNumericValue': 1230000}], + 'interactive': ['warn', {'maxNumericValue': 1540000}], + 'max-potential-fid': ['warn', {'maxNumericValue': 130000}], + 'uses-responsive-images': ['error', {'minScore': 1}], + 'uses-optimized-images': ['error', {'minScore': 1}], + 'uses-rel-preconnect': ['error', {'minScore': 0.5}], + 'uses-rel-preload': ['error', {'minScore': 1}], + 'efficient-animated-content': ['error',{'minScore': 1}], + 'offscreen-images': ['error', {'minScore': 0.45}], + 'time-to-first-byte': ['off', {}], + // Best practices category. + 'appcache-manifest': ['error', {'minScore': 1}], + 'errors-in-console': ['error', {'minScore': 1}], + 'no-document-write': ['error', {'minScore': 1}], + 'external-anchors-use-rel-noopener': ['error', {'minScore': 1}], + 'geolocation-on-start': ['error', {'minScore': 1}], + 'doctype': ['error', {'minScore': 1}], + 'no-vulnerable-libraries': ['off', {'minScore': 1}], + 'js-libraries': ['error', {'minScore': 1}], + 'notification-on-start': ['error', {'minScore': 1}], + 'password-inputs-can-be-pasted-into': ['error', {'minScore': 1}], + 'image-aspect-ratio': ['error', {'minScore': 1}], + 'is-on-https': ['off', {}], + 'uses-http2': ['off', {}], + } + }, + basePerformanceAssertions: { + 'uses-webp-images': [ + 'error', {'maxLength': 0, 'strategy': 'pessimistic'} + ], + 'uses-passive-event-listeners': ['error', {'minScore': 1}], + 'deprecations': ['error', {'minScore': 1}], + 'redirects': ['error', {'minScore': 1}] + }, + baseAccessibilityAssertions: { + 'categories:accessibility': ['error', {'minScore': 1}] + } +}; diff --git a/.lighthouserc.js b/.lighthouserc.js deleted file mode 100644 index 51e7390e9c31..000000000000 --- a/.lighthouserc.js +++ /dev/null @@ -1,429 +0,0 @@ -// Copyright 2020 The Oppia Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS-IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/** - * @fileoverview Configuration for lighthouse-ci. - */ - -module.exports = { - 'ci': { - 'collect': { - 'numberOfRuns': 3, - 'puppeteerScript': 'puppeteer-login-script.js', - 'url': [ - 'http://127.0.0.1:8181/', - 'http://127.0.0.1:8181/about', - 'http://127.0.0.1:8181/admin', - 'http://127.0.0.1:8181/community-library', - 'http://127.0.0.1:8181/contact', - 'http://127.0.0.1:8181/contributor-dashboard', - 'http://127.0.0.1:8181/creator-dashboard', - 'http://127.0.0.1:8181/creator-guidelines', - 'http://127.0.0.1:8181/delete-account', - 'http://127.0.0.1:8181/donate', - 'http://127.0.0.1:8181/emaildashboard', - 'http://127.0.0.1:8181/get-started', - 'http://127.0.0.1:8181/learner-dashboard', - 'http://127.0.0.1:8181/license', - 'http://127.0.0.1:8181/nonprofits', - 'http://127.0.0.1:8181/moderator', - 'http://127.0.0.1:8181/parents', - 'http://127.0.0.1:8181/partners', - 'http://127.0.0.1:8181/preferences', - 'http://127.0.0.1:8181/privacy-policy', - 'http://127.0.0.1:8181/profile/username1', - 'http://127.0.0.1:8181/signup?return_url=%2F', - 'http://127.0.0.1:8181/teach', - 'http://127.0.0.1:8181/teachers', - 'http://127.0.0.1:8181/topics-and-skills-dashboard', - 'http://127.0.0.1:8181/terms', - 'http://127.0.0.1:8181/thanks', - 'http://127.0.0.1:8181/volunteers', - `http://127.0.0.1:8181/create/${process.env.exploration_id}`, - `http://127.0.0.1:8181/collection_editor/create/${process.env.collection_id}`, - `http://127.0.0.1:8181/topic_editor/${process.env.topic_id}`, - `http://127.0.0.1:8181/skill_editor/${process.env.skill_id}`, - ] - }, - 'assert': { - 'assertMatrix': [ - { - 'matchingUrlPattern': '.*', - 'assertions': { - // Performance category. - 'first-contentful-paint': [ 'warn', {'maxNumericValue': 1230000}], - 'first-meaningful-paint': ['warn', {'maxNumericValue': 1280000}], - 'first-cpu-idle': ['warn', {'maxNumericValue': 1460000}], - 'speed-index': ['warn', {'maxNumericValue': 1230000}], - 'interactive': ['warn', {'maxNumericValue': 1540000}], - 'max-potential-fid': ['warn', {'maxNumericValue': 130000}], - 'uses-responsive-images': ['error', {'minScore': 1}], - 'uses-optimized-images': ['error', {'minScore': 1}], - 'uses-rel-preconnect': ['error', {'minScore': 0.5}], - 'redirects': ['error', {'minScore': 1}], - 'uses-rel-preload': ['error', {'minScore': 1}], - 'efficient-animated-content': ['error',{'minScore': 1}], - 'offscreen-images': ['error', {'minScore': 0.45}], - 'time-to-first-byte': ['off', {}], - // Best practices category. - 'appcache-manifest': ['error', {'minScore': 1}], - 'errors-in-console': ['error', {'minScore': 1}], - 'no-document-write': ['error', {'minScore': 1}], - 'external-anchors-use-rel-noopener': ['error', {'minScore': 1}], - 'geolocation-on-start': ['error', {'minScore': 1}], - 'doctype': ['error', {'minScore': 1}], - 'no-vulnerable-libraries': ['off', {'minScore': 1}], - 'js-libraries': ['error', {'minScore': 1}], - 'notification-on-start': ['error', {'minScore': 1}], - 'password-inputs-can-be-pasted-into': ['error', {'minScore': 1}], - 'image-aspect-ratio': ['error', {'minScore': 1}], - 'is-on-https': ['off', {}], - 'uses-http2': ['off', {}], - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/admin$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/about$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/community-library$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/contact$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/contributor-dashboard$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/creator-dashboard$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/creator-guidelines$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/delete-account$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/donate$', - 'assertions': { - // The YouTube embed on donate page loads images in jpg format, thus - // we need to allow one image. - 'uses-webp-images': [ - 'error', {'maxLength': 1, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/emaildashboard$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/get-started$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/learner-dashboard$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - // We need to use passive event listeners on this page so that - // the page works correctly. - 'uses-passive-event-listeners': ['error', {'minScore': 0}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/license$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/nonprofits$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/moderator$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/parents$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/partners$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/preferences$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/privacy-policy$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/profile/username1$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/signup?return_url=%2F$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/teach$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/teachers$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/terms$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/thanks$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/volunteers$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/topics-and-skills-dashboard$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/create/.*$', - 'assertions': { - // TODO(#13465): Change this maxLength to 0 once images are migrated. - 'uses-webp-images': [ - 'error', {'maxLength': 3, 'strategy': 'pessimistic'} - ], - // We need to use passive event listeners on this page so that - // the page works correctly. - 'uses-passive-event-listeners': ['error', {'minScore': 0}], - // MIDI library uses some deprecated API. - 'deprecations': ['error', {'minScore': 0}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/collection_editor/create/.*$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/topic_editor/.*$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - { - 'matchingUrlPattern': 'http://[^/]+/skill_editor/.*$', - 'assertions': { - 'uses-webp-images': [ - 'error', {'maxLength': 0, 'strategy': 'pessimistic'} - ], - 'uses-passive-event-listeners': ['error', {'minScore': 1}], - 'deprecations': ['error', {'minScore': 1}] - } - }, - ] - }, - 'upload': { - 'target': 'temporary-public-storage' - } - } -}; diff --git a/.pylintrc b/.pylintrc index 8da2d38e0b76..a99f8b6d288b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -27,7 +27,7 @@ method-rgx=^([_a-z][a-z0-9_]*|__[a-z0-9]+__)$ const-rgx=^(([A-Z_][A-Z0-9_]*)|(__.*__)|([a-z_]+_models)|([a-z_]+_services))$ # Good variable names which should always be accepted, separated by a comma -good-names=e,_,d,f,i,l,p,w,fn,fs,id,pc,sc,zf,setUp,tearDown,longMessage,maxDiff +good-names=e,_,d,f,i,l,p,w,fn,fs,id,pc,sc,zf,setUp,tearDown,longMessage,maxDiff,T # Regex for dummy variables (to prevent 'unused argument' errors) dummy-variables-rgx=_|unused_* @@ -107,7 +107,52 @@ ignore-imports=yes [MESSAGES CONTROL] -disable=consider-using-ternary,locally-disabled,locally-enabled,logging-not-lazy,abstract-method,arguments-differ,broad-except,duplicate-code,fixme,inconsistent-return-statements,len-as-condition,missing-raises-doc,multiple-constructor-doc,no-else-return,no-member,no-self-use,not-context-manager,redefined-variable-type,redundant-returns-doc,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-public-methods,too-many-nested-blocks,too-many-statements,wrong-import-order +disable=abstract-method, + arguments-differ, + broad-except, + consider-using-ternary, + duplicate-code, + fixme, + inconsistent-return-statements, + len-as-condition, + locally-disabled, + locally-enabled, + logging-not-lazy, + multiple-constructor-doc, + no-else-return, + no-member, + no-self-use, + not-context-manager, + redefined-variable-type, + redundant-returns-doc, + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-statements, + wrong-import-order, +# TODO(#12912): Remove these after the Python 3 migration. + arg-name-for-non-keyword-arg, + arguments-renamed, + consider-using-dict-items, + consider-using-with, + cyclic-import, + deprecated-method, + import-outside-toplevel, + missing-raises-doc, + no-else-break, + no-else-continue, + no-else-raise, + non-explicit-keyword-args, + not-an-iterable, + raise-missing-from, + super-with-arguments, + unnecessary-pass, + useless-object-inheritance [REPORTS] diff --git a/android_validation_constants_test.py b/android_validation_constants_test.py index 08a404cc8f13..07f9868b7f42 100644 --- a/android_validation_constants_test.py +++ b/android_validation_constants_test.py @@ -70,12 +70,12 @@ def test_exploration_constants_in_both_files_are_equal(self): for obj in constants_languages_list: language_ids_in_constants.append(obj['code']) - self.assertItemsEqual( + self.assertItemsEqual( # type: ignore[no-untyped-call] interaction_ids_in_constants, android_validation_constants.VALID_INTERACTION_IDS) - self.assertItemsEqual( + self.assertItemsEqual( # type: ignore[no-untyped-call] constants.VALID_RTE_COMPONENTS_FOR_ANDROID, android_validation_constants.VALID_RTE_COMPONENTS) - self.assertItemsEqual( + self.assertItemsEqual( # type: ignore[no-untyped-call] language_ids_in_constants, android_validation_constants.SUPPORTED_LANGUAGES) diff --git a/app_dev.yaml b/app_dev.yaml index 1d8013106fdb..d69388c28897 100644 --- a/app_dev.yaml +++ b/app_dev.yaml @@ -1,15 +1,9 @@ -runtime: python27 -api_version: 1 -threadsafe: false +runtime: python37 instance_class: F2 # The "version" line is added here so that MR jobs can run locally (see issue # #6534 on oppia/oppia). version: default -builtins: -- deferred: on -- remote_api: on - inbound_services: - warmup - mail @@ -49,12 +43,10 @@ handlers: - url: /webpack_bundles static_dir: webpack_bundles secure: always - application_readable: true expiration: "0" - url: /assets static_dir: assets secure: always - application_readable: true expiration: "0" # Serve js scripts and css files under core/templates. # This regex allows us to recursively serve js scripts. @@ -74,7 +66,6 @@ handlers: upload: core/templates/(.*\.(html))$ secure: always expiration: "0" - application_readable: true - url: /third_party/generated static_dir: third_party/generated secure: always @@ -119,44 +110,6 @@ handlers: secure: always expiration: "0" -# DYNAMIC -- url: /mapreduce(/.*)? - script: mapreduce.main.APP - login: admin - secure: always -- url: /mapreduce/worker(/.*)? - script: mapreduce.main.APP - login: admin - secure: always -- url: /cron/.* - login: admin - script: main_cron.app - secure: always -- url: /task/.* - login: admin - script: main_taskqueue.app - secure: always -- url: /_ah/mail/.* - login: admin - script: main_mail.app - secure: always - -# STATIC PAGES. -- url: /.* - script: main.app - secure: always - -libraries: -- name: PIL - version: "1.1.7" -# This is needed for sending requests to a mailgun HTTPS URL. -- name: ssl - version: "2.7.11" -- name: grpcio - version: 1.0.0 -- name: setuptools - version: 36.6.0 - # This is an access connector that connects to the Google Cloud Redis instance # used for memory caching. Please replace PROJECT_ID with the correct project id # for oppia on the production server. @@ -176,167 +129,12 @@ env_variables: # with the Firebase emulator. THIS MUST NOT BE DEPLOYED TO PRODUCTION. We # protect against this in the build script. FIREBASE_AUTH_EMULATOR_HOST: "localhost:9099" - -skip_files: -# .pyc and .pyo files -- ^(.*/)?.*\.py[co]$ -# Unix hidden files whose names begin with a dot -- ^(.*/)?\..*$ -# Karma test files -- ^(.*/)Spec.js$ -# Typescript files -- ^core/(.*/)?.*\.ts$ -# Typescript output log file -- ^(.*/)tsc_output_log.txt$ -# Python test files -- ^(.*/)?.*_test\.py$ -# Other folders to ignore -- core/tests/ -- node_modules/ -- scripts/ -# Some third_party static scripts are directly imported, namely: jquery, -# jqueryui, angularjs, jqueryui-touch-punch, MathJax, code-mirror, -# ui-codemirror, d3js, midi-js, ui-map, guppy, skulpt, math-expressions. -# We exclude some of the scripts that are not directly imported in order to -# reduce deployed file count. -# TODO(sll): Find a more structured way of doing this. -# -# Please do not remove line below this, as it is used for testing purpose. -# Third party files: -- third_party/static/bootstrap-4.3.1/ -- third_party/static/bower-angular-translate-2.18.1/ -- third_party/static/bower-angular-translate-interpolation-messageformat-2.18.1/ -- third_party/static/bower-angular-translate-loader-partial-2.18.1/ -- third_party/static/bower-angular-translate-loader-static-files-2.18.1/ -- third_party/static/bower-angular-translate-storage-cookie-2.18.1/ -- third_party/static/bower-material-1.1.19/ -- third_party/python_libs/google/appengine/ -- third_party/python_libs/google/net/ -- third_party/python_libs/google/pyglib/ -- third_party/python_libs/grpc/ -# CKEditor-4.12.1 plugins in the download from the CKEditor website include -# only a11yhelp, about, clipboard, colordialog, copyformatting, dialog, div, -# find, flash, forms, iframe, image, link, liststyle, magicline, pagebreak, -# pastefromword, preview, scayt, showblocks, smiley, specialchar, table, -# tableselection, tabletools, templates, widget and wsc. Our code is also using -# the sharedspace plugin. So, for now, we exclude all others, as well as flash, -# a11yhelp, about, colordialog, iframe, and anything related to tables, which -# we definitely don't use. -- third_party/static/ckeditor-4.12.1/plugins/a11yhelp/ -- third_party/static/ckeditor-4.12.1/plugins/about/ -- third_party/static/ckeditor-4.12.1/plugins/adobeair/ -- third_party/static/ckeditor-4.12.1/plugins/ajax/ -- third_party/static/ckeditor-4.12.1/plugins/autocomplete/ -- third_party/static/ckeditor-4.12.1/plugins/autoembed/ -- third_party/static/ckeditor-4.12.1/plugins/autogrow/ -- third_party/static/ckeditor-4.12.1/plugins/autolink/ -- third_party/static/ckeditor-4.12.1/plugins/balloonpanel/ -- third_party/static/ckeditor-4.12.1/plugins/balloontoolbar/ -- third_party/static/ckeditor-4.12.1/plugins/bbcode/ -- third_party/static/ckeditor-4.12.1/plugins/bidi/ -- third_party/static/ckeditor-4.12.1/plugins/cloudservices/ -- third_party/static/ckeditor-4.12.1/plugins/codesnippet/ -- third_party/static/ckeditor-4.12.1/plugins/codesnippetgeshi/ -- third_party/static/ckeditor-4.12.1/plugins/colorbutton/ -- third_party/static/ckeditor-4.12.1/plugins/colordialog/ -- third_party/static/ckeditor-4.12.1/plugins/devtools/ -- third_party/static/ckeditor-4.12.1/plugins/dialogadvtab/ -- third_party/static/ckeditor-4.12.1/plugins/divarea/ -- third_party/static/ckeditor-4.12.1/plugins/docprops/ -- third_party/static/ckeditor-4.12.1/plugins/easyimage/ -- third_party/static/ckeditor-4.12.1/plugins/embed/ -- third_party/static/ckeditor-4.12.1/plugins/embedbase/ -- third_party/static/ckeditor-4.12.1/plugins/embedsemantic/ -- third_party/static/ckeditor-4.12.1/plugins/emoji/ -- third_party/static/ckeditor-4.12.1/plugins/flash/ -- third_party/static/ckeditor-4.12.1/plugins/font/ -- third_party/static/ckeditor-4.12.1/plugins/iframe/ -- third_party/static/ckeditor-4.12.1/plugins/iframedialog/ -- third_party/static/ckeditor-4.12.1/plugins/image2/ -- third_party/static/ckeditor-4.12.1/plugins/imagebase/ -- third_party/static/ckeditor-4.12.1/plugins/indentblock/ -- third_party/static/ckeditor-4.12.1/plugins/justify/ -- third_party/static/ckeditor-4.12.1/plugins/language/ -- third_party/static/ckeditor-4.12.1/plugins/mathjax/ -- third_party/static/ckeditor-4.12.1/plugins/mentions/ -- third_party/static/ckeditor-4.12.1/plugins/newpage/ -- third_party/static/ckeditor-4.12.1/plugins/panelbutton/ -- third_party/static/ckeditor-4.12.1/plugins/placeholder/ -- third_party/static/ckeditor-4.12.1/plugins/print/ -- third_party/static/ckeditor-4.12.1/plugins/save/ -- third_party/static/ckeditor-4.12.1/plugins/selectall/ -- third_party/static/ckeditor-4.12.1/plugins/sourcedialog/ -- third_party/static/ckeditor-4.12.1/plugins/stylesheetparser/ -- third_party/static/ckeditor-4.12.1/plugins/table/ -- third_party/static/ckeditor-4.12.1/plugins/tableresize/ -- third_party/static/ckeditor-4.12.1/plugins/tabletools/ -- third_party/static/ckeditor-4.12.1/plugins/textmatch/ -- third_party/static/ckeditor-4.12.1/plugins/textwatcher/ -- third_party/static/ckeditor-4.12.1/plugins/uicolor/ -- third_party/static/ckeditor-4.12.1/plugins/uploadfile/ -- third_party/static/ckeditor-4.12.1/plugins/xml/ -- third_party/static/ckeditor-4.12.1/samples/ -- third_party/static/ckeditor-4.12.1/skins/kama/ -- third_party/static/ckeditor-4.12.1/skins/moono/ -- third_party/static/ckeditor-bootstrapck-1.0.0/core/ -- third_party/static/ckeditor-bootstrapck-1.0.0/lang/ -- third_party/static/ckeditor-bootstrapck-1.0.0/plugins/ -- third_party/static/ckeditor-bootstrapck-1.0.0/skins/bootstrapck-dev/ -- third_party/static/ckeditor-bootstrapck-1.0.0/skins/moono/ -- third_party/static/ckeditor-bootstrapck-1.0.0/skins/ckbuilder.jar/ -- third_party/static/ckeditor-bootstrapck-1.0.0/skins/bootstrapck/sample/ -- third_party/static/ckeditor-bootstrapck-1.0.0/skins/bootstrapck/scss/ -- third_party/static/fontawesome-free-5.9.0-web/ -- third_party/static/guppy-7509f3/site/ -- third_party/static/guppy-7509f3/test/ -- third_party/static/MathJax-2.7.5/docs/ -- third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Gyre-Pagella/ -- third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Gyre-Termes/ -- third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Latin-Modern/ -- third_party/static/MathJax-2.7.5/fonts/HTML-CSS/Neo-Euler/ -- third_party/static/MathJax-2.7.5/fonts/HTML-CSS/TeX/png/ -- third_party/static/MathJax-2.7.5/localization/ast/ -- third_party/static/MathJax-2.7.5/localization/bcc/ -- third_party/static/MathJax-2.7.5/localization/bg/ -- third_party/static/MathJax-2.7.5/localization/br/ -- third_party/static/MathJax-2.7.5/localization/ca/ -- third_party/static/MathJax-2.7.5/localization/cdo/ -- third_party/static/MathJax-2.7.5/localization/cs/ -- third_party/static/MathJax-2.7.5/localization/cy/ -- third_party/static/MathJax-2.7.5/localization/da/ -- third_party/static/MathJax-2.7.5/localization/de/ -- third_party/static/MathJax-2.7.5/localization/eo/ -- third_party/static/MathJax-2.7.5/localization/es/ -- third_party/static/MathJax-2.7.5/localization/fa/ -- third_party/static/MathJax-2.7.5/localization/fi/ -- third_party/static/MathJax-2.7.5/localization/fr/ -- third_party/static/MathJax-2.7.5/localization/gl/ -- third_party/static/MathJax-2.7.5/localization/he/ -- third_party/static/MathJax-2.7.5/localization/ia/ -- third_party/static/MathJax-2.7.5/localization/it/ -- third_party/static/MathJax-2.7.5/localization/ja/ -- third_party/static/MathJax-2.7.5/localization/kn/ -- third_party/static/MathJax-2.7.5/localization/ko/ -- third_party/static/MathJax-2.7.5/localization/lb/ -- third_party/static/MathJax-2.7.5/localization/lki/ -- third_party/static/MathJax-2.7.5/localization/lt/ -- third_party/static/MathJax-2.7.5/localization/mk/ -- third_party/static/MathJax-2.7.5/localization/nl/ -- third_party/static/MathJax-2.7.5/localization/oc/ -- third_party/static/MathJax-2.7.5/localization/pl/ -- third_party/static/MathJax-2.7.5/localization/pt/ -- third_party/static/MathJax-2.7.5/localization/pt-br/ -- third_party/static/MathJax-2.7.5/localization/qqq/ -- third_party/static/MathJax-2.7.5/localization/ru/ -- third_party/static/MathJax-2.7.5/localization/scn/ -- third_party/static/MathJax-2.7.5/localization/sco/ -- third_party/static/MathJax-2.7.5/localization/sl/ -- third_party/static/MathJax-2.7.5/localization/sv/ -- third_party/static/MathJax-2.7.5/localization/tr/ -- third_party/static/MathJax-2.7.5/localization/uk/ -- third_party/static/MathJax-2.7.5/localization/vi/ -- third_party/static/MathJax-2.7.5/localization/zh-hans/ -- third_party/static/MathJax-2.7.5/test/ -- third_party/static/MathJax-2.7.5/unpacked/ -- third_party/static/messageformat-2.0.5/ -- third_party/static/popperJs-1.15.0/ +# These environment variables allows Cloud NDB services to communicate with the +# Cloud Datastore Emulator. THESE MUST NOT BE DEPLOYED TO PRODUCTION! We strip +# them from the deployed app.yaml in scripts.build. + DATASTORE_DATASET: "dev-project-id" + DATASTORE_EMULATOR_HOST: "localhost:8089" + DATASTORE_EMULATOR_HOST_PATH: "localhost:8089/datastore" + DATASTORE_HOST: "http://localhost:8089" + DATASTORE_PROJECT_ID: "dev-project-id" + DATASTORE_USE_PROJECT_ID_AS_APP_ID: "true" diff --git a/appengine_config.py b/appengine_config.py deleted file mode 100644 index 28029322defc..000000000000 --- a/appengine_config.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2014 The Oppia Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS-IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Configuration for App Engine.""" - -from __future__ import absolute_import -from __future__ import unicode_literals - -import os -import sys - -import python_utils - -from google.appengine.ext import vendor -import pkg_resources - -from typing import Any, Text # isort:skip # pylint: disable=unused-import - -# Root path of the app. -ROOT_PATH = os.path.dirname(__file__) -THIRD_PARTY_PATH = os.path.join(ROOT_PATH, 'third_party') -_PARENT_DIR = os.path.abspath(os.path.join(os.getcwd(), os.pardir)) -OPPIA_TOOLS_PATH = os.path.join(_PARENT_DIR, 'oppia_tools') -THIRD_PARTY_PYTHON_LIBS_PATH = os.path.join(THIRD_PARTY_PATH, 'python_libs') - - -# oppia_tools/ is available locally (in both dev and prod mode). However, -# on the GAE production server, oppia_tools/ is not available, and the default -# PIL third-party library is used instead. -# -# We cannot special-case this using DEV_MODE because it is possible to run -# Oppia in production mode locally, where a built-in PIL won't be available. -# Hence the check for oppia_tools instead. -if os.path.isdir(OPPIA_TOOLS_PATH): - PIL_PATH = os.path.join(OPPIA_TOOLS_PATH, 'Pillow-6.2.2') - if not os.path.isdir(PIL_PATH): - raise Exception('Invalid path for oppia_tools library: %s' % PIL_PATH) - sys.path.insert(0, PIL_PATH) # type: ignore[arg-type] - - -# Google App Engine (GAE) uses its own virtual environment that sets up the -# python library system path using their third party python library, vendor. In -# order to inform GAE of the packages that are required for Oppia, we need to -# add it using the vendor library. More information can be found here: -# https://cloud.google.com/appengine/docs/standard/python/tools/using-libraries-python-27 -vendor.add(THIRD_PARTY_PYTHON_LIBS_PATH) -pkg_resources.working_set.add_entry(THIRD_PARTY_PYTHON_LIBS_PATH) - -# It is necessary to reload the six module because of a bug in the google cloud -# ndb imports. More details can be found here: -# https://github.com/googleapis/python-ndb/issues/249. -# We need to reload at the very end of this file because we have to add the -# six python path to the app engine vendor first. -import six # isort:skip pylint: disable=wrong-import-position, wrong-import-order -reload(six) # pylint: disable=reload-builtin - - -# For some reason, pkg_resources.get_distribution returns an empty list in the -# prod env. We need to monkeypatch this function so that prod deployments -# work. Otherwise, pkg_resources.get_distribution('google-api-core').version in -# google/api_core/__init__.py throws a DistributionNotFound error, and -# similarly for pkg_resources.get_distribution('google-cloud-tasks').version in -# google/cloud/tasks_v2/gapic/cloud_tasks_client.py, which results in the -# entire application being broken on production. -import requests_toolbelt.adapters.appengine # isort:skip pylint: disable=wrong-import-position, wrong-import-order -requests_toolbelt.adapters.appengine.monkeypatch() -old_get_distribution = pkg_resources.get_distribution # pylint: disable=invalid-name - -# Disables "AppEnginePlatformWarning" because it is a "warning" that occurs -# frequently and tends to bury other important logs. -# It should be fine to ignore this, see https://stackoverflow.com/a/47494229 -# and https://github.com/urllib3/urllib3/issues/1138#issuecomment-290325277. -import requests # isort:skip pylint: disable=wrong-import-position, wrong-import-order -requests.packages.urllib3.disable_warnings( # type: ignore[no-untyped-call] - requests.packages.urllib3.contrib.appengine.AppEnginePlatformWarning # type: ignore[attr-defined] -) - - -class MockDistribution(python_utils.OBJECT): - """Mock distribution object for the monkeypatching function.""" - - def __init__(self, version): - # type: (Text) -> None - self.version = version - - -def monkeypatched_get_distribution(distribution_name): - # type: (Text) -> Any - """Monkeypatched version of pkg_resources.get_distribution. - - This approach is inspired by the discussion at: - - https://github.com/googleapis/google-cloud-python/issues/1893#issuecomment-396983379 - - Args: - distribution_name: str. The name of the distribution to get the - Distribution object for. - - Returns: - *. An object that contains the version attribute needed for App Engine - third-party libs to successfully initialize. This is a hack to get the - new Python 3 libs working in a Python 2 environment. - - Raises: - Exception. The mock is used in a context apart from the google-api-core - and google-cloud-tasks initializations. - """ - try: - return old_get_distribution(distribution_name) - except pkg_resources.DistributionNotFound: - if distribution_name == 'google-cloud-tasks': - return MockDistribution('1.5.0') - if distribution_name == 'google-cloud-translate': - return MockDistribution('2.0.1') - else: - raise - - -pkg_resources.get_distribution = monkeypatched_get_distribution diff --git a/appengine_config_test.py b/appengine_config_test.py deleted file mode 100644 index 533222fe92ee..000000000000 --- a/appengine_config_test.py +++ /dev/null @@ -1,85 +0,0 @@ -# coding: utf-8 -# -# Copyright 2014 The Oppia Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS-IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for appengine_config.py.""" - -from __future__ import absolute_import -from __future__ import unicode_literals - -import appengine_config -from core.tests import test_utils -import pkg_resources - -from typing import Text # isort:skip # pylint: disable=unused-import - - -class AppengineConfigTests(test_utils.GenericTestBase): - """Test the appengine config mock methods.""" - - def _mock_get_distribution_which_raises_error(self, distribution_name): - # type: (Text) -> None - """Mock function for pkg_resources.get_distribution(). - - Args: - distribution_name: str. The name of the distribution to get the - Distribution object for. - - Raises: - DistributionNotFound. This is always raised, in order to simulate - the case where the distribution does not exist (which is what - currently happens in a prod environment). - """ - raise pkg_resources.DistributionNotFound(distribution_name, 'tests') - - def test_monkeypatched_get_distribution_for_google_cloud_tasks(self): - # type: () -> None - """Test that the monkey-patched get_distribution() method returns an - object with a suitable version string for google-cloud-tasks. - """ - with self.swap( - appengine_config, 'old_get_distribution', - self._mock_get_distribution_which_raises_error - ): - mock_distribution = appengine_config.monkeypatched_get_distribution( - 'google-cloud-tasks') - self.assertEqual(mock_distribution.version, '1.5.0') - - def test_monkeypatched_get_distribution_for_google_cloud_translate(self): - # type: () -> None - """Test that the monkey-patched get_distribution() method returns an - object with a suitable version string for google-cloud-tasks. - """ - with self.swap( - appengine_config, 'old_get_distribution', - self._mock_get_distribution_which_raises_error - ): - mock_distribution = appengine_config.monkeypatched_get_distribution( - 'google-cloud-translate') - self.assertEqual(mock_distribution.version, '2.0.1') - - def test_monkeypatched_get_distribution_behaves_like_existing_one(self): - # type: () -> None - """Test that the monkey-patched get_distribution() method returns an - object with a suitable version string for google-cloud-tasks. - """ - assert_raises_regexp_context_manager = self.assertRaisesRegexp( # type: ignore[no-untyped-call] - pkg_resources.DistributionNotFound, 'invalid-lib') - with self.swap( - appengine_config, 'old_get_distribution', - self._mock_get_distribution_which_raises_error - ): - with assert_raises_regexp_context_manager: - appengine_config.monkeypatched_get_distribution('invalid-lib') diff --git a/constants_test.py b/constants_test.py index 4fd721ee2354..1bf1416372e5 100644 --- a/constants_test.py +++ b/constants_test.py @@ -52,7 +52,7 @@ def test_difficulty_values_are_matched(self): constants.constants.SKILL_DIFFICULTY_EASY, constants.constants.SKILL_DIFFICULTY_MEDIUM, constants.constants.SKILL_DIFFICULTY_HARD]) - self.assertItemsEqual( + self.assertItemsEqual( # type: ignore[no-untyped-call] list(constants.constants.SKILL_DIFFICULTY_LABEL_TO_FLOAT.keys()), constants.constants.SKILL_DIFFICULTIES) self.assertEqual( diff --git a/core/controllers/acl_decorators.py b/core/controllers/acl_decorators.py index 5ed278dc5fdb..440536afc0f2 100644 --- a/core/controllers/acl_decorators.py +++ b/core/controllers/acl_decorators.py @@ -1997,6 +1997,48 @@ def test_can_modify(self, exploration_id, **kwargs): return test_can_modify +def can_perform_tasks_in_taskqueue(handler): + """Decorator to ensure that the handler is being called by task scheduler or + by a superadmin of the application. + + Args: + handler: function. The function to be decorated. + + Returns: + function. The newly decorated function that now also ensures that + the handler can only be executed if it is called by task scheduler or by + a superadmin of the application. + """ + + def test_can_perform(self, **kwargs): + """Checks if the handler is called by task scheduler or by a superadmin + of the application. + + Args: + **kwargs: *. Keyword arguments. + + Returns: + *. The return value of the decorated function. + + Raises: + UnauthorizedUserException. The user does not have + credentials to access the page. + """ + # The X-AppEngine-QueueName header is set inside AppEngine and if + # a request from outside comes with this header AppEngine will get + # rid of it. + # https://cloud.google.com/tasks/docs/creating-appengine-handlers#reading_app_engine_task_request_headers + if (self.request.headers.get('X-AppEngine-QueueName') is None and + not self.current_user_is_super_admin): + raise self.UnauthorizedUserException( + 'You do not have the credentials to access this page.') + else: + return handler(self, **kwargs) + test_can_perform.__wrapped__ = True + + return test_can_perform + + def can_perform_cron_tasks(handler): """Decorator to ensure that the handler is being called by cron or by a superadmin of the application. @@ -2024,12 +2066,15 @@ def test_can_perform(self, **kwargs): UnauthorizedUserException. The user does not have credentials to access the page. """ + # The X-AppEngine-Cron header is set inside AppEngine and if a request + # from outside comes with this header AppEngine will get rid of it. + # https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#validating_cron_requests if (self.request.headers.get('X-AppEngine-Cron') is None and not self.current_user_is_super_admin): raise self.UnauthorizedUserException( 'You do not have the credentials to access this page.') - else: - return handler(self, **kwargs) + + return handler(self, **kwargs) test_can_perform.__wrapped__ = True return test_can_perform diff --git a/core/controllers/acl_decorators_test.py b/core/controllers/acl_decorators_test.py index 8e78c968f3cf..acc970fea299 100644 --- a/core/controllers/acl_decorators_test.py +++ b/core/controllers/acl_decorators_test.py @@ -4071,6 +4071,59 @@ def test_guest_cannot_change_topic_publication_status(self): 'You must be logged in to access this resource.') +class PerformTasksInTaskqueueTests(test_utils.GenericTestBase): + """Tests for decorator can_perform_tasks_in_taskqueue.""" + + viewer_username = 'viewer' + viewer_email = 'viewer@example.com' + + class MockHandler(base.BaseHandler): + GET_HANDLER_ERROR_RETURN_TYPE = feconf.HANDLER_TYPE_JSON + URL_PATH_ARGS_SCHEMAS = {} + HANDLER_ARGS_SCHEMAS = {'GET': {}} + + @acl_decorators.can_perform_tasks_in_taskqueue + def get(self): + self.render_json({}) + + def setUp(self): + super(PerformTasksInTaskqueueTests, self).setUp() + self.signup(self.CURRICULUM_ADMIN_EMAIL, self.CURRICULUM_ADMIN_USERNAME) + + self.admin_id = self.get_user_id_from_email(self.CURRICULUM_ADMIN_EMAIL) + self.admin = user_services.get_user_actions_info(self.admin_id) + self.signup(self.viewer_email, self.viewer_username) + + self.mock_testapp = webtest.TestApp(webapp2.WSGIApplication( + [webapp2.Route( + '/mock_perform_tasks_in_taskqueue', self.MockHandler)], + debug=feconf.DEBUG, + )) + + def test_super_admin_can_perform_tasks_in_taskqueue(self): + self.login(self.CURRICULUM_ADMIN_EMAIL, is_super_admin=True) + with self.swap(self, 'testapp', self.mock_testapp): + self.get_json('/mock_perform_tasks_in_taskqueue') + self.logout() + + def test_normal_user_cannot_perform_tasks_in_taskqueue(self): + self.login(self.viewer_email) + with self.swap(self, 'testapp', self.mock_testapp): + response = self.get_json( + '/mock_perform_tasks_in_taskqueue', expected_status_int=401) + self.assertEqual( + response['error'], + 'You do not have the credentials to access this page.') + self.logout() + + def test_request_with_appropriate_header_can_perform_tasks_in_taskqueue( + self): + with self.swap(self, 'testapp', self.mock_testapp): + self.get_json( + '/mock_perform_tasks_in_taskqueue', + headers={'X-AppEngine-QueueName': 'name'}) + + class PerformCronTaskTests(test_utils.GenericTestBase): """Tests for decorator can_perform_cron_tasks.""" @@ -4116,6 +4169,11 @@ def test_normal_user_cannot_perform_cron_tasks(self): 'You do not have the credentials to access this page.') self.logout() + def test_request_with_appropriate_header_can_perform_cron_tasks(self): + with self.swap(self, 'testapp', self.mock_testapp): + self.get_json( + '/mock_perform_cron_task', headers={'X-AppEngine-Cron': 'true'}) + class EditSkillDecoratorTests(test_utils.GenericTestBase): """Tests permissions for accessing the skill editor.""" diff --git a/core/controllers/admin.py b/core/controllers/admin.py index c93a14e813cd..1c9c8f7bf7ff 100644 --- a/core/controllers/admin.py +++ b/core/controllers/admin.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import io import logging import random @@ -304,11 +305,10 @@ def _reload_exploration(self, exploration_id): logging.info( '[ADMIN] %s reloaded exploration %s' % (self.user_id, exploration_id)) - exp_services.load_demo(python_utils.convert_to_bytes( - exploration_id)) + exp_services.load_demo(python_utils.UNICODE(exploration_id)) rights_manager.release_ownership_of_exploration( - user_services.get_system_user(), python_utils.convert_to_bytes( - exploration_id)) + user_services.get_system_user(), + python_utils.UNICODE(exploration_id)) else: raise Exception('Cannot reload an exploration in production.') @@ -612,11 +612,9 @@ def _reload_collection(self, collection_id): logging.info( '[ADMIN] %s reloaded collection %s' % (self.user_id, collection_id)) - collection_services.load_demo( - python_utils.convert_to_bytes(collection_id)) + collection_services.load_demo(collection_id) rights_manager.release_ownership_of_collection( - user_services.get_system_user(), python_utils.convert_to_bytes( - collection_id)) + user_services.get_system_user(), collection_id) else: raise Exception('Cannot reload a collection in production.') @@ -969,9 +967,16 @@ class AdminTopicsCsvFileDownloader(base.BaseHandler): @acl_decorators.can_access_admin_page def get(self): + topic_similarities = ( + recommendations_services.get_topic_similarities_as_csv() + ) + # Downloadable file accepts only bytes, so we need to encode + # topic_similarities to bytes. self.render_downloadable_file( - recommendations_services.get_topic_similarities_as_csv(), - 'topic_similarities.csv', 'text/csv') + io.BytesIO(topic_similarities.encode('utf-8')), + 'topic_similarities.csv', + 'text/csv' + ) class DataExtractionQueryHandler(base.BaseHandler): diff --git a/core/controllers/admin_test.py b/core/controllers/admin_test.py index 2d271f8a4a39..d640065a05ac 100644 --- a/core/controllers/admin_test.py +++ b/core/controllers/admin_test.py @@ -62,7 +62,6 @@ BOTH_MODERATOR_AND_ADMIN_EMAIL = 'moderator.and.admin@example.com' BOTH_MODERATOR_AND_ADMIN_USERNAME = 'moderatorandadm1n' - PARAM_NAMES = python_utils.create_enum('test_feature_1') # pylint: disable=invalid-name FEATURE_STAGES = platform_parameter_domain.FEATURE_STAGES @@ -395,11 +394,11 @@ def test_admin_topics_csv_download_handler(self): 'attachment; filename=topic_similarities.csv') self.assertIn( - 'Architecture,Art,Biology,Business,Chemistry,Computing,Economics,' - 'Education,Engineering,Environment,Geography,Government,Hobbies,' - 'Languages,Law,Life Skills,Mathematics,Medicine,Music,Philosophy,' - 'Physics,Programming,Psychology,Puzzles,Reading,Religion,Sport,' - 'Statistics,Welcome', + b'Architecture,Art,Biology,Business,Chemistry,Computing,Economics,' + b'Education,Engineering,Environment,Geography,Government,Hobbies,' + b'Languages,Law,Life Skills,Mathematics,Medicine,Music,Philosophy,' + b'Physics,Programming,Psychology,Puzzles,Reading,Religion,Sport,' + b'Statistics,Welcome', response.body) self.logout() @@ -723,6 +722,9 @@ def test_update_flag_rules_with_rules_of_non_list_of_dict_type_returns_400( self.login(self.CURRICULUM_ADMIN_EMAIL, is_super_admin=True) csrf_token = self.get_new_csrf_token() + error_msg = ( + 'Schema validation for \'new_rules\' failed: \'int\' ' + 'object is not subscriptable') response = self.post_json( '/adminhandler', { 'action': 'update_feature_flag_rules', @@ -733,9 +735,6 @@ def test_update_flag_rules_with_rules_of_non_list_of_dict_type_returns_400( csrf_token=csrf_token, expected_status_int=400 ) - error_msg = ( - 'Schema validation for \'new_rules\' failed: \'int\' ' - 'object has no attribute \'__getitem__\'') self.assertEqual(response['error'], error_msg) self.logout() @@ -957,17 +956,17 @@ def test_handler_raises_error_with_non_int_num_dummy_exps_to_generate(self): self.login(self.CURRICULUM_ADMIN_EMAIL, is_super_admin=True) csrf_token = self.get_new_csrf_token() + response = self.post_json( + '/adminhandler', { + 'action': 'generate_dummy_explorations', + 'num_dummy_exps_to_publish': 1, + 'num_dummy_exps_to_generate': 'invalid_type' + }, csrf_token=csrf_token, expected_status_int=400) + error_msg = ( 'Schema validation for \'num_dummy_exps_to_generate\' failed: ' - 'Could not convert unicode to int: invalid_type') - with self.assertRaisesRegexp(Exception, error_msg): - self.post_json( - '/adminhandler', { - 'action': 'generate_dummy_explorations', - 'num_dummy_exps_to_publish': 1, - 'num_dummy_exps_to_generate': 'invalid_type' - }, csrf_token=csrf_token) - + 'Could not convert str to int: invalid_type') + self.assertEqual(response['error'], error_msg) generated_exps = exp_services.get_all_exploration_summaries() published_exps = exp_services.get_recently_published_exp_summaries(5) self.assertEqual(generated_exps, {}) @@ -979,17 +978,17 @@ def test_handler_raises_error_with_non_int_num_dummy_exps_to_publish(self): self.login(self.CURRICULUM_ADMIN_EMAIL, is_super_admin=True) csrf_token = self.get_new_csrf_token() + response = self.post_json( + '/adminhandler', { + 'action': 'generate_dummy_explorations', + 'num_dummy_exps_to_publish': 'invalid_type', + 'num_dummy_exps_to_generate': 1 + }, csrf_token=csrf_token, expected_status_int=400) + error_msg = ( 'Schema validation for \'num_dummy_exps_to_publish\' failed: ' - 'Could not convert unicode to int: invalid_type') - with self.assertRaisesRegexp(Exception, error_msg): - self.post_json( - '/adminhandler', { - 'action': 'generate_dummy_explorations', - 'num_dummy_exps_to_publish': 'invalid_type', - 'num_dummy_exps_to_generate': 1 - }, csrf_token=csrf_token) - + 'Could not convert str to int: invalid_type') + self.assertEqual(response['error'], error_msg) generated_exps = exp_services.get_all_exploration_summaries() published_exps = exp_services.get_recently_published_exp_summaries(5) self.assertEqual(generated_exps, {}) @@ -1111,7 +1110,7 @@ def test_cannot_view_role_with_invalid_view_filter_criterion(self): error_msg = ( 'Schema validation for \'filter_criterion\' failed: Received ' 'invalid which is not in the allowed range of choices: ' - '[u\'role\', u\'username\']') + '[\'role\', \'username\']') self.assertEqual(response['error'], error_msg) def test_replacing_user_role_from_topic_manager_to_moderator(self): @@ -1654,16 +1653,14 @@ def test_handler_when_exp_version_is_not_int_throws_exception(self): 'num_answers': 0 } + error_msg = ( + 'Schema validation for \'exp_version\' failed: ' + 'Could not convert str to int: a') response = self.get_json( '/explorationdataextractionhandler', params=payload, - expected_status_int=400 - ) - error_msg = ( - 'Schema validation for \'exp_version\' failed: ' - 'Could not convert unicode to int: a') - self.assertEqual( - response['error'], error_msg) + expected_status_int=400) + self.assertEqual(response['error'], error_msg) def test_that_handler_raises_exception(self): self.login(self.CURRICULUM_ADMIN_EMAIL, is_super_admin=True) @@ -1852,7 +1849,7 @@ def test_update_username_with_long_new_username(self): expected_status_int=400) error_msg = ( 'Schema validation for \'new_username\' failed: Validation failed' - ': has_length_at_most ({u\'max_value\': %s}) for object %s' + ': has_length_at_most ({\'max_value\': %s}) for object %s' % (constants.MAX_USERNAME_LENGTH, long_username)) self.assertEqual(response['error'], error_msg) diff --git a/core/controllers/base.py b/core/controllers/base.py index f90e16c67266..5bfd5243e10c 100755 --- a/core/controllers/base.py +++ b/core/controllers/base.py @@ -254,8 +254,7 @@ def dispatch(self): # If the request is to the old demo server, redirect it permanently to # the new demo server. if self.request.uri.startswith('https://oppiaserver.appspot.com'): - self.redirect( - b'https://oppiatestserver.appspot.com', permanent=True) + self.redirect('https://oppiatestserver.appspot.com', permanent=True) return if not self._is_requested_path_currently_accessible_to_user(): @@ -355,7 +354,7 @@ def vaidate_and_normalize_args(self): elif arg == 'payload': payload_args = self.payload if payload_args is not None: - payload_arg_keys = payload_args.keys() + payload_arg_keys = list(payload_args.keys()) handler_args.update(payload_args) else: request_arg_keys.append(arg) @@ -459,7 +458,6 @@ def get(self, *args, **kwargs): # pylint: disable=unused-argument self._render_exception( 404, { 'error': 'Could not find the page %s.' % self.request.uri}) - return def post(self, *args): # pylint: disable=unused-argument """Base method to handle POST requests. @@ -491,31 +489,35 @@ def render_json(self, values): Args: values: dict. The key-value pairs to encode in the JSON response. """ - self.response.content_type = b'application/json; charset=utf-8' - self.response.headers[b'Content-Disposition'] = ( - b'attachment; filename="oppia-attachment.txt"') - self.response.headers[b'Strict-Transport-Security'] = ( - b'max-age=31536000; includeSubDomains') - self.response.headers[b'X-Content-Type-Options'] = b'nosniff' - self.response.headers[b'X-Xss-Protection'] = b'1; mode=block' + self.response.content_type = 'application/json; charset=utf-8' + self.response.headers['Content-Disposition'] = ( + 'attachment; filename="oppia-attachment.txt"') + self.response.headers['Strict-Transport-Security'] = ( + 'max-age=31536000; includeSubDomains') + self.response.headers['X-Content-Type-Options'] = 'nosniff' + self.response.headers['X-Xss-Protection'] = '1; mode=block' json_output = json.dumps(values, cls=utils.JSONEncoderForHTML) - self.response.write('%s%s' % (feconf.XSSI_PREFIX, json_output)) + # Write expects bytes, thus we need to encode the JSON output. + self.response.write( + b'%s%s' % (feconf.XSSI_PREFIX, json_output.encode('utf-8'))) - def render_downloadable_file(self, values, filename, content_type): + def render_downloadable_file(self, file, filename, content_type): """Prepares downloadable content to be sent to the client. Args: - values: dict. The key-value pairs to include in the response. + file: BytesIO. The data of the downloadable file. filename: str. The name of the file to be rendered. content_type: str. The type of file to be rendered. """ - self.response.headers[b'Content-Type'] = python_utils.convert_to_bytes( - content_type) - self.response.headers[ - b'Content-Disposition'] = python_utils.convert_to_bytes( - 'attachment; filename=%s' % filename) - self.response.write(values) + self.response.headers['Content-Type'] = content_type + self.response.headers['Content-Disposition'] = ( + 'attachment; filename=%s' % filename) + self.response.charset = 'utf-8' + # We use this super in order to bypass the write method + # in webapp2.Response, since webapp2.Response doesn't support writing + # bytes. + super(webapp2.Response, self.response).write(file.getvalue()) # pylint: disable=bad-super-call def render_template(self, filepath, iframe_restriction='DENY'): """Prepares an HTML response to be sent to the client. @@ -534,16 +536,15 @@ def render_template(self, filepath, iframe_restriction='DENY'): # deploy a new version, using only 'no-cache' doesn't work properly. self.response.cache_control.no_store = True self.response.cache_control.must_revalidate = True - self.response.headers[b'Strict-Transport-Security'] = ( - b'max-age=31536000; includeSubDomains') - self.response.headers[b'X-Content-Type-Options'] = b'nosniff' - self.response.headers[b'X-Xss-Protection'] = b'1; mode=block' + self.response.headers['Strict-Transport-Security'] = ( + 'max-age=31536000; includeSubDomains') + self.response.headers['X-Content-Type-Options'] = 'nosniff' + self.response.headers['X-Xss-Protection'] = '1; mode=block' if iframe_restriction is not None: if iframe_restriction in ['SAMEORIGIN', 'DENY']: - self.response.headers[ - b'X-Frame-Options'] = python_utils.convert_to_bytes( - iframe_restriction) + self.response.headers['X-Frame-Options'] = ( + python_utils.UNICODE(iframe_restriction)) else: raise Exception( 'Invalid X-Frame-Options: %s' % iframe_restriction) @@ -579,8 +580,8 @@ def _render_exception_json_or_html(self, return_type, values): self.render_template( 'error-page-%s.mainpage.html' % values['status_code']) else: - if return_type != feconf.HANDLER_TYPE_JSON and ( - return_type != feconf.HANDLER_TYPE_DOWNLOADABLE): + if return_type not in ( + feconf.HANDLER_TYPE_JSON, feconf.HANDLER_TYPE_DOWNLOADABLE): logging.warning( 'Not a recognized return type: defaulting to render JSON.') self.render_json(values) @@ -654,31 +655,31 @@ def handle_exception(self, exception, unused_debug_mode): if isinstance(exception, self.UnauthorizedUserException): self.error(401) - self._render_exception(401, {'error': python_utils.convert_to_bytes( - exception)}) + self._render_exception( + 401, {'error': python_utils.UNICODE(exception)}) return if isinstance(exception, self.InvalidInputException): self.error(400) - self._render_exception(400, {'error': python_utils.convert_to_bytes( - exception)}) + self._render_exception( + 400, {'error': python_utils.UNICODE(exception)}) return if isinstance(exception, self.InternalErrorException): self.error(500) - self._render_exception(500, {'error': python_utils.convert_to_bytes( - exception)}) + self._render_exception( + 500, {'error': python_utils.UNICODE(exception)}) return if isinstance(exception, self.TemporaryMaintenanceException): self.error(503) - self._render_exception(503, {'error': python_utils.convert_to_bytes( - exception)}) + self._render_exception( + 503, {'error': python_utils.UNICODE(exception)}) return self.error(500) self._render_exception( - 500, {'error': python_utils.convert_to_bytes(exception)}) + 500, {'error': python_utils.UNICODE(exception)}) InternalErrorException = UserFacingExceptions.InternalErrorException InvalidInputException = UserFacingExceptions.InvalidInputException @@ -736,15 +737,18 @@ def _create_token(cls, user_id, issued_on): user_id = cls._USER_ID_DEFAULT # Round time to seconds. - issued_on = int(issued_on) + issued_on = python_utils.UNICODE(int(issued_on)) - digester = hmac.new(python_utils.convert_to_bytes(CSRF_SECRET.value)) - digester.update(python_utils.convert_to_bytes(user_id)) - digester.update(':') - digester.update(python_utils.convert_to_bytes(issued_on)) + digester = hmac.new(CSRF_SECRET.value.encode('utf-8')) + digester.update(user_id.encode('utf-8')) + digester.update(b':') + digester.update(issued_on.encode('utf-8')) digest = digester.digest() - token = '%s/%s' % (issued_on, base64.urlsafe_b64encode(digest)) + # The b64encode returns bytes, so we first need to decode the returned + # bytes to string. + token = '%s/%s' % ( + issued_on, base64.urlsafe_b64encode(digest).decode('utf-8')) return token diff --git a/core/controllers/base_test.py b/core/controllers/base_test.py index a9dba242d049..898e70ac4db8 100644 --- a/core/controllers/base_test.py +++ b/core/controllers/base_test.py @@ -19,9 +19,8 @@ from __future__ import absolute_import from __future__ import unicode_literals -import datetime -import importlib import inspect +import io import json import logging import os @@ -42,18 +41,17 @@ from core.domain import rights_manager from core.domain import user_services from core.platform import models -from core.platform.auth import firebase_auth_services from core.tests import test_utils import feconf import main import python_utils import utils -from mapreduce import main as mapreduce_main import webapp2 import webtest auth_services = models.Registry.import_auth_services() +datastore_services = models.Registry.import_datastore_services() (user_models,) = models.Registry.import_models([models.NAMES.user]) FORTY_EIGHT_HOURS_IN_SECS = 48 * 60 * 60 @@ -373,7 +371,9 @@ def test_renders_error_page_with_iframed(self): '/mock_iframed', expected_status_int=500) self.assertIn( - '', response.body) + b'', + response.body + ) def test_dev_mode_cannot_be_true_on_production(self): # We need to delete the existing module else the re-importing @@ -390,160 +390,6 @@ def test_dev_mode_cannot_be_true_on_production(self): # 'unused-import' would appear if this line was not disabled. import feconf # pylint: disable-all - def test_valid_pillow_path(self): - # We need to re-import appengine_config here to make it look like a - # local variable so that we can again re-import appengine_config later. - import appengine_config - assert_raises_regexp_context_manager = self.assertRaisesRegexp( - Exception, 'Invalid path for oppia_tools library: invalid_path') - - def mock_os_path_join_for_pillow(*args): - """Mocks path for 'Pillow' with an invalid path. This is done by - substituting os.path.join to return an invalid path. This is - needed to test the scenario where the 'Pillow' path points - to a non-existent directory. - """ - path = '' - if args[1] == 'Pillow-6.2.2': - return 'invalid_path' - else: - path = '/'.join(args) - return path - - pil_path_swap = self.swap(os.path, 'join', mock_os_path_join_for_pillow) - # We need to delete the existing module else the re-importing - # would just call the existing module. - del sys.modules['appengine_config'] - - with assert_raises_regexp_context_manager, pil_path_swap: - # This pragma is needed since we are re-importing under - # invalid conditions. The pylint error messages - # 'reimported', 'unused-variable', 'redefined-outer-name' and - # 'unused-import' would appear if this line was not disabled. - import appengine_config # pylint: disable-all - - def test_valid_third_party_library_path(self): - # We need to re-import appengine_config here to make it look like a - # local variable so that we can again re-import appengine_config later. - import appengine_config - # This exception is generated by Google App Engine (GAE). Since GAE - # operates its own special virtual environment, when we attempt to - # modify its system path with an invalid path, it throws the error - # below. - assert_raises_regexp_context_manager = self.assertRaisesRegexp( - Exception, - 'virtualenv: cannot access invalid_path/python_libs: No such ' - 'virtualenv or site directory') - - def mock_os_path_join_for_third_party_lib(*args): - """Mocks path for third_party libs with an invalid path. This is - done by substituting os.path.join to return an invalid path. This is - needed to test the scenario where the third_party libs path points - to a non-existent directory. - """ - path = '' - if args[1] == 'third_party': - return 'invalid_path' - else: - path = '/'.join(args) - return path - - third_party_lib_path_swap = self.swap( - os.path, 'join', mock_os_path_join_for_third_party_lib) - # We need to delete the existing module else the re-importing - # would just call the existing module. - del sys.modules['appengine_config'] - - with assert_raises_regexp_context_manager, third_party_lib_path_swap: - # This pragma is needed since we are re-importing under - # invalid conditions. The pylint error messages - # 'reimported', 'unused-variable', 'redefined-outer-name' and - # 'unused-import' would appear if this line was not disabled. - import appengine_config # pylint: disable-all - - def test_authorization_wrapper_with_x_app_engine_task_name(self): - self.testapp = webtest.TestApp(webapp2.WSGIApplication( - [webapp2.Route( - '/mock', self.MockHandlerForTestingAuthorizationWrapper, - name='MockHandlerForTestingAuthorizationWrapper')], - debug=feconf.DEBUG, - )) - - def mock_create_handlers_map(): - return [('/mock', self.MockHandlerForTestingAuthorizationWrapper)] - - # We need to delete the existing module else the re-importing - # would just call the existing module. - del sys.modules['main'] - with self.swap( - mapreduce_main, 'create_handlers_map', mock_create_handlers_map): - # This pragma is needed since we are re-importing under - # different conditions. The pylint error messages - # 'reimported', 'unused-variable', 'redefined-outer-name' and - # 'unused-import' would appear if this line was not disabled. - import main # pylint: disable-all - - headers_dict = { - 'X-AppEngine-TaskName': b'taskname' - } - self.assertEqual(len(main.MAPREDUCE_HANDLERS), 1) - self.assertEqual(main.MAPREDUCE_HANDLERS[0][0], '/mock') - - response = self.testapp.get('/mock', headers=headers_dict) - self.assertEqual(response.status_int, 200) - - def test_authorization_wrapper_without_x_app_engine_task_name(self): - self.testapp = webtest.TestApp(webapp2.WSGIApplication( - [webapp2.Route( - '/mock', self.MockHandlerForTestingAuthorizationWrapper, - name='MockHandlerForTestingAuthorizationWrapper')], - debug=feconf.DEBUG, - )) - - def mock_create_handlers_map(): - return [('/mock', self.MockHandlerForTestingAuthorizationWrapper)] - - # We need to delete the existing module else the re-importing - # would just call the existing module. - del sys.modules['main'] - with self.swap( - mapreduce_main, 'create_handlers_map', mock_create_handlers_map): - # This pragma is needed since we are re-importing under - # different conditions. The pylint error messages - # 'reimported', 'unused-variable', 'redefined-outer-name' and - # 'unused-import' would appear if this line was not disabled. - import main # pylint: disable-all - - self.assertEqual(len(main.MAPREDUCE_HANDLERS), 1) - self.assertEqual(main.MAPREDUCE_HANDLERS[0][0], '/mock') - self.get_html_response('/mock', expected_status_int=403) - - def test_ui_access_wrapper(self): - self.testapp = webtest.TestApp(webapp2.WSGIApplication( - [webapp2.Route( - '/ui', self.MockHandlerForTestingUiAccessWrapper, - name='MockHandlerForTestingUiAccessWrapper')], - debug=feconf.DEBUG, - )) - - def mock_create_handlers_map(): - return [('/ui', self.MockHandlerForTestingUiAccessWrapper)] - - # We need to delete the existing module else the re-importing - # would just call the existing module. - del sys.modules['main'] - with self.swap( - mapreduce_main, 'create_handlers_map', mock_create_handlers_map): - # This pragma is needed since we are re-importing under - # different conditions. The pylint error messages - # 'reimported', 'unused-variable', 'redefined-outer-name' and - # 'unused-import' would appear if this line was not disabled. - import main # pylint: disable-all - - self.assertEqual(len(main.MAPREDUCE_HANDLERS), 1) - self.assertEqual(main.MAPREDUCE_HANDLERS[0][0], '/ui') - self.get_html_response('/ui') - def test_frontend_error_handler(self): observed_log_messages = [] @@ -603,6 +449,33 @@ def test_unauthorized_user_exception_raised_when_session_is_invalid(self): response.location, 'http://localhost/login?return_url=http%3A%2F%2Flocalhost%2F') + def test_signup_attempt_on_wrong_page_fails(self): + with python_utils.ExitStack() as exit_stack: + call_counter = exit_stack.enter_context(self.swap_with_call_counter( + auth_services, 'destroy_auth_session')) + logs = exit_stack.enter_context( + self.capture_logging(min_level=logging.ERROR)) + exit_stack.enter_context(self.swap_to_always_return( + auth_services, + 'get_auth_claims_from_request', + auth_domain.AuthClaims( + 'auth_id', self.NEW_USER_EMAIL, role_is_super_admin=False) + )) + response = self.get_html_response('/', expected_status_int=200) + self.assertIn( + b'', + response.body + ) + + self.assert_matches_regexps( + logs, + [ + 'Cannot find user auth_id with email %s on ' + 'page http://localhost/\nNoneType: None' % self.NEW_USER_EMAIL + ] + ) + self.assertEqual(call_counter.times_called, 1) + class MaintenanceModeTests(test_utils.GenericTestBase): """Tests BaseHandler behavior when maintenance mode is enabled. @@ -633,8 +506,8 @@ def test_html_response_is_rejected(self): response = self.get_html_response( '/community-library', expected_status_int=503) - self.assertIn('', response.body) - self.assertNotIn('', response.body) + self.assertIn(b'', response.body) + self.assertNotIn(b'', response.body) self.assertEqual(destroy_auth_session_call_counter.times_called, 1) def test_html_response_is_not_rejected_when_user_is_super_admin(self): @@ -644,8 +517,8 @@ def test_html_response_is_not_rejected_when_user_is_super_admin(self): response = self.get_html_response('/community-library') - self.assertIn('', response.body) - self.assertNotIn('', response.body) + self.assertIn(b'', response.body) + self.assertNotIn(b'', response.body) self.assertEqual(destroy_auth_session_call_counter.times_called, 0) def test_html_response_is_not_rejected_when_user_is_release_coordinator( @@ -657,8 +530,8 @@ def test_html_response_is_not_rejected_when_user_is_release_coordinator( response = self.get_html_response('/community-library') - self.assertIn('', response.body) - self.assertNotIn('', response.body) + self.assertIn(b'', response.body) + self.assertNotIn(b'', response.body) self.assertEqual(destroy_auth_session_call_counter.times_called, 0) def test_json_response_is_rejected(self): @@ -834,9 +707,9 @@ def test_special_char_escaping(self): self.assertEqual(response.status_int, 200) self.assertTrue(response.body.startswith(feconf.XSSI_PREFIX)) - self.assertIn('\\n\\u003cscript\\u003e\\u9a6c={{', response.body) - self.assertNotIn('