diff --git a/.gitignore b/.gitignore index 55de48a72..d20d05da4 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,3 @@ docs/collections/people.rst docs/collections/projects.rst docs/collections/proposals.rst docs/collections/students.rst - diff --git a/src/regolith/builders/coabuilder.py b/src/regolith/builders/coabuilder.py index 386a8171e..122251853 100644 --- a/src/regolith/builders/coabuilder.py +++ b/src/regolith/builders/coabuilder.py @@ -18,11 +18,7 @@ from src.regolith.builders.basebuilder import BuilderBase from src.regolith.dates import get_dates, is_after, month_to_int from src.regolith.sorters import position_key -from src.regolith.tools import ( - all_docs_from_collection, - filter_publications, - fuzzy_retrieval, -) +from src.regolith.tools import all_docs_from_collection, filter_publications, fuzzy_retrieval NUM_COAUTHOR_MONTHS = 48 NUM_POSTDOC_MONTHS = None diff --git a/src/regolith/builders/cpbuilder.py b/src/regolith/builders/cpbuilder.py index c3a19de98..1365fa2c0 100644 --- a/src/regolith/builders/cpbuilder.py +++ b/src/regolith/builders/cpbuilder.py @@ -9,12 +9,7 @@ from src.regolith.dates import get_dates, is_current from src.regolith.fsclient import _id_key from src.regolith.sorters import position_key -from src.regolith.tools import ( - all_docs_from_collection, - filter_grants, - fuzzy_retrieval, - merge_collections_all, -) +from src.regolith.tools import all_docs_from_collection, filter_grants, fuzzy_retrieval, merge_collections_all def is_pending(status): diff --git a/src/regolith/builders/grantreportbuilder.py b/src/regolith/builders/grantreportbuilder.py index 236cc2cd1..50f8b3632 100644 --- a/src/regolith/builders/grantreportbuilder.py +++ b/src/regolith/builders/grantreportbuilder.py @@ -10,11 +10,7 @@ from src.regolith.builders.basebuilder import LatexBuilderBase from src.regolith.dates import get_dates, is_current from src.regolith.fsclient import _id_key -from src.regolith.tools import ( - all_docs_from_collection, - filter_presentations, - fuzzy_retrieval, -) +from src.regolith.tools import all_docs_from_collection, filter_presentations, fuzzy_retrieval class GrantReportBuilder(LatexBuilderBase): diff --git a/src/regolith/builders/preslistbuilder.py b/src/regolith/builders/preslistbuilder.py index 81f24c476..4485347e8 100644 --- a/src/regolith/builders/preslistbuilder.py +++ b/src/regolith/builders/preslistbuilder.py @@ -22,11 +22,7 @@ from src.regolith.fsclient import _id_key from src.regolith.sorters import position_key from src.regolith.stylers import month_fullnames, sentencecase -from src.regolith.tools import ( - all_docs_from_collection, - filter_presentations, - group_member_ids, -) +from src.regolith.tools import all_docs_from_collection, filter_presentations, group_member_ids class PresListBuilder(LatexBuilderBase): diff --git a/src/regolith/builders/publistbuilder.py b/src/regolith/builders/publistbuilder.py index 67a1dffe3..9233b8155 100644 --- a/src/regolith/builders/publistbuilder.py +++ b/src/regolith/builders/publistbuilder.py @@ -4,11 +4,7 @@ from src.regolith.builders.basebuilder import LatexBuilderBase from src.regolith.sorters import ene_date_key, position_key -from src.regolith.tools import ( - all_docs_from_collection, - filter_publications, - make_bibtex_file, -) +from src.regolith.tools import all_docs_from_collection, filter_publications, make_bibtex_file LATEX_OPTS = ["-halt-on-error", "-file-line-error"] diff --git a/src/regolith/helpers/a_todohelper.py b/src/regolith/helpers/a_todohelper.py index b9d70880f..7c403e165 100644 --- a/src/regolith/helpers/a_todohelper.py +++ b/src/regolith/helpers/a_todohelper.py @@ -9,12 +9,7 @@ from src.regolith.chained_db import _convert_to_dict from src.regolith.fsclient import _id_key from src.regolith.helpers.basehelper import DbHelperBase -from src.regolith.tools import ( - all_docs_from_collection, - fragment_retrieval, - get_pi_id, - get_uuid, -) +from src.regolith.tools import all_docs_from_collection, fragment_retrieval, get_pi_id, get_uuid TARGET_COLL = "todos" ALLOWED_IMPORTANCE = [3, 2, 1, 0] diff --git a/src/regolith/helpers/l_abstracthelper.py b/src/regolith/helpers/l_abstracthelper.py index 6f694bfb5..02db14b1e 100644 --- a/src/regolith/helpers/l_abstracthelper.py +++ b/src/regolith/helpers/l_abstracthelper.py @@ -5,12 +5,7 @@ from src.regolith.dates import get_dates from src.regolith.fsclient import _id_key from src.regolith.helpers.basehelper import SoutHelperBase -from src.regolith.tools import ( - all_docs_from_collection, - dereference_institution, - get_person_contact, - get_pi_id, -) +from src.regolith.tools import all_docs_from_collection, dereference_institution, get_person_contact, get_pi_id TARGET_COLL = "presentations" HELPER_TARGET = "l_abstract" diff --git a/src/regolith/helpers/l_contactshelper.py b/src/regolith/helpers/l_contactshelper.py index 596b623c0..2f9064a64 100644 --- a/src/regolith/helpers/l_contactshelper.py +++ b/src/regolith/helpers/l_contactshelper.py @@ -9,12 +9,7 @@ from src.regolith.dates import get_dates, is_current from src.regolith.fsclient import _id_key from src.regolith.helpers.basehelper import SoutHelperBase -from src.regolith.tools import ( - all_docs_from_collection, - fuzzy_retrieval, - get_pi_id, - search_collection, -) +from src.regolith.tools import all_docs_from_collection, fuzzy_retrieval, get_pi_id, search_collection TARGET_COLL = "contacts" HELPER_TARGET = "l_contacts" diff --git a/src/regolith/helpers/l_currentappointmentshelper.py b/src/regolith/helpers/l_currentappointmentshelper.py index 8abe47a44..7855417d9 100644 --- a/src/regolith/helpers/l_currentappointmentshelper.py +++ b/src/regolith/helpers/l_currentappointmentshelper.py @@ -6,11 +6,7 @@ from src.regolith.helpers.basehelper import SoutHelperBase from src.regolith.helpers.makeappointmentshelper import _future_grant from src.regolith.sorters import position_key -from src.regolith.tools import ( - all_docs_from_collection, - fuzzy_retrieval, - merge_collections_superior, -) +from src.regolith.tools import all_docs_from_collection, fuzzy_retrieval, merge_collections_superior SEMESTER_START_MONTH = {"fall": (9, 12), "spring": (1, 5), "summer": (6, 8)} diff --git a/src/regolith/helpers/l_generalhelper.py b/src/regolith/helpers/l_generalhelper.py index 4a8d11004..811fa21af 100644 --- a/src/regolith/helpers/l_generalhelper.py +++ b/src/regolith/helpers/l_generalhelper.py @@ -2,12 +2,7 @@ from src.regolith.fsclient import _id_key from src.regolith.helpers.basehelper import SoutHelperBase -from src.regolith.tools import ( - all_docs_from_collection, - collection_str, - get_pi_id, - search_collection, -) +from src.regolith.tools import all_docs_from_collection, collection_str, get_pi_id, search_collection HELPER_TARGET = "lister" diff --git a/src/regolith/helpers/l_milestoneshelper.py b/src/regolith/helpers/l_milestoneshelper.py index 86ec0139c..441189785 100644 --- a/src/regolith/helpers/l_milestoneshelper.py +++ b/src/regolith/helpers/l_milestoneshelper.py @@ -16,12 +16,7 @@ PROJECTUM_PAUSED_STATI, alloweds, ) -from regolith.tools import ( - all_docs_from_collection, - collection_str, - get_pi_id, - key_value_pair_filter, -) +from regolith.tools import all_docs_from_collection, collection_str, get_pi_id, key_value_pair_filter PROJECTUM_STATI = alloweds.get("PROJECTUM_STATI") TARGET_COLL = "projecta" diff --git a/src/regolith/helpers/l_projectahelper.py b/src/regolith/helpers/l_projectahelper.py index c9b6f3a8c..e8a3b34c4 100644 --- a/src/regolith/helpers/l_projectahelper.py +++ b/src/regolith/helpers/l_projectahelper.py @@ -9,6 +9,7 @@ import dateutil.parser as date_parser from gooey import GooeyParser +from regolith.tools import all_docs_from_collection, collection_str, get_pi_id, key_value_pair_filter from src.regolith.fsclient import _id_key from src.regolith.helpers.basehelper import SoutHelperBase from src.regolith.schemas import ( @@ -17,12 +18,6 @@ PROJECTUM_FINISHED_STATI, PROJECTUM_PAUSED_STATI, ) -from regolith.tools import ( - all_docs_from_collection, - collection_str, - get_pi_id, - key_value_pair_filter, -) TARGET_COLL = "projecta" HELPER_TARGET = "l_projecta" diff --git a/src/regolith/helpers/makeappointmentshelper.py b/src/regolith/helpers/makeappointmentshelper.py index f49dccfa1..4d1ae5a8f 100644 --- a/src/regolith/helpers/makeappointmentshelper.py +++ b/src/regolith/helpers/makeappointmentshelper.py @@ -15,8 +15,6 @@ from dateutil.relativedelta import relativedelta from gooey import GooeyParser -from src.regolith.dates import get_dates - # from regolith.schemas import APPOINTMENTS_TYPES from regolith.fsclient import _id_key from regolith.helpers.basehelper import SoutHelperBase @@ -30,6 +28,7 @@ is_fully_appointed, merge_collections_superior, ) +from src.regolith.dates import get_dates TARGET_COLL = "people" HELPER_TARGET = "makeappointments" diff --git a/src/regolith/tests/conftest.py b/src/regolith/tests/conftest.py index e3b63139b..dcfe2ae00 100644 --- a/src/regolith/tests/conftest.py +++ b/src/regolith/tests/conftest.py @@ -1,19 +1,511 @@ +"""Copyright (c) 2017, Anthony Scopatz +All rights reserved.""" + import json -from pathlib import Path +import os +import tempfile +from copy import deepcopy +from subprocess import CalledProcessError import pytest +from pymongo import MongoClient +from pymongo import errors as mongo_errors +from xonsh.lib import subprocess +from xonsh.lib.os import rmtree + +from src.regolith.fsclient import dump_yaml +from src.regolith.schemas import EXEMPLARS + +OUTPUT_FAKE_DB = False # always turn it to false after you used it +# Currently the first two must be named test solely to match the helper map test output text +REGOLITH_MONGODB_NAME = "test" +FS_DB_NAME = "test" +ALTERNATE_REGOLITH_MONGODB_NAME = "mongo_test" + + +@pytest.fixture(scope="session") +def make_db(): + """A test fixutre that creates and destroys a git repo in a temporary + directory. + This will yield the path to the repo. + """ + cwd = os.getcwd() + name = "regolith_fake" + repo = os.path.join(tempfile.gettempdir(), name) + if os.path.exists(repo): + rmtree(repo) + subprocess.run(["git", "init", repo]) + os.chdir(repo) + with open("README", "w") as f: + f.write("testing " + name) + with open("regolithrc.json", "w") as f: + json.dump( + { + "default_user_id": "sbillinge", + "groupname": "ERGS", + "databases": [ + { + "name": "test", + "url": repo, + "public": True, + "path": "db", + "local": True, + "backend": "filesystem", + } + ], + "repos": [ + { + "_id": "talk_repo", + "params": {"namespace_id": "35", "initialize_with_readme": "true", "name": "repo name"}, + "url": "https://example.com", + "api_route": "/url/example", + "namespace_name": "talks", + } + ], + "stores": [ + { + "name": "store", + "url": repo, + "path": repo, + "public": True, + } + ], + "tokens": [{"_id": "gitlab_private_token", "token": ""}], + }, + f, + ) + fspath = os.path.join(repo, "db") + os.mkdir(fspath) + exemplars_to_fs(fspath) + subprocess.run(["git", "add", "."]) + subprocess.run(["git", "commit", "-am", "Initial readme"]) + yield repo + os.chdir(cwd) + if not OUTPUT_FAKE_DB: + rmtree(repo) + + +@pytest.fixture(scope="module") +def make_mongodb(): + """A test fixture that creates and destroys a git repo in a temporary + directory, as well as a mongo database. + This will yield the path to the repo. + """ + forked = False + name = "regolith_mongo_fake" + repo = os.path.join(tempfile.gettempdir(), name) + if os.path.exists(repo): + rmtree(repo) + subprocess.run(["git", "init", repo]) + os.chdir(repo) + with open("README", "w") as f: + f.write("testing " + name) + mongodbpath = os.path.join(repo, "dbs") + os.mkdir(mongodbpath) + with open("regolithrc.json", "w") as f: + json.dump( + { + "groupname": "ERGS", + "databases": [ + { + "name": REGOLITH_MONGODB_NAME, + "url": "localhost", + "path": repo, + "public": True, + "local": True, + "backend": "mongodb", + } + ], + "stores": [ + { + "name": "store", + "url": repo, + "path": repo, + "public": True, + } + ], + "mongodbpath": mongodbpath, + }, + f, + ) + if os.name == "nt": + # If on windows, the mongod command cannot be run with the fork or syslog options. + # Instead, it is installed as a service and the exceptions that would typically be log outputs + # are handled by the exception handlers below. In addition, the database must always be manually + # deleted from the windows mongo instance before running a fresh test. + cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"] + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + "Mongodb likely has not been installed as a service. In order to run mongodb tests, make sure\n" + "to install the mongodb community edition with the following link: \n" + "https://docs.mongodb.com/manual/installation/" + ) + yield False + return + cmd = ["mongostat", "--host", "localhost", "-n", "1"] + else: + cmd = ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath] + forked = True + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + "If using linux or mac, Mongod command failed to execute. " + "If using windows, the status of mongo could \n not be retrieved. " + "In order to run mongodb tests, make sure to install the mongodb community edition with " + "\nthe following link:\n" + "https://docs.mongodb.com/manual/installation/" + ) + yield False + return + try: + exemplars_to_mongo(REGOLITH_MONGODB_NAME) + except ConnectionError: + yield False + return + yield repo + cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"] + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + f'Deleting the test database failed, insert "mongo {REGOLITH_MONGODB_NAME} --eval ' + f'"db.dropDatabase()"" into command line manually' + ) + shut_down_fork(forked, repo) + if not OUTPUT_FAKE_DB: + rmtree(repo) + + +@pytest.fixture(scope="module") +def make_mixed_db(): + """A test fixture that creates and destroys a git repo in a temporary + directory, as well as a mongo database. + This will yield the path to the repo. + + This specific test fixture points to a repo that mixes mongo and filesystem backends for the assignments and + abstracts test collections in EXEMPLARS respectively. + """ + cwd = os.getcwd() + forked = False + name = "regolith_mongo_fake" + repo = os.path.join(tempfile.gettempdir(), name) + if os.path.exists(repo): + rmtree(repo) + subprocess.run(["git", "init", repo]) + os.chdir(repo) + with open("README", "w") as f: + f.write("testing " + name) + mongodbpath = os.path.join(repo, "dbs") + os.mkdir(mongodbpath) + fspath = os.path.join(repo, "db") + os.mkdir(fspath) + with open("regolithrc.json", "w") as f: + json.dump( + { + "groupname": "ERGS", + "databases": [ + { + "name": REGOLITH_MONGODB_NAME, + "url": "localhost", + "path": repo, + "public": True, + "local": True, + "backend": "mongodb", + }, + { + "name": FS_DB_NAME, + "url": repo, + "public": True, + "path": "db", + "local": True, + "backend": "filesystem", + }, + ], + "stores": [ + { + "name": "store", + "url": repo, + "path": repo, + "public": True, + } + ], + "mongodbpath": mongodbpath, + }, + f, + ) + if os.name == "nt": + # If on windows, the mongod command cannot be run with the fork or syslog options. + # Instead, it is installed as a service and the exceptions that would typically be log outputs + # are handled by the exception handlers below. In addition, the database must always be manually + # deleted from the windows mongo instance before running a fresh test. + cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"] + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + "Mongod likely has not been installed as a service. In order to run mongodb tests, make sure\n" + "to install the mongodb community edition with the following link: \n" + "https://docs.mongodb.com/manual/installation/" + ) + yield False + return + cmd = ["mongostat", "--host", "localhost", "-n", "1"] + else: + cmd = ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath] + forked = True + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + "If on linux/mac, Mongod command failed to execute. If on windows, the status of mongo could not be\n" + "retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with\n" + "the following link:\n" + "https://docs.mongodb.com/manual/installation/" + ) + yield False + return + # Write one collection doc in mongo + mongo_coll = "assignments" + try: + exemplars_to_mongo(REGOLITH_MONGODB_NAME, collection_list=[mongo_coll]) + except ConnectionError: + yield False + return + # Write one collection doc in file system + fs_coll = "abstracts" + exemplars_to_fs(fspath, collection_list=[fs_coll]) + yield repo, fs_coll, mongo_coll + cmd = ["mongo", REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"] + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + f'Deleting the test database failed, insert "mongo {REGOLITH_MONGODB_NAME} --eval ' + f'"db.dropDatabase()"" into command line manually' + ) + shut_down_fork(forked, repo) + os.chdir(cwd) + if not OUTPUT_FAKE_DB: + rmtree(repo) + + +@pytest.fixture(scope="session") +def make_bad_db(): + """A test fixutre that creates and destroys a git repo in a temporary + directory. + This will yield the path to the repo. + """ + cwd = os.getcwd() + name = "regolith_fake_bad" + repo = os.path.join(tempfile.gettempdir(), name) + if os.path.exists(repo): + rmtree(repo) + subprocess.run(["git", "init", repo]) + os.chdir(repo) + with open("README", "w") as f: + f.write("testing " + name) + with open("regolithrc.json", "w") as f: + json.dump( + { + "groupname": "ERGS", + "databases": [ + { + "name": "test", + "url": repo, + "public": True, + "path": "db", + "local": True, + } + ], + }, + f, + ) + os.mkdir("db") + # Write collection docs + for coll, example in deepcopy(EXEMPLARS).items(): + if isinstance(example, list): + d = {dd["_id"]: dd for dd in example} + else: + d = {example["_id"]: example} + d.update({"bad": {"_id": "bad", "bad": True}}) + if coll == "presentations": + d.update( + { + "bad_inst": { + "_id": "bad_inst", + "institution": "noinstitution", + "department": "nodept", + } + } + ) + dump_yaml("db/{}.yaml".format(coll), d) + subprocess.run(["git", "add", "."]) + subprocess.run(["git", "commit", "-am", "Initial readme"]) + yield repo + os.chdir(cwd) + rmtree(repo) + + +@pytest.fixture(scope="function") +def make_fs_to_mongo_migration_db(): + """A test fixture that creates and destroys a git repo in a temporary + directory, as well as a mongo database. + This will yield the path to the repo. + + This specific test fixture points to a repo that contains mongo and filesystem backends with only the + filesystem containing the exemplars. This is meant for use in migration testing. + """ + yield from make_migration_db(True) + + +@pytest.fixture(scope="function") +def make_mongo_to_fs_backup_db(): + """A test fixture that creates and destroys a git repo in a temporary + directory, as well as a mongo database. + This will yield the path to the repo. + + This specific test fixture points to a repo that contains mongo and filesystem backends with only the + filesystem containing the exemplars. This is meant for use in migration testing. + """ + yield from make_migration_db(False) + + +def make_migration_db(fs_to_mongo_true__mongo_to_fs_false): + cwd = os.getcwd() + forked = False + name = "regolith_mongo_fake" + repo = os.path.join(tempfile.gettempdir(), name) + if os.path.exists(repo): + rmtree(repo) + subprocess.run(["git", "init", repo]) + os.chdir(repo) + with open("README", "w") as f: + f.write("testing " + name) + mongodbpath = os.path.join(repo, "dbs") + os.mkdir(mongodbpath) + fspath = os.path.join(repo, "db") + os.mkdir(fspath) + with open("regolithrc.json", "w") as f: + json.dump( + { + "groupname": "ERGS", + "databases": [ + { + "name": ALTERNATE_REGOLITH_MONGODB_NAME, + "dst_url": "localhost", + "url": repo, + "path": "db", + "public": True, + "local": True, + "backend": "mongodb", + } + ], + "mongodbpath": mongodbpath, + }, + f, + ) + if os.name == "nt": + # If on windows, the mongod command cannot be run with the fork or syslog options. + # Instead, it is installed as a service and the exceptions that would typically be log outputs + # are handled by the exception handlers below. In addition, the database must always be manually + # deleted from the windows mongo instance before running a fresh test. + cmd = ["mongo", ALTERNATE_REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"] + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + "Mongod likely has not been installed as a service. In order to run mongodb tests, make sure\n" + "to install the mongodb community edition with the following link: \n" + "https://docs.mongodb.com/manual/installation/" + ) + yield False + return + cmd = ["mongostat", "--host", "localhost", "-n", "1"] + else: + cmd = ["mongod", "--fork", "--syslog", "--dbpath", mongodbpath] + forked = True + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + "If on linux/mac, Mongod command failed to execute. If on windows, the status of mongo could not be\n" + "retrieved. In order to run mongodb tests, make sure to install the mongodb community edition with\n" + "the following link:\n" + "https://docs.mongodb.com/manual/installation/" + ) + yield False + return + if fs_to_mongo_true__mongo_to_fs_false: + exemplars_to_fs(fspath) + else: + exemplars_to_mongo(ALTERNATE_REGOLITH_MONGODB_NAME) + yield repo + cmd = ["mongo", ALTERNATE_REGOLITH_MONGODB_NAME, "--eval", "db.dropDatabase()"] + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + f'Deleting the test database failed, insert "mongo {ALTERNATE_REGOLITH_MONGODB_NAME} --eval ' + f'"db.dropDatabase()"" into command line manually' + ) + shut_down_fork(forked, repo) + os.chdir(cwd) + if not OUTPUT_FAKE_DB: + rmtree(repo) + + +def shut_down_fork(forked, repo): + if forked: + cmd = ["mongo", "admin", "--eval", "db.shutdownServer()"] + try: + subprocess.check_call(cmd, cwd=repo) + except CalledProcessError: + print( + 'Deleting the test database failed, insert "mongo admin --eval ' + '"db.shutdownServer()"" into command line manually' + ) -@pytest.fixture -def user_filesystem(tmp_path): - base_dir = Path(tmp_path) - home_dir = base_dir / "home_dir" - home_dir.mkdir(parents=True, exist_ok=True) - cwd_dir = base_dir / "cwd_dir" - cwd_dir.mkdir(parents=True, exist_ok=True) +def exemplars_to_fs(fspath, collection_list=None): + exemplars_copy = deepcopy(EXEMPLARS) + if collection_list is None: + exemplars = exemplars_copy + else: + exemplars = {k: exemplars_copy[k] for k in collection_list if k in exemplars_copy} + cwd = os.getcwd() + os.chdir(fspath) + for coll, example in exemplars.items(): + if isinstance(example, list): + d = {dd["_id"]: dd for dd in example} + else: + d = {example["_id"]: example} + dump_yaml("{}.yaml".format(coll), d) + os.chdir(cwd) - home_config_data = {"username": "home_username", "email": "home@email.com"} - with open(home_dir / "diffpyconfig.json", "w") as f: - json.dump(home_config_data, f) - yield tmp_path +def exemplars_to_mongo(mongo_db_name, collection_list=None): + exemplars_copy = deepcopy(EXEMPLARS) + if collection_list is None: + exemplars = exemplars_copy + else: + exemplars = {k: exemplars_copy[k] for k in collection_list if k in exemplars_copy} + client = MongoClient("localhost", serverSelectionTimeoutMS=2000) + client.server_info() + for col_name, example in exemplars.items(): + db = client[mongo_db_name] + col = db[col_name] + try: + if isinstance(example, list): + for doc in example: + doc["_id"].replace(".", "") + col.insert_many(example) + else: + example["_id"].replace(".", "") + col.insert_one(example) + except mongo_errors.DuplicateKeyError: + print("Duplicate key error, check exemplars for duplicates if tests fail") + except mongo_errors.BulkWriteError: + print("Duplicate key error, check exemplars for duplicates if tests fail") diff --git a/src/regolith/tests/test_commands.py b/src/regolith/tests/test_commands.py index 01d93767b..4b937b3d7 100644 --- a/src/regolith/tests/test_commands.py +++ b/src/regolith/tests/test_commands.py @@ -6,13 +6,13 @@ from pathlib import Path import pytest +from tests.conftest import ALTERNATE_REGOLITH_MONGODB_NAME, FS_DB_NAME from src.regolith.database import connect from src.regolith.dates import convert_doc_iso_to_date from src.regolith.main import main from src.regolith.mongoclient import load_mongo_col from src.regolith.runcontrol import DEFAULT_RC, load_rcfile -from tests.conftest import ALTERNATE_REGOLITH_MONGODB_NAME, FS_DB_NAME BILLINGE_TEST = False # special tests for Billinge group, switch it to False before push to remote