From 63c18eac477782aed0795e665afa3d47908d411e Mon Sep 17 00:00:00 2001 From: farisdurrani Date: Fri, 15 Sep 2023 00:18:14 +0000 Subject: [PATCH 1/2] :art: Auto-generated directory tree for repository in Architecture.md --- .github/Architecture.md | 278 ++++++++++++++++++++-------------------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/.github/Architecture.md b/.github/Architecture.md index b126d7e66..0e59b4cf4 100644 --- a/.github/Architecture.md +++ b/.github/Architecture.md @@ -4,44 +4,44 @@ ``` 📦 backend -| |- 📂 aws_helpers: -| | |- 📂 dynamo_db_utils: -| | | |- 📜 DynamoUnitTests.md -| | | |- 📜 dynamo_db_utils.py -| | | |- 📜 trainspace_db.py -| | | |- 📜 constants.py : list of helpful constants -| | | |- 📜 userprogress_db.py -| | |- 📜 __init__.py -| |- 📂 ml: -| | |- 📜 ml_trainer.py : train a classical machine learning learning model on the dataset -| | |- 📜 __init__.py -| | |- 📜 ml_model_parser.py | |- 📂 dl: -| | |- 📜 __init__.py +| | |- 📜 dl_model_parser.py : parse the user specified pytorch model | | |- 📜 dl_trainer.py : train a deep learning model on the dataset +| | |- 📜 __init__.py | | |- 📜 dl_model.py : torch model based on user specifications from drag and drop -| | |- 📜 dl_model_parser.py : parse the user specified pytorch model | | |- 📜 detection.py | | |- 📜 dl_eval.py : Evaluation functions for deep learning models in Pytorch (eg: accuracy, loss, etc) +| |- 📂 ml: +| | |- 📜 ml_trainer.py : train a classical machine learning learning model on the dataset +| | |- 📜 ml_model_parser.py +| | |- 📜 __init__.py | |- 📂 common: -| | |- 📜 default_datasets.py : store logic to load in default datasets from scikit-learn +| | |- 📜 optimizer.py : what optimizer to use (ie: SGD or Adam for now) +| | |- 📜 dataset.py : read in the dataset through URL or file upload +| | |- 📜 loss_functions.py : loss function enum | | |- 📜 __init__.py | | |- 📜 preprocessing.py -| | |- 📜 dataset.py : read in the dataset through URL or file upload | | |- 📜 constants.py : list of helpful constants -| | |- 📜 optimizer.py : what optimizer to use (ie: SGD or Adam for now) | | |- 📜 ai_drive.py +| | |- 📜 default_datasets.py : store logic to load in default datasets from scikit-learn +| | |- 📜 utils.py : utility functions that could be helpful | | |- 📜 email_notifier.py : Endpoint to send email notification of training results via API Gateway + AWS SES -| | |- 📜 loss_functions.py : loss function enum | | |- 📜 kernel.py -| | |- 📜 utils.py : utility functions that could be helpful -| |- 📜 __init__.py -| |- 📜 middleware.py -| |- 📜 poetry.lock +| |- 📂 aws_helpers: +| | |- 📂 dynamo_db_utils: +| | | |- 📜 DynamoUnitTests.md +| | | |- 📜 dynamo_db_utils.py +| | | |- 📜 constants.py : list of helpful constants +| | | |- 📜 trainspace_db.py +| | | |- 📜 userprogress_db.py +| | |- 📜 __init__.py | |- 📜 epoch_times.csv | |- 📜 app.py : run the backend (entrypoint script) -| |- 📜 data.csv : data csv file for use in the playground +| |- 📜 __init__.py | |- 📜 pyproject.toml +| |- 📜 data.csv : data csv file for use in the playground +| |- 📜 poetry.lock +| |- 📜 middleware.py ``` ## Frontend Architecture @@ -49,177 +49,177 @@ ``` 📦 frontend | |- 📂 src: -| | |- 📂 backend_outputs: -| | | |- 📜 model.pt : Last model.pt output -| | | |- 📜 my_deep_learning_model.onnx : Last ONNX file output -| | | |- 📜 model.pkl -| | |- 📂 pages: -| | | |- 📂 train: -| | | | |- 📜 [train_space_id].tsx -| | | | |- 📜 index.tsx -| | | |- 📜 feedback.tsx -| | | |- 📜 settings.tsx -| | | |- 📜 _app.tsx -| | | |- 📜 learn.tsx -| | | |- 📜 login.tsx -| | | |- 📜 wiki.tsx -| | | |- 📜 _document.tsx -| | | |- 📜 about.tsx -| | | |- 📜 forgot.tsx -| | | |- 📜 dashboard.tsx -| | | |- 📜 LearnContent.tsx | | |- 📂 features: | | | |- 📂 Train: -| | | | |- 📂 constants: -| | | | | |- 📜 trainConstants.ts +| | | | |- 📂 redux: +| | | | | |- 📜 trainspaceSlice.ts +| | | | | |- 📜 trainspaceApi.ts | | | | |- 📂 features: -| | | | | |- 📂 Image: -| | | | | | |- 📂 constants: -| | | | | | | |- 📜 imageConstants.ts -| | | | | | |- 📂 components: -| | | | | | | |- 📜 ImageFlow.tsx -| | | | | | | |- 📜 ImageTrainspace.tsx -| | | | | | | |- 📜 ImageDatasetStep.tsx -| | | | | | | |- 📜 ImageReviewStep.tsx -| | | | | | | |- 📜 ImageParametersStep.tsx -| | | | | | |- 📂 types: -| | | | | | | |- 📜 imageTypes.ts -| | | | | | |- 📂 redux: -| | | | | | | |- 📜 imageActions.ts -| | | | | | | |- 📜 imageApi.ts -| | | | | | |- 📜 index.ts | | | | | |- 📂 Tabular: +| | | | | | |- 📂 redux: +| | | | | | | |- 📜 tabularApi.ts +| | | | | | | |- 📜 tabularActions.ts | | | | | | |- 📂 constants: | | | | | | | |- 📜 tabularConstants.ts +| | | | | | |- 📂 types: +| | | | | | | |- 📜 tabularTypes.ts | | | | | | |- 📂 components: -| | | | | | | |- 📜 TabularParametersStep.tsx +| | | | | | | |- 📜 TabularReviewStep.tsx | | | | | | | |- 📜 TabularDatasetStep.tsx -| | | | | | | |- 📜 TabularFlow.tsx +| | | | | | | |- 📜 TabularParametersStep.tsx | | | | | | | |- 📜 TabularTrainspace.tsx -| | | | | | | |- 📜 TabularReviewStep.tsx -| | | | | | |- 📂 types: -| | | | | | | |- 📜 tabularTypes.ts +| | | | | | | |- 📜 TabularFlow.tsx +| | | | | | |- 📜 index.ts +| | | | | |- 📂 Image: | | | | | | |- 📂 redux: -| | | | | | | |- 📜 tabularActions.ts -| | | | | | | |- 📜 tabularApi.ts +| | | | | | | |- 📜 imageActions.ts +| | | | | | | |- 📜 imageApi.ts +| | | | | | |- 📂 constants: +| | | | | | | |- 📜 imageConstants.ts +| | | | | | |- 📂 types: +| | | | | | | |- 📜 imageTypes.ts +| | | | | | |- 📂 components: +| | | | | | | |- 📜 ImageParametersStep.tsx +| | | | | | | |- 📜 ImageFlow.tsx +| | | | | | | |- 📜 ImageTrainspace.tsx +| | | | | | | |- 📜 ImageReviewStep.tsx +| | | | | | | |- 📜 ImageDatasetStep.tsx | | | | | | |- 📜 index.ts -| | | | |- 📂 components: -| | | | | |- 📜 CreateTrainspace.tsx -| | | | | |- 📜 DatasetStepLayout.tsx -| | | | | |- 📜 TrainspaceLayout.tsx +| | | | |- 📂 constants: +| | | | | |- 📜 trainConstants.ts | | | | |- 📂 types: | | | | | |- 📜 trainTypes.ts -| | | | |- 📂 redux: -| | | | | |- 📜 trainspaceApi.ts -| | | | | |- 📜 trainspaceSlice.ts -| | | |- 📂 Dashboard: | | | | |- 📂 components: -| | | | | |- 📜 TrainDoughnutChart.tsx -| | | | | |- 📜 TrainBarChart.tsx -| | | | | |- 📜 TrainDataGrid.tsx +| | | | | |- 📜 TrainspaceLayout.tsx +| | | | | |- 📜 DatasetStepLayout.tsx +| | | | | |- 📜 CreateTrainspace.tsx +| | | |- 📂 Feedback: | | | | |- 📂 redux: -| | | | | |- 📜 dashboardApi.ts +| | | | | |- 📜 feedbackApi.ts +| | | |- 📂 OpenAi: +| | | | |- 📜 openAiUtils.ts | | | |- 📂 LearnMod: -| | | | |- 📜 LearningModulesContent.tsx -| | | | |- 📜 Exercise.tsx -| | | | |- 📜 ImageComponent.tsx | | | | |- 📜 ModulesSideBar.tsx +| | | | |- 📜 ImageComponent.tsx | | | | |- 📜 MCQuestion.tsx -| | | | |- 📜 FRQuestion.tsx | | | | |- 📜 ClassCard.tsx -| | | |- 📂 Feedback: +| | | | |- 📜 FRQuestion.tsx +| | | | |- 📜 Exercise.tsx +| | | | |- 📜 LearningModulesContent.tsx +| | | |- 📂 Dashboard: | | | | |- 📂 redux: -| | | | | |- 📜 feedbackApi.ts -| | | |- 📂 OpenAi: -| | | | |- 📜 openAiUtils.ts +| | | | | |- 📜 dashboardApi.ts +| | | | |- 📂 components: +| | | | | |- 📜 TrainDoughnutChart.tsx +| | | | | |- 📜 TrainDataGrid.tsx +| | | | | |- 📜 TrainBarChart.tsx | | |- 📂 common: +| | | |- 📂 redux: +| | | | |- 📜 backendApi.ts +| | | | |- 📜 userLogin.ts +| | | | |- 📜 train.ts +| | | | |- 📜 store.ts +| | | | |- 📜 hooks.ts | | | |- 📂 styles: | | | | |- 📜 globals.css | | | | |- 📜 Home.module.css | | | |- 📂 components: | | | | |- 📜 ClientOnlyPortal.tsx | | | | |- 📜 EmailInput.tsx +| | | | |- 📜 NavBarMain.tsx +| | | | |- 📜 Spacer.tsx | | | | |- 📜 HtmlTooltip.tsx | | | | |- 📜 TitleText.tsx -| | | | |- 📜 NavBarMain.tsx -| | | | |- 📜 Footer.tsx | | | | |- 📜 DlpTooltip.tsx -| | | | |- 📜 Spacer.tsx +| | | | |- 📜 Footer.tsx | | | |- 📂 utils: +| | | | |- 📜 dndHelpers.ts | | | | |- 📜 firebase.ts | | | | |- 📜 dateFormat.ts -| | | | |- 📜 dndHelpers.ts -| | | |- 📂 redux: -| | | | |- 📜 backendApi.ts -| | | | |- 📜 userLogin.ts -| | | | |- 📜 hooks.ts -| | | | |- 📜 store.ts -| | | | |- 📜 train.ts +| | |- 📂 pages: +| | | |- 📂 train: +| | | | |- 📜 [train_space_id].tsx +| | | | |- 📜 index.tsx +| | | |- 📜 _document.tsx +| | | |- 📜 dashboard.tsx +| | | |- 📜 _app.tsx +| | | |- 📜 LearnContent.tsx +| | | |- 📜 forgot.tsx +| | | |- 📜 learn.tsx +| | | |- 📜 settings.tsx +| | | |- 📜 feedback.tsx +| | | |- 📜 about.tsx +| | | |- 📜 login.tsx +| | | |- 📜 wiki.tsx +| | |- 📂 backend_outputs: +| | | |- 📜 model.pkl +| | | |- 📜 model.pt : Last model.pt output +| | | |- 📜 my_deep_learning_model.onnx : Last ONNX file output | | |- 📜 iris.csv : Sample CSV data -| | |- 📜 next-env.d.ts | | |- 📜 constants.ts | | |- 📜 GlobalStyle.ts -| |- 📂 layer_docs: -| | |- 📜 Linear.md : Doc for Linear layer -| | |- 📜 Softmax.md : Doc for Softmax layer -| | |- 📜 softmax_equation.png : PNG file of Softmax equation -| | |- 📜 ReLU.md : Doc for ReLU later +| | |- 📜 next-env.d.ts | |- 📂 public: | | |- 📂 images: +| | | |- 📂 wiki_images: +| | | | |- 📜 maxpool2d.gif +| | | | |- 📜 tanh_plot.png +| | | | |- 📜 avgpool_maxpool.gif +| | | | |- 📜 batchnorm_diagram.png +| | | | |- 📜 softmax_equation.png : PNG file of Softmax equation +| | | | |- 📜 conv2d.gif +| | | | |- 📜 conv2d2.gif +| | | | |- 📜 tanh_equation.png +| | | | |- 📜 dropout_diagram.png +| | | | |- 📜 sigmoid_equation.png +| | | |- 📂 learn_mod_images: +| | | | |- 📜 ReLUactivation.png +| | | | |- 📜 binarystepactivation.png +| | | | |- 📜 lossExampleEquation.png +| | | | |- 📜 sigmoidactivation.png +| | | | |- 📜 sigmoidfunction.png +| | | | |- 📜 robotImage.jpg +| | | | |- 📜 lossExample.png +| | | | |- 📜 lossExampleTable.png +| | | | |- 📜 neuronWithEquation.png +| | | | |- 📜 LeakyReLUactivation.png +| | | | |- 📜 neuron.png +| | | | |- 📜 neuralnet.png +| | | | |- 📜 tanhactivation.png | | | |- 📂 logos: | | | | |- 📂 dlp_branding: | | | | | |- 📜 dlp-logo.svg : DLP Logo, duplicate of files in public, but essential as the frontend can't read public | | | | | |- 📜 dlp-logo.png : DLP Logo, duplicate of files in public, but essential as the frontend can't read public | | | | |- 📜 dsgt-logo-dark.png | | | | |- 📜 github.png -| | | | |- 📜 dsgt-logo-light.png -| | | | |- 📜 pandas-logo.png -| | | | |- 📜 react-logo.png +| | | | |- 📜 google.png | | | | |- 📜 python-logo.png +| | | | |- 📜 dsgt-logo-white-back.png +| | | | |- 📜 pandas-logo.png +| | | | |- 📜 aws-logo.png | | | | |- 📜 flask-logo.png +| | | | |- 📜 react-logo.png +| | | | |- 📜 dsgt-logo-light.png | | | | |- 📜 pytorch-logo.png -| | | | |- 📜 aws-logo.png -| | | | |- 📜 google.png -| | | | |- 📜 dsgt-logo-white-back.png -| | | |- 📂 learn_mod_images: -| | | | |- 📜 lossExampleTable.png -| | | | |- 📜 tanhactivation.png -| | | | |- 📜 sigmoidactivation.png -| | | | |- 📜 LeakyReLUactivation.png -| | | | |- 📜 ReLUactivation.png -| | | | |- 📜 robotImage.jpg -| | | | |- 📜 lossExample.png -| | | | |- 📜 binarystepactivation.png -| | | | |- 📜 lossExampleEquation.png -| | | | |- 📜 neuronWithEquation.png -| | | | |- 📜 neuron.png -| | | | |- 📜 neuralnet.png -| | | | |- 📜 sigmoidfunction.png -| | | |- 📂 wiki_images: -| | | | |- 📜 avgpool_maxpool.gif -| | | | |- 📜 conv2d2.gif -| | | | |- 📜 dropout_diagram.png -| | | | |- 📜 batchnorm_diagram.png -| | | | |- 📜 tanh_plot.png -| | | | |- 📜 softmax_equation.png : PNG file of Softmax equation -| | | | |- 📜 maxpool2d.gif -| | | | |- 📜 tanh_equation.png -| | | | |- 📜 conv2d.gif -| | | | |- 📜 sigmoid_equation.png | | | |- 📜 demo_video.gif : GIF tutorial of a simple classification training session +| | |- 📜 dlp-logo.ico : DLP Logo | | |- 📜 dlp-logo.svg : DLP Logo, duplicate of files in public, but essential as the frontend can't read public -| | |- 📜 robots.txt +| | |- 📜 index.html : Base HTML file that will be initially rendered | | |- 📜 dlp-logo.png : DLP Logo, duplicate of files in public, but essential as the frontend can't read public | | |- 📜 manifest.json : Default React file for choosing icon based on -| | |- 📜 index.html : Base HTML file that will be initially rendered -| | |- 📜 dlp-logo.ico : DLP Logo +| | |- 📜 robots.txt +| |- 📂 layer_docs: +| | |- 📜 Softmax.md : Doc for Softmax layer +| | |- 📜 Linear.md : Doc for Linear layer +| | |- 📜 softmax_equation.png : PNG file of Softmax equation +| | |- 📜 ReLU.md : Doc for ReLU later +| |- 📜 .eslintrc.json +| |- 📜 .eslintignore | |- 📜 package.json | |- 📜 jest.config.js -| |- 📜 .eslintrc.json | |- 📜 tsconfig.json +| |- 📜 next.config.js | |- 📜 yarn.lock -| |- 📜 .eslintignore | |- 📜 next-env.d.ts -| |- 📜 next.config.js ``` From 8e63d8a1e56686aa1b1fcbe511beca221aa40ffd Mon Sep 17 00:00:00 2001 From: prishavall <124836866+prishavall@users.noreply.github.com> Date: Thu, 14 Sep 2023 20:24:03 -0400 Subject: [PATCH 2/2] removed yml (#947) Co-authored-by: Faris Durrani --- yml_to_req.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 yml_to_req.py diff --git a/yml_to_req.py b/yml_to_req.py deleted file mode 100644 index e4fb05d9a..000000000 --- a/yml_to_req.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Utility file to convert our environment.yml file to a requirements.txt for deployment purpose. - -Credit to: https://gist.github.com/pemagrg1/f959c19ec18fee3ce2ff9b3b86b67c16 - -""" - -import ruamel.yaml - -yaml = ruamel.yaml.YAML() -data = yaml.load(open("conda/environment.yml")) - -requirements = [] -for dep in data["dependencies"]: - if isinstance(dep, str): - if dep == "pytorch": - requirements.append("torch") - elif dep.startswith("python"): - continue - else: - parsed_dep = dep.split("=") - if parsed_dep[0] == dep: - requirements.append(dep) - elif len(parsed_dep) > 2: - raise ValueError( - "Misformat in environment.yml file. Make sure that you have = or format" - ) - else: - package_name, package_version = parsed_dep[0], parsed_dep[1] - requirements.append(f"{parsed_dep[0]}=={parsed_dep[1]}") - elif isinstance(dep, dict): - for preq in dep.get("pip", []): - requirements.append(preq) - -with open("requirements.txt", "w") as fp: - for requirement in requirements: - fp.write(requirement + "\n")