Agents

.coveragerc

@@ -0,0 +1,19 @@
+[run]
+source = foundry
+omit = 
+    */tests/*
+    */site-packages/*
+    setup.py
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    pass
+    raise ImportError
+    except ImportError:
+
+[html]
+directory = coverage_html 

.github/workflows/tests.yml

@@ -1,55 +1,57 @@
-name: tests
+name: Tests
 
-on:
-  pull_request:
-    branches:
-      - dev
-      - main
+on: [push, pull_request]
 
 jobs:
-
-  build:
+  test:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-
+        python-version: ['3.9', '3.10', '3.11']
+        install-type: [core, all]
     env:
       CLIENT_ID: ${{ secrets.CLIENT_ID }}
       CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
-    name: build
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache : 'pip'
-
-      - name: Globus auth
-        run: 'echo "$GLOBUS_CONFIG" > ~/.globus-native-apps.cfg'
-        shell: bash
-        env:
-          GLOBUS_CONFIG: "${{ secrets.GLOBUS_CONFIG }}"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install -r test-requirements.txt
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are any-flake8 comments
-          flake8 foundry
-
-      - name: Test with pytest
-        run: |
-          pytest -s -v tests/ --cov=./foundry --cov-report=xml
-      - name: Upload coverage to Codecov
-        run: |
-          curl -Os https://uploader.codecov.io/v0.1.0_4653/linux/codecov
-
-          chmod +x codecov
-          ./codecov -t ${{ secrets.CODECOV_TOKEN }}
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        # Install base package
+        if [ "${{ matrix.install-type }}" = "core" ]; then
+          pip install -e .
+        else
+          pip install -e .[all]
+        fi
+        # Install test dependencies
+        pip install -r requirements-dev.txt
+        pip install -r test-requirements.txt
+        # Install optional dependencies for testing
+        pip install rdkit pandas numpy h5py
+
+    - name: Run tests with coverage
+      run: |
+        if [ "${{ matrix.install-type }}" = "core" ]; then
+          # Skip tests that require optional dependencies
+          pytest --cov=foundry --cov-report=xml --cov-report=term-missing \
+            --ignore=tests/test_molecular.py \
+            --ignore=tests/test_ml_frameworks.py \
+            --ignore=tests/test_loaders_specialized.py
+        else
+          pytest --cov=foundry --cov-report=xml --cov-report=term-missing
+        fi
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: ./coverage.xml
+        fail_ci_if_error: false
+        verbose: true

README.md

@@ -1,4 +1,3 @@
-
 <picture>
   <source srcset="https://raw.githubusercontent.com/MLMI2-CSSI/foundry/main/assets/foundry-white.png" height=175" media="(prefers-color-scheme: dark)">
   <img src="https://raw.githubusercontent.com/MLMI2-CSSI/foundry/main/assets/foundry-black.png" height="175">
@@ -9,6 +8,7 @@
 [![Tests](https://github.com/MLMI2-CSSI/foundry/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MLMI2-CSSI/foundry/actions/workflows/python-publish.yml)
 [![NSF-1931306](https://img.shields.io/badge/NSF-1931306-blue)](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1931306&HistoricalAwards=false)
 [<img src="https://img.shields.io/badge/view-documentation-blue">](https://ai-materials-and-chemistry.gitbook.io/foundry/)
+[![Coverage](https://codecov.io/gh/MLMI2-CSSI/foundry/branch/main/graph/badge.svg)](https://codecov.io/gh/MLMI2-CSSI/foundry)
 
 
 Foundry-ML simplifies the discovery and usage of ML-ready datasets in materials science and chemistry providing a simple API to access even complex datasets. 
@@ -30,42 +30,61 @@ DLHub documentation for model publication and running information can be found [
 Install Foundry-ML via command line with:
 `pip install foundry_ml`
 
-You can use the following code to import and instantiate Foundry-ML, then load a dataset.
+You can use the following code to import and instantiate Foundry-ML, then load a dataset:
 
 ```python
 from foundry import Foundry
+
 f = Foundry(index="mdf")
+results_df = f.search(query="materials science", limit=10)
+print(results_df.head())
+```
 
+Below is an example of publishing your own dataset with Foundry:
 
-f = f.load("10.18126/e73h-3w6n", globus=True)
+```python
+# Let's assume you have a local folder of data you'd like to publish
+from foundry.foundry_dataset import FoundryDataset
+
+dataset = FoundryDataset(dataset_name="MyNewDataset")
+dataset.add_data("/path/to/local_data_folder")  # Make sure to have the correct structure
+
+# Then publish the dataset
+res = f.publish_dataset(dataset, update=False, test=False)
+print("Dataset submitted with response:", res)
 ```
-*NOTE*: If you run locally and don't want to install the [Globus Connect Personal endpoint](https://www.globus.org/globus-connect-personal), just set the `globus=False`.
 
-If running this code in a notebook, a table of metadata for the dataset will appear:
+If you run locally and don't want to install the [Globus Connect Personal endpoint](https://www.globus.org/globus-connect-personal), just set the `globus=False` when loading datasets.
 
-<img width="903" alt="metadata" src="https://user-images.githubusercontent.com/16869564/197038472-0b6ae559-4a6b-4b20-88e5-679bb6eb4f5c.png">
+# How to Contribute
 
-We can use the data with `f.load_data()` and specifying splits such as `train` for different segments of the dataset, then use matplotlib to visualize it.
+We welcome contributions from the community to enhance Foundry-ML. Whether you want to fix a bug, propose a new feature, or improve documentation, follow the steps below to get started:
 
-```python
-res = f.load_data()
+1. Fork the repository:
+   - Click the "Fork" button at the top of this repository's page.
 
-imgs = res['train']['input']['imgs']
-desc = res['train']['input']['metadata']
-coords = res['train']['target']['coords']
+2. Clone your fork locally:
+   - git clone https://github.com/your-username/foundry.git
 
-n_images = 3
-offset = 150
-key_list = list(res['train']['input']['imgs'].keys())[0+offset:n_images+offset]
+3. Create a new branch for your feature or fix:
+   - git checkout -b feature/my-new-feature
 
-fig, axs = plt.subplots(1, n_images, figsize=(20,20))
-for i in range(n_images):
-    axs[i].imshow(imgs[key_list[i]])
-    axs[i].scatter(coords[key_list[i]][:,0], coords[key_list[i]][:,1], s = 20, c = 'r', alpha=0.5)
-```
-<img width="595" alt="Screen Shot 2022-10-20 at 2 22 43 PM" src="https://user-images.githubusercontent.com/16869564/197039252-6d9c78ba-dc09-4037-aac2-d6f7e8b46851.png">
+4. Install dependencies and set up a virtual environment if needed:
+   - pip install -r requirements.txt
+
+5. Make your changes and write tests:
+   - For code-related changes, add or update tests under tests/ to ensure ongoing stability.
+
+6. Run tests to confirm everything works:
+   - pytest
+
+7. Commit your changes and push the branch to GitHub:
+   - git push origin feature/my-new-feature
 
-[See full examples](./examples)
+8. Create a Pull Request:
+   - On GitHub, open a Pull Request from your branch to the main branch of MLMI2-CSSI/foundry.
+
+Our team will review your submission and provide feedback. Thank you for helping us grow Foundry-ML!
 
 # How to Cite
 If you find Foundry-ML useful, please cite the following [paper](https://doi.org/10.21105/joss.05467)
@@ -108,3 +127,45 @@ https://www.dlhub.org
 ## The Materials Data Facility
 This work was performed under financial assistance award 70NANB14H012 from U.S. Department of Commerce, National Institute of Standards and Technology as part of the [Center for Hierarchical Material Design (CHiMaD)](http://chimad.northwestern.edu). This work was performed under the following financial assistance award 70NANB19H005 from U.S. Department of Commerce, National Institute of Standards and Technology as part of the Center for Hierarchical Materials Design (CHiMaD). This work was also supported by the National Science Foundation as part of the [Midwest Big Data Hub](http://midwestbigdatahub.org) under NSF Award Number: 1636950 "BD Spokes: SPOKE: MIDWEST: Collaborative: Integrative Materials Design (IMaD): Leverage, Innovate, and Disseminate".
 https://www.materialsdatafacility.org
+
+## Installation
+
+Basic installation:
+```bash
+pip install foundry_ml
+```
+
+With optional features:
+```bash
+# For molecular data support
+pip install foundry_ml[molecular]
+
+# For PyTorch integration
+pip install foundry_ml[torch]
+
+# For TensorFlow integration
+pip install foundry_ml[tensorflow]
+
+# Install all optional dependencies
+pip install foundry_ml[all]
+```
+
+## Development Installation
+
+For development, you can install all dependencies including testing tools:
+
+```bash
+# Clone the repository
+git clone https://github.com/MLMI2-CSSI/foundry.git
+cd foundry
+
+# Install in development mode with all extras
+pip install -e .[all]
+
+# Install development dependencies
+pip install -r requirements-dev.txt
+
+# Run tests
+pytest  # Run all tests
+pytest --ignore=tests/test_molecular.py  # Skip tests requiring optional dependencies
+```

data/https_test/test_data.json

@@ -0,0 +1 @@
+"[{\"A\":0.643870455,\"B\":0.6091335397,\"C\":0.7822690824,\"D\":0.8268479159},{\"A\":0.9456540646,\"B\":0.3326369296,\"C\":0.8841205492,\"D\":0.2556220774},{\"A\":0.6291401542,\"B\":0.2247617025,\"C\":0.8462314772,\"D\":0.5304308964},{\"A\":0.975547697,\"B\":0.5416567183,\"C\":0.9508777456,\"D\":0.2569899117},{\"A\":0.4010386755,\"B\":0.4127109481,\"C\":0.017865727,\"D\":0.421698277},{\"A\":0.7083439555,\"B\":0.0440585057,\"C\":0.8736216744,\"D\":0.5628843021},{\"A\":0.3236709921,\"B\":0.074779633,\"C\":0.5776595747,\"D\":0.2022646601},{\"A\":0.4839563307,\"B\":0.425772453,\"C\":0.8387236919,\"D\":0.4334753751},{\"A\":0.5215194358,\"B\":0.2194216689,\"C\":0.6183393944,\"D\":0.797299772},{\"A\":0.0713460804,\"B\":0.3466899498,\"C\":0.6522131807,\"D\":0.1778979284},{\"A\":0.2707517899,\"B\":0.095771952,\"C\":0.1290605884,\"D\":0.9248904319},{\"A\":0.8716852911,\"B\":0.9444690268,\"C\":0.1294852031,\"D\":0.5068766826},{\"A\":0.5843460535,\"B\":0.9755514645,\"C\":0.9818506546,\"D\":0.3279229361},{\"A\":0.3007062433,\"B\":0.6231023141,\"C\":0.7639033834,\"D\":0.8988798453},{\"A\":0.7695470502,\"B\":0.0501137742,\"C\":0.6176087152,\"D\":0.1445764886},{\"A\":0.2675470175,\"B\":0.4640297176,\"C\":0.8329810114,\"D\":0.6570328862},{\"A\":0.7674771752,\"B\":0.1118872796,\"C\":0.7793321285,\"D\":0.1543847316},{\"A\":0.6721579139,\"B\":0.6553073897,\"C\":0.2456219058,\"D\":0.4863060138},{\"A\":0.8931107755,\"B\":0.4741801354,\"C\":0.7188146988,\"D\":0.3763431712},{\"A\":0.5469096954,\"B\":0.8118171604,\"C\":0.4255865379,\"D\":0.2218850497},{\"A\":0.9271195751,\"B\":0.8887750214,\"C\":0.7481395646,\"D\":0.8030491361},{\"A\":0.2463806012,\"B\":0.7847482354,\"C\":0.8679944882,\"D\":0.9079702233},{\"A\":0.6210355879,\"B\":0.9386518629,\"C\":0.701089623,\"D\":0.2504838653},{\"A\":0.8492633109,\"B\":0.3586585934,\"C\":0.0629805626,\"D\":0.9214151715},{\"A\":0.5424288564,\"B\":0.4999920911,\"C\":0.3261323566,\"D\":0.1618166139},{\"A\":0.2888237847,\"B\":0.9141131506,\"C\":0.0343475971,\"D\":0.8115089618},{\"A\":0.2522191092,\"B\":0.345685169,\"C\":0.7884565197,\"D\":0.0355617466},{\"A\":0.4457942938,\"B\":0.6971631221,\"C\":0.5781304242,\"D\":0.1200273581},{\"A\":0.6002382923,\"B\":0.4484248841,\"C\":0.7621932221,\"D\":0.7981014515},{\"A\":0.3691042403,\"B\":0.5542810312,\"C\":0.1286744678,\"D\":0.0136204246},{\"A\":0.4810123215,\"B\":0.6406599704,\"C\":0.6024308844,\"D\":0.1024778434},{\"A\":0.8038373911,\"B\":0.9133361012,\"C\":0.3962806696,\"D\":0.0077623626},{\"A\":0.0780094832,\"B\":0.9021410901,\"C\":0.6492799769,\"D\":0.6116689937},{\"A\":0.3045182842,\"B\":0.8557673608,\"C\":0.1466120067,\"D\":0.5995064907},{\"A\":0.6430901418,\"B\":0.2708746313,\"C\":0.1527527593,\"D\":0.3692156059},{\"A\":0.5106498724,\"B\":0.0371026187,\"C\":0.1790488679,\"D\":0.6432881157},{\"A\":0.0016580644,\"B\":0.6681410215,\"C\":0.4205885498,\"D\":0.8184233958},{\"A\":0.0964162811,\"B\":0.3384898573,\"C\":0.8300644236,\"D\":0.4340685704},{\"A\":0.8950115298,\"B\":0.9260489775,\"C\":0.0742238435,\"D\":0.2144982415},{\"A\":0.8115382564,\"B\":0.1321103032,\"C\":0.0943974514,\"D\":0.0431889336},{\"A\":0.7288958605,\"B\":0.9392151718,\"C\":0.7556298045,\"D\":0.1455506361},{\"A\":0.1091433253,\"B\":0.1931513453,\"C\":0.6288272134,\"D\":0.8938803543},{\"A\":0.5666071526,\"B\":0.6866406665,\"C\":0.3236813586,\"D\":0.905914349},{\"A\":0.0742907865,\"B\":0.5449429136,\"C\":0.6803403721,\"D\":0.2261924153},{\"A\":0.9670714577,\"B\":0.3705962079,\"C\":0.6221720406,\"D\":0.9565497286},{\"A\":0.8795892767,\"B\":0.503880071,\"C\":0.6675467877,\"D\":0.8789986354},{\"A\":0.2085745935,\"B\":0.2422616603,\"C\":0.6609245125,\"D\":0.5574753552},{\"A\":0.3901471816,\"B\":0.4636722689,\"C\":0.6276594441,\"D\":0.9157642311},{\"A\":0.4989495397,\"B\":0.8789409087,\"C\":0.8639646997,\"D\":0.4271356786},{\"A\":0.6699409813,\"B\":0.1002664854,\"C\":0.2746497369,\"D\":0.9139472198},{\"A\":0.9931311364,\"B\":0.8188230319,\"C\":0.0256276884,\"D\":0.7103069326},{\"A\":0.651548713,\"B\":0.564519124,\"C\":0.2685295664,\"D\":0.5504980797},{\"A\":0.6783159928,\"B\":0.8582392923,\"C\":0.0106426152,\"D\":0.343881835},{\"A\":0.9125146347,\"B\":0.8098421837,\"C\":0.1570679367,\"D\":0.7426108926},{\"A\":0.5605342219,\"B\":0.5531496696,\"C\":0.9762101759,\"D\":0.9150743664},{\"A\":0.4589324575,\"B\":0.3824806997,\"C\":0.4509337436,\"D\":0.0977040707},{\"A\":0.6860159363,\"B\":0.3468538712,\"C\":0.4075233003,\"D\":0.8304722601},{\"A\":0.2042029477,\"B\":0.8559035537,\"C\":0.1758115017,\"D\":0.2806214757},{\"A\":0.3396679638,\"B\":0.108741023,\"C\":0.0471056095,\"D\":0.549568292},{\"A\":0.3966874957,\"B\":0.5983624282,\"C\":0.7548218378,\"D\":0.6888001455},{\"A\":0.2496667117,\"B\":0.9411866804,\"C\":0.5447466736,\"D\":0.6027201657},{\"A\":0.6803464783,\"B\":0.6756867895,\"C\":0.1776041426,\"D\":0.8260296751},{\"A\":0.5324327065,\"B\":0.8164624535,\"C\":0.0345776987,\"D\":0.6946683903},{\"A\":0.3278234112,\"B\":0.7767899593,\"C\":0.4236985558,\"D\":0.7855641902},{\"A\":0.1871835033,\"B\":0.7185915704,\"C\":0.8892584241,\"D\":0.9041425981},{\"A\":0.0252279047,\"B\":0.1327784456,\"C\":0.4012167679,\"D\":0.6827824908},{\"A\":0.3242069607,\"B\":0.5819640332,\"C\":0.8668542152,\"D\":0.1434934146},{\"A\":0.8318364286,\"B\":0.0538128288,\"C\":0.465820118,\"D\":0.920627827},{\"A\":0.4278724113,\"B\":0.1968834312,\"C\":0.0518638606,\"D\":0.6344022855},{\"A\":0.8190435048,\"B\":0.8062720847,\"C\":0.2882240224,\"D\":0.865150816},{\"A\":0.2061553601,\"B\":0.2758599846,\"C\":0.5853995293,\"D\":0.6143356356},{\"A\":0.3124749213,\"B\":0.8534254958,\"C\":0.2368244264,\"D\":0.0347652254},{\"A\":0.2462624136,\"B\":0.7669510032,\"C\":0.4371029225,\"D\":0.8295093479},{\"A\":0.4809361558,\"B\":0.794014646,\"C\":0.7514307895,\"D\":0.5129345564},{\"A\":0.3361090064,\"B\":0.0592829927,\"C\":0.2856413958,\"D\":0.4455347327},{\"A\":0.650641595,\"B\":0.0793956841,\"C\":0.928293866,\"D\":0.4721011054},{\"A\":0.2117624986,\"B\":0.0142221126,\"C\":0.2005417756,\"D\":0.8059841184},{\"A\":0.4660735224,\"B\":0.5215475799,\"C\":0.7693073705,\"D\":0.8356910391},{\"A\":0.0885559958,\"B\":0.0119811352,\"C\":0.3977151269,\"D\":0.1626498842},{\"A\":0.4535536676,\"B\":0.2507941702,\"C\":0.8279387846,\"D\":0.1019767725},{\"A\":0.503070451,\"B\":0.6063300971,\"C\":0.4454432603,\"D\":0.0517681478},{\"A\":0.6022218039,\"B\":0.7411709954,\"C\":0.9960575021,\"D\":0.9662997087},{\"A\":0.3314992037,\"B\":0.9145246624,\"C\":0.3891086247,\"D\":0.8709633507},{\"A\":0.65170299,\"B\":0.220147899,\"C\":0.4263250241,\"D\":0.8495339182},{\"A\":0.0890642543,\"B\":0.697432947,\"C\":0.59118971,\"D\":0.932403869},{\"A\":0.8927340507,\"B\":0.9653811365,\"C\":0.0352710207,\"D\":0.8088212468},{\"A\":0.44895611,\"B\":0.5863929011,\"C\":0.4340695146,\"D\":0.5461338805},{\"A\":0.8779709692,\"B\":0.7346527936,\"C\":0.5166408555,\"D\":0.7090164912},{\"A\":0.1301537384,\"B\":0.038524793,\"C\":0.1086061957,\"D\":0.5054981954},{\"A\":0.4676610596,\"B\":0.0464128774,\"C\":0.0701372197,\"D\":0.804096783},{\"A\":0.2637587773,\"B\":0.570283762,\"C\":0.8375562828,\"D\":0.6459677418},{\"A\":0.7537321074,\"B\":0.5692118474,\"C\":0.3920713273,\"D\":0.6739361886},{\"A\":0.9293691973,\"B\":0.0942572446,\"C\":0.5260171998,\"D\":0.9192443409},{\"A\":0.0272109634,\"B\":0.905314036,\"C\":0.8146917598,\"D\":0.612996192},{\"A\":0.8257428339,\"B\":0.3017979784,\"C\":0.8840314119,\"D\":0.389652628},{\"A\":0.3656587178,\"B\":0.0713495029,\"C\":0.0356915025,\"D\":0.2670063823},{\"A\":0.2022555318,\"B\":0.0913283538,\"C\":0.8383074628,\"D\":0.3797109131},{\"A\":0.1898597875,\"B\":0.3885636091,\"C\":0.0757421967,\"D\":0.3362250295},{\"A\":0.5289049892,\"B\":0.5188115576,\"C\":0.1688304016,\"D\":0.845068647},{\"A\":0.4126513276,\"B\":0.2554108881,\"C\":0.015179739,\"D\":0.5686325064}]"