Skip to content

Commit

Permalink
format:
Browse files Browse the repository at this point in the history 
Signed-off-by: Nasfame <[email protected]>
  • Loading branch information
Nasfame committed Sep 30, 2023
1 parent cef728d commit 9b81241
Show file tree
Hide file tree
Showing 10 changed files with 139 additions and 138 deletions.
41 changes: 13 additions & 28 deletions decenter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@ def json(self) -> str:
return json_str

@staticmethod
def loads(json_str: str) -> 'App':
def loads(json_str: str) -> "App":
args = json.loads(json_str)
return App(**args)

# def to_dict(self):
# return {"train_cmd": self.train_cmd, "t": self.t, }

Expand All @@ -49,14 +50,9 @@ def _decenter(params: str):

return {
"APIVersion": "V1beta1",
"Metadata": {
"CreatedAt": "0001-01-01T00:00:00Z",
"Requester": {}
},
"Metadata": {"CreatedAt": "0001-01-01T00:00:00Z", "Requester": {}},
"Spec": {
"Deal": {
"Concurrency": 1
},
"Deal": {"Concurrency": 1},
"Docker": {
"Entrypoint": [
# "bash", "-c",
Expand All @@ -72,35 +68,24 @@ def _decenter(params: str):
"OUTPUT_DIR=/outputs/",
"DATA_DIR=/data",
# FIXME: use appropriate values , this is the default val

]
],
},
"Engine": "Docker",
"Language": {
"JobContext": {}
},
"Network": {
"Type": "None"
},
"PublisherSpec": {
"Type": "Estuary"
},
"Resources": {
"GPU": "1"
},
"Language": {"JobContext": {}},
"Network": {"Type": "None"},
"PublisherSpec": {"Type": "Estuary"},
"Resources": {"GPU": "1"},
"Timeout": 1800,
"Verifier": "Noop",
"Wasm": {
"EntryModule": {}
},
"Wasm": {"EntryModule": {}},
"outputs": [
{
"Name": "outputs",
"StorageSource": "IPFS",
"path": "/outputs"
"path": "/outputs",
}
]
}
],
},
}


Expand Down
39 changes: 22 additions & 17 deletions helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,20 @@

def get_notebook_cmd(starter_script: str, python_repl=sys.executable):
# It will save executed notebook to your-notebook.nbconvert.ipynb file. You can specify the custom output name and custom output director
cmd_string = 'jupyter nbconvert --execute --to notebook --output custom-name --output-dir /custom/path/ your-notebook.ipynb'
cmd_string = "jupyter nbconvert --execute --to notebook --output custom-name --output-dir /custom/path/ your-notebook.ipynb"

cmd_string = "jupyter nbconvert --execute --to notebook --allow-errors your-notebook.ipynb"
# You can execute the notebook and save output into PDF or HTML format. Additionally, you can hide code in the final notebook. The example command that will execute notebook and save it as HTML file with code hidden.
cmd_string = (
'jupyter nbconvert --execute --to notebook --allow-errors your-notebook.ipynb'
f"jupyter nbconvert --execute --to html --no-input {starter_script}"
)
# You can execute the notebook and save output into PDF or HTML format. Additionally, you can hide code in the final notebook. The example command that will execute notebook and save it as HTML file with code hidden.
cmd_string = f'jupyter nbconvert --execute --to html --no-input {starter_script}'
cmd_string = f'jupyter nbconvert --execute --to html --output {starter_script} {starter_script}'
cmd_string = f"jupyter nbconvert --execute --to html --output {starter_script} {starter_script}"

# cmd_string = f'jupyter nbconvert --execute --to notebook {starter_notebook}'
command = cmd_string.split(' ')
command = cmd_string.split(" ")

if python_repl is not None:
command = [python_repl, '-m'] + command
command = [python_repl, "-m"] + command

return command

Expand All @@ -37,11 +37,11 @@ def install_deps(python_repl=sys.executable, requirements: list = None):
if not requirements:
return

print('install_deps', requirements)
print("install_deps", requirements)

def install(package):
subprocess.check_call(
[python_repl, '-m', 'pip', 'install', package],
[python_repl, "-m", "pip", "install", package],
# stdout=st.info,
# stderr=st.error,
universal_newlines=True,
Expand All @@ -53,22 +53,24 @@ def install(package):


def install_dependencies(
python_repl=sys.executable, requirements_path=None, requirements=None,
python_repl=sys.executable,
requirements_path=None,
requirements=None,
):
if requirements:
logging.info('install_dependencies:')
logging.info("install_dependencies:")
install_deps(python_repl, requirements)

if not requirements_path:
logging.warning('install_dependencies:requirements_path not found')
logging.warning("install_dependencies:requirements_path not found")
return

print('installing dependencies: for ', python_repl)
command = [python_repl, '-m', 'pip', 'install', '-r', requirements_path]
print("installing dependencies: for ", python_repl)
command = [python_repl, "-m", "pip", "install", "-r", requirements_path]
result = subprocess.run(
command,
capture_output=True,
encoding='UTF-8',
encoding="UTF-8",
)

logging.info(result.stdout)
Expand All @@ -78,11 +80,14 @@ def install_dependencies(


def archive_directory(
archive_path_name: str, src_dir: str, base_dir: str = None, format='zip',
archive_path_name: str,
src_dir: str,
base_dir: str = None,
format="zip",
) -> str:
# with tempfile.TemporaryDirectory() as temp_dir:
# subprocess.run(command, cwd=temp_dir)
print('zipping', os.listdir(src_dir))
print("zipping", os.listdir(src_dir))

created_archive_loc = shutil.make_archive(
archive_path_name,
Expand Down
46 changes: 25 additions & 21 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@
from web3.cid import is_cid

python_repl = sys.executable
DATA_DIR = os.getenv('DATA_DIR', '/data')
DATA_DIR = os.getenv("DATA_DIR", "/data")

JUPYTER_NOTEBOOK: Final[str] = '.ipynb'
PYTHON: Final[str] = '.py'
JUPYTER_NOTEBOOK: Final[str] = ".ipynb"
PYTHON: Final[str] = ".py"

output_dir: Final[str] = os.getenv('OUTPUT_DIR','/outputs')
output_dir: Final[str] = os.getenv("OUTPUT_DIR", "/outputs")

EXECUTION_FRAMEWORK: str


def path_not_found(path: str):
if not os.path.exists(path):
logging.warning(f'{path} not found')
logging.warning(f"{path} not found")
return True

return False
Expand All @@ -46,7 +46,8 @@ def train(train_script: str, requirements_txt: str = None, data_dir=DATA_DIR):
)
if path_not_found(requirements_path):
logging.critical(
f"requirements path- {requirements_path} not found")
f"requirements path- {requirements_path} not found"
)
else:
install_dependencies(
python_repl,
Expand All @@ -66,12 +67,12 @@ def train(train_script: str, requirements_txt: str = None, data_dir=DATA_DIR):
training_cmd = [python_repl, train_script]
case ".ipynb":
EXECUTION_FRAMEWORK = JUPYTER_NOTEBOOK
cmd_string = f'jupyter nbconvert --execute --to html --output {train_script} {train_script}'
training_cmd = cmd_string.split(' ')
training_cmd = [python_repl, '-m'] + training_cmd
cmd_string = f"jupyter nbconvert --execute --to html --output {train_script} {train_script}"
training_cmd = cmd_string.split(" ")
training_cmd = [python_repl, "-m"] + training_cmd

case _:
logging.critical('invalid training script')
logging.critical("invalid training script")
sys.exit(1)

logging.info(f"train cmd - {training_cmd}")
Expand All @@ -86,31 +87,37 @@ def train(train_script: str, requirements_txt: str = None, data_dir=DATA_DIR):
logging.info(result.stdout)
logging.error(result.stderr)

with open(os.path.join(data_dir, 'stdout'), 'w') as f1, open(os.path.join(data_dir, 'stderr'), 'w') as f2:
with open(os.path.join(data_dir, "stdout"), "w") as f1, open(
os.path.join(data_dir, "stderr"), "w"
) as f2:
f1.write(result.stdout)
f2.write(result.stderr)
# TODO: pass files directly to subprocess...

return True


def train_v2(train_script: str, input_archive: str, requirements_txt: str = None):
def train_v2(
train_script: str, input_archive: str, requirements_txt: str = None
):
logging.info(f"start {datetime.datetime.utcnow()}")

data_dir = DATA_DIR
if not os.path.exists(data_dir):
logging.warning(f'data dir {data_dir} doesnt exists')
logging.warning(f"data dir {data_dir} doesnt exists")

logging.info('creating temp directory for data dir')
logging.info("creating temp directory for data dir")

temp_dir = tempfile.TemporaryDirectory(prefix="decenter-ai-",
suffix="-training-working-dir", )
temp_dir = tempfile.TemporaryDirectory(
prefix="decenter-ai-",
suffix="-training-working-dir",
)
data_dir = temp_dir.name

print("data_dir is ", data_dir)

if is_cid(input_archive):
new_archive = os.path.join(data_dir, f'{input_archive}.zip')
new_archive = os.path.join(data_dir, f"{input_archive}.zip")
f2 = lighthouse.download(input_archive, new_archive)
input_archive = os.path.join(data_dir, f2.name)
os.rename(new_archive, input_archive)
Expand Down Expand Up @@ -154,7 +161,4 @@ def train_v2(train_script: str, input_archive: str, requirements_txt: str = None

# fire.Fire(train, 'train', 'Train')
# fire.Fire(train_v2, 'train_v2', 'Train v2')
fire.Fire({
'train': train,
'train_v2': train_v2
})
fire.Fire({"train": train, "train_v2": train_v2})
46 changes: 18 additions & 28 deletions refs/lilypad_module_sample.py
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,25 @@
import yaml


def _sdxl(params: str):
if params.startswith("{"):
params = yaml.safe_load(params)
else:
prompt = params
params = {"prompt": prompt, "seed": 0}
if not isinstance(params, dict):
raise Exception("Please set params to a dict like {'prompt': 'astronaut riding a horse', 'seed': 42}")
raise Exception(
"Please set params to a dict like {'prompt': 'astronaut riding a horse', 'seed': 42}"
)
return {
"APIVersion": "V1beta1",
"Metadata": {
"CreatedAt": "0001-01-01T00:00:00Z",
"Requester": {}
},
"Metadata": {"CreatedAt": "0001-01-01T00:00:00Z", "Requester": {}},
"Spec": {
"Deal": {
"Concurrency": 1
},
"Deal": {"Concurrency": 1},
"Docker": {
"Entrypoint": [
"bash", "-c",
"bash",
"-c",
# stderr logging is nondeterministic (includes timing information)
"python3 inference.py 2>/dev/null",
],
Expand All @@ -30,36 +29,27 @@ def _sdxl(params: str):
f"RANDOM_SEED={params.get('seed', 0)}",
f"OUTPUT_DIR=/outputs/",
"HF_HUB_OFFLINE=1",
]
],
},
"Engine": "Docker",
"Language": {
"JobContext": {}
},
"Network": {
"Type": "None"
},
"PublisherSpec": {
"Type": "Estuary"
},
"Resources": {
"GPU": "1"
},
"Language": {"JobContext": {}},
"Network": {"Type": "None"},
"PublisherSpec": {"Type": "Estuary"},
"Resources": {"GPU": "1"},
"Timeout": 1800,
"Verifier": "Noop",
"Wasm": {
"EntryModule": {}
},
"Wasm": {"EntryModule": {}},
"outputs": [
{
"Name": "outputs",
"StorageSource": "IPFS",
"path": "/outputs"
"path": "/outputs",
}
]
}
],
},
}


if __name__ == "__main__":
print(_sdxl("{prompt: hello, seed: 99}"))
print(_sdxl("{prompt: 'hello world', seed: 99}"))
Loading

0 comments on commit 9b81241

Please sign in to comment.