Skip to content

Commit 33ba8cf

Browse files
arjunsuresharjunsuresh
committed
Sync with dev (new)
Former-commit-id: d0d09aeb1e298e778f0f66252db735001d3fb22c
1 parent 47e7c3e commit 33ba8cf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+1449
-0
lines changed

‎LICENSE.third-party.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This CM repository may contain CM scripts with third-party files licensed under Apache2, BSD or MIT license.

‎automation/cfg/README-extra.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Examples:
2+
3+
```bash
4+
cm set cfg default
5+
cm set cfg default --key.script.silent
6+
cm set cfg default --key.script.silent-
7+
8+
```

‎automation/script/docker_repro_example/ubuntu-23.04.Dockerfile

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
FROM ubuntu:23.04
2+
3+
# Automatically generated by the CM workflow automation meta-framework
4+
# https://github.com/mlcommons/ck
5+
6+
LABEL github=""
7+
LABEL maintainer=""
8+
LABEL license=""
9+
10+
SHELL ["/bin/bash", "-c"]
11+
12+
ARG CM_GH_TOKEN
13+
ARG CM_ADD_DOCKER_GROUP_ID=""
14+
15+
# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes
16+
# Install system dependencies
17+
RUN apt-get update -y
18+
RUN apt-get install -y python3 python3-pip git sudo wget
19+
20+
# Setup docker environment
21+
ENTRYPOINT ["/bin/bash", "-c"]
22+
ENV TZ="US/Pacific"
23+
ENV PATH="${PATH}:/home/cmuser/.local/bin"
24+
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone
25+
26+
# Setup docker user
27+
RUN groupadd ${CM_ADD_DOCKER_GROUP_ID} cm
28+
RUN useradd -g cm --create-home --shell /bin/bash cmuser
29+
RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
30+
USER cmuser:cm
31+
WORKDIR /home/cmuser
32+
33+
# Install python packages
34+
RUN python3 -m pip install --user cmind requests giturlparse tabulate --break-system-packages

‎automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

‎automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

‎automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

‎automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

‎cfg/docker-basic-configurations/_cm.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
alias: docker-basic-configurations
2+
uid: d2a0c5bb17664c93
3+
4+
automation_alias: cfg
5+
automation_uid: 88dce9c160324c5d
6+
7+
tags:
8+
- docker
9+
- basic
10+
- configurations

‎cfg/docker-basic-configurations/basic-archlinux.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
uid: 9960e9fb3cb24cb3
2+
3+
name: "Basic ArchLinux"
4+
5+
input:
6+
docker_base_image: 'archlinux'
7+
docker_os: arch
8+
docker_os_version: 'latest'
9+

‎cfg/docker-basic-configurations/basic-rhel-9.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
uid: 27b4afcdd8e042e8
2+
3+
name: "Basic RHEL 9"
4+
5+
input:
6+
docker_base_image: 'registry.access.redhat.com/ubi9'
7+
docker_os: 'rhel'
8+
docker_os_version: '9'
9+

‎cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
uid: 59311e6098c14b21
2+
3+
name: "Basic Ubuntu 20.04"
4+
5+
input:
6+
docker_base_image: 'ubuntu:20.04'
7+
docker_os: ubuntu
8+
docker_os_version: '20.04'
9+

‎cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
uid: 614aa48d90724835
2+
3+
name: "Basic Ubuntu 22.04"
4+
5+
input:
6+
docker_base_image: 'ubuntu:22.04'
7+
docker_os: ubuntu
8+
docker_os_version: '22.04'
9+

‎cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
uid: 276bd8ab39324f5f
2+
3+
name: "Basic Ubuntu 23.04"
4+
5+
input:
6+
docker_base_image: 'ubuntu:23.04'
7+
docker_os: ubuntu
8+
docker_os_version: '23.04'
9+

‎script/app-image-corner-detection/_cm.yaml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
alias: app-image-corner-detection
2+
uid: 998ffee0bc534d0a
3+
4+
automation_alias: script
5+
automation_uid: 5b4e0237da074764
6+
7+
category: Modular application pipeline
8+
9+
deps:
10+
- tags: detect,os
11+
12+
- tags: detect,cpu
13+
14+
- tags: download,file,_url.https://cKnowledge.org/ai/data/data.pgm
15+
md5sum: 0af279e557a8de252d7ff0751a999379
16+
force_cache: false
17+
18+
19+
posthook_deps:
20+
- skip_if_env:
21+
CM_SKIP_COMPILE:
22+
- 'on'
23+
tags: compile,cpp-program
24+
- skip_if_env:
25+
CM_SKIP_RUN:
26+
- 'on'
27+
tags: benchmark-program
28+
29+
tags:
30+
- app
31+
- image
32+
- corner-detection

‎script/benchmark-program/README-extra.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
This is a universal script to run and profile programs.
2+
3+
It is a part of our universal benchmarking and optimization roadmap: https://github.com/mlcommons/cm4mlops/issues/23

‎script/build-docker-image/_cm.yaml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
alias: build-docker-image
2+
uid: 2c3c4ba2413442e7
3+
4+
automation_alias: script
5+
automation_uid: 5b4e0237da074764
6+
7+
cache: false
8+
9+
category: Docker automation
10+
11+
tags:
12+
- build
13+
- docker
14+
- image
15+
- docker-image
16+
- dockerimage
17+
18+
default_env:
19+
CM_DOCKER_IMAGE_REPO: local
20+
CM_DOCKER_IMAGE_TAG: latest
21+
22+
input_mapping:
23+
cache: CM_DOCKER_CACHE
24+
cm_repo: CM_MLOPS_REPO
25+
docker_os: CM_DOCKER_OS
26+
docker_os_version: CM_DOCKER_OS_VERSION
27+
dockerfile: CM_DOCKERFILE_WITH_PATH
28+
gh_token: CM_GH_TOKEN
29+
image_name: CM_DOCKER_IMAGE_NAME
30+
image_repo: CM_DOCKER_IMAGE_REPO
31+
image_tag: CM_DOCKER_IMAGE_TAG
32+
post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
33+
pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
34+
real_run: CM_REAL_RUN
35+
script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
36+
push_image: CM_DOCKER_PUSH_IMAGE
37+
38+
new_env_keys:
39+
- CM_DOCKER_*
40+
41+
prehook_deps:
42+
- enable_if_env:
43+
CM_BUILD_DOCKERFILE:
44+
- 'yes'
45+
- '1'
46+
tags: build,dockerfile

‎script/build-dockerfile/_cm.yaml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
alias: build-dockerfile
2+
uid: e66a7483230d4641
3+
4+
automation_alias: script
5+
automation_uid: 5b4e0237da074764
6+
7+
tags:
8+
- build
9+
- dockerfile
10+
11+
cache: false
12+
13+
category: Docker automation
14+
15+
default_env:
16+
CM_DOCKER_BUILD_SLIM: 'no'
17+
CM_DOCKER_IMAGE_EOL: '
18+
19+
'
20+
CM_DOCKER_OS: ubuntu
21+
22+
input_mapping:
23+
build: CM_BUILD_DOCKER_IMAGE
24+
cache: CM_DOCKER_CACHE
25+
cm_repo: CM_MLOPS_REPO
26+
cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO
27+
cm_repos: CM_DOCKER_EXTRA_CM_REPOS
28+
comments: CM_DOCKER_RUN_COMMENTS
29+
copy_files: CM_DOCKER_COPY_FILES
30+
docker_base_image: CM_DOCKER_IMAGE_BASE
31+
docker_os: CM_DOCKER_OS
32+
docker_os_version: CM_DOCKER_OS_VERSION
33+
extra_sys_deps: CM_DOCKER_EXTRA_SYS_DEPS
34+
fake_docker_deps: CM_DOCKER_FAKE_DEPS
35+
fake_run_option: CM_DOCKER_FAKE_RUN_OPTION
36+
file_path: CM_DOCKERFILE_WITH_PATH
37+
gh_token: CM_GH_TOKEN
38+
image_repo: CM_DOCKER_IMAGE_REPO
39+
image_tag: CM_DOCKER_IMAGE_TAG
40+
package_manager_update_cmd: CM_PACKAGE_MANAGER_UPDATE_CMD
41+
pip_extra_flags: CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS
42+
post_file: DOCKER_IMAGE_POST_FILE
43+
post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
44+
pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
45+
real_run: CM_REAL_RUN
46+
run_cmd: CM_DOCKER_RUN_CMD
47+
run_cmd_extra: CM_DOCKER_RUN_CMD_EXTRA
48+
script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
49+
skip_cm_sys_upgrade: CM_DOCKER_SKIP_CM_SYS_UPGRADE
50+
push_image: CM_DOCKER_PUSH_IMAGE
51+
52+
new_env_keys:
53+
- CM_DOCKERFILE_*
54+
55+
post_deps:
56+
- enable_if_env:
57+
CM_BUILD_DOCKER_IMAGE:
58+
- 'yes'
59+
- '1'
60+
names:
61+
- build-docker-image
62+
tags: build,docker,image
63+
64+
variations:
65+
slim:
66+
env:
67+
CM_DOCKER_BUILD_SLIM: 'yes'

‎script/create-custom-cache-entry/_cm.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
alias: create-custom-cache-entry
2+
uid: 485741440fbe4236
3+
4+
automation_alias: script
5+
automation_uid: 5b4e0237da074764
6+
7+
tags:
8+
- create
9+
- custom
10+
- cache
11+
- entry
12+
13+
category: CM automation
14+
15+
cache: true
16+
17+
input_mapping:
18+
env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY
19+
env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2
20+
path: CM_CUSTOM_CACHE_ENTRY_PATH
21+
to: CM_CUSTOM_CACHE_ENTRY_PATH
22+
23+
new_env_keys:
24+
- CM_CUSTOM_CACHE_ENTRY*
25+
26+
print_env_at_the_end:
27+
CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry"

‎script/create-custom-cache-entry/customize.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
from cmind import utils
2+
import os
3+
import shutil
4+
5+
def preprocess(i):
6+
7+
# CM script internal variables
8+
env = i['env']
9+
10+
extra_cache_tags = []
11+
if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '':
12+
print ('')
13+
extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ')
14+
15+
extra_cache_tags = extra_cache_tags_str.strip().split(',')
16+
17+
return {'return':0, 'add_extra_cache_tags':extra_cache_tags}
18+
19+
def postprocess(i):
20+
21+
env = i['env']
22+
23+
path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip()
24+
25+
if path!='':
26+
if not os.path.isdir(path):
27+
os.makedirs(path)
28+
else:
29+
path = os.getcwd()
30+
31+
x = ''
32+
env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '')
33+
if env_key != '': x = env_key+'_'
34+
35+
env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path
36+
env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path
37+
38+
env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '')
39+
v = env.get(env_key2, '')
40+
real_path = v if v != '' else path
41+
42+
env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path
43+
44+
return {'return': 0}

‎script/generate-docs-for-all-scripts.cmd

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
cm doc script --repos=mlcommons@cm4mlops --output_dir=..

‎script/get-cuda-devices/_cm.yaml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
alias: get-cuda-devices
2+
uid: 7a3ede4d3558427a
3+
4+
automation_alias: script
5+
automation_uid: 5b4e0237da074764
6+
7+
tags:
8+
- get
9+
- cuda-devices
10+
11+
cache: false
12+
13+
can_force_cache: true
14+
15+
category: CUDA automation
16+
17+
clean_files:
18+
- tmp-run.out
19+
20+
deps:
21+
- names:
22+
- cuda
23+
tags: get,cuda,_toolkit
24+
25+
docker:
26+
run: false
27+
all_gpus: 'yes'
28+
skip_run_cmd: 'no'
29+
skip_cm_sys_upgrade: 'yes'
30+
cm_repo_flags: '--checkout=dev'
31+
use_host_group_id: 'yes'
32+
image_tag_extra: '-cm-dev'
33+
34+
new_env_keys:
35+
- CM_CUDA_DEVICE_*
36+
37+
new_state_keys:
38+
- cm_cuda_device_prop
39+
40+
print_files_if_script_error:
41+
- tmp-run.out

0 commit comments

Comments
 (0)