From df4013c1fd08639c343058e227aea0d17f5066f8 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 10:54:18 +0200 Subject: [PATCH 01/18] remove sys deps from script "python hello world" --- script/print-hello-world-py/_cm.json | 3 --- script/print-hello-world-py/run.bat | 2 +- script/print-hello-world-py/run.sh | 4 ++-- script/print-hello-world/_cm.json | 32 ---------------------------- 4 files changed, 3 insertions(+), 38 deletions(-) delete mode 100644 script/print-hello-world/_cm.json diff --git a/script/print-hello-world-py/_cm.json b/script/print-hello-world-py/_cm.json index 5e6a0629c7..7bb36e4b67 100644 --- a/script/print-hello-world-py/_cm.json +++ b/script/print-hello-world-py/_cm.json @@ -7,9 +7,6 @@ { "tags": "detect,os" }, - { - "tags": "get,sys-utils-cm" - }, { "names": [ "python", diff --git a/script/print-hello-world-py/run.bat b/script/print-hello-world-py/run.bat index d1881d3a35..b4f5e3a7ac 100644 --- a/script/print-hello-world-py/run.bat +++ b/script/print-hello-world-py/run.bat @@ -1,6 +1,6 @@ IF NOT DEFINED CM_TMP_CURRENT_SCRIPT_PATH SET CM_TMP_CURRENT_SCRIPT_PATH=%CD% -%CM_PYTHON_BIN_WITH_PATH% --version +rem %CM_PYTHON_BIN_WITH_PATH% --version %CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\code.py IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/script/print-hello-world-py/run.sh b/script/print-hello-world-py/run.sh index b7c69c7906..26b72ba626 100644 --- a/script/print-hello-world-py/run.sh +++ b/script/print-hello-world-py/run.sh @@ -2,8 +2,8 @@ CM_TMP_CURRENT_SCRIPT_PATH=${CM_TMP_CURRENT_SCRIPT_PATH:-$PWD} -which ${CM_PYTHON_BIN_WITH_PATH} -${CM_PYTHON_BIN_WITH_PATH} --version +#which ${CM_PYTHON_BIN_WITH_PATH} +#${CM_PYTHON_BIN_WITH_PATH} --version ${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/code.py test $? -eq 0 || exit $? diff --git a/script/print-hello-world/_cm.json b/script/print-hello-world/_cm.json deleted file mode 100644 index c8ccffe533..0000000000 --- a/script/print-hello-world/_cm.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "alias": "print-hello-world", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "category": "Tests", - "default_env": { - "CM_ENV_TEST1": "TEST1" - }, - "env": { - "CM_ENV_TEST2": "TEST2" - }, - "input_mapping": { - "test1": "CM_ENV_TEST1" - }, - "new_env_keys": [ - "CM_ENV_TEST*" - ], - "new_state_keys": [ - "hello_world*" - ], - "tags": [ - "print", - "hello-world", - "hello world", - "hello", - "world", - "native-script", - "native", - "script" - ], - "uid": "b9f0acba4aca4baa" -} From 9a7057e467b6ae76defb9c5fbb8123a6d27e250a Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 11:52:40 +0200 Subject: [PATCH 02/18] * added "dummy" script to test Docker containers * added more standard Nvidia Docker configuration for PyTorch --- CHANGES.md | 4 ++++ ...-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml | 2 +- ...4-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml | 2 +- ...4-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml | 2 +- ...4-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml | 11 ++++++++++ ...4-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml | 11 ++++++++++ script/dummy/README-extra.md | 1 + script/dummy/_cm.yaml | 10 +++++++++ script/dummy/customize.py | 22 +++++++++++++++++++ script/dummy/run.bat | 1 + script/dummy/run.sh | 2 ++ 11 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml create mode 100644 cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml create mode 100644 script/dummy/README-extra.md create mode 100644 script/dummy/_cm.yaml create mode 100644 script/dummy/customize.py create mode 100644 script/dummy/run.bat create mode 100644 script/dummy/run.sh diff --git a/CHANGES.md b/CHANGES.md index c497559773..34d887627e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,7 @@ +### 20240927 + * added "dummy" script to test Docker containers + * added more standard Nvidia Docker configuration for PyTorch + ### 20240916 * fixed "cm add script" diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml index 1e71c67ce7..c72268b15a 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml @@ -1,6 +1,6 @@ uid: 854e65fb31584d63 -name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0" +name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0 (pytorch:22.10)" input: docker_base_image: 'nvcr.io/nvidia/pytorch:22.10-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml index 4e146ca665..de9a1bc1db 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml @@ -1,6 +1,6 @@ uid: e0e7167139a74e36 -name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0" +name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0 (pytorch:23.05)" input: docker_base_image: 'nvcr.io/nvidia/pytorch:23.05-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml index a671d699a7..f23bba2eb4 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml @@ -1,6 +1,6 @@ uid: 49fc51f2999b4545 -name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0" +name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0 (pytorch:24.03)" input: docker_base_image: 'nvcr.io/nvidia/pytorch:24.03-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml new file mode 100644 index 0000000000..b4e45d348f --- /dev/null +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml @@ -0,0 +1,11 @@ +uid: 81879736ae5842f4 + +name: "Nvidia Ubuntu 22.04 CUDA 12.5 cuDNN 9.1.0 PyTorch 2.4.0 (pytorch:24.06)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-06.html + +input: + docker_base_image: 'nvcr.io/nvidia/pytorch:24.06-py3' + docker_os: ubuntu + docker_os_version: '22.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml new file mode 100644 index 0000000000..a9e2229ead --- /dev/null +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml @@ -0,0 +1,11 @@ +uid: 203a68df99d44137 + +name: "Nvidia Ubuntu 22.04 CUDA 12.6 cuDNN 9.3.0 PyTorch 2.5.0 (pytorch:24.08)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html + +input: + docker_base_image: 'nvcr.io/nvidia/pytorch:24.08-py3' + docker_os: ubuntu + docker_os_version: '22.04' + \ No newline at end of file diff --git a/script/dummy/README-extra.md b/script/dummy/README-extra.md new file mode 100644 index 0000000000..582991f6d2 --- /dev/null +++ b/script/dummy/README-extra.md @@ -0,0 +1 @@ +# CM script diff --git a/script/dummy/_cm.yaml b/script/dummy/_cm.yaml new file mode 100644 index 0000000000..6e73e8ca8d --- /dev/null +++ b/script/dummy/_cm.yaml @@ -0,0 +1,10 @@ +alias: dummy +uid: 3ef5d69f929349bc + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: false + +tags: +- dummy diff --git a/script/dummy/customize.py b/script/dummy/customize.py new file mode 100644 index 0000000000..d12f9b3e1d --- /dev/null +++ b/script/dummy/customize.py @@ -0,0 +1,22 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = (env.get('CM_QUIET', False) == 'yes') + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + return {'return':0} diff --git a/script/dummy/run.bat b/script/dummy/run.bat new file mode 100644 index 0000000000..648302ca71 --- /dev/null +++ b/script/dummy/run.bat @@ -0,0 +1 @@ +rem native script diff --git a/script/dummy/run.sh b/script/dummy/run.sh new file mode 100644 index 0000000000..05a7907cf5 --- /dev/null +++ b/script/dummy/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash + From 6b02a9db8b0b0213014cad49f8a2cd9e26e162ac Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 12:31:03 +0200 Subject: [PATCH 03/18] * added better support to select Docker configurations via UID * renamed dummy script to test-dummy --- CHANGES.md | 3 +- automation/script/module_misc.py | 13 +++-- automation/utils/module_cfg.py | 52 +++++++++++-------- .../_cm.yaml | 39 ++++++++++++++ script/{dummy => test-dummy}/README-extra.md | 0 script/{dummy => test-dummy}/_cm.yaml | 3 +- script/{dummy => test-dummy}/customize.py | 0 script/{dummy => test-dummy}/run.bat | 0 script/{dummy => test-dummy}/run.sh | 0 9 files changed, 84 insertions(+), 26 deletions(-) create mode 100644 cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml rename script/{dummy => test-dummy}/README-extra.md (100%) rename script/{dummy => test-dummy}/_cm.yaml (81%) rename script/{dummy => test-dummy}/customize.py (100%) rename script/{dummy => test-dummy}/run.bat (100%) rename script/{dummy => test-dummy}/run.sh (100%) diff --git a/CHANGES.md b/CHANGES.md index 34d887627e..d640f53eec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ ### 20240927 - * added "dummy" script to test Docker containers + * added "test dummy" script to test Docker containers * added more standard Nvidia Docker configuration for PyTorch + * added better support to select Docker configurations via UID ### 20240916 * fixed "cm add script" diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index b193ff5dfe..e143d52042 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1641,6 +1641,7 @@ def docker(i): (docker_save_script) (str): if !='' name of script to save docker command (docker_interactive) (bool): if True, run in interactive mode (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one + (docker_cfg_uid) (str): if True, select docker configuration with this UID Returns: (CM return dict): @@ -1693,13 +1694,19 @@ def docker(i): # Check available configurations docker_cfg = i.get('docker_cfg', '') - if docker_cfg != '': + docker_cfg_uid = i.get('docker_cfg_uid', '') + + if docker_cfg != '' or docker_cfg_uid != '': # Check if docker_cfg is turned on but not selected if type(docker_cfg) == bool or str(docker_cfg).lower() in ['true','yes']: docker_cfg= '' - r = self_module.cmind.access({'action':'select_cfg', 'automation':'utils,dc2743f8450541e3', - 'tags':'basic,docker,configurations', 'title':'docker', 'alias':docker_cfg}) + r = self_module.cmind.access({'action':'select_cfg', + 'automation':'utils,dc2743f8450541e3', + 'tags':'basic,docker,configurations', + 'title':'docker', + 'alias':docker_cfg, + 'uid':docker_cfg_uid}) if r['return'] > 0: if r['return'] == 16: return {'return':1, 'error':'Docker configuration {} was not found'.format(docker_cfg)} diff --git a/automation/utils/module_cfg.py b/automation/utils/module_cfg.py index 145c388f2a..994aa43290 100644 --- a/automation/utils/module_cfg.py +++ b/automation/utils/module_cfg.py @@ -230,6 +230,7 @@ def select_cfg(i): self_module = i['self_module'] tags = i['tags'] alias = i.get('alias', '') + uid = i.get('uid', '') title = i.get('title', '') # Check if alias is not provided @@ -240,6 +241,7 @@ def select_cfg(i): selector = [] + # Do coarse-grain search for CM artifacts for l in lst: p = l.path @@ -257,42 +259,50 @@ def select_cfg(i): if not f.startswith('_cm') and (f.endswith('.json') or f.endswith('.yaml')): selector.append({'path':os.path.join(p, f), 'alias':f[:-5]}) - if len(selector) == 0: + + # Load meta for name and UID + selector_with_meta = [] + for s in range(0, len(selector)): + ss = selector[s] + + path = ss['path'] + + full_path_without_ext = path[:-5] + + r = cmind.utils.load_yaml_and_json(full_path_without_ext) + if r['return']>0: + print ('Warning: problem loading configuration file {}'.format(path)) + + meta = r['meta'] + + if uid == '' or meta.get('uid', '') == uid: + ss['meta'] = meta + selector_with_meta.append(ss) + + # Quit if no configurations found + if len(selector_with_meta) == 0: return {'return':16, 'error':'configuration was not found'} select = 0 - if len(selector) > 1: + if len(selector_with_meta) > 1: xtitle = ' ' + title if title!='' else '' print ('') print ('Available{} configurations:'.format(xtitle)) print ('') - for s in range(0, len(selector)): - ss = selector[s] - - path = ss['path'] - - full_path_without_ext = path[:-5] - - r = cmind.utils.load_yaml_and_json(full_path_without_ext) - if r['return']>0: - print ('Warning: problem loading configuration file {}'.format(path)) - - meta = r['meta'] - ss['meta'] = meta - - selector = sorted(selector, key = lambda x: x['meta'].get('name','')) + selector_with_meta = sorted(selector_with_meta, key = lambda x: x['meta'].get('name','')) s = 0 for ss in selector: alias = ss['alias'] - name = ss['meta'].get('name','') + uid = ss['meta'].get('uid', '') + name = ss['meta'].get('name', '') x = name if x!='': x+=' ' - x += '('+alias+')' + x += '(' + uid + ')' - print ('{}) {}'.format(s, x)) + print (f'{s}) {x}'.format(s, x)) s+=1 @@ -306,6 +316,6 @@ def select_cfg(i): if select<0 or select>=len(selector): return {'return':1, 'error':'selection is out of range'} - ss = selector[select] + ss = selector_with_meta[select] return {'return':0, 'selection':ss} diff --git a/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml b/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml new file mode 100644 index 0000000000..716adc20b3 --- /dev/null +++ b/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml @@ -0,0 +1,39 @@ +alias: benchmark-run-mlperf-inference-v4.1 +uid: b7e89771987d4168 + +automation_alias: cfg +automation_uid: 88dce9c160324c5d + +tags: +- benchmark +- run +- mlperf +- inference +- v4.1 + +name: "MLPerf inference - v4.1" + +supported_compute: +- ee8c568e0ac44f2b +- fe379ecd1e054a00 +- d8f06040f7294319 + +bench_uid: 39877bb63fb54725 + +view_dimensions: +- - input.device + - "MLPerf device" +- - input.implementation + - "MLPerf implementation" +- - input.backend + - "MLPerf backend" +- - input.model + - "MLPerf model" +- - input.scenario + - "MLPerf scenario" +- - input.host_os + - "Host OS" +- - output.state.cm-mlperf-inference-results-last.performance + - "Got performance" +- - output.state.cm-mlperf-inference-results-last.accuracy + - "Got accuracy" diff --git a/script/dummy/README-extra.md b/script/test-dummy/README-extra.md similarity index 100% rename from script/dummy/README-extra.md rename to script/test-dummy/README-extra.md diff --git a/script/dummy/_cm.yaml b/script/test-dummy/_cm.yaml similarity index 81% rename from script/dummy/_cm.yaml rename to script/test-dummy/_cm.yaml index 6e73e8ca8d..09ba01f929 100644 --- a/script/dummy/_cm.yaml +++ b/script/test-dummy/_cm.yaml @@ -1,4 +1,4 @@ -alias: dummy +alias: test-dummy uid: 3ef5d69f929349bc automation_alias: script @@ -7,4 +7,5 @@ automation_uid: 5b4e0237da074764 cache: false tags: +- test - dummy diff --git a/script/dummy/customize.py b/script/test-dummy/customize.py similarity index 100% rename from script/dummy/customize.py rename to script/test-dummy/customize.py diff --git a/script/dummy/run.bat b/script/test-dummy/run.bat similarity index 100% rename from script/dummy/run.bat rename to script/test-dummy/run.bat diff --git a/script/dummy/run.sh b/script/test-dummy/run.sh similarity index 100% rename from script/dummy/run.sh rename to script/test-dummy/run.sh From 5c7492587da1ec2e3ac6e7c9bf87f6f5b46fb8e9 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 13:50:11 +0000 Subject: [PATCH 04/18] fixing docker cfg selection --- automation/script/module_misc.py | 9 ++++----- automation/utils/module_cfg.py | 10 +++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index e143d52042..4718efc74b 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1677,7 +1677,7 @@ def docker(i): # CAREFUL -> artifacts and parsed_artifacts are not supported in input (and should not be?) if 'artifacts' in i: del(i['artifacts']) if 'parsed_artifacts' in i: del(i['parsed_artifacts']) - + # Prepare "clean" input to replicate command r = self_module.cmind.access({'action':'prune_input', 'automation':'utils', 'input':i, 'extra_keys_starts_with':['docker_']}) i_run_cmd_arc = r['new_input'] @@ -1700,7 +1700,7 @@ def docker(i): # Check if docker_cfg is turned on but not selected if type(docker_cfg) == bool or str(docker_cfg).lower() in ['true','yes']: docker_cfg= '' - + r = self_module.cmind.access({'action':'select_cfg', 'automation':'utils,dc2743f8450541e3', 'tags':'basic,docker,configurations', @@ -1715,10 +1715,9 @@ def docker(i): selection = r['selection'] docker_input_update = selection['meta']['input'] - + i.update(docker_input_update) - ######################################################################################## # Run dockerfile if not noregenerate_docker_file: @@ -1729,7 +1728,7 @@ def docker(i): cur_dir = os.getcwd() console = i.get('out') == 'con' - + # Search for script(s) r = aux_search({'self_module': self_module, 'input': i}) if r['return']>0: return r diff --git a/automation/utils/module_cfg.py b/automation/utils/module_cfg.py index 994aa43290..04ab0a9ad1 100644 --- a/automation/utils/module_cfg.py +++ b/automation/utils/module_cfg.py @@ -236,7 +236,7 @@ def select_cfg(i): # Check if alias is not provided r = self_module.cmind.access({'action':'find', 'automation':'cfg', 'tags':'basic,docker,configurations'}) if r['return'] > 0: return r - + lst = r['list'] selector = [] @@ -288,12 +288,12 @@ def select_cfg(i): xtitle = ' ' + title if title!='' else '' print ('') print ('Available{} configurations:'.format(xtitle)) - + print ('') selector_with_meta = sorted(selector_with_meta, key = lambda x: x['meta'].get('name','')) s = 0 - for ss in selector: + for ss in selector_with_meta: alias = ss['alias'] uid = ss['meta'].get('uid', '') name = ss['meta'].get('name', '') @@ -301,11 +301,11 @@ def select_cfg(i): x = name if x!='': x+=' ' x += '(' + uid + ')' - + print (f'{s}) {x}'.format(s, x)) s+=1 - + print ('') select = input ('Enter configuration number of press Enter for 0: ') From 8dbc03812f5cf7cc39f286234a789c4de645a1ed Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 15:52:31 +0200 Subject: [PATCH 05/18] clean up --- automation/script/module_misc.py | 49 +++++++++++++++++++++------- script/build-dockerfile/customize.py | 2 +- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index e143d52042..7c9552ea21 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1335,15 +1335,9 @@ def dockerfile(i): Args: (CM input dict): - (out) (str): if 'con', output to console - - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - (repos) (str): list of repositories to search for automations - - (output_dir) (str): output directory (./ by default) + (out) (str): if 'con', output to console + (repos) (str): list of repositories to search for automations + (output_dir) (str): output directory (./ by default) Returns: (CM return dict): @@ -1636,12 +1630,45 @@ def docker(i): (docker_noregenerate) (bool): do not generate Dockerfiles (docker_norecreate) (bool): do not recreate Docker image + (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one + (docker_cfg_uid) (str): if True, select docker configuration with this UID + (docker_path) (str): where to create or find Dockerfile (docker_gh_token) (str): GitHub token for private repositories (docker_save_script) (str): if !='' name of script to save docker command (docker_interactive) (bool): if True, run in interactive mode - (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one - (docker_cfg_uid) (str): if True, select docker configuration with this UID + (docker_it) (bool): the same as `docker_interactive` + (docker_detached) (bool): detach Docker + (docker_dt) (bool) the same as `docker_detached` + + (docker_base_image) (str): force base image + (docker_os) (str): force docker OS (default: ubuntu) + (docker_os_version) (str): force docker OS version (default: 22.04) + (docker_image_tag_extra) (str): add extra tag (default:-latest) + + (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops) + (docker_cm_repos) + (docker_cm_repo_flags) + + (dockerfile_env) + + (docker_skip_cm_sys_upgrade) + + (docker_extra_sys_deps) + + (fake_run_deps) + (docker_run_final_cmds) + + (all_gpus) + (num_gpus) + + (docker_device) + + (docker_port_maps) + + (docker_shm_size) + + (docker_extra_run_args) Returns: (CM return dict): diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 41300cddef..d99e243921 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -55,7 +55,7 @@ def preprocess(i): if env.get("CM_MLOPS_REPO", "") != "": cm_mlops_repo = env["CM_MLOPS_REPO"] else: - cm_mlops_repo = "mlcommons@ck" + cm_mlops_repo = "mlcommons@cm4mlops" if env.get("CM_MLOPS_REPO_BRANCH", '') != '': cm_mlops_repo_branch_string = f" --branch {env['CM_MLOPS_REPO_BRANCH']}" From 2087704d00aa012b3478c62e3313106004400e40 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 14:00:57 +0000 Subject: [PATCH 06/18] clean up --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 777340d39b..92f7f07fe2 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,11 @@ cm run script \ [Apache 2.0](LICENSE.md) +## CM concept + +* https://doi.org/10.5281/zenodo.8105339 +* https://arxiv.org/abs/2406.16791 + ## Authors [Grigori Fursin](https://cKnowledge.org/gfursin) and [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) From 773ef26f9fdf18c2446dab0d5204ccd64697029c Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 14:02:11 +0000 Subject: [PATCH 07/18] removed ^M from setup.py --- setup.py | 294 +++++++++++++++++++++++++++---------------------------- 1 file changed, 147 insertions(+), 147 deletions(-) diff --git a/setup.py b/setup.py index e7c96a97cd..2ff4112866 100644 --- a/setup.py +++ b/setup.py @@ -1,147 +1,147 @@ -# setup.py -from setuptools import setup -from setuptools._distutils.dist import Distribution -from setuptools.command.install import install -import subprocess -import sys -import importlib.util -import platform -import os - -# Try to use importlib.metadata for Python 3.8+ -try: - if sys.version_info >= (3, 8): - from importlib.metadata import version, PackageNotFoundError - else: - # Fallback to pkg_resources for Python < 3.8 - import pkg_resources - PackageNotFoundError = pkg_resources.DistributionNotFound -except ImportError: - # If importlib.metadata is unavailable, fall back to pkg_resources - import pkg_resources - PackageNotFoundError = pkg_resources.DistributionNotFound - - - -class CustomInstallCommand(install): - def run(self): - self.get_sys_platform() - self.install_system_packages() - - # Call the standard run method - install.run(self) - - # Call the custom function - return self.custom_function() - - def is_package_installed(self, package_name): - try: - if sys.version_info >= (3, 8): - version(package_name) # Tries to get the version of the package - else: - pkg_resources.get_distribution(package_name) # Fallback for < 3.8 - return True - except PackageNotFoundError: - return False - - def install_system_packages(self): - # List of packages to install via system package manager - packages = [] - - git_status = self.command_exists('git') - if not git_status: - packages.append("git") - wget_status = self.command_exists('wget') - if not wget_status: - packages.append("wget") - curl_status = self.command_exists('curl') - if not curl_status: - packages.append("curl") - - name='venv' - - if name in sys.modules: - pass #nothing needed - elif self.is_package_installed(name): - pass - else: - packages.append("python3-venv") - - if packages: - if self.system == 'Linux' or self.system == 'Darwin': - manager, details = self.get_package_manager_details() - if manager: - if manager == "apt-get": - subprocess.check_call(['sudo', 'apt-get', 'update']) - subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) - elif self.system == 'Windows': - print(f"Please install the following packages manually: {packages}") - - - - def detect_package_manager(self): - package_managers = { - 'apt-get': '/usr/bin/apt-get', - 'yum': '/usr/bin/yum', - 'dnf': '/usr/bin/dnf', - 'pacman': '/usr/bin/pacman', - 'zypper': '/usr/bin/zypper', - 'brew': '/usr/local/bin/brew' - } - - for name, path in package_managers.items(): - if os.path.exists(path): - return name - - return None - - def get_package_manager_details(self): - manager = self.detect_package_manager() - if manager: - try: - version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') - return manager, version_output.split('\n')[0] - except subprocess.CalledProcessError: - return manager, 'Version information not available' - else: - return None, 'No supported package manager found' - - # Checks if command exists(for installing required packages). - # If the command exists, which returns 0, making the function return True. - # If the command does not exist, which returns a non-zero value, making the function return False. - # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. - def command_exists(self, command): - if self.system == "Linux" or self.system == 'Darwin': - return subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 - elif self.system == "Windows": - return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 - - def custom_function(self): - import cmind - #r = cmind.access({'action':'rm', 'automation':'repo', 'data_uoa':'mlcommons@cm4mlops', 'force': True}) - r = cmind.access({'action':'pull', 'automation':'repo', 'artifact':'mlcommons@cm4mlops', 'branch': 'mlperf-inference'}) - print(r) - if r['return'] > 0: - return r['return'] - - def get_sys_platform(self): - self.system = platform.system() - -setup( - name='cm4mlops', - version='0.1', - long_description='CM automations and scripts for MLOps', - long_description_content_type='text/x-rst', - packages=[], - install_requires=[ - "setuptools>=60", - "wheel", - "cmind", - "giturlparse", - "requests", - "pyyaml" - ], - cmdclass={ - 'install': CustomInstallCommand, - }, -) +# setup.py +from setuptools import setup +from setuptools._distutils.dist import Distribution +from setuptools.command.install import install +import subprocess +import sys +import importlib.util +import platform +import os + +# Try to use importlib.metadata for Python 3.8+ +try: + if sys.version_info >= (3, 8): + from importlib.metadata import version, PackageNotFoundError + else: + # Fallback to pkg_resources for Python < 3.8 + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound +except ImportError: + # If importlib.metadata is unavailable, fall back to pkg_resources + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound + + + +class CustomInstallCommand(install): + def run(self): + self.get_sys_platform() + self.install_system_packages() + + # Call the standard run method + install.run(self) + + # Call the custom function + return self.custom_function() + + def is_package_installed(self, package_name): + try: + if sys.version_info >= (3, 8): + version(package_name) # Tries to get the version of the package + else: + pkg_resources.get_distribution(package_name) # Fallback for < 3.8 + return True + except PackageNotFoundError: + return False + + def install_system_packages(self): + # List of packages to install via system package manager + packages = [] + + git_status = self.command_exists('git') + if not git_status: + packages.append("git") + wget_status = self.command_exists('wget') + if not wget_status: + packages.append("wget") + curl_status = self.command_exists('curl') + if not curl_status: + packages.append("curl") + + name='venv' + + if name in sys.modules: + pass #nothing needed + elif self.is_package_installed(name): + pass + else: + packages.append("python3-venv") + + if packages: + if self.system == 'Linux' or self.system == 'Darwin': + manager, details = self.get_package_manager_details() + if manager: + if manager == "apt-get": + subprocess.check_call(['sudo', 'apt-get', 'update']) + subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) + elif self.system == 'Windows': + print(f"Please install the following packages manually: {packages}") + + + + def detect_package_manager(self): + package_managers = { + 'apt-get': '/usr/bin/apt-get', + 'yum': '/usr/bin/yum', + 'dnf': '/usr/bin/dnf', + 'pacman': '/usr/bin/pacman', + 'zypper': '/usr/bin/zypper', + 'brew': '/usr/local/bin/brew' + } + + for name, path in package_managers.items(): + if os.path.exists(path): + return name + + return None + + def get_package_manager_details(self): + manager = self.detect_package_manager() + if manager: + try: + version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') + return manager, version_output.split('\n')[0] + except subprocess.CalledProcessError: + return manager, 'Version information not available' + else: + return None, 'No supported package manager found' + + # Checks if command exists(for installing required packages). + # If the command exists, which returns 0, making the function return True. + # If the command does not exist, which returns a non-zero value, making the function return False. + # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. + def command_exists(self, command): + if self.system == "Linux" or self.system == 'Darwin': + return subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + elif self.system == "Windows": + return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 + + def custom_function(self): + import cmind + #r = cmind.access({'action':'rm', 'automation':'repo', 'data_uoa':'mlcommons@cm4mlops', 'force': True}) + r = cmind.access({'action':'pull', 'automation':'repo', 'artifact':'mlcommons@cm4mlops', 'branch': 'mlperf-inference'}) + print(r) + if r['return'] > 0: + return r['return'] + + def get_sys_platform(self): + self.system = platform.system() + +setup( + name='cm4mlops', + version='0.1', + long_description='CM automations and scripts for MLOps', + long_description_content_type='text/x-rst', + packages=[], + install_requires=[ + "setuptools>=60", + "wheel", + "cmind", + "giturlparse", + "requests", + "pyyaml" + ], + cmdclass={ + 'install': CustomInstallCommand, + }, +) From de55cc158ad14fd0276526f80aa3736c6cc7dd2a Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 16:28:30 +0200 Subject: [PATCH 08/18] improving docker docs --- automation/script/module.py | 49 +++++++++++++++++++++++++++++--- automation/script/module_misc.py | 43 ---------------------------- 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index e40cf8b353..9747639cb7 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4028,14 +4028,55 @@ def docker(self, i): (out) (str): if 'con', output to console - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - (repos) (str): list of repositories to search for automations (output_dir) (str): output directory (./ by default) + (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) + (docker_noregenerate) (bool): do not generate Dockerfiles + (docker_norecreate) (bool): do not recreate Docker image + + (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one + (docker_cfg_uid) (str): if True, select docker configuration with this UID + + (docker_path) (str): where to create or find Dockerfile + (docker_gh_token) (str): GitHub token for private repositories + (docker_save_script) (str): if !='' name of script to save docker command + (docker_interactive) (bool): if True, run in interactive mode + (docker_it) (bool): the same as `docker_interactive` + (docker_detached) (bool): detach Docker + (docker_dt) (bool) the same as `docker_detached` + + (docker_base_image) (str): force base image + (docker_os) (str): force docker OS (default: ubuntu) + (docker_os_version) (str): force docker OS version (default: 22.04) + (docker_image_tag_extra) (str): add extra tag (default:-latest) + + (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops) + (docker_cm_repos) + (docker_cm_repo_flags) + + (dockerfile_env) + + (docker_skip_cm_sys_upgrade) (bool): if True, do not install CM sys deps + + (docker_extra_sys_deps) + + (fake_run_deps) + (docker_run_final_cmds) + + (all_gpus) + (num_gpus) + + (docker_device) + + (docker_port_maps) + + (docker_shm_size) + + (docker_extra_run_args) + + Returns: (CM return dict): diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index c2a1cf740e..98a4c667d9 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1626,49 +1626,6 @@ def docker(i): (out) (str): if 'con', output to console - (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) - (docker_noregenerate) (bool): do not generate Dockerfiles - (docker_norecreate) (bool): do not recreate Docker image - - (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one - (docker_cfg_uid) (str): if True, select docker configuration with this UID - - (docker_path) (str): where to create or find Dockerfile - (docker_gh_token) (str): GitHub token for private repositories - (docker_save_script) (str): if !='' name of script to save docker command - (docker_interactive) (bool): if True, run in interactive mode - (docker_it) (bool): the same as `docker_interactive` - (docker_detached) (bool): detach Docker - (docker_dt) (bool) the same as `docker_detached` - - (docker_base_image) (str): force base image - (docker_os) (str): force docker OS (default: ubuntu) - (docker_os_version) (str): force docker OS version (default: 22.04) - (docker_image_tag_extra) (str): add extra tag (default:-latest) - - (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops) - (docker_cm_repos) - (docker_cm_repo_flags) - - (dockerfile_env) - - (docker_skip_cm_sys_upgrade) - - (docker_extra_sys_deps) - - (fake_run_deps) - (docker_run_final_cmds) - - (all_gpus) - (num_gpus) - - (docker_device) - - (docker_port_maps) - - (docker_shm_size) - - (docker_extra_run_args) Returns: (CM return dict): From c5aeb3dc4c3d213f9838fed63b32b4f69f4545ea Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 27 Sep 2024 16:37:59 +0200 Subject: [PATCH 09/18] minor fix in get-cuda-devices --- script/get-cuda-devices/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/get-cuda-devices/run.sh b/script/get-cuda-devices/run.sh index 91d0a9e1b0..3d208dd6b3 100644 --- a/script/get-cuda-devices/run.sh +++ b/script/get-cuda-devices/run.sh @@ -29,8 +29,7 @@ echo "" echo "Running program ..." echo "" -cd - -#${CM_TMP_CURRENT_PATH} +cd ${CM_TMP_CURRENT_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/a.out > tmp-run.out test $? -eq 0 || exit 1 From 9b79c55b671686cd864a4848cb696da1cf8da394 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 29 Sep 2024 15:14:35 +0200 Subject: [PATCH 10/18] added --docker.key = value to cm docker script --- automation/script/module.py | 3 +++ automation/script/module_misc.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/automation/script/module.py b/automation/script/module.py index 9747639cb7..9494e7a39a 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4032,6 +4032,9 @@ def docker(self, i): (output_dir) (str): output directory (./ by default) + (docker) (dict): convert keys into docker_{key} strings for CM >= 2.3.8.1 + + (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) (docker_noregenerate) (bool): do not generate Dockerfiles (docker_norecreate) (bool): do not recreate Docker image diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 98a4c667d9..18ea9b826a 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1638,6 +1638,18 @@ def docker(i): import copy import re + from cmind import __version__ as current_cm_version + + self_module = i['self_module'] + + if utils.compare_versions(current_cm_version, '2.3.8.1') >= 0 and type(i.get('docker', None)) == dict: + # Grigori started cleaning and refactoring this code on 20240929 + # + # 1. use --docker dictionary instead of --docker_{keys} + + docker_params = utils.convert_dictionary(i['docker'], 'docker') + i.update(docker_params) + quiet = i.get('quiet', False) detached = i.get('docker_detached', '') @@ -1655,7 +1667,6 @@ def docker(i): # Check simplified CMD: cm docker script "python app image-classification onnx" # If artifact has spaces, treat them as tags! - self_module = i['self_module'] self_module.cmind.access({'action':'detect_tags_in_artifact', 'automation':'utils', 'input':i}) # CAREFUL -> artifacts and parsed_artifacts are not supported in input (and should not be?) From 07ef382427132b55c1656754cc4a259d7603d2f8 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 29 Sep 2024 15:33:26 +0200 Subject: [PATCH 11/18] docker fixes --- automation/script/module_misc.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 18ea9b826a..b68d90e279 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1642,13 +1642,15 @@ def docker(i): self_module = i['self_module'] - if utils.compare_versions(current_cm_version, '2.3.8.1') >= 0 and type(i.get('docker', None)) == dict: + if type(i.get('docker', None)) == dict: # Grigori started cleaning and refactoring this code on 20240929 # # 1. use --docker dictionary instead of --docker_{keys} - - docker_params = utils.convert_dictionary(i['docker'], 'docker') - i.update(docker_params) + + if utils.compare_versions(current_cm_version, '2.3.8.1') >= 0: + docker_params = utils.convert_dictionary(i['docker'], 'docker') + i.update(docker_params) + del(i['docker']) quiet = i.get('quiet', False) From 2924ce68597283aa5d7172714515869553cca811 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 29 Sep 2024 15:46:18 +0200 Subject: [PATCH 12/18] added ubuntu 24.04 config --- cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml | 9 +++++++++ ...buntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml | 2 ++ ...ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml | 2 ++ ...ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml | 2 ++ 4 files changed, 15 insertions(+) create mode 100644 cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml diff --git a/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml b/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml new file mode 100644 index 0000000000..d949d5519b --- /dev/null +++ b/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml @@ -0,0 +1,9 @@ +uid: 12e86eb386314866 + +name: "Basic Ubuntu 24.04" + +input: + docker_base_image: 'ubuntu:24.04' + docker_os: ubuntu + docker_os_version: '24.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml index c72268b15a..16107d8d5c 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml @@ -2,6 +2,8 @@ name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0 (pytorch:22.10)" +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-22-10.html + input: docker_base_image: 'nvcr.io/nvidia/pytorch:22.10-py3' docker_os: ubuntu diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml index de9a1bc1db..66b9efd0d9 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml @@ -2,6 +2,8 @@ name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0 (pytorch:23.05)" +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-23-05.html + input: docker_base_image: 'nvcr.io/nvidia/pytorch:23.05-py3' docker_os: ubuntu diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml index f23bba2eb4..38bcff6942 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml @@ -2,6 +2,8 @@ name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0 (pytorch:24.03)" +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-03.html + input: docker_base_image: 'nvcr.io/nvidia/pytorch:24.03-py3' docker_os: ubuntu From e3b43b852c739a2daa3d988e03c45f13c75dffde Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 30 Sep 2024 10:39:33 +0200 Subject: [PATCH 13/18] removed sys deps from image-classification examples --- script/app-image-classification-onnx-py/_cm.yaml | 2 +- script/app-image-classification-torch-py/_cm.json | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/script/app-image-classification-onnx-py/_cm.yaml b/script/app-image-classification-onnx-py/_cm.yaml index 825be9128d..2e2241a07e 100644 --- a/script/app-image-classification-onnx-py/_cm.yaml +++ b/script/app-image-classification-onnx-py/_cm.yaml @@ -22,7 +22,7 @@ default_env: deps: - tags: detect,os -- tags: get,sys-utils-cm +#- tags: get,sys-utils-cm - names: - python - python3 diff --git a/script/app-image-classification-torch-py/_cm.json b/script/app-image-classification-torch-py/_cm.json index a6a78a6798..4e30d3f5fe 100644 --- a/script/app-image-classification-torch-py/_cm.json +++ b/script/app-image-classification-torch-py/_cm.json @@ -11,9 +11,6 @@ { "tags": "detect,os" }, - { - "tags": "get,sys-utils-cm" - }, { "names": [ "python", From 62c38ef9e9e6214360a0da14a9a868ae33c18344 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 30 Sep 2024 10:47:27 +0200 Subject: [PATCH 14/18] removed sys deps from image-classification examples and added YAML --- .../_cm.json | 73 ------------------- .../_cm.yaml | 48 ++++++++++++ 2 files changed, 48 insertions(+), 73 deletions(-) delete mode 100644 script/app-image-classification-tvm-onnx-py/_cm.json create mode 100644 script/app-image-classification-tvm-onnx-py/_cm.yaml diff --git a/script/app-image-classification-tvm-onnx-py/_cm.json b/script/app-image-classification-tvm-onnx-py/_cm.json deleted file mode 100644 index 1ae2e5c320..0000000000 --- a/script/app-image-classification-tvm-onnx-py/_cm.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "alias": "app-image-classification-tvm-onnx-py", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "category": "Modular AI/ML application pipeline", - "default_env": { - "CM_BATCH_COUNT": "1", - "CM_BATCH_SIZE": "1" - }, - "deps": [ - { - "tags": "detect,os" - }, - { - "tags": "detect,cpu" - }, - { - "names": [ - "python", - "python3" - ], - "tags": "get,python3" - }, - { - "tags": "get,dataset,image-classification,original" - }, - { - "tags": "get,dataset-aux,image-classification" - }, - { - "tags": "get,raw,ml-model,image-classification,resnet50,_onnx" - }, - { - "tags": "get,generic-python-lib,_onnxruntime" - }, - { - "names": [ "tvm" ], - "tags": "get,tvm" - } - ], - "tags": [ - "app", - "image-classification", - "tvm-onnx", - "python" - ], - "tags_help":"app image-classification python tvm-onnx", - "uid": "63080407db4d4ac4", - "variations": { - "llvm": { - "add_deps_recursive": { - "tvm": { - "tags": "_llvm" - } - } - }, - "cuda": { - "add_deps_recursive": { - "tvm": { - "tags": "_cuda" - } - }, - "env": { - "USE_CUDA": "yes" - }, - "deps": [ - { - "tags": "get,cuda" - } - ] - } - } -} diff --git a/script/app-image-classification-tvm-onnx-py/_cm.yaml b/script/app-image-classification-tvm-onnx-py/_cm.yaml new file mode 100644 index 0000000000..2b5cc9cca5 --- /dev/null +++ b/script/app-image-classification-tvm-onnx-py/_cm.yaml @@ -0,0 +1,48 @@ +alias: app-image-classification-tvm-onnx-py +uid: 63080407db4d4ac4 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Modular AI/ML application pipeline + +default_env: + CM_BATCH_COUNT: '1' + CM_BATCH_SIZE: '1' + +deps: +- tags: detect,os +- tags: detect,cpu +- names: + - python + - python3 + tags: get,python3 +- tags: get,dataset,image-classification,original +- tags: get,dataset-aux,image-classification +- tags: get,raw,ml-model,image-classification,resnet50,_onnx +- tags: get,generic-python-lib,_onnxruntime +- names: + - tvm + tags: get,tvm + +tags: +- app +- image-classification +- tvm-onnx +- python + +tags_help: app image-classification python tvm-onnx + +variations: + cuda: + add_deps_recursive: + tvm: + tags: _cuda + deps: + - tags: get,cuda + env: + USE_CUDA: 'yes' + llvm: + add_deps_recursive: + tvm: + tags: _llvm From f8a4e7a2ec59758197183bec01cebd115cc629c8 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 30 Sep 2024 17:37:59 +0200 Subject: [PATCH 15/18] fixing debug examples for customize.py and wrapped Python code (external to CM) --- COPYRIGHT.txt | 2 +- README.md | 2 +- debug.py | 9 +++++++++ script/print-hello-world-py/app.py | 16 ++++++++++++++++ script/print-hello-world-py/code.py | 6 ------ script/print-hello-world-py/customize.py | 19 +++++++++++++++++++ script/print-hello-world-py/run.bat | 2 +- script/print-hello-world-py/run.sh | 2 +- script/test-debug/_demo.py | 2 +- 9 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 debug.py create mode 100644 script/print-hello-world-py/app.py delete mode 100644 script/print-hello-world-py/code.py create mode 100644 script/print-hello-world-py/customize.py diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 80e8c506b8..7b62e067df 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -1,5 +1,5 @@ Copyright (c) 2021-2024 MLCommons -The cTuning foundation and OctoML donated this project to MLCommons to benefit everyone. +Grigori Fursin, the cTuning foundation and OctoML donated this project to MLCommons to benefit everyone. Copyright (c) 2014-2021 cTuning foundation diff --git a/README.md b/README.md index 92f7f07fe2..06c2cd2bcf 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ cm run script \ [Apache 2.0](LICENSE.md) -## CM concept +## CM concepts * https://doi.org/10.5281/zenodo.8105339 * https://arxiv.org/abs/2406.16791 diff --git a/debug.py b/debug.py new file mode 100644 index 0000000000..9fd54803c3 --- /dev/null +++ b/debug.py @@ -0,0 +1,9 @@ +# Developer: Grigori Fursin + +import cmind +import sys + +print(sys.executable) + +r = cmind.access('run script "print hello-world python" --debug_uid=f52670e5f3f345a2') +print(r) diff --git a/script/print-hello-world-py/app.py b/script/print-hello-world-py/app.py new file mode 100644 index 0000000000..9dfd48b33a --- /dev/null +++ b/script/print-hello-world-py/app.py @@ -0,0 +1,16 @@ +def main(): + print ('') + + # Import cmind to test break points + import cmind.utils + import os + if os.environ.get('CM_TMP_DEBUG_UID', '') == 'f52670e5f3f345a2': + cmind.utils.debug_here(__file__, port=5678, text='Debugging main.py!').breakpoint() + + print ('HELLO WORLD from Python') + + x = 1 + print (x) + +if __name__ == '__main__': + main() diff --git a/script/print-hello-world-py/code.py b/script/print-hello-world-py/code.py deleted file mode 100644 index 735a890622..0000000000 --- a/script/print-hello-world-py/code.py +++ /dev/null @@ -1,6 +0,0 @@ -def main(): - print ('') - print ('HELLO WORLD from Python') - -if __name__ == '__main__': - main() diff --git a/script/print-hello-world-py/customize.py b/script/print-hello-world-py/customize.py new file mode 100644 index 0000000000..8308693677 --- /dev/null +++ b/script/print-hello-world-py/customize.py @@ -0,0 +1,19 @@ +# Developer(s): Grigori Fursin + +import os + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + + return {'return':0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + return {'return':0} diff --git a/script/print-hello-world-py/run.bat b/script/print-hello-world-py/run.bat index b4f5e3a7ac..c0980c59bb 100644 --- a/script/print-hello-world-py/run.bat +++ b/script/print-hello-world-py/run.bat @@ -2,7 +2,7 @@ IF NOT DEFINED CM_TMP_CURRENT_SCRIPT_PATH SET CM_TMP_CURRENT_SCRIPT_PATH=%CD% rem %CM_PYTHON_BIN_WITH_PATH% --version -%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\code.py +%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\app.py IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% echo CM_NEW_VAR_FROM_RUN=XYZ > tmp-run-env.out diff --git a/script/print-hello-world-py/run.sh b/script/print-hello-world-py/run.sh index 26b72ba626..bc7e2c3016 100644 --- a/script/print-hello-world-py/run.sh +++ b/script/print-hello-world-py/run.sh @@ -5,7 +5,7 @@ CM_TMP_CURRENT_SCRIPT_PATH=${CM_TMP_CURRENT_SCRIPT_PATH:-$PWD} #which ${CM_PYTHON_BIN_WITH_PATH} #${CM_PYTHON_BIN_WITH_PATH} --version -${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/code.py +${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/app.py test $? -eq 0 || exit $? echo "CM_NEW_VAR_FROM_RUN=$MLPERF_XYZ" > tmp-run-env.out diff --git a/script/test-debug/_demo.py b/script/test-debug/_demo.py index 781bed321b..878249a744 100644 --- a/script/test-debug/_demo.py +++ b/script/test-debug/_demo.py @@ -5,5 +5,5 @@ print(sys.executable) -r = cmind.access('run "cm-debug"') +r = cmind.access('run script "test cm-debug"') print(r) From 79871c0fd7f941f319bea767c969ed106d42d031 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Tue, 1 Oct 2024 13:25:59 +0200 Subject: [PATCH 16/18] adding latest wget.exe to get-sys-utils-cm --- script/get-sys-utils-cm/_cm.json | 38 ------------------------------- script/get-sys-utils-cm/_cm.yaml | 3 ++- script/get-sys-utils-min/_cm.json | 2 +- 3 files changed, 3 insertions(+), 40 deletions(-) delete mode 100644 script/get-sys-utils-cm/_cm.json diff --git a/script/get-sys-utils-cm/_cm.json b/script/get-sys-utils-cm/_cm.json deleted file mode 100644 index a496f42c91..0000000000 --- a/script/get-sys-utils-cm/_cm.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "alias": "get-sys-utils-cm", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "cache": true, - "category": "Detection or installation of tools and artifacts", - "deps": [ - { - "tags": "detect,os" - } - ], - "env": { - "CM_CLEAN_DIRS": "bin", - "CM_PACKAGE_WIN_URL": "https://zenodo.org/record/6501550/files/cm-artifact-os-windows-32.zip ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1", - "CM_SUDO": "sudo" - }, - "input_mapping": { - "skip": "CM_SKIP_SYS_UTILS" - }, - "new_env_keys": [ - "+PATH" - ], - "tags": [ - "get", - "sys-utils-cm" - ], - "uid": "bc90993277e84b8e", - "variations": { - "user": { - "env": { - "CM_PYTHON_PIP_USER": "--user" - } - } - }, - "warnings": [ - "This CM script will install extra OS system utils required for CM automation workflows!" - ] -} diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml index 555fe7c6a0..1bbfac9ee5 100644 --- a/script/get-sys-utils-cm/_cm.yaml +++ b/script/get-sys-utils-cm/_cm.yaml @@ -13,8 +13,9 @@ deps: env: CM_CLEAN_DIRS: bin - CM_PACKAGE_WIN_URL: https://zenodo.org/record/6501550/files/cm-artifact-os-windows-32.zip + CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 + ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip CM_SUDO: sudo input_mapping: diff --git a/script/get-sys-utils-min/_cm.json b/script/get-sys-utils-min/_cm.json index d033f8d1c5..19ce3f21a3 100644 --- a/script/get-sys-utils-min/_cm.json +++ b/script/get-sys-utils-min/_cm.json @@ -8,7 +8,7 @@ ], "env": { "CM_CLEAN_DIRS": "bin", - "CM_PACKAGE_WIN_URL": "https://zenodo.org/records/10379926/files/cm-artifact-os-windows-32.zip ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip", + "CM_PACKAGE_WIN_URL": "https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip", "CM_SUDO": "sudo" }, "new_env_keys": [ From 5419871b768f63a89e2018cfb5f0f295408a674b Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Tue, 1 Oct 2024 14:48:53 +0200 Subject: [PATCH 17/18] turn on tests on Windows --- .github/workflows/test-mlperf-inference-resnet50.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 797c6f2a0b..69ce2722a1 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -27,7 +27,7 @@ jobs: implementation: cpp - os: macos-latest backend: tf - - os: windows-latest +# - os: windows-latest steps: - uses: actions/checkout@v4 From a92f8d506b07ab3538f1e16864ee4c678d26509e Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Tue, 1 Oct 2024 17:39:44 +0200 Subject: [PATCH 18/18] * removed windows test for MLPerf (requires interaction) * added windows test for image classification --- .github/workflows/test-image-classification-onnx.yml | 6 +++--- .github/workflows/test-mlperf-inference-resnet50.yml | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-image-classification-onnx.yml b/.github/workflows/test-image-classification-onnx.yml index 62284a2a96..44f27a6167 100644 --- a/.github/workflows/test-image-classification-onnx.yml +++ b/.github/workflows/test-image-classification-onnx.yml @@ -13,12 +13,12 @@ on: jobs: build: - - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [ "3.12"] + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: [ "3.10", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-mlperf-inference-resnet50.yml b/.github/workflows/test-mlperf-inference-resnet50.yml index 69ce2722a1..acbe88b906 100644 --- a/.github/workflows/test-mlperf-inference-resnet50.yml +++ b/.github/workflows/test-mlperf-inference-resnet50.yml @@ -27,7 +27,9 @@ jobs: implementation: cpp - os: macos-latest backend: tf -# - os: windows-latest + - os: windows-latest +# MLPerf requires interaction when installing LLVM on Windows - that's why we excluded it here + steps: - uses: actions/checkout@v4