diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..c49b5453 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug jarvis rg build", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/bin/jarvis", + "args": ["rg", "show"], + "console": "integratedTerminal", + "justMyCode": false, + "env": { + "PATH": "/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/sbin:/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/bin:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/sbin:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/python-venv-1.0-uqbvonvo4asvhbhk6f6jcdtp7jwf7fj2/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mercury-2.3.1-5366qq4fzz53ysefxpvn664pnrgucb72/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pybind11-2.12.0-swuxn5ipnpebl4vkrnsycsg224eyohmm/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pip-23.1.2-7yp5n7qqvtb3tmetty3gotomqz4f6xmg/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-margo-0.15.0-gens72h2rgcnjrsjywyqtslrt235p3vg/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hdf5-1.14.0-zgpju3oqetv3mnwzsufyandndsqvkrku/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/openmpi-5.0.3-ldoaami4sixoo52mtedxikc3jljo6mp5/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/pmix-5.0.1-egvacayyzcck5tezheyjmekjydpg7mnj/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/numactl-2.0.14-r4u7xt5ereaelnt2k7lg5wxdfqsglisy/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libevent-2.1.12-q7ftyq7xm5elcqru6tqbh2m237wsxvrj/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hwloc-2.9.1-7a44yc37rybmbkve2xtfy263qvwursmf/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/ncurses-6.5-ef6nhyqjhebwwj2ijt7qr7vlxau7t6ln/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libxml2-2.10.3-f3h245df2lkrj5iksszkubi7go6ug3cc/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/xz-5.4.6-3pcaf2nbeq4mybsc43qjy46y4anb2vxe/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/doxygen-1.9.8-vooamtsrldlxdfusdocpupsdijcbdbbc/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libiconv-1.17-ygdufncpsmylhcgvwuzosengvr4x6fhu/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lz4-1.9.4-mslrz5yag5xzm6lgqapcfa6dcgfg7qmb/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lizard-1.0-6wko5vrv3r3euppxhql4tzchkn3axgnn/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/bzip2-1.0.8-7s5gj6xpxj7pmtjdf6zbtzovawnxhpsp/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/brotli-1.1.0-oq55g2a6oy3b56u2woxhjcp43u3bh4cd/bin:/opt/rocm/bin:/home/llogan/Documents/Projects/scspkg/packages/rocm/sbin:/home/llogan/Documents/Projects/scspkg/packages/rocm/bin:/home/llogan/.venv/bin:/home/llogan/.local/bin:/home/llogan/Documents/Projects/spack/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin", + "LD_LIBRARY_PATH": "/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/lib64:/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/lib:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/lib64:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/lib:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o/lib64:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o/lib:/home/llogan/Documents/Projects/scspkg/packages/rocm/lib64:/home/llogan/Documents/Projects/scspkg/packages/rocm/lib:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hdf5-1.14.0-zgpju3oqetv3mnwzsufyandndsqvkrku/lib", + "LIBRARY_PATH": "/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/lib64:/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/lib:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/lib64:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/lib:/home/llogan/Documents/Projects/scspkg/packages/rocm/lib64:/home/llogan/Documents/Projects/scspkg/packages/rocm/lib", + "INCLUDE": "/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/include:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/include:/home/llogan/Documents/Projects/scspkg/packages/rocm/include", + "CPATH": "/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/include:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/include:/home/llogan/Documents/Projects/scspkg/packages/rocm/include", + "PKG_CONFIG_PATH": "/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-1.3.1-iijm3e64hy36ppnyq5gtyemtepvzou2s/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/yaml-cpp-0.7.0-2x7cxdqbjnhth2va4o4p4fzganeqjxdc/share/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pybind11-2.12.0-swuxn5ipnpebl4vkrnsycsg224eyohmm/share/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-thallium-0.10.1-bzk2smurrx4lm4y3s3vdtrgjprsbl6d2/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-margo-0.15.0-gens72h2rgcnjrsjywyqtslrt235p3vg/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mercury-2.3.1-5366qq4fzz53ysefxpvn664pnrgucb72/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/json-c-0.16-hslrhkgu6ead546whhemkbprilyfifsj/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lzo-2.10-elgla3744fkvg4wnsntnsw43wdg732hq/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libelf-0.8.13-qbpkxbhedrmoysx5gcqaecj3bvdwnm6r/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hdf5-1.14.0-zgpju3oqetv3mnwzsufyandndsqvkrku/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/openmpi-5.0.3-ldoaami4sixoo52mtedxikc3jljo6mp5/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/pmix-5.0.1-egvacayyzcck5tezheyjmekjydpg7mnj/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/numactl-2.0.14-r4u7xt5ereaelnt2k7lg5wxdfqsglisy/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libevent-2.1.12-q7ftyq7xm5elcqru6tqbh2m237wsxvrj/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hwloc-2.9.1-7a44yc37rybmbkve2xtfy263qvwursmf/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/ncurses-6.5-ef6nhyqjhebwwj2ijt7qr7vlxau7t6ln/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libxml2-2.10.3-f3h245df2lkrj5iksszkubi7go6ug3cc/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/xz-5.4.6-3pcaf2nbeq4mybsc43qjy46y4anb2vxe/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libpciaccess-0.17-bbwqfswunbdrs6xz7mwrbv266vlxyuaj/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/catch2-3.0.1-lxlzp3z364g7zdshjj7zuqxuduz6vg2x/share/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/c-blosc2-2.11.1-w3nw5xv23dyki6fzqegejl6ohbfwlvgp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zstd-1.5.6-g3k37rx7k7fzrisp4ti4p4bb6zlw2bfp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-ng-2.1.6-cij6bkvliadwfgoxwbnwg7oijuewgv2i/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/snappy-1.1.10-kl5iqgbkuizjpdj4qxhtpbg3ueypctvi/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lz4-1.9.4-mslrz5yag5xzm6lgqapcfa6dcgfg7qmb/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lizard-1.0-6wko5vrv3r3euppxhql4tzchkn3axgnn/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/bzip2-1.0.8-7s5gj6xpxj7pmtjdf6zbtzovawnxhpsp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/brotli-1.1.0-oq55g2a6oy3b56u2woxhjcp43u3bh4cd/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/argobots-1.1-afsjfcqmgjscsts2jmpmmiwriznludo4/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-1.3.1-iijm3e64hy36ppnyq5gtyemtepvzou2s/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/yaml-cpp-0.7.0-2x7cxdqbjnhth2va4o4p4fzganeqjxdc/share/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pybind11-2.12.0-swuxn5ipnpebl4vkrnsycsg224eyohmm/share/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-thallium-0.10.1-bzk2smurrx4lm4y3s3vdtrgjprsbl6d2/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-margo-0.15.0-gens72h2rgcnjrsjywyqtslrt235p3vg/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mercury-2.3.1-5366qq4fzz53ysefxpvn664pnrgucb72/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/json-c-0.16-hslrhkgu6ead546whhemkbprilyfifsj/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lzo-2.10-elgla3744fkvg4wnsntnsw43wdg732hq/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libelf-0.8.13-qbpkxbhedrmoysx5gcqaecj3bvdwnm6r/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hdf5-1.14.0-zgpju3oqetv3mnwzsufyandndsqvkrku/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/openmpi-5.0.3-ldoaami4sixoo52mtedxikc3jljo6mp5/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/pmix-5.0.1-egvacayyzcck5tezheyjmekjydpg7mnj/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/numactl-2.0.14-r4u7xt5ereaelnt2k7lg5wxdfqsglisy/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libevent-2.1.12-q7ftyq7xm5elcqru6tqbh2m237wsxvrj/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hwloc-2.9.1-7a44yc37rybmbkve2xtfy263qvwursmf/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/ncurses-6.5-ef6nhyqjhebwwj2ijt7qr7vlxau7t6ln/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libxml2-2.10.3-f3h245df2lkrj5iksszkubi7go6ug3cc/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/xz-5.4.6-3pcaf2nbeq4mybsc43qjy46y4anb2vxe/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libpciaccess-0.17-bbwqfswunbdrs6xz7mwrbv266vlxyuaj/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/catch2-3.0.1-lxlzp3z364g7zdshjj7zuqxuduz6vg2x/share/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/c-blosc2-2.11.1-w3nw5xv23dyki6fzqegejl6ohbfwlvgp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zstd-1.5.6-g3k37rx7k7fzrisp4ti4p4bb6zlw2bfp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-ng-2.1.6-cij6bkvliadwfgoxwbnwg7oijuewgv2i/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/snappy-1.1.10-kl5iqgbkuizjpdj4qxhtpbg3ueypctvi/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lz4-1.9.4-mslrz5yag5xzm6lgqapcfa6dcgfg7qmb/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lizard-1.0-6wko5vrv3r3euppxhql4tzchkn3axgnn/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/bzip2-1.0.8-7s5gj6xpxj7pmtjdf6zbtzovawnxhpsp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/brotli-1.1.0-oq55g2a6oy3b56u2woxhjcp43u3bh4cd/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/argobots-1.1-afsjfcqmgjscsts2jmpmmiwriznludo4/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-1.3.1-iijm3e64hy36ppnyq5gtyemtepvzou2s/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/yaml-cpp-0.7.0-2x7cxdqbjnhth2va4o4p4fzganeqjxdc/share/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pybind11-2.12.0-swuxn5ipnpebl4vkrnsycsg224eyohmm/share/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-thallium-0.10.1-bzk2smurrx4lm4y3s3vdtrgjprsbl6d2/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-margo-0.15.0-gens72h2rgcnjrsjywyqtslrt235p3vg/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mercury-2.3.1-5366qq4fzz53ysefxpvn664pnrgucb72/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/json-c-0.16-hslrhkgu6ead546whhemkbprilyfifsj/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lzo-2.10-elgla3744fkvg4wnsntnsw43wdg732hq/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libelf-0.8.13-qbpkxbhedrmoysx5gcqaecj3bvdwnm6r/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hdf5-1.14.0-zgpju3oqetv3mnwzsufyandndsqvkrku/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/openmpi-5.0.3-ldoaami4sixoo52mtedxikc3jljo6mp5/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/pmix-5.0.1-egvacayyzcck5tezheyjmekjydpg7mnj/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/numactl-2.0.14-r4u7xt5ereaelnt2k7lg5wxdfqsglisy/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libevent-2.1.12-q7ftyq7xm5elcqru6tqbh2m237wsxvrj/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hwloc-2.9.1-7a44yc37rybmbkve2xtfy263qvwursmf/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/ncurses-6.5-ef6nhyqjhebwwj2ijt7qr7vlxau7t6ln/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libxml2-2.10.3-f3h245df2lkrj5iksszkubi7go6ug3cc/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/xz-5.4.6-3pcaf2nbeq4mybsc43qjy46y4anb2vxe/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libpciaccess-0.17-bbwqfswunbdrs6xz7mwrbv266vlxyuaj/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/catch2-3.0.1-lxlzp3z364g7zdshjj7zuqxuduz6vg2x/share/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/c-blosc2-2.11.1-w3nw5xv23dyki6fzqegejl6ohbfwlvgp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zstd-1.5.6-g3k37rx7k7fzrisp4ti4p4bb6zlw2bfp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-ng-2.1.6-cij6bkvliadwfgoxwbnwg7oijuewgv2i/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/snappy-1.1.10-kl5iqgbkuizjpdj4qxhtpbg3ueypctvi/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lz4-1.9.4-mslrz5yag5xzm6lgqapcfa6dcgfg7qmb/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lizard-1.0-6wko5vrv3r3euppxhql4tzchkn3axgnn/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/bzip2-1.0.8-7s5gj6xpxj7pmtjdf6zbtzovawnxhpsp/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/brotli-1.1.0-oq55g2a6oy3b56u2woxhjcp43u3bh4cd/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/argobots-1.1-afsjfcqmgjscsts2jmpmmiwriznludo4/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o/lib/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig", + "CMAKE_PREFIX_PATH": "/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/cmake:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/cmake:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hermes-priv-b4wemj4qw3eonanvhxtglxelnoy6ck7g:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/chimaera-dev-manbdgcofvgnyfmr5anbwwqft2cyvzve:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hermes_shm-dev-df6ysqyvgtq6ts5zv7f3z3mdz7vpmsxe:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-1.3.1-iijm3e64hy36ppnyq5gtyemtepvzou2s:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/yaml-cpp-0.7.0-2x7cxdqbjnhth2va4o4p4fzganeqjxdc:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pybind11-2.12.0-swuxn5ipnpebl4vkrnsycsg224eyohmm:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pip-23.1.2-7yp5n7qqvtb3tmetty3gotomqz4f6xmg:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/python-venv-1.0-uqbvonvo4asvhbhk6f6jcdtp7jwf7fj2:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-thallium-0.10.1-bzk2smurrx4lm4y3s3vdtrgjprsbl6d2:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mochi-margo-0.15.0-gens72h2rgcnjrsjywyqtslrt235p3vg:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/mercury-2.3.1-5366qq4fzz53ysefxpvn664pnrgucb72:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libfabric-1.21.0-3kke226ocwnkcuc5js2x5vtllefhqh6o:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/json-c-0.16-hslrhkgu6ead546whhemkbprilyfifsj:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lzo-2.10-elgla3744fkvg4wnsntnsw43wdg732hq:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libelf-0.8.13-qbpkxbhedrmoysx5gcqaecj3bvdwnm6r:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hdf5-1.14.0-zgpju3oqetv3mnwzsufyandndsqvkrku:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/openmpi-5.0.3-ldoaami4sixoo52mtedxikc3jljo6mp5:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/pmix-5.0.1-egvacayyzcck5tezheyjmekjydpg7mnj:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/numactl-2.0.14-r4u7xt5ereaelnt2k7lg5wxdfqsglisy:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libevent-2.1.12-q7ftyq7xm5elcqru6tqbh2m237wsxvrj:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/hwloc-2.9.1-7a44yc37rybmbkve2xtfy263qvwursmf:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/ncurses-6.5-ef6nhyqjhebwwj2ijt7qr7vlxau7t6ln:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libxml2-2.10.3-f3h245df2lkrj5iksszkubi7go6ug3cc:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/xz-5.4.6-3pcaf2nbeq4mybsc43qjy46y4anb2vxe:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libpciaccess-0.17-bbwqfswunbdrs6xz7mwrbv266vlxyuaj:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/doxygen-1.9.8-vooamtsrldlxdfusdocpupsdijcbdbbc:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/libiconv-1.17-ygdufncpsmylhcgvwuzosengvr4x6fhu:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/cereal-1.3.2-wxfe4s6x7vi2oan2cc3rovskk4ppudap:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/catch2-3.0.1-lxlzp3z364g7zdshjj7zuqxuduz6vg2x:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/c-blosc2-2.11.1-w3nw5xv23dyki6fzqegejl6ohbfwlvgp:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zstd-1.5.6-g3k37rx7k7fzrisp4ti4p4bb6zlw2bfp:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/zlib-ng-2.1.6-cij6bkvliadwfgoxwbnwg7oijuewgv2i:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/snappy-1.1.10-kl5iqgbkuizjpdj4qxhtpbg3ueypctvi:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lz4-1.9.4-mslrz5yag5xzm6lgqapcfa6dcgfg7qmb:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/lizard-1.0-6wko5vrv3r3euppxhql4tzchkn3axgnn:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/bzip2-1.0.8-7s5gj6xpxj7pmtjdf6zbtzovawnxhpsp:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/brotli-1.1.0-oq55g2a6oy3b56u2woxhjcp43u3bh4cd:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/boost-1.85.0-54fn3htnrjht6xgyjvvaxeigl4zmogan:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/argobots-1.1-afsjfcqmgjscsts2jmpmmiwriznludo4:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/gcc-runtime-13.3.0-vfdjm6c7xp7xss6zypjbuta7zd7jtqh7:/opt/rocm/lib/cmake/hip:/home/llogan/Documents/Projects/scspkg/packages/rocm/cmake", + "JAVA_HOME": "", + "PYTHONPATH": "/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/lib64:/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/lib:/home/llogan/Documents/Projects/scspkg/packages/chi_nettest/bin:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/lib64:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/lib:/home/llogan/Documents/Projects/scspkg/packages/hermes_shm/bin:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pybind11-2.12.0-swuxn5ipnpebl4vkrnsycsg224eyohmm/lib/python3.12/site-packages:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/py-pip-23.1.2-7yp5n7qqvtb3tmetty3gotomqz4f6xmg/lib/python3.12/site-packages:/home/llogan/Documents/Projects/spack/opt/spack/linux-linuxmint22-zen2/gcc-13.3.0/python-venv-1.0-uqbvonvo4asvhbhk6f6jcdtp7jwf7fj2/local/lib/python3.12/dist-packages:/home/llogan/Documents/Projects/scspkg/packages/rocm/lib64:/home/llogan/Documents/Projects/scspkg/packages/rocm/lib:/home/llogan/Documents/Projects/scspkg/packages/rocm/bin" + } + + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 043f893f..52b59109 100644 --- a/README.md +++ b/README.md @@ -6,35 +6,55 @@ We provide a builtin repo which contains various applications to deploy. We refer to applications as "jarivs pkgs" which can be connected to form "deployment pipelines". -# 0.1 Dependencies +# 1. Installation -## 0.1.1. Jarvis-Util +Get the GRC spack repo: +```bash +git clone https://github.com/grc-iit/grc-repo +spack repo add grc-repo +``` + +Install jarvis-cd: +```bash +spack external find python +spack install py-jarvis-cd +``` + +Spack packages must be loaded to use them. +You'll have to do this for each new terminal. +```bash +spack load py-jarvis-cd +``` + +# 2. Manual Installation + +## 2.1. Jarvis-Util Jarvis-CD depends on jarvis-util. jarvis-util contains functions to execute binaries in python and collect their output. ```bash -git clone https://github.com/scs-lab/jarvis-util.git +git clone https://github.com/grc-iit/jarvis-util.git cd jarvis-util python3 -m pip install -r requirements.txt python3 -m pip install -e . ``` -## 0.1.2. Scspkg +## 2.2. Scspkg Scspkg is a tool for building modulefiles using a CLI. It's not strictly necessary for Jarvis to function, but many of the readmes use it to provide structure to manual installations. ```bash -git clone https://github.com/scs-lab/scspkg.git +git clone https://github.com/grc-iit/scspkg.git python3 -m pip install -r requirements.txt python3 -m pip install -e . echo "module use \`scspkg module dir\`" >> ~/.bashrc ``` -The wiki for scspkg is [here](https://github.com/scs-lab/scspkg.git). +The wiki for scspkg is [here](https://github.com/grc-iit/scspkg.git). -# 0.2. Installation +# 2.3. Jarvis-CD ```bash cd /path/to/jarvis-cd @@ -42,11 +62,27 @@ python3 -m pip install -r requirements.txt python3 -m pip install -e . ``` -# 0.3. Configuring Jarvis +# 2.4. Net Test + +Network test tool for identifying valid networks. +```bash +spack install chi-nettest +``` + +# 3. Configuring Jarvis -## 0.3.1. Bootstrapping from a specific machine +## 3.1. Bootstrapping for a single-node machine -Jarivs has been pre-configured on some machines. To bootstrap from +You may be trying to test things on just a single node. + +In this case, run: +```bash +jarvis bootstrap from local +``` + +## 3.2. Bootstrapping from a specific machine + +Jarvis has been pre-configured on some machines. To bootstrap from one of them, run the following: ```bash @@ -60,7 +96,7 @@ To check the set of available machines to bootstrap from, run: jarvis bootstrap list ``` -## 0.3.2. Creating a new configuration +## 3.3. Creating a new configuration A configuration can be generated as follows: ```bash @@ -78,3 +114,10 @@ require this, but on machines without a global filesystem (e.g., Chameleon Cloud this parameter can be set later. For a personal machine, these directories can be the same directory. + +# 4. Building the Resource Graph + +Python jarvis: +```bash +jarvis rg build +``` \ No newline at end of file diff --git a/bin/jarvis b/bin/jarvis index c55fa851..bb9fc7c4 100755 --- a/bin/jarvis +++ b/bin/jarvis @@ -50,8 +50,8 @@ class JarvisArgs(ArgParse): 'pos': True } ]) - - # jarvis celan + + # jarvis reset self.add_cmd('reset', msg='Clean all pipelines and configurations') @@ -110,10 +110,6 @@ class JarvisArgs(ArgParse): msg='Resources to build a resource graph for a machine', aliases=['rg']) - # jarvis resource-graph init - self.add_cmd('resource-graph init', - msg='Create an empty resource graph') - # jarvis resource-graph show self.add_cmd('resource-graph show', msg='Show the resource graph') @@ -127,23 +123,29 @@ class JarvisArgs(ArgParse): msg='Introspect resource graph for this machine') self.add_args([ { - 'name': 'walkthrough', - 'msg': 'A guide for building a resource graph', - 'type': bool, - 'default': False + 'name': 'net_sleep', + 'msg': 'How long to sleep in network tests', + 'type': float, + 'default': 5, + 'pos': False, + 'required': False }, + ]) + + # jarvis resource-graph modify + self.add_cmd('resource-graph modify', + msg='Modify the resource graph to introspect new resources') + self.add_args([ { - 'name': 'introspect', - 'msg': 'Whether or not to do an introspect before building', - 'type': bool, - 'default': True + 'name': 'net_sleep', + 'msg': 'How long to sleep in network tests', + 'type': float, + 'default': 5, + 'pos': False, + 'required': False }, ]) - - # jarvis resource-graph build - self.add_cmd('resource-graph prune', - msg='An interactive CLI for modifying the resource graph') - + # jarvis resource-graph add storage self.add_cmd('resource-graph add storage', msg='Add a storage device or PFS to track') @@ -1122,15 +1124,10 @@ class JarvisArgs(ArgParse): def hostfile_set(self): self.jarvis.set_hostfile(self.kwargs['path']) - pipelines = self.jarvis.list_pipelines() - self.jarvis.save() - - def resource_graph_init(self): - self.jarvis.resource_graph_init() self.jarvis.save() def resource_graph_show(self): - self.jarvis.resource_graph_show() + self.jarvis.load().resource_graph_show() self.jarvis.save() def resource_graph_path(self): @@ -1138,14 +1135,12 @@ class JarvisArgs(ArgParse): self.jarvis.save() def resource_graph_build(self): - walkthrough = self.kwargs['walkthrough'] - introspect = self.kwargs['introspect'] - if walkthrough: - self.jarvis.resource_graph.walkthrough_build( - PsshExecInfo(hostfile=self.jarvis.hostfile), - introspect) - else: - self.jarvis.resource_graph_build() + net_sleep = self.kwargs['net_sleep'] + self.jarvis.resource_graph_build(net_sleep) + self.jarvis.save() + + def resource_graph_modify(self): + self.jarvis.resource_graph_modify() self.jarvis.save() def resource_graph_prune(self): diff --git a/builtin/builtin/hermes/pkg.py b/builtin/builtin/hermes/pkg.py index 4a70fdfe..8ae3e0e6 100644 --- a/builtin/builtin/hermes/pkg.py +++ b/builtin/builtin/hermes/pkg.py @@ -86,11 +86,12 @@ def _configure_server(self): if len(self.config['devices']) == 0: # Get all the fastest storage device mount points on machine - dev_df = rg.find_storage() + dev_df = rg.find_storage(needs_root=False) else: # Get the storage devices for the user dev_list = [rg.find_storage(dev_types=dev_type, - count_per_pkg=count) + count_per_pkg=count, + needs_root=False) for dev_type, count in self.config['devices']] dev_df = sdf.concat(dev_list) if len(dev_df) == 0: diff --git a/builtin/builtin/hermes_run/pkg.py b/builtin/builtin/hermes_run/pkg.py index 52f30901..e1dbc97d 100644 --- a/builtin/builtin/hermes_run/pkg.py +++ b/builtin/builtin/hermes_run/pkg.py @@ -336,11 +336,12 @@ def _configure(self, **kwargs): # Get storage info if len(self.config['devices']) == 0: # Get all the fastest storage device mount points on machine - dev_df = rg.find_storage() + dev_df = rg.find_storage(needs_root=False) else: # Get the storage devices for the user dev_list = [rg.find_storage(dev_types=dev_type, - count_per_pkg=count) + count_per_pkg=count, + needs_root=False) for dev_type, count in self.config['devices']] dev_df = sdf.concat(dev_list) if len(dev_df) == 0: diff --git a/builtin/builtin/orangefs/fuse.py b/builtin/builtin/orangefs/fuse.py index 42e55fa0..d5772c2a 100644 --- a/builtin/builtin/orangefs/fuse.py +++ b/builtin/builtin/orangefs/fuse.py @@ -103,12 +103,14 @@ def _configure(self, **kwargs): dev_types = ['hdd', 'ssd', 'nvme', 'dimm'] for dev_type in dev_types: dev_df = rg.find_storage(dev_types=[dev_type], - shared=False) + shared=False, + needs_root=False) if len(dev_df) != 0: break else: dev_df = rg.find_storage(dev_types=[self.config['dev_type']], - shared=False) + shared=False, + needs_root=False) if len(dev_df) == 0: raise Exception('Could not find any storage devices :(') diff --git a/builtin/builtin/orangefs/pkg.py b/builtin/builtin/orangefs/pkg.py index 9ef562c2..685dee9c 100644 --- a/builtin/builtin/orangefs/pkg.py +++ b/builtin/builtin/orangefs/pkg.py @@ -54,7 +54,7 @@ def _configure_menu(self): { 'name': 'name', 'msg': 'The name of the orangefs installation', - 'type': int, + 'type': str, 'default': 'orangefs', }, { @@ -107,12 +107,14 @@ def _configure(self, **kwargs): dev_types = ['hdd', 'ssd', 'nvme', 'dimm'] for dev_type in dev_types: dev_df = rg.find_storage(dev_types=[dev_type], - shared=False) + shared=False, + needs_root=False) if len(dev_df) != 0: break else: dev_df = rg.find_storage(dev_types=[self.config['dev_type']], - shared=False) + shared=False, + needs_root=False) if len(dev_df) == 0: raise Exception('Could not find any storage devices :(') storage_dir = os.path.expandvars(dev_df.rows[0]['mount']) diff --git a/builtin/resource_graph/ares.yaml b/builtin/resource_graph/ares.yaml index b591aafd..0369acbf 100644 --- a/builtin/resource_graph/ares.yaml +++ b/builtin/resource_graph/ares.yaml @@ -3,48 +3,45 @@ fs: dev_type: ssd device: /dev/sdb1 fs_type: xfs - host: localhost model: Samsung SSD 860 mount: /mnt/ssd/${USER} parent: /dev/sdb shared: false uuid: 45b6abb3-7786-4b68-95d0-a8fac92e0d70 + needs_root: false - avail: 500GB dev_type: hdd device: null fs_type: xfs - host: localhost model: null mount: ${HOME} parent: null shared: true uuid: null + needs_root: false - avail: 240GB dev_type: nvme device: /dev/nvme0n1p1 fs_type: xfs - host: localhost model: TOSHIBA-RD400 mount: /mnt/nvme/${USER} parent: /dev/nvme0n1 shared: false uuid: cbc9b18c-fbb0-4ab7-a9fd-8cc719a93f64 + needs_root: false - avail: 900GB dev_type: hdd device: /dev/sdc1 fs_type: xfs - host: localhost model: ST1000LM049-2GH1 mount: /mnt/hdd/${USER} parent: /dev/sdc shared: false uuid: 7857cbad-2e46-40c2-835a-b297bc5ee1d2 -hosts: -- localhost + needs_root: false net: - domain: lo fabric: 127.0.0.1/32 - host: localhost protocol: FI_PROTO_RXM provider: tcp;ofi_rxm shared: false @@ -53,7 +50,6 @@ net: version: '114.10' - domain: eno1 fabric: 172.20.0.0/16 - host: localhost protocol: FI_PROTO_UDP provider: udp shared: true @@ -62,7 +58,6 @@ net: version: '114.10' - domain: lo fabric: 127.0.0.1/32 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: tcp shared: false @@ -71,7 +66,6 @@ net: version: '114.10' - domain: lo fabric: 127.0.0.1/32 - host: localhost protocol: FI_PROTO_UDP provider: udp shared: false @@ -80,7 +74,6 @@ net: version: '114.10' - domain: enp47s0np0 fabric: 172.25.0.0/16 - host: localhost protocol: FI_PROTO_RXD provider: udp;ofi_rxd shared: true @@ -89,7 +82,6 @@ net: version: '114.10' - domain: eno1 fabric: 172.20.0.0/16 - host: localhost protocol: FI_PROTO_RXM provider: tcp;ofi_rxm shared: true @@ -98,7 +90,6 @@ net: version: '114.10' - domain: lo fabric: 127.0.0.1/32 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: sockets shared: false @@ -107,7 +98,6 @@ net: version: '114.10' - domain: lo fabric: 127.0.0.1/32 - host: localhost protocol: FI_PROTO_RXD provider: udp;ofi_rxd shared: false @@ -116,7 +106,6 @@ net: version: '114.10' - domain: eno1 fabric: 172.20.0.0/16 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: tcp shared: true @@ -125,7 +114,6 @@ net: version: '114.10' - domain: enp47s0np0 fabric: 172.25.0.0/16 - host: localhost protocol: FI_PROTO_RXM provider: tcp;ofi_rxm shared: true @@ -134,7 +122,6 @@ net: version: '114.10' - domain: enp47s0np0 fabric: 172.25.0.0/16 - host: localhost protocol: FI_PROTO_UDP provider: udp shared: true @@ -143,7 +130,6 @@ net: version: '114.10' - domain: enp47s0np0 fabric: 172.25.0.0/16 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: sockets shared: true @@ -152,7 +138,6 @@ net: version: '114.10' - domain: enp47s0np0 fabric: 172.25.0.0/16 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: tcp shared: true @@ -161,7 +146,6 @@ net: version: '114.10' - domain: eno1 fabric: 172.20.0.0/16 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: sockets shared: true @@ -170,7 +154,6 @@ net: version: '114.10' - domain: eno1 fabric: 172.20.0.0/16 - host: localhost protocol: FI_PROTO_RXD provider: udp;ofi_rxd shared: true @@ -179,7 +162,6 @@ net: version: '114.10' - domain: '' fabric: 172.25.0.0/16 - host: localhost protocol: '' provider: ofi+verbs;ofi_rxm shared: true diff --git a/builtin/resource_graph/deception.yaml b/builtin/resource_graph/deception.yaml index c0d67b3b..430e25da 100644 --- a/builtin/resource_graph/deception.yaml +++ b/builtin/resource_graph/deception.yaml @@ -3,7 +3,7 @@ fs: dev_type: nvme device: nvme0n1 fs_type: null - host: localhost + needs_root: false model: INTEL SSDPEKKA512G8 mount: /scratch/${USER} parent: null @@ -13,7 +13,7 @@ fs: dev_type: hdd device: null fs_type: null - host: localhost + needs_root: false model: null mount: /rcfs/projects/chess/${USER} parent: null @@ -24,7 +24,6 @@ hosts: net: - domain: mlx5_0:1 fabric: 172.17.0.0/16 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: ucx+rc_verbs shared: true diff --git a/builtin/resource_graph/g2-standard-4.yaml b/builtin/resource_graph/g2-standard-4.yaml new file mode 100644 index 00000000..09f6fead --- /dev/null +++ b/builtin/resource_graph/g2-standard-4.yaml @@ -0,0 +1,30 @@ +fs: +- avail: 107238916608 + dev_type: null + device: nvme0n1p1 + fs_mount: null + fs_size: null + fs_type: null + needs_root: false + model: nvme_card-pd + mount: ${HOME} + parent: nvme0n1 + rota: 0 + sector_size: 512 + shared: false + size: 107238916608 + tran: nvme + use%: null + used: null + uuid: null +hosts: +- localhost +net: +- domain: lo + fabric: '' + protocol: FI_PROTO_SOCK_TCP + provider: sockets + shared: null + speed: 0 + type: FI_EP_DGRAM + version: '121.0' \ No newline at end of file diff --git a/builtin/resource_graph/polaris.yaml b/builtin/resource_graph/polaris.yaml index 227f3a9c..b13b03a7 100644 --- a/builtin/resource_graph/polaris.yaml +++ b/builtin/resource_graph/polaris.yaml @@ -3,7 +3,7 @@ fs: dev_type: hdd device: null fs_type: null - host: localhost + needs_root: false model: null mount: /lus/grand/projects/VeloC/${USER}/SCS_lab parent: null @@ -14,7 +14,6 @@ hosts: net: - domain: mlx5_3-dgram fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_IB_UD provider: verbs shared: true @@ -23,7 +22,6 @@ net: version: '111.0' - domain: mlx5_bond_0 fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RDMA_CM_IB_RC provider: verbs shared: true @@ -32,7 +30,6 @@ net: version: '111.0' - domain: mlx5_3 fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RDMA_CM_IB_RC provider: verbs shared: true @@ -41,7 +38,6 @@ net: version: '111.0' - domain: mlx5_bond_0-xrc fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RXM provider: verbs;ofi_rxm shared: true @@ -50,7 +46,6 @@ net: version: '111.0' - domain: mlx5_3-xrc fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RDMA_CM_IB_XRC provider: verbs shared: true @@ -59,7 +54,6 @@ net: version: '111.0' - domain: mlx5_3-xrc fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RXM provider: verbs;ofi_rxm shared: true @@ -68,7 +62,6 @@ net: version: '111.0' - domain: mlx5_0-xrc fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RDMA_CM_IB_XRC provider: verbs shared: true @@ -77,7 +70,6 @@ net: version: '111.0' - domain: mlx5_0 fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RDMA_CM_IB_RC provider: verbs shared: true @@ -86,7 +78,6 @@ net: version: '111.0' - domain: mlx5_0 fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RXM provider: verbs;ofi_rxm shared: true @@ -95,7 +86,6 @@ net: version: '111.0' - domain: mlx5_0-xrc fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RXM provider: verbs;ofi_rxm shared: true @@ -104,7 +94,6 @@ net: version: '111.0' - domain: mlx5_bond_0-dgram fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_IB_UD provider: verbs shared: true @@ -113,7 +102,6 @@ net: version: '111.0' - domain: hsn0 fabric: 10.201.0.0/16 - host: localhost protocol: FI_PROTO_SOCK_TCP provider: sockets shared: true @@ -122,7 +110,6 @@ net: version: '111.0' - domain: mlx5_0-dgram fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_IB_UD provider: verbs shared: true @@ -131,7 +118,6 @@ net: version: '111.0' - domain: mlx5_3 fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RXM provider: verbs;ofi_rxm shared: true @@ -140,7 +126,6 @@ net: version: '111.0' - domain: mlx5_bond_0-xrc fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RDMA_CM_IB_XRC provider: verbs shared: true @@ -149,7 +134,6 @@ net: version: '111.0' - domain: mlx5_bond_0 fabric: IB-0xfe80000000000000 - host: localhost protocol: FI_PROTO_RXM provider: verbs;ofi_rxm shared: true diff --git a/jarvis_cd/basic/jarvis_manager.py b/jarvis_cd/basic/jarvis_manager.py index 9c16e552..50a237b6 100644 --- a/jarvis_cd/basic/jarvis_manager.py +++ b/jarvis_cd/basic/jarvis_manager.py @@ -47,15 +47,15 @@ def __init__(self): self.shared_dir = None # The current pipeline (per-user) self.cur_pipeline = None + # Path to local jarvis configuration directory + self.local_config_dir = os.path.join(os.environ['HOME'], '.jarvis') # The path to the global jarvis configuration (root user) - self.jarvis_conf_path = os.path.join(self.jarvis_root, - 'config', + self.jarvis_conf_path = os.path.join(self.local_config_dir, 'jarvis_config.yaml') # The Jarvis configuration (per-user) self.jarvis_conf = None # The path to the jarvis resource graph (global across users) - self.resource_graph_path = os.path.join(self.jarvis_root, - 'config', + self.resource_graph_path = os.path.join(self.local_config_dir, 'resource_graph.yaml') # The Jarvis resource graph (global across users) self.resource_graph = None @@ -92,24 +92,9 @@ def create(self, config_dir, private_dir, shared_dir=None): self.add_repo(f'{self.jarvis_root}/builtin') self.resource_graph = ResourceGraph() self.hostfile = Hostfile() - os.makedirs(f'{self.jarvis_root}/config', exist_ok=True) + os.makedirs(self.local_config_dir, exist_ok=True) self.save() - def save(self): - """ - Save the jarvis config to config/jarvis_config.yaml - - :return: None - """ - # Update jarvis conf - self.jarvis_conf['CUR_PIPELINE'] = self.cur_pipeline - self.jarvis_conf['REPOS'] = self.repos - self.jarvis_conf['HOSTFILE'] = self.hostfile.path - # Save global resource graph - self.resource_graph.save(self.resource_graph_path) - # Save global and per-user conf - YamlFile(self.jarvis_conf_path).save(self.jarvis_conf) - def load(self): """ Load the jarvis config from config/jarvis_config.yaml @@ -117,6 +102,7 @@ def load(self): :return: None """ if not os.path.exists(self.jarvis_conf_path): + print('No configuration was found. Run jarvis init or bootstrap.') return self.jarvis_conf = {} # Read global jarvis conf @@ -143,6 +129,25 @@ def load(self): except Exception as e: print(f'Failed to open hostfile {self.jarvis_conf["HOSTFILE"]}') self.hostfile = Hostfile() + return self + + def save(self): + """ + Save the jarvis config to config/jarvis_config.yaml + + :return: None + """ + # Update jarvis conf + if self.jarvis_conf: + self.jarvis_conf['CUR_PIPELINE'] = self.cur_pipeline + self.jarvis_conf['REPOS'] = self.repos + self.jarvis_conf['HOSTFILE'] = self.hostfile.path + # Save global resource graph + if self.resource_graph: + self.resource_graph.save(self.resource_graph_path) + # Save global and per-user conf + if self.jarvis_conf: + YamlFile(self.jarvis_conf_path).save(self.jarvis_conf) def set_hostfile(self, path): """ @@ -162,17 +167,25 @@ def bootstrap_from(self, machine): :param machine: The machine config to copy :return: None """ - os.makedirs(f'{self.jarvis_root}/config', exist_ok=True) + if machine == 'local': + # + self.create( + os.path.join(self.local_config_dir, 'config'), + os.path.join(self.local_config_dir, 'private'), + os.path.join(self.local_config_dir, 'shared')) + self.save() + return + os.makedirs(self.local_config_dir, exist_ok=True) config_path = f'{self.jarvis_root}/builtin/config/{machine}.yaml' if os.path.exists(config_path): config = expand_env(YamlFile(config_path).load()) - new_config_path = f'{self.jarvis_root}/config/jarvis_config.yaml' + new_config_path = f'{self.local_config_dir}/jarvis_config.yaml' YamlFile(new_config_path).save(config) rg_path = f'{self.jarvis_root}/builtin/resource_graph/{machine}.yaml' if os.path.exists(rg_path): self.resource_graph = ResourceGraph().load(rg_path) - new_rg_path = f'{self.jarvis_root}/config/resource_graph.yaml' + new_rg_path = f'{self.local_config_dir}/resource_graph.yaml' self.resource_graph.save(new_rg_path) def bootstrap_list(self): @@ -201,14 +214,6 @@ def print_config(self): def print_config_path(self): print(self.jarvis_conf_path) - def resource_graph_init(self): - """ - Create an empty resource graph - - :return: None - """ - self.resource_graph = ResourceGraph() - def resource_graph_show(self): """ Print the resource graph @@ -220,7 +225,7 @@ def resource_graph_show(self): print("net:") self.resource_graph.print_df(self.resource_graph.net) - def resource_graph_build(self): + def resource_graph_build(self, net_sleep): """ Introspect the system and construct a resource graph. @@ -228,7 +233,14 @@ def resource_graph_build(self): """ self.resource_graph = ResourceGraph() self.resource_graph.build( - PsshExecInfo(hostfile=self.hostfile)) + PsshExecInfo(hostfile=self.hostfile), net_sleep=net_sleep) + + def resource_graph_modify(self, net_sleep): + """ + Modify the resource graph to retest resources + """ + self.resource_graph.modify( + PsshExecInfo(hostfile=self.hostfile), net_sleep=net_sleep) def list_pipelines(self): """ @@ -268,11 +280,16 @@ def add_repo(self, path): for repo in self.repos: if repo['name'] == repo_name: repo['path'] = path + print(f'Updated the {repo_name} to path {path}') return + if not os.path.exists(os.path.join(path, repo_name)): + print('Error: repo must have a subdirectory with the same name') + exit(1) self.repos.insert(0, { 'path': path, 'name': repo_name }) + print(f'Added the {repo_name} repo') def create_pkg(self, pkg_cls, pkg_type): """ diff --git a/requirements.txt b/requirements.txt index 18d3ab31..c41f383a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ pyyaml pandas +jarvis-util @ git+https://github.com/scs-lab/jarvis-util.git#egg=jarvis-util #pylint==2.15.0 #coverage==5.5 #coverage-lcov==0.2.4 diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 00000000..18d3ab31 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,6 @@ +pyyaml +pandas +#pylint==2.15.0 +#coverage==5.5 +#coverage-lcov==0.2.4 +#pytest==6.2.5 \ No newline at end of file diff --git a/setup.py b/setup.py index 3508ffa0..8fc05751 100644 --- a/setup.py +++ b/setup.py @@ -23,11 +23,11 @@ "Topic :: Application Configuration", ], install_requires=[ - 'pyyaml', + # 'pyyaml', # 'pylint==2.15.0', # 'coverage==5.5', # 'coverage-lcov==0.2.4', # 'pytest==6.2.5', - 'jarvis-util @ git+https://github.com/scs-lab/jarvis-util.git#egg=jarvis-util' + #'jarvis-util @ git+https://github.com/scs-lab/jarvis-util.git#egg=jarvis-util' ] )