Python Virtual Env

.gitignore

@@ -166,5 +166,8 @@ runner.yml
 # command scripts for runner
 servers/execution/runner/lifecycle*
 
+# Python virtual-env
+dtaas-venv/
+
 # certs
 *.pem

docs/developer/index.md

@@ -21,7 +21,6 @@ Please use the steps given here to install the required software packages.
 
 ```bash
 bash script/env.sh
-bash script/configure-git-hooks.sh
 bash script/docker.sh
 ```
 
@@ -68,14 +67,16 @@ following commands in the given order:
 ```bash
 powershell -F script/base.ps1
 powershell -F script/env.ps1
-powershell -F script/configure-git-hooks.ps1
 powershell -F script/docker.ps1
 ```
 
 :warning: The docker images are large and are likely to consume
 about 5GB of bandwidth and 15GB of space.
 You will have to download the docker images on a really good network.
 
+<!-- 
+TODO insert when githooks works
+
 ### git hooks
 
 The git-hooks will ensure that your commits are formatted
@@ -108,22 +109,30 @@ you can use the `--no-verify` flag
 on `git commit` or `git push`. Please use this
 option with care.
 
+-->
+
 ## :building_construction: Development Workflow
 
 To manage collaboration by multiple developers on the software,
 a development workflow is in place. Each developer should follow these steps:
 
+<!--
+TODO Insert before Use fork,branch ... when githooks works
+Install git-hooks for the project.
+-->
+
 1. Fork of the main repository into your github account.
 1. Setup
    [Code Climate](https://docs.codeclimate.com/docs/getting-started-with-code-climate)
    and
    [Codecov](https://docs.codecov.com/docs/quick-start)
    for your fork. The codecov does not require secret token
    for public repositories.
-1. Install git-hooks for the project.
+1. nvm use 22 (if nvm or node isn't installed)
 1. Use
    [Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/)
    workflow.
+1. Active the Python Virtual Enviornment by calling (`./dtaas-venv/bin/activate`)
 1. Work in your fork and open a PR from your working
    branch to your `feature/distributed-demo` branch.
    The PR will run all the github actions, code climate and codecov checks.

script/base.ps1

@@ -28,6 +28,10 @@ if (-not (Get-Command docker-desktop -ErrorAction SilentlyContinue)) {
     # https://docs.docker.com/engine/install/ubuntu/
     choco install -y docker-desktop
 
+    $dockerDesktopPath = Join-Path $env:ProgramFiles "Docker\Docker\resources\cli-plugins"
+    # Ensure docker-desktop is in the system path
+    [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$dockerDesktopPath", [EnvironmentVariableTarget]::Machine)
+
     # Check if docker group exists, if not, create it
     $dockerGroup = 'docker'
     if (-not (Get-LocalGroup -Name $dockerGroup -ErrorAction SilentlyContinue)) {
@@ -61,21 +65,22 @@ if (-not (Get-Command docker-desktop -ErrorAction SilentlyContinue)) {
 
     # Enable Docker service
     Set-Service -Name "com.docker.service" -StartupType Automatic
-
-    # Install docker-compose from https://docs.docker.com/compose/install/other/
-    $url = "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-windows-x86_64.exe"
-    $outputPath = Join-Path $env:ProgramFiles "Docker\docker-compose.exe"
-    Invoke-WebRequest -Uri $url -OutFile $outputPath
-    # Ensure docker-compose is in the system path
-    [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$outputPath", [EnvironmentVariableTarget]::Machine)
 }
 
 # Check if Node.js is already installed
 if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
     # Install Node.js environment if not already installed
-    choco install -y nodejs-lts
+    choco install -y nvm
+    $nvmInstallPath = Join-Path "C:\ProgramData" "nvm"
+    [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$nvmInstallPath", [EnvironmentVariableTarget]::Machine)
+    Write-Host "installing nvm"
 }
 
+# Install node
+nvm install 22
+nvm use 22
+
+
 # Install Yarn
 choco install -y yarn
 

script/base.sh

@@ -14,7 +14,8 @@ sudo apt-get install -y \
     apache2-utils \
     net-tools \
     python3-dev \
-    python3-pip
+    python3-pip \
+    python3-venv
 
 sudo mkdir -p /etc/apt/keyrings
 if [ ! -f /etc/apt/keyrings/docker.gpg ]
@@ -26,7 +27,7 @@ then
 fi
 
 sudo apt-get update -y
-sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 sudo groupadd docker || true
 sudo usermod -aG docker "$USER" || true
 
@@ -52,27 +53,15 @@ docker run hello-world
 sudo systemctl enable docker.service
 sudo systemctl enable containerd.service
 
-
-#install docker-compose from https://docs.docker.com/compose/install/other/
-sudo curl -SL "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-x86_64" \
-  -o /usr/local/bin/docker-compose
-sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
-sudo chmod 755 /usr/local/bin/docker-compose /usr/bin/docker-compose
-
-
 # Install nodejs environment
 sudo apt-get update
 sudo apt-get install -y ca-certificates curl gnupg
-sudo mkdir -p /etc/apt/keyrings
-if [ ! -f /etc/apt/keyrings/nodesource.gpg ]
-then
-  curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \
-    sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
-fi
-NODE_MAJOR=20
-printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \
-  https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \
-  sudo tee /etc/apt/sources.list.d/nodesource.list
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+
+nvm install 22
+nvm use 22
 
 sudo apt-get update
 sudo apt-get install -y nodejs
@@ -90,4 +79,4 @@ fi
 sudo apt-get update -y
 sudo apt-get install -y yarn
 sudo npm install -g serve
-sudo npm install -g pm2
+sudo npm install -g pm2

script/env.ps1

@@ -21,9 +21,6 @@ if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarg
     [Environment]::SetEnvironmentVariable("Path", "$env:Path;$(Get-Command python3 | Select-Object -ExpandProperty Directory)", [EnvironmentVariableTarget]::Machine)
 }
 
-# Install mkdocs
-pip install mkdocs
-
 # Install Ruby: https://github.com/oneclick/rubyinstaller2/releases/download/RubyInstaller-3.1.2-1/rubyinstaller-devkit-3.1.2-1-x64.exe
 choco install -y ruby
 
@@ -36,9 +33,18 @@ if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarg
 # Install markdownlint
 gem install mdl
 
+python -m venv ./dtaas-venv
+./dtaas-venv/Scripts/Activate
+
+# Install mkdocs
+pip install mkdocs
+
 # Install mkdocs plugins
 pip install mkdocs-material python-markdown-math mkdocs-open-in-new-tab mkdocs-with-pdf qrcode
 
+# Deativate venv for python
+deactivate
+
 # Ensure Python scripts directory is added to PATH (for pip)
 if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) -like "*Python3*")) {
     Write-Host "Adding Python scripts to PATH (for pip)..."

script/env.sh

@@ -11,12 +11,20 @@ npx --yes playwright install-deps
 #-------------
 printf "\n\n Installing required python packages...."
 sudo apt install -y python3-pip
-sudo -H pip3 install mkdocs
-sudo -H pip3 install mkdocs-material
-sudo -H pip3 install python-markdown-math
-sudo -H pip3 install mkdocs-open-in-new-tab
-sudo -H pip3 install mkdocs-with-pdf
-sudo -H pip3 install qrcode
+
+python3 -m venv ./dtaas-venv
+
+chmod +x ./dtaas-venv/bin/activate
+./dtaas-venv/bin/activate
+
+./dtaas-venv/bin/pip3 install mkdocs
+./dtaas-venv/bin/pip3 install mkdocs-material
+./dtaas-venv/bin/pip3 install python-markdown-math
+./dtaas-venv/bin/pip3 install mkdocs-open-in-new-tab
+./dtaas-venv/bin/pip3 install mkdocs-with-pdf
+./dtaas-venv/bin/pip3 install qrcode
+
+deactivate
 
 # Install markdownlint
 sudo apt-get install -y rubygems