Merge pull request #89 from ignis-sec/dev

Release 0.6.0 - Building and Linking Improvements with CMake

.dockerignore

@@ -0,0 +1,12 @@
+**/CmakeFiles/*
+docs/*
+documentation/*
+out/*
+Testing/*
+models/*
+dataset/*
+
+.idea/*
+.vs/*
+.vscode/*
+.scannerwork/*

.github/action_helpers/actions_linux_python_libs.sh

@@ -1,18 +1,22 @@
 #!/bin/bash
 
+pwd
+whoami
+ls /root/ -alR
+ls /opt/ -alR
 # work inside include folder
 cd /opt/hostedtoolcache/Python/$1*/x64/include;
 
 echo "Working with python$1"
 
 # Create symlink without "m" in name
-[[ -f "python$1" ]] || sudo ln -s python$1m python$1;
+[[ -f "python$1" ]] || ln -s python$1m python$1;
 
 # copy libraries and includes to global paths
-sudo cp -r /opt/hostedtoolcache/Python/$1*/x64/lib/* /usr/local/lib/;
-sudo cp -r /opt/hostedtoolcache/Python/$1*/x64/include/py* /usr/include/;
-sudo ln -s /usr/include/$1m /usr/include/$1;
+cp -r /opt/hostedtoolcache/Python/$1*/x64/lib/* /usr/local/lib/;
+cp -r /opt/hostedtoolcache/Python/$1*/x64/include/py* /usr/include/;
+ln -s /usr/include/$1m /usr/include/$1;
 
 # Create symlink without "m" in name
-[[ -f "/usr/local/lib/libpython$1.so.1.0" ]] ||  sudo ln -s /usr/local/lib/libpython$1m.so.1.0 /usr/local/lib/libpython$1.so.1.0;
-[[ -f "/usr/local/lib/libpython$1.so" ]] ||  sudo ln -s /usr/local/lib/libpython$1m.so /usr/local/lib/libpython$1.so;
+[[ -f "/usr/local/lib/libpython$1.so.1.0" ]] ||  ln -s /usr/local/lib/libpython$1m.so.1.0 /usr/local/lib/libpython$1.so.1.0;
+[[ -f "/usr/local/lib/libpython$1.so" ]] ||  ln -s /usr/local/lib/libpython$1m.so /usr/local/lib/libpython$1.so;

.github/action_helpers/actions_linux_setup_boost.sh

@@ -5,4 +5,4 @@ cd /home/runner/;
 tar -xf /home/runner/boost_1_76_0.tar.gz;
 cd /home/runner/boost_1_76_0/;
 ./bootstrap.sh --with-python=$(which python$1) --with-python-version=$1;
-sudo ./b2 --with-python --with-program_options stage;
+./b2 --with-python --with-program_options stage;

.github/action_helpers/actions_linux_setup_libboost.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-sudo cp -r /home/runner/boost_1_76_0/stage/lib/* /usr/local/lib;
-sudo cp -r /home/runner/boost_1_76_0/boost /usr/include/boost;
+cp -r /home/runner/boost_1_76_0/stage/lib/* /usr/local/lib;
+cp -r /home/runner/boost_1_76_0/boost /usr/include/boost;

.github/action_helpers/actions_windows_libpython.ps1

@@ -3,6 +3,12 @@ param (
    [string]$pythonver = "3.8"
 )
 
+$a = $pythonver.substring(0,1)
+$b = $pythonver.substring(2,1)
+echo "$a$b"
+echo "set-output name=python_ver::$a$b"
+echo "::set-output name=python_ver::$a$b"
+
 which python;
 mkdir D:\a\python$pythonver;
 

.github/action_helpers/build.py

@@ -52,18 +52,20 @@ def set_action_outputs(**kwargs):
     if(args.step == "version_matrix"):
 
         if(a_type == "minor"):
-            lm = "[3.6, 3.7, 3.8, 3.9]"
-            wm = "[3.8, 3.9]"
+            lm = "[36, 37, 38, 39]"
+            wm = "[36, 37, 38, 39]"
+            wms = "[3.6, 3.7, 3.8, 3.9]"
 
         elif(a_type == "patch"):
-            lm = "[3.8]"
-            wm = "[3.8]"
-
+            lm = "[38]"
+            wm = "[38]"
+            wms = "[3.8]"
         else:
-            lm = "[3.8]"
-            wm = "[3.8]"
+            lm = "[38]"
+            wm = "[38]"
+            wms = "[3.8]"
 
-        set_action_outputs(linux_matrix=lm, windows_matrix=wm)  
+        set_action_outputs(linux_matrix=lm, windows_matrix=wm, windows_matrix_sep=wms)  
 
     if(args.step == "linux_python_libs"):
         os.system(f".github/action_helpers/actions_linux_python_libs.sh {args.pyversion}")

.github/workflows/build.yml

@@ -48,6 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       upload_url: ${{ steps.create_release.outputs.upload_url }}
+      tag: ${{ steps.tag_version.outputs.new_tag }} 
     steps:   
 
     - uses: actions/checkout@v2    
@@ -81,98 +82,141 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       windows_matrix: ${{ steps.core.outputs.windows_matrix }}
+      windows_matrix_sep: ${{ steps.core.outputs.windows_matrix_sep }}
       linux_matrix: ${{ steps.core.outputs.linux_matrix }}
     steps:
     - uses: actions/checkout@v2
     - name: Configure Version Matrix for Python
       id: core
       run: python3 ./.github/action_helpers/build.py version_matrix
+
+
 
   build_linux:
-    name: Build All on Linux with Makefile
+    name: Build python libraries on linux with CMake
     needs: [reversion, setup_build_matrix]
     runs-on: ubuntu-latest
+    container: ahakcil/markopy:0.5.0
     strategy:
       matrix:
         python-version: ${{fromJson(needs.setup_build_matrix.outputs.linux_matrix) }}
     steps:
-    # Check cache for boost libraries
-    - name: Cache Boost
-      uses: actions/cache@v2
-      id: cache-boost
-      with:
-        path: |
-          /home/runner/boost_1_76_0/stage
-          /home/runner/boost_1_76_0/boost
-        key: 'nix-boost-src-latest-py${{ matrix.python-version }}'
-
-    # Checkout current commit
     - uses: actions/checkout@v2
-
-    # Set up Python 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+    - name: Compile CPP
+      run: |
+        ls -al; 
+        cmake . -DPYTHON_VER=${{ matrix.python-version }} -DPARTIAL=1 -DB_LIBS=1 -DB_CUDA=1 -DPROJ_VERSION="${{ needs.reversion.outputs.tag }}" -DCMAKE_BUILD_TYPE=Release; 
+        cmake --build . --config Release; 
+        ls -alR out/;
+        ls -al /usr/local/lib;
 
-    # Link Python includes and libraries
-    - name: Set up libraries for Python ${{ matrix.python-version }}
+    - name: Zip library files # This would actually build your project, using zip for an example artifact
+      if: matrix.python-version == '38'
       run: |
-        chmod 755 ./.github/action_helpers/*;
-        ./.github/action_helpers/actions_linux_python_libs.sh ${{ matrix.python-version }}
+        zip --junk-paths libmarkov-${{ needs.reversion.outputs.tag }}-linux.zip out/lib/libmarkov.so out/bin/markovp
+        zip --junk-paths libcudamarkov-${{ needs.reversion.outputs.tag }}-linux.zip out/lib/libcudamarkov.so out/bin/cudamarkovp
     
-    # Setup boost
-    - name: Setup boost
-      if: steps.cache-boost.outputs.cache-hit != 'true'
-      run: ./.github/action_helpers/actions_linux_setup_boost.sh ${{ matrix.python-version }}
-
+    - name: Zip models files # This would actually build your project, using zip for an example artifact
+      if: matrix.python-version == '38'
+      run: |
+        zip -r models-${{ needs.reversion.outputs.tag }}.zip models/*
+  
+    - name: Zip python files
+      run: |
+        mkdir markopy-linux;
+        mkdir cudamarkopy-linux;
+        cp out/lib/markopy.so out/lib/libmarkov.so Markopy/src/CLI/*.py  /usr/local/lib/libboost_python${{ matrix.python-version }}*so.1* markopy-linux/
+        zip -r markopy-${{ needs.reversion.outputs.tag }}-linux-py${{ matrix.python-version }}.so.zip markopy-linux
+        cp out/lib/markopy.so out/lib/libmarkov.so out/lib/libcudamarkov.so out/lib/cudamarkopy.so Markopy/src/CLI/*.py CudaMarkopy/src/CLI/*.py  /usr/local/lib/libboost_python${{ matrix.python-version }}*so.1* cudamarkopy-linux/
+        zip -r cudamarkopy-${{ needs.reversion.outputs.tag }}-linux-py${{ matrix.python-version }}.so.zip cudamarkopy-linux
+        
+    
+    - name: Upload LibMarkov
+      if: matrix.python-version == '38'
+      id: upload-release-asset-libmarkov
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.reversion.outputs.upload_url }} 
+        asset_path: ./libmarkov-${{ needs.reversion.outputs.tag }}-linux.zip
+        asset_name:   libmarkov-${{ needs.reversion.outputs.tag }}-linux.zip
+        asset_content_type: application/zip
 
-    - name: Link boost libraries
-      run: ./.github/action_helpers/actions_linux_setup_libboost.sh ${{ matrix.python-version }} 
+    - name: Upload LibCudaMarkov
+      if: matrix.python-version == '38'
+      id: upload-release-asset-libcudamarkov
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.reversion.outputs.upload_url }} 
+        asset_path: ./libcudamarkov-${{ needs.reversion.outputs.tag }}-linux.zip
+        asset_name:   libcudamarkov-${{ needs.reversion.outputs.tag }}-linux.zip
+        asset_content_type: application/zip
 
-    - name: Compile CPP
-      run: make PYTHON_VERSION=${{ matrix.python-version }} all
+    - name: Upload Markopy
+      id: upload-release-asset-markopy
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.reversion.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
+        asset_path: ./markopy-${{ needs.reversion.outputs.tag }}-linux-py${{ matrix.python-version }}.so.zip
+        asset_name:   markopy-${{ needs.reversion.outputs.tag }}-linux-py${{ matrix.python-version }}.so.zip
+        asset_content_type: application/zip
 
-    - name: Zip built files # This would actually build your project, using zip for an example artifact
-      run: zip --junk-paths markopy-linux-py${{ matrix.python-version }}.so.zip Markopy/src/CLI/markopy_cli.py bin/markopy.so /usr/local/lib/libboost_python*.so.1.76.0
-
-    - name: Upload Release Asset
-      id: upload-release-asset 
+    - name: Upload CudaMarkopy
+      id: upload-release-asset-cudamarkopy
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ needs.reversion.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
-        asset_path: ./markopy-linux-py${{ matrix.python-version }}.so.zip
-        asset_name:   markopy-linux-py${{ matrix.python-version }}.so.zip
+        asset_path: ./cudamarkopy-${{ needs.reversion.outputs.tag }}-linux-py${{ matrix.python-version }}.so.zip
+        asset_name:   cudamarkopy-${{ needs.reversion.outputs.tag }}-linux-py${{ matrix.python-version }}.so.zip
         asset_content_type: application/zip
 
-  MSBuild: 
-    name: Build All on Windows with MSBuild 
+    - name: Upload Models
+      id: upload-release-asset-models
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.reversion.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
+        asset_path: ./models-${{ needs.reversion.outputs.tag }}.zip
+        asset_name:   models-${{ needs.reversion.outputs.tag }}.zip
+        asset_content_type: application/zip
+
+
+  build_windows: 
+    name: Build All on Windows with CMake 
     needs: [reversion, setup_build_matrix]
     runs-on: windows-latest
     strategy:
         fail-fast: false
-        max-parallel: 3
+        max-parallel: 4
         matrix: 
-          python-version: ${{ fromJson(needs.setup_build_matrix.outputs.windows_matrix) }}
+          #python-version: ${{ fromJson(needs.setup_build_matrix.outputs.windows_matrix) }}
+          python-version-sep: ${{ fromJson(needs.setup_build_matrix.outputs.windows_matrix_sep) }}
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python ${{ matrix.python-version-sep }}
       uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python-version-sep }}
 
     - name: Link python libraries And clean boost ones
-      run: .\.github\action_helpers\actions_windows_libpython.ps1 -pythonver ${{ matrix.python-version }}
+      id: link_python
+      run: .\.github\action_helpers\actions_windows_libpython.ps1 -pythonver ${{ matrix.python-version-sep }}
 
     - name: Cache Boost
       uses: actions/cache@v2
       id: cache-boost
       with:
         path: ${{env.GITHUB_ACTION_WIN_PROJECT}}\boost
-        key: 'windows-boost-1.76-markopy-${{ matrix.python-version }}'
+        key: 'windows-boost-1.76-markopy-${{ matrix.python-version-sep }}'
 
     - name: Build Boost
       id: boost
@@ -186,39 +230,71 @@ jobs:
 
     - name: Rename boost static and dynamic libraries
       if: steps.cache-boost.outputs.cache-hit != 'true'
-      run: .\.github\action_helpers\actions_windows_libboost.ps1 -pythonver ${{ matrix.python-version }}
+      run: .\.github\action_helpers\actions_windows_libboost.ps1 -pythonver ${{ matrix.python-version-sep }}
+
+    # These work, but take a damn hour to complete, if only github actions allowed custom containers
+    #- uses: Jimver/cuda-toolkit@v0.2.2
+    #  id: cuda-toolkit
+    #  with:
+    #    cuda: '11.2.2'
+
+    #- name: Cache Qt
+    #  id: cache-qt
+    #  uses: actions/cache@v1
+    #  with:
+    #    path: ../Qt
+    #    key: ${{ runner.os }}-QtCache
 
-    - name: Add MSBuild to PATH
-      uses: microsoft/setup-msbuild@v1
+    #- name: Install Qt
+    #  uses: jurplel/install-qt-action@v2
+    #  with:
+    #    modules: 'qtwebengine'
+    #    cached: ${{ steps.cache-qt.outputs.cache-hit }}
 
-    #- name: Restore NuGet packages
-    #  working-directory: ${{env.GITHUB_WORKSPACE}}
-    #  run: nuget restore ${{env.SOLUTION_FILE_PATH}}
 
     - name: Build
       working-directory: ${{env.GITHUB_WORKSPACE}}
-
       run: |
-        $env:PYTHON_PATH = "D:\a\python${{ matrix.python-version }}";
+        $env:PYTHON_PATH = "D:\a\python${{ matrix.python-version-sep }}";
         $env:BOOST_ROOT = "${{env.GITHUB_ACTION_WIN_PROJECT}}\boost";
-        msbuild /m /p:Configuration=${{env.BUILD_CONFIGURATION}} ${{env.SOLUTION_FILE_PATH}}
+        cmake . -DPYTHON_VER=${{ steps.link_python.outputs.python_ver}} -DPARTIAL=1 -DBOOST_VER="1.76.0" -DPROJ_VERSION="${{ needs.reversion.outputs.tag }}" -DCMAKE_BUILD_TYPE=MinSizeRel -DPY_DEV_PATH="D:\a\python${{ matrix.python-version-sep }}" 
+        cmake --build . --config Release; 
 
     - name: Zip built files # This would actually build your project, using zip for an example artifact
-      run: 
-        powershell "Compress-Archive Markopy/src/CLI/markopy_cli.py,x64/Release/markopy.pyd,${{env.GITHUB_ACTION_WIN_PROJECT}}\boost\stage\lib\boost_python*.dll markopy-windows-py${{ matrix.python-version }}.pyd.zip"
+      run: |
+        dir .;
+        dir out/;
+        dir out/lib/Release;
+        mkdir markopy-windows
+        copy-item -Path Markopy\src\CLI\*.py,out\lib\Release\markopy.pyd,${{env.GITHUB_ACTION_WIN_PROJECT}}\boost\stage\lib\boost_python*.dll -Destination markopy-windows\
+        powershell "Compress-Archive markopy-windows markopy-${{ needs.reversion.outputs.tag }}-win64-py${{ steps.link_python.outputs.python_ver}}.pyd.zip"
+        powershell "Compress-Archive out\lib\Release\markov.lib libmarkov-${{ needs.reversion.outputs.tag }}-win64.zip"
+      
+    - name: Upload LibMarkov
+      if: matrix.python-version-sep == '3.8'
+      id: upload-release-asset-libmarkov
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.reversion.outputs.upload_url }} 
+        asset_path: ./libmarkov-${{ needs.reversion.outputs.tag }}-win64.zip
+        asset_name:   libmarkov-${{ needs.reversion.outputs.tag }}-win64.zip
+        asset_content_type: application/zip<
 
-
-    - name: Upload Markopy Release Assets
-      id: upload-release-asset 
+    - name: Upload Markopy
+      id: upload-release-asset-markopy
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         upload_url: ${{ needs.reversion.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
-        asset_path: ./markopy-windows-py${{ matrix.python-version }}.pyd.zip
-        asset_name:   markopy-windows-py${{ matrix.python-version }}.pyd.zip
+        asset_path: ./markopy-${{ needs.reversion.outputs.tag }}-win64-py${{ steps.link_python.outputs.python_ver}}.pyd.zip
+        asset_name:   markopy-${{ needs.reversion.outputs.tag }}-win64-py${{ steps.link_python.outputs.python_ver}}.pyd.zip
         asset_content_type: application/zip
 
+
+
   #Integrity check on pull requests with sonar
   sonar:
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -244,4 +320,4 @@ jobs:
       - name: Delete drafts
         uses: hugo19941994/delete-draft-releases@v0.1.0
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}