Use pants managed venv in Makefile

.github/workflows/lint.yaml

@@ -72,7 +72,7 @@ jobs:
           gha-cache-key: cache0
           # This hash should include all of our lockfiles so that the pip/pex caches
           # get invalidated on any transitive dependency update.
-          named-caches-hash: ${{ hashFiles('requirements.txt') }}
+          named-caches-hash: ${{ hashFiles('lockfiles/*.lock') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
           # install whatever version of python we need for our in-repo pants-plugins

.github/workflows/orquesta-integration-tests.yaml

@@ -121,31 +121,35 @@ jobs:
 
       # GitHub is juggling how to set vars for multiple shells. Protect our PATH assumptions.
       PATH: /home/runner/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          # a test uses a submodule, and pants needs access to it to calculate deps.
+          submodules: 'true'
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
       - name: 'Set up Python (${{ matrix.python-version }})'
         uses: actions/setup-python@v5
         with:
           python-version: '${{ matrix.python-version }}'
-      - name: Cache Python Dependencies
-        uses: actions/cache@v4
-        with:
-          path: |
-            ~/.cache/pip
-            virtualenv
-            ~/virtualenv
-          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
-          # !virtualenv/lib/python*/site-packages/st2*
-          # !virtualenv/bin/st2*
-          key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
-          # Don't use alternative key as if requirements.txt has altered we
-          # don't want to retrieve previous cache
-          #restore-keys: |
-          #  ${{ runner.os }}-v5-python-${{ matrix.python }}-
+      #- name: Cache Python Dependencies
+      #  uses: actions/cache@v4
+      #  with:
+      #    path: |
+      #      ~/.cache/pip
+      #      virtualenv
+      #      ~/virtualenv
+      #    # TODO: maybe make the virtualenv a partial cache to exclude st2*?
+      #    # !virtualenv/lib/python*/site-packages/st2*
+      #    # !virtualenv/bin/st2*
+      #    key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }}
+      #    # Don't use alternative key as if requirements.txt has altered we
+      #    # don't want to retrieve previous cache
+      #    #restore-keys: |
+      #    #  ${{ runner.os }}-v5-python-${{ matrix.python }}-
       - name: Cache APT Dependencies
         id: cache-apt-deps
         uses: actions/cache@v4
@@ -162,6 +166,29 @@ jobs:
           # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap
           ./scripts/github/install-apt-packages-use-cache.sh
+
+      - name: Initialize Pants and its GHA caches
+        uses: pantsbuild/actions/init-pants@v6-scie-pants
+        # This action adds an env var to make pants use both pants.ci.toml & pants.toml.
+        # This action also creates 3 GHA caches (1 is optional).
+        # - `pants-setup` has the bootsrapped pants install
+        # - `pants-named-caches` has pip/wheel and PEX caches
+        # - `pants-lmdb-store` has the fine-grained process cache.
+        #   If we ever use a remote cache, then we can drop this.
+        #   Otherwise, we may need an additional workflow or job to delete old caches
+        #   if they are not expiring fast enough, and we hit the GHA 10GB per repo max.
+        with:
+          base-branch: master
+          # To ignore a bad cache, bump the cache* integer.
+          gha-cache-key: cache0
+          # This hash should include all of our lockfiles so that the pip/pex caches
+          # get invalidated on any transitive dependency update.
+          named-caches-hash: ${{ hashFiles('lockfiles/*.lock') }}
+          # enable the optional lmdb_store cache since we're not using remote caching.
+          cache-lmdb-store: 'true'
+          # install whatever version of python we need for our in-repo pants-plugins
+          setup-python-for-plugins: 'true'
+
       - name: Install virtualenv
         run: |
           ./scripts/github/install-virtualenv.sh
@@ -214,21 +241,29 @@ jobs:
       - name: Compress Service Logs Before upload
         if: ${{ failure() }}
         run: |
+          ./tools/launchdev.sh stop # stop st2 before collecting logs
           tar cvzpf logs.tar.gz logs/*
       - name: Upload StackStorm services Logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
-          name: logs-py${{ matrix.python-version }}
+          name: logs-py${{ matrix.python-version }}-nose-${{ matrix.nosetests_node_index }}
           path: logs.tar.gz
           retention-days: 7
       - name: Stop Redis Service Container
         if: "${{ always() }}"
         run: docker rm --force redis || true
 
+
+      - name: Upload pants log
+        uses: actions/upload-artifact@v2
+        with:
+          name: pants-log-py${{ matrix.python-version }}-nose-${{ matrix.nosetests_node_index }}
+          path: .pants.d/pants.log
+        if: always()  # We want the log even on failures.
+
   slack-notification:
     name: Slack notification for failed master builds
-    if: always()
     needs:
       - integration-tests
     runs-on: ubuntu-20.04

.github/workflows/pants.yaml

@@ -45,7 +45,7 @@ jobs:
           gha-cache-key: cache0-BUILD
           # This hash should include all of our lockfiles so that the pip/pex caches
           # get invalidated on any transitive dependency update.
-          named-caches-hash: ${{ hashFiles('requirements.txt') }}
+          named-caches-hash: ${{ hashFiles('lockfiles/*.lock') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
           # install whatever version of python we need for our in-repo pants-plugins

.github/workflows/test.yaml

@@ -117,7 +117,7 @@ jobs:
           gha-cache-key: cache0-py${{ matrix.python-version }}
           # This hash should include all of our lockfiles so that the pip/pex caches
           # get invalidated on any transitive dependency update.
-          named-caches-hash: ${{ hashFiles('requirements.txt') }}
+          named-caches-hash: ${{ hashFiles('lockfiles/*.lock') }}
           # enable the optional lmdb_store cache since we're not using remote caching.
           cache-lmdb-store: 'true'
           # install whatever version of python we need for our in-repo pants-plugins

Makefile

@@ -5,19 +5,22 @@ OS := $(shell uname)
 # We separate the OSX X and Linux virtualenvs so we can run in a Docker
 # container (st2devbox) while doing things on our host Mac machine
 ifeq ($(OS),Darwin)
-	VIRTUALENV_DIR ?= virtualenv-osx
+	#VIRTUALENV_DIR ?= virtualenv-osx
 	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client-osx
 	VIRTUALENV_ST2CLIENT_PYPI_DIR ?= virtualenv-st2client-pypi-osx
 	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components-osx
 else
-	VIRTUALENV_DIR ?= virtualenv
+	#VIRTUALENV_DIR ?= virtualenv
 	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client
 	VIRTUALENV_ST2CLIENT_PYPI_DIR ?= virtualenv-st2client-pypi
 	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components
 endif
 
 # Assign PYTHON_VERSION if it doesn't already exist
 PYTHON_VERSION ?= python3
+PYTHON_BINARY = $(shell which $(PYTHON_VERSION))
+PYTHON_BINARY_VERSION = $(shell $(PYTHON_BINARY) -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}")')
+VIRTUALENV_DIR = dist/export/python/virtualenvs/st2/$(PYTHON_BINARY_VERSION)
 
 BINARIES := bin
 
@@ -173,7 +176,7 @@ install-runners:
 	@echo "================== INSTALL RUNNERS ===================="
 	@echo ""
 	# NOTE: We use xargs to speed things up by installing runners in parallel
-	echo -e "$(COMPONENTS_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd {} ; python setup.py develop --no-deps"
+#	echo -e "$(COMPONENTS_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd {} ; python setup.py develop --no-deps"
 	#@for component in $(COMPONENTS_RUNNERS); do \
 	#	echo "==========================================================="; \
 	#	echo "Installing runner:" $$component; \
@@ -187,7 +190,7 @@ install-mock-runners:
 	@echo "================== INSTALL MOCK RUNNERS ===================="
 	@echo ""
 	# NOTE: We use xargs to speed things up by installing runners in parallel
-	echo -e "$(MOCK_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd {} ; python setup.py develop --no-deps"
+#	echo -e "$(MOCK_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c ". $(VIRTUALENV_DIR)/bin/activate; cd {} ; python setup.py develop --no-deps"
 	#@for component in $(MOCK_RUNNERS); do \
 	#	echo "==========================================================="; \
 	#	echo "Installing mock runner:" $$component; \
@@ -646,10 +649,10 @@ distclean: clean
 .PHONY: .sdist-requirements
 .sdist-requirements:
 	# Copy over shared dist utils module which is needed by setup.py
-	@for component in $(COMPONENTS_WITH_RUNNERS); do\
-		cp -f ./scripts/dist_utils.py $$component/dist_utils.py;\
-		scripts/write-headers.sh $$component/dist_utils.py || break;\
-	done
+#	@for component in $(COMPONENTS_WITH_RUNNERS); do\
+#		cp -f ./scripts/dist_utils.py $$component/dist_utils.py;\
+#		scripts/write-headers.sh $$component/dist_utils.py || break;\
+#	done
 
 	# Copy over CHANGELOG.RST, CONTRIBUTING.RST and LICENSE file to each component directory
 	#@for component in $(COMPONENTS_TEST); do\
@@ -660,18 +663,18 @@ distclean: clean
 
 .PHONY: .requirements
 .requirements: virtualenv
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
+#	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
 	# Print out pip version
 	$(VIRTUALENV_DIR)/bin/pip --version
 	# Generate all requirements to support current CI pipeline.
-	$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt
+#	$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt
 
 	# Remove any *.egg-info files which polute PYTHONPATH
-	rm -rf *.egg-info*
+#	rm -rf *.egg-info*
 
 	# Generate finall requirements.txt file for each component
 	# NOTE: We use xargs to speed things up by running commands in parallel
-	echo -e "$(COMPONENTS_WITH_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c "$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s {}/in-requirements.txt -f fixed-requirements.txt -o {}/requirements.txt"
+#	echo -e "$(COMPONENTS_WITH_RUNNERS)" | tr -d "\n" | xargs -P $(XARGS_CONCURRENCY) -d " " -n1 -i sh -c "$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s {}/in-requirements.txt -f fixed-requirements.txt -o {}/requirements.txt"
 
 	#@for component in $(COMPONENTS_WITH_RUNNERS); do\
 	#	echo "==========================================================="; \
@@ -693,40 +696,41 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 
 	# Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
 	#       only have to update it one place when we change the version
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt)
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
+#	$(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt)
+#	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
 
 	# Install requirements
-	for req in $(REQUIREMENTS); do \
-		echo "Installing $$req..." ; \
-		$(VIRTUALENV_DIR)/bin/pip install $(PIP_OPTIONS) -r $$req ; \
-	done
+#	for req in $(REQUIREMENTS); do \
+#		echo "Installing $$req..." ; \
+#		$(VIRTUALENV_DIR)/bin/pip install $(PIP_OPTIONS) -r $$req ; \
+#	done
 
 	# Install st2common package to load drivers defined in st2common setup.py
 	# NOTE: We pass --no-deps to the script so we don't install all the
 	# package dependencies which are already installed as part of "requirements"
 	# make targets. This speeds up the build
-	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
+#	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 
 	# Install st2common to register metrics drivers
 	# NOTE: We pass --no-deps to the script so we don't install all the
 	# package dependencies which are already installed as part of "requirements"
 	# make targets. This speeds up the build
-	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
+#	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 
 	# Install st2auth to register SSO drivers
 	# NOTE: We pass --no-deps to the script so we don't install all the
 	# package dependencies which are already installed as part of "requirements"
 	# make targets. This speeds up the build
-	(cd ${ROOT_DIR}/st2auth; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
+#	(cd ${ROOT_DIR}/st2auth; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 
 	# Some of the tests rely on submodule so we need to make sure submodules are check out
 	git submodule update --init --recursive --remote
+	git submodule foreach --recursive git fetch --tags --verbose
 
 	# Show currently install requirements
-	echo ""
-	$(VIRTUALENV_DIR)/bin/pip list
-	echo ""
+#	echo ""
+#	$(VIRTUALENV_DIR)/bin/pip list
+#	echo ""
 
 .PHONY: check-dependency-conflicts
 check-dependency-conflicts:
@@ -745,27 +749,32 @@ virtualenv:
 	@echo
 	@echo "==================== virtualenv ===================="
 	@echo
-	test -f $(VIRTUALENV_DIR)/bin/activate || $(PYTHON_VERSION) -m venv $(VIRTUALENV_DIR)
+#	test -f $(VIRTUALENV_DIR)/bin/activate || $(PYTHON_VERSION) -m venv $(VIRTUALENV_DIR)
+	test -f $(VIRTUALENV_DIR)/bin/activate || pants export --resolve=st2 \
+		--python-bootstrap-search-path=[] --python-bootstrap-search-path=$(PYTHON_BINARY)
+	# workaround pants+pex default of hermetic scripts so we can run nosetest with PYTHONPATH
+	# sed -i -e 's/^#!\(.*\) -sE$$/#!\1 -s/' $(VIRTUALENV_DIR)/bin/*
+	test -L virtualenv || (rm -rf virtualenv && ln -s $(VIRTUALENV_DIR) virtualenv)
 
 	# Setup PYTHONPATH in bash activate script...
 	# Delete existing entries (if any)
-ifeq ($(OS),Darwin)
-	echo 'Setting up virtualenv on $(OS)...'
-	sed -i '' '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate
-	sed -i '' '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate
-	sed -i '' '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate
-else
-	echo 'Setting up virtualenv on $(OS)...'
-	sed -i '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate
-	sed -i '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate
-	sed -i '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate
-endif
-
-	echo '_OLD_PYTHONPATH=$$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate
-	#echo 'PYTHONPATH=$$_OLD_PYTHONPATH:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate
-	echo 'PYTHONPATH=${ROOT_DIR}:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate
-	echo 'export PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate
-	touch $(VIRTUALENV_DIR)/bin/activate
+#ifeq ($(OS),Darwin)
+#	echo 'Setting up virtualenv on $(OS)...'
+#	sed -i '' '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate
+#	sed -i '' '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate
+#	sed -i '' '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate
+#else
+#	echo 'Setting up virtualenv on $(OS)...'
+#	sed -i '/_OLD_PYTHONPATHp/d' $(VIRTUALENV_DIR)/bin/activate
+#	sed -i '/PYTHONPATH=/d' $(VIRTUALENV_DIR)/bin/activate
+#	sed -i '/export PYTHONPATH/d' $(VIRTUALENV_DIR)/bin/activate
+#endif
+
+#	echo '_OLD_PYTHONPATH=$$PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate
+#	#echo 'PYTHONPATH=$$_OLD_PYTHONPATH:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate
+#	echo 'PYTHONPATH=${ROOT_DIR}:$(COMPONENT_PYTHONPATH)' >> $(VIRTUALENV_DIR)/bin/activate
+#	echo 'export PYTHONPATH' >> $(VIRTUALENV_DIR)/bin/activate
+#	touch $(VIRTUALENV_DIR)/bin/activate
 
 	# Setup PYTHONPATH in fish activate script...
 	#echo '' >> $(VIRTUALENV_DIR)/bin/activate.fish
@@ -1052,7 +1061,7 @@ packs-tests: requirements .packs-tests
 	@echo "==================== packs-tests ===================="
 	@echo
 	# Install st2common to register metrics drivers
-	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
+#	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 	. $(VIRTUALENV_DIR)/bin/activate; find ${ROOT_DIR}/contrib/* -maxdepth 0 -type d -print0 | xargs -0 -I FILENAME ./st2common/bin/st2-run-pack-tests -c -t -x -p FILENAME
 
 

lockfiles/st2-constraints.txt

@@ -74,7 +74,7 @@ dnspython>=1.16.0,<2.0.0
 #         greenlet 3 adds py3.12 support, drops py3.6 support, fixes various crash conditions
 # NOTE: If constrained, bump carefully. Tests seem to be passing without this constraint.
 # DROPS RESOLVED VERSION: 1.1.3.post0
-#greenlet==1.0.0
+#greenlet<2
 
 # REQUIRED BY: argcomplete, click, debtcollector, kombu, pluggy, prettytable,
 #             pytest, virtualenv

requirements-pants.txt

@@ -24,6 +24,20 @@ gunicorn
 jinja2
 jsonpath-rw
 jsonschema>=3,<4
+# Kombu 5.1:
+#   - dropped py2
+#   - increase pickle protocol from 2 to 4 (only affects serialization. deserialization detects version)
+#   - from 5.1 up to at least 5.3.5 use pickle protocol 4
+#   - switches login_method from AMQPLAIN to PLAIN (rabbitmq supports both by default, AMQPLAIN only for backwards compat)
+#   - adds named_tuple_as_object=False to simplejson.dump kwargs
+# Kombu 5.2:
+#   - dropped py3.6
+#   - drops support for librabbitmq
+#   - switches from simplejson to json
+# Kombu 5.3:
+#   - dropped py3.7
+#   - drops simplejson support
+#kombu>=5.0,<5.1
 kombu
 lockfile
 mock

scripts/github/prepare-integration.sh

@@ -27,7 +27,7 @@ cat conf/st2.ci.conf || true
 echo ""
 
 # install st2 client
-python ./st2client/setup.py develop
+#python ./st2client/setup.py develop
 st2 --version
 
 # Clean up old st2 log files

setup.cfg

@@ -1,2 +1,3 @@
 [nosetests]
-logging-level=INFO
+logging-level=DEBUG
+detailed-errors=1

st2common/bin/st2-run-pack-tests

@@ -322,7 +322,7 @@ fi
 echo "Running tests..."
 # Note: We run nosetests with "--exe" option so it also runs test files which are executable
 # (pack install command automatically makes all the files, including test files executable)
-NOSE_OPTS=(-s -v --exe --rednose --immediate)
+NOSE_OPTS=(-s -v --exe --rednose --immediate --detailed-error --logging-level=DEBUG)
 
 # Is test coverage reporting enabled?
 if [ "${ENABLE_COVERAGE}" = true ]; then