ingest/ledgerbackend: Add version check for protocol 21

.github/workflows/horizon.yml

@@ -13,7 +13,7 @@ jobs:
         os: [ubuntu-20.04, ubuntu-22.04]
         go: ["1.21", "1.22"]
         pg: [12, 16]
-        protocol-version: [20, 21]
+        protocol-version: [21]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
@@ -36,9 +36,6 @@ jobs:
       PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
       PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21
       PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
-      PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
-      PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21
-      PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
       PGHOST: localhost
       PGPORT: 5432
       PGUSER: postgres
@@ -101,7 +98,7 @@ jobs:
       - name: Calculate the source hash
         id: calculate_source_hash
         run: |
-          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_20_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_19_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
+          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
           echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV"
 
       - name: Restore Horizon binary and integration tests source hash to cache

exp/services/ledgerexporter/internal/config.go

@@ -5,8 +5,6 @@ import (
 	_ "embed"
 	"fmt"
 	"os"
-	"os/exec"
-	"strings"
 
 	"github.com/stellar/go/historyarchive"
 	"github.com/stellar/go/ingest/ledgerbackend"
@@ -194,26 +192,11 @@ func (config *Config) GenerateCaptiveCoreConfig(coreBinFromPath string) (ledgerb
 	}, nil
 }
 
-// By default, it points to exec.Command, overridden for testing purpose
-var execCommand = exec.Command
-
-// Executes the "stellar-core version" command and parses its output to extract
-// the core version
-// The output of the "version" command is expected to be a multi-line string where the
-// first line is the core version in format "vX.Y.Z-*".
 func (c *Config) setCoreVersionInfo() (err error) {
-	versionCmd := execCommand(c.StellarCoreConfig.StellarCoreBinaryPath, "version")
-	versionOutput, err := versionCmd.Output()
+	c.CoreVersion, err = ledgerbackend.GetCoreBuildVersionFunc(c.StellarCoreConfig.StellarCoreBinaryPath)
 	if err != nil {
-		return fmt.Errorf("failed to execute stellar-core version command: %w", err)
-	}
-
-	// Split the output into lines
-	rows := strings.Split(string(versionOutput), "\n")
-	if len(rows) == 0 || len(rows[0]) == 0 {
-		return fmt.Errorf("stellar-core version not found")
+		return fmt.Errorf("failed to set stellar-core version: %w", err)
 	}
-	c.CoreVersion = rows[0]
 	logger.Infof("stellar-core version: %s", c.CoreVersion)
 	return nil
 }

exp/services/ledgerexporter/internal/config_test.go

@@ -3,17 +3,15 @@ package ledgerexporter
 import (
 	"context"
 	"fmt"
-	"os"
-	"os/exec"
 	"testing"
 
+	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/support/datastore"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/stellar/go/historyarchive"
-	"github.com/stellar/go/support/errors"
 )
 
 func TestNewConfig(t *testing.T) {
@@ -95,7 +93,7 @@ func TestDefaultCaptiveCoreBin(t *testing.T) {
 		RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"})
 	require.NoError(t, err)
 
-	cmdOut = "v20.2.0-2-g6e73c0a88\n"
+	ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }
 	ccConfig, err := cfg.GenerateCaptiveCoreConfig("/test/default/stellar-core")
 	require.NoError(t, err)
 	require.Equal(t, ccConfig.BinaryPath, "/test/default/stellar-core")
@@ -116,7 +114,7 @@ func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) {
 	require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
 	require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs)
 
-	cmdOut = "v20.2.0-2-g6e73c0a88\n"
+	ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }
 	ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
 	require.NoError(t, err)
 
@@ -137,7 +135,7 @@ func TestValidCaptiveCoreManualNetwork(t *testing.T) {
 	require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, "test")
 	require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"})
 
-	cmdOut = "v20.2.0-2-g6e73c0a88\n"
+	ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }
 	ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
 	require.NoError(t, err)
 
@@ -157,7 +155,7 @@ func TestValidCaptiveCoreOverridenToml(t *testing.T) {
 	require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
 	require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs)
 
-	cmdOut = "v20.2.0-2-g6e73c0a88\n"
+	ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }
 	ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
 	require.NoError(t, err)
 
@@ -179,7 +177,7 @@ func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) {
 	require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
 	require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"})
 
-	cmdOut = "v20.2.0-2-g6e73c0a88\n"
+	ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88\n", nil }
 	ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
 	require.NoError(t, err)
 
@@ -438,72 +436,3 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
 	}
 	mockArchive.AssertExpectations(t)
 }
-
-var cmdOut = ""
-
-func fakeExecCommand(command string, args ...string) *exec.Cmd {
-	cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...)
-	cmd := exec.Command(os.Args[0], cs...)
-	cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+cmdOut)
-	return cmd
-}
-
-func init() {
-	execCommand = fakeExecCommand
-}
-
-func TestExecCmdHelperProcess(t *testing.T) {
-	if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" {
-		return
-	}
-	fmt.Fprint(os.Stdout, os.Getenv("CMD_OUT"))
-	os.Exit(0)
-}
-
-func TestSetCoreVersionInfo(t *testing.T) {
-	execCommand = fakeExecCommand
-	tests := []struct {
-		name            string
-		commandOutput   string
-		expectedError   error
-		expectedCoreVer string
-	}{
-		{
-			name: "version found",
-			commandOutput: "v20.2.0-2-g6e73c0a88\n" +
-				"rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" +
-				"soroban-env-host: \n" +
-				"    curr:\n" +
-				"       package version: 20.2.0\n" +
-				"       git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" +
-				"       ledger protocol version: 20\n" +
-				"       pre-release version: 0\n" +
-				"       rs-stellar-xdr:\n" +
-				"           package version: 20.1.0\n" +
-				"           git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" +
-				"           base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n",
-			expectedError:   nil,
-			expectedCoreVer: "v20.2.0-2-g6e73c0a88",
-		},
-		{
-			name:            "core version not found",
-			commandOutput:   "",
-			expectedError:   errors.New("stellar-core version not found"),
-			expectedCoreVer: "",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			config := Config{}
-			cmdOut = tt.commandOutput
-			err := config.setCoreVersionInfo()
-
-			if tt.expectedError != nil {
-				require.EqualError(t, err, tt.expectedError.Error())
-			} else {
-				require.NoError(t, err)
-				require.Equal(t, tt.expectedCoreVer, config.CoreVersion)
-			}
-		})
-	}
-}

ingest/ledgerbackend/captive_core_backend.go

@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"os/exec"
-	"strings"
 	"sync"
 	"time"
 
@@ -22,6 +20,8 @@ import (
 	"github.com/stellar/go/xdr"
 )
 
+var minProtocolVersionSupported uint = 21
+
 // Ensure CaptiveStellarCore implements LedgerBackend
 var _ LedgerBackend = (*CaptiveStellarCore)(nil)
 
@@ -168,6 +168,17 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) {
 		config.Log.SetLevel(logrus.InfoLevel)
 	}
 
+	protocolVersion, err := GetCoreProtocolVersionFunc(config.BinaryPath)
+	if err != nil {
+		return nil, fmt.Errorf("error determining stellar-core protocol version: %w", err)
+	}
+
+	if protocolVersion < minProtocolVersionSupported {
+		return nil, fmt.Errorf("stellar-core version not supported. Installed stellar-core version is at protocol %d, but minimum "+
+			"required version is %d. Please upgrade stellar-core to a version that supports protocol version %d or higher",
+			protocolVersion, minProtocolVersionSupported, minProtocolVersionSupported)
+	}
+
 	parentCtx := config.Context
 	if parentCtx == nil {
 		parentCtx = context.Background()
@@ -250,28 +261,12 @@ func (c *CaptiveStellarCore) coreVersionMetric() float64 {
 	return float64(info.Info.ProtocolVersion)
 }
 
-// By default, it points to exec.Command, overridden for testing purpose
-var execCommand = exec.Command
-
-// Executes the "stellar-core version" command and parses its output to extract
-// the core version
-// The output of the "version" command is expected to be a multi-line string where the
-// first line is the core version in format "vX.Y.Z-*".
 func (c *CaptiveStellarCore) setCoreVersion() {
-	versionCmd := execCommand(c.config.BinaryPath, "version")
-	versionOutput, err := versionCmd.Output()
+	var err error
+	c.captiveCoreVersion, err = GetCoreBuildVersionFunc(c.config.BinaryPath)
 	if err != nil {
-		c.config.Log.Errorf("failed to execute stellar-core version command: %s", err)
+		c.config.Log.Errorf("Failed to set stellar-core version: %s", err)
 	}
-
-	// Split the output into lines
-	rows := strings.Split(string(versionOutput), "\n")
-	if len(rows) == 0 || len(rows[0]) == 0 {
-		c.config.Log.Error("stellar-core version not found")
-		return
-	}
-
-	c.captiveCoreVersion = rows[0]
 	c.config.Log.Infof("stellar-core version: %s", c.captiveCoreVersion)
 }
 

ingest/ledgerbackend/captive_core_backend_test.go

@@ -151,6 +151,8 @@ func TestCaptiveNew(t *testing.T) {
 	networkPassphrase := network.PublicNetworkPassphrase
 	historyURLs := []string{server.URL}
 
+	GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil }
+
 	captiveStellarCore, err := NewCaptive(
 		CaptiveCoreConfig{
 			BinaryPath:         executablePath,
@@ -169,6 +171,35 @@ func TestCaptiveNew(t *testing.T) {
 	assert.Equal(t, "uatest", userAgent)
 }
 
+func TestCaptiveNewUnsupportedProtocolVersion(t *testing.T) {
+	storagePath, err := os.MkdirTemp("", "captive-core-*")
+	require.NoError(t, err)
+	defer os.RemoveAll(storagePath)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer server.Close()
+
+	executablePath := "/etc/stellar-core"
+	networkPassphrase := network.PublicNetworkPassphrase
+	historyURLs := []string{server.URL}
+
+	GetCoreProtocolVersionFunc = func(string) (uint, error) { return 20, nil }
+
+	_, err = NewCaptive(
+		CaptiveCoreConfig{
+			BinaryPath:         executablePath,
+			NetworkPassphrase:  networkPassphrase,
+			HistoryArchiveURLs: historyURLs,
+			StoragePath:        storagePath,
+			UserAgent:          "uatest",
+		},
+	)
+
+	assert.EqualError(t, err, "stellar-core version not supported. Installed stellar-core version is at protocol 20, but minimum required version is 21. Please upgrade stellar-core to a version that supports protocol version 21 or higher")
+}
+
 func TestCaptivePrepareRange(t *testing.T) {
 	metaChan := make(chan metaResult, 100)
 
@@ -984,6 +1015,8 @@ func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) {
 	captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{})
 	assert.NoError(t, err)
 
+	GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil }
+
 	captiveStellarCore, err := NewCaptive(
 		CaptiveCoreConfig{
 			BinaryPath:         executablePath,

ingest/ledgerbackend/stellar_core_version.go

@@ -0,0 +1,63 @@
+package ledgerbackend
+
+import (
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// By default, it points to exec.Command, overridden for testing purpose
+var execCommand = exec.Command
+var GetCoreProtocolVersionFunc = GetCoreProtocolVersion
+var GetCoreBuildVersionFunc = GetCoreBuildVersion
+
+// GetCoreBuildVersion executes the "stellar-core version" command and parses its output to extract
+// the core version
+// The output of the "version" command is expected to be a multi-line string where the
+// first line is the core version in format "vX.Y.Z-*".
+func GetCoreBuildVersion(coreBinaryPath string) (string, error) {
+	versionCmd := execCommand(coreBinaryPath, "version")
+	versionOutput, err := versionCmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("failed to execute stellar-core version command: %w", err)
+	}
+
+	// Split the output into lines
+	rows := strings.Split(string(versionOutput), "\n")
+	if len(rows) == 0 || len(rows[0]) == 0 {
+		return "", fmt.Errorf("stellar-core version not found")
+	}
+
+	return rows[0], nil
+}
+
+// GetCoreProtocolVersion retrieves the ledger protocol version from the specified stellar-core binary.
+// It executes the "stellar-core version" command and parses the output to extract the protocol version.
+func GetCoreProtocolVersion(coreBinaryPath string) (uint, error) {
+	if coreBinaryPath == "" {
+		return 0, fmt.Errorf("stellar-core binary path is empty")
+	}
+
+	versionBytes, err := execCommand(coreBinaryPath, "version").Output()
+	if err != nil {
+		return 0, fmt.Errorf("error executing stellar-core version command (%s): %w", coreBinaryPath, err)
+	}
+
+	versionRows := strings.Split(string(versionBytes), "\n")
+	re := regexp.MustCompile(`^\s*ledger protocol version: (\d*)`)
+	var ledgerProtocolStrings []string
+	for _, line := range versionRows {
+		ledgerProtocolStrings = re.FindStringSubmatch(line)
+		if len(ledgerProtocolStrings) == 2 {
+			val, err := strconv.Atoi(ledgerProtocolStrings[1])
+			if err != nil {
+				return 0, fmt.Errorf("error parsing protocol version from stellar-core output: %w", err)
+			}
+			return uint(val), nil
+		}
+	}
+
+	return 0, fmt.Errorf("error parsing protocol version from stellar-core output")
+}