From 176a86c14f6fedbbf7e3562fc3fa6c08c187485d Mon Sep 17 00:00:00 2001 From: Bogdan Matican Date: Fri, 19 Jul 2019 23:59:54 +0000 Subject: [PATCH] [#1886] Adapt release code to work with the new tar.gz format without -ce/-ee Summary: Some of the devops related code for release formatting was still relying on the -ce/-ee components. Ensured the YW code still handles -ee packages (for backwards compatibility with pre-existing releases), as well as new packages without the -ee. Also made all the `version.txt` files be symlinks: ``` ls -la {,managed/,managed/devops/}version.txt managed/devops/version.txt -> ../../version.txt managed/version.txt -> src/main/resources/version.txt version.txt -> managed/version.txt ``` The source of truth for `version.txt` lives in `managed/src/main/resources/version.txt`, due to YW build requirements. Test Plan: Adapted our integration tests to run over this. This depends on D6918. Reviewers: ram, mikhail Reviewed By: mikhail Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D6944 --- managed/devops/opscli/ybops/utils/__init__.py | 34 +++++++++---------- .../yugabyte/yw/common/ReleaseManager.java | 5 ++- managed/src/main/resources/version.txt | 2 +- .../yw/common/ReleaseManagerTest.java | 19 +++++++++-- managed/ui/package.json | 2 +- managed/yb_release | 2 +- version.txt | 2 +- 7 files changed, 40 insertions(+), 26 deletions(-) mode change 100644 => 120000 version.txt diff --git a/managed/devops/opscli/ybops/utils/__init__.py b/managed/devops/opscli/ybops/utils/__init__.py index 7eda21db97a1..4569967c6d56 100644 --- a/managed/devops/opscli/ybops/utils/__init__.py +++ b/managed/devops/opscli/ybops/utils/__init__.py @@ -96,39 +96,37 @@ def extract_components_from_package_name(self, package_name, is_official_release eg: --b--.tar.gz """ # Always expect -. - pattern = "^([^-]+)-({})".format(RELEASE_VERSION_PATTERN) + pattern = "^(?P[^-]+)-(?P{})".format(RELEASE_VERSION_PATTERN) # If this is an official release, we expect a commit hash and maybe a build_type, else we # expect a "-b" and a build number. if is_official_release: # Add build number. - pattern += "-b([0-9]+)" + pattern += "-b(?P[0-9]+)" else: # Add commit hash and maybe build type. - pattern += "-([^-]+)(-([^-]+))?" - pattern += "-([^-]+)-([^-]+)\.tar\.gz$" + pattern += "-(?P[^-]+)-(?P([^-]+))?" + pattern += "-(?P[^-]+)-(?P[^-]+)\.tar\.gz$" match = re.match(pattern, package_name) if not match: raise YBOpsRuntimeError("Invalid package name format: {}".format(package_name)) - self.repo, self.version, commit_or_build_number = match.group(1, 2, 3) - if is_official_release: - self.build_number = commit_or_build_number - self.commit = None - else: - self.build_number = None - self.commit = commit_or_build_number - if is_official_release: - self.system, self.machine = match.group(4, 5) - else: - # build_type should be None except for yugabyte. We ignore match 4 as -release - self.build_type, self.system, self.machine = match.group(5, 6, 7) + self.repo = match.group("repo") + self.version = match.group("version") + self.build_number = match.group("build_number") if is_official_release else None + self.commit = match.group("commit_hash") if not is_official_release else None + self.build_type = match.group("build_type") if not is_official_release else None + self.system = match.group("system") + self.machine = match.group("machine") def validate(self): if self.repo not in RELEASE_REPOS: raise YBOpsRuntimeError("Invalid repo {}".format(self.repo)) def get_release_package_name(self): - return "{}-{}-{}-{}.tar.gz".format( - self.repo, self.get_release_name(), self.system, self.machine) + return "{repo}-{release_name}-{system}-{machine}.tar.gz".format( + repo=self.repo, + release_name=self.get_release_name(), + system=self.system, + machine=self.machine) def get_release_name(self): # If we have a build number set, prioritize that to get the release version name, rather diff --git a/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java b/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java index 254f4859c70a..5cbbf0a5c430 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java @@ -74,7 +74,10 @@ public static ReleaseMetadata create(String version) { FileSystems.getDefault().getPathMatcher("glob:**yugabyte*.tar.gz"); Predicate packagesFilter = p -> Files.isRegularFile(p) && ybPackageMatcher.matches(p); - final Pattern ybPackagePattern = Pattern.compile("[^.]+yugabyte-ee-(.*)-centos(.*).tar.gz"); + // This regex needs to support old style packages with -ee as well as new style packages without. + // There are previously existing YW deployments that will have the old packages and users will + // need to still be able to use said universes and their existing YB releases. + final Pattern ybPackagePattern = Pattern.compile("[^.]+yugabyte-(?:ee-)?(.*)-centos(.*).tar.gz"); public Map getLocalReleases(String localReleasePath) { Map releaseMap = new HashMap<>(); diff --git a/managed/src/main/resources/version.txt b/managed/src/main/resources/version.txt index 55a1c0b49363..83375d66d999 100644 --- a/managed/src/main/resources/version.txt +++ b/managed/src/main/resources/version.txt @@ -1 +1 @@ -1.3.0.0-b0 +1.3.1.0-b0 diff --git a/managed/src/test/java/com/yugabyte/yw/common/ReleaseManagerTest.java b/managed/src/test/java/com/yugabyte/yw/common/ReleaseManagerTest.java index 7beba6246fa3..4e9af7e053de 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/ReleaseManagerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/ReleaseManagerTest.java @@ -63,15 +63,24 @@ public void tearDown() throws IOException { } private void createDummyReleases(List versions, boolean multipleRepos, boolean inDockerPath) { + createDummyReleases(versions, multipleRepos, inDockerPath, true); + } + + private void createDummyReleases( + List versions, boolean multipleRepos, boolean inDockerPath, + boolean hasEnterpriseStr) { versions.forEach((version) -> { String versionPath = String.format("%s/%s", TMP_STORAGE_PATH, version); new File(versionPath).mkdirs(); if (inDockerPath) { versionPath = TMP_DOCKER_STORAGE_PATH; } - createTempFile(versionPath, "yugabyte-ee-" + version + "-centos-x86_64.tar.gz", "Sample data"); + String eeStr = hasEnterpriseStr ? "ee-" : ""; + createTempFile( + versionPath, "yugabyte-" + eeStr + version + "-centos-x86_64.tar.gz", "Sample data"); if (multipleRepos) { - createTempFile(versionPath, "devops.xyz." + version + "-centos-x86_64.tar.gz", "Sample data"); + createTempFile( + versionPath, "devops.xyz." + version + "-centos-x86_64.tar.gz", "Sample data"); } }); } @@ -149,6 +158,8 @@ public void testLoadReleasesWithReleaseAndDockerPath() { createDummyReleases(versions, false, false); List dockerVersions = ImmutableList.of("0.0.2-b2"); createDummyReleases(dockerVersions, false, true); + List dockerVersionsWithoutEe = ImmutableList.of("0.0.3-b3"); + createDummyReleases(dockerVersionsWithoutEe, false, true, false); releaseManager.importLocalReleases(); ArgumentCaptor configType; ArgumentCaptor releaseMap; @@ -157,7 +168,9 @@ public void testLoadReleasesWithReleaseAndDockerPath() { Mockito.verify(configHelper, times(1)).loadConfigToDB(configType.capture(), releaseMap.capture()); Map expectedMap = ImmutableMap.of( "0.0.1", TMP_STORAGE_PATH + "/0.0.1/yugabyte-ee-0.0.1-centos-x86_64.tar.gz", - "0.0.2-b2", TMP_STORAGE_PATH + "/0.0.2-b2/yugabyte-ee-0.0.2-b2-centos-x86_64.tar.gz"); + "0.0.2-b2", TMP_STORAGE_PATH + "/0.0.2-b2/yugabyte-ee-0.0.2-b2-centos-x86_64.tar.gz", + "0.0.3-b3", TMP_STORAGE_PATH + "/0.0.3-b3/yugabyte-0.0.3-b3-centos-x86_64.tar.gz" + ); assertEquals(SoftwareReleases, configType.getValue()); assertReleases(expectedMap, releaseMap.getValue()); diff --git a/managed/ui/package.json b/managed/ui/package.json index ac43ad71e8c0..a24d0d6cfdab 100644 --- a/managed/ui/package.json +++ b/managed/ui/package.json @@ -6,7 +6,7 @@ "start": "react-app-rewired start", "start:prod": "pushstate-server build", "cleanup": "rimraf build && rimraf ../src/main/public", - "fetch-map": "aws s3 sync s3://no-such-url build/static/map", + "fetch-map": "aws s3 sync s3://$YB_MAP_URL build/static/map", "build": "npm run cleanup && react-app-rewired build && npm run fetch-map", "build-and-copy": "npm run build && ncp build ../src/main/public", "eject": "react-app-rewired eject", diff --git a/managed/yb_release b/managed/yb_release index 3507ab95bc47..662a8be75e13 100755 --- a/managed/yb_release +++ b/managed/yb_release @@ -4,7 +4,7 @@ set -euo pipefail # TODO: move this to common file. if [[ -z ${DEVOPS_HOME:-} ]]; then - devops_home_candidate_dir="${BASH_SOURCE%/*}"/../devops + devops_home_candidate_dir="${BASH_SOURCE%/*}"/devops if [[ -d $devops_home_candidate_dir ]]; then export DEVOPS_HOME=$( cd "$devops_home_candidate_dir" && pwd ) else diff --git a/version.txt b/version.txt deleted file mode 100644 index 83375d66d999..000000000000 --- a/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.3.1.0-b0 diff --git a/version.txt b/version.txt new file mode 120000 index 000000000000..0a82fe5b5f50 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +managed/version.txt \ No newline at end of file