Merge pull request #349 from Shopify/build-and-release-on-arm

Build and release on arm64

.github/workflows/build-deb.sh

@@ -6,25 +6,20 @@ sudo apt-get update
 sudo apt-get install -y make
 
 # We set this here, so it's the same between the copydb and sharding debian
-# package.
-DATETIME=$(date -u +%Y%m%d%H%M%S)
+# package, and between different arch builds
+DATETIME=$(date -u +%Y%m%d)
 
 git status
 
 make copydb-deb DATETIME=${DATETIME}
 make sharding-deb DATETIME=${DATETIME}
 
 cd build
-sha256sum *.deb > ghostferry-$GITHUB_SHA.sha256sum
-
 set +x
 
 echo "Debian package built successfully as follows:"
 ls -l ghostferry*
 
-echo "sha256sum:"
-cat ghostferry-$GITHUB_SHA.sha256sum
-
 # Make sure the we didn't release a dirty build by accident
 if ls | grep -q dirty; then
   echo "ERROR: source directory is not clean! refused to release. showing git status below:"

.github/workflows/release.yml

@@ -6,8 +6,11 @@ on:
       - main
 
 jobs:
-  release:
-    runs-on: ubuntu-latest
+  build-debs:
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, shopify-ubuntu-arm64]
+    runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v2
 
@@ -19,7 +22,28 @@ jobs:
       - name: Building Ghostferry
         run: .github/workflows/build-deb.sh
 
+      - name: Upload debs
+        uses: actions/upload-artifact@v3
+        with:
+          name: debs-${{ github.sha }}
+          path: build/ghostferry*
+
+  release:
+    runs-on: ubuntu-latest
+    needs: build-debs
+    steps:
+      - name: Fetch uploaded artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: debs-${{ github.sha }}
+      - name: shasum
+        shell: bash
+        run: |
+          sha256sum *.deb > ghostferry-$GITHUB_SHA.sha256sum
+          echo "sha256sum:"
+          cat ghostferry-$GITHUB_SHA.sha256sum
+
       - name: Releasing Ghostferry to Github
-        run: gh release create --target ${GITHUB_REF#refs/heads/} -p release-${GITHUB_SHA::7} build/ghostferry*
+        run: gh release create --target ${GITHUB_REF#refs/heads/} -p release-${GITHUB_SHA::7} ghostferry*
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yml

@@ -71,4 +71,44 @@ jobs:
 
       - name: Running Ruby tests
         run: bundle exec ruby test/main.rb
+  build-debs:
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, shopify-ubuntu-arm64]
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
 
+      - name: Building Ghostferry
+        run: .github/workflows/build-deb.sh
+
+      - name: Upload debs
+        uses: actions/upload-artifact@v3
+        with:
+          name: debs-${{ github.sha }}
+          path: build/ghostferry*
+
+  checksum-debs:
+    runs-on: ubuntu-latest
+    needs: build-debs
+    steps:
+      - name: Fetch uploaded artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: debs-${{ github.sha }}
+      - name: shasum
+        shell: bash
+        run: |
+          sha256sum *.deb > ghostferry-$GITHUB_SHA.sha256sum
+          echo "sha256sum:"
+          cat ghostferry-$GITHUB_SHA.sha256sum
+      - name: Upload checksum
+        uses: actions/upload-artifact@v3
+        with:
+          name: debs-${{ github.sha }}
+          path: "*.sha256sum"

Makefile

@@ -26,15 +26,15 @@ BIN_DIR         := usr/bin
 PROJECTS        := copydb sharding
 PROJECT_DEBS    := $(foreach name,$(PROJECTS),$(name)-deb)
 
+PLATFORM        := $(shell uname -s | tr A-Z a-z)
+ARCH            := $(shell uname -m | sed 's/x86_64/amd64/g' | sed 's/aarch64/arm64/g')
+GOTESTSUM_URL   := "https://github.com/gotestyourself/gotestsum/releases/download/v1.7.0/gotestsum_1.7.0_$(PLATFORM)_$(ARCH).tar.gz"
+
 # Target specific variable, set proj to have a valid value.
 PROJECT_PKG      = ./$(proj)/cmd
 PROJECT_BIN      = ghostferry-$(proj)
 BIN_TARGET       = $(GOBIN)/$(PROJECT_BIN)
-DEB_TARGET       = $(BUILD_DIR)/$(PROJECT_BIN)_$(VERSION_STR).deb
-
-PLATFORM        := $(shell uname -s | tr A-Z a-z)
-ARCH            := $(shell uname -m | sed 's/x86_64/amd64/g' | sed 's/aarch64/arm64/g')
-GOTESTSUM_URL   := "https://github.com/gotestyourself/gotestsum/releases/download/v1.7.0/gotestsum_1.7.0_$(PLATFORM)_$(ARCH).tar.gz"
+DEB_TARGET       = $(BUILD_DIR)/$(PROJECT_BIN)_$(VERSION_STR)_$(ARCH).deb
 
 .PHONY: test test-go test-ruby clean reset-deb-dir $(PROJECTS) $(PROJECT_DEBS)
 .DEFAULT_GOAL := test
@@ -46,7 +46,7 @@ $(PROJECTS): $(GOBIN)
 $(PROJECT_DEBS): LDFLAGS += -X github.com/Shopify/ghostferry.WebUiBasedir=/$(SHARE_DIR)
 $(PROJECT_DEBS): reset-deb-dir
 	$(eval proj := $(subst -deb,,$@))
-	sed -e "s/{version}/$(VERSION_STR)/" $(proj)/debian/control > $(DEB_PREFIX)/DEBIAN/control
+	sed -e "s/{version}/$(VERSION_STR)/" -e "s/{arch}/$(ARCH)/" $(proj)/debian/control > $(DEB_PREFIX)/DEBIAN/control
 	cp $(proj)/debian/copyright $(DEB_PREFIX)/DEBIAN/copyright
 	go build -ldflags "$(LDFLAGS)" -o $(DEB_PREFIX)/$(BIN_DIR)/$(PROJECT_BIN) $(PROJECT_PKG)
 	cp -ar webui $(DEB_PREFIX)/$(SHARE_DIR)

copydb/debian/control

@@ -2,7 +2,7 @@ Package: ghostferry-copydb
 Version: {version}-1
 Section: misc
 Priority: optional
-Architecture: amd64
+Architecture: {arch}
 Maintainer: Shuhao Wu <shuhao.wu@shopify.com>
 Description: Migrate MySQL data from one server to another
   without downtime via ghostferry.

sharding/debian/control

@@ -2,7 +2,7 @@ Package: ghostferry-sharding
 Version: {version}
 Section: misc
 Priority: optional
-Architecture: amd64
+Architecture: {arch}
 Maintainer: Shuhao Wu <shuhao.wu@shopify.com>
 Uploaders: Justin Li <justin.li@shopify.com>,Hormoz Kheradmand <hormoz.kheradmand@shopify.com>
 Description: Migrate MySQL sharded data from one server to