diff --git a/.github/workflows/build-server.yml b/.github/workflows/build-server.yml new file mode 100644 index 0000000..e1ab1b6 --- /dev/null +++ b/.github/workflows/build-server.yml @@ -0,0 +1,159 @@ +# build.yml +on: + # pull_request: + # paths: + # - desci-server/** + push: + paths: + - .github/workflows/** + - desci-server/** + - desci-contracts/** + - Dockerfile + branches: # array of glob patterns matching against refs/heads. Optional; defaults to all + - main # triggers on pushes that contain changes + - develop + + name: Build automating-metadata + + # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html + env: + AWS_DEFAULT_REGION: us-east-2 + AWS_DEFAULT_OUTPUT: json + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + CONTAINER_IMAGE: automating-metadata + DOCKER_BUILDKIT: 1 + + jobs: + build-and-push-images: + # we build and push for every commit, even if tests pass, that way when tests pass deployment is short (run test + build in parallel) + name: Build and push images + runs-on: ubuntu-latest + steps: + - uses: hashicorp/setup-terraform@v1 + - name: Checkout + uses: actions/checkout@v4 + + # Add steps here like linting, testing, minification, etc. + - id: install-aws-cli + uses: unfor19/install-aws-cli-action@v1 + with: + version: 1 + + - uses: prepor/action-aws-iam-authenticator@master + - run: aws-iam-authenticator version + + - name: Install Kubectl + run: | + #$(curl -Ls https://dl.k8s.io/release/stable.txt) + version=v1.23.6 + echo "using kubectl@$version" + curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl + chmod +x kubectl + mv kubectl /usr/local/bin + mkdir $HOME/.kube + sudo apt-get update + sudo apt-get install less + echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config + aws sts get-caller-identity + + - name: Build and tag the image (DEV) + if: github.ref == 'refs/heads/develop' + run: | + # Build and tag the image + docker build \ + -t $CONTAINER_IMAGE-dev:${{ github.sha }} \ + -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev \ + . + + - name: Build and tag the image (PROD) + if: github.ref == 'refs/heads/main' + run: | + # Build and tag the image + docker build \ + -t $CONTAINER_IMAGE-prod:${{ github.sha }} \ + -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod \ + . + + - name: Push (DEV) + if: github.ref == 'refs/heads/develop' + run: | + # Push image to AWS ECR + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + docker tag $CONTAINER_IMAGE-dev:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} + docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} + + - name: Push (PROD) + if: github.ref == 'refs/heads/main' + run: | + # Push image to AWS ECR + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + docker tag $CONTAINER_IMAGE-prod:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} + docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} + + deploy: + name: Deploy automating-metadata + needs: + - build-and-push-images + + runs-on: ubuntu-latest + steps: + - uses: hashicorp/setup-terraform@v1 + - name: Checkout + uses: actions/checkout@v4 + + # Add steps here like linting, testing, minification, etc. + - id: install-aws-cli + uses: unfor19/install-aws-cli-action@v1 + with: + version: 1 + + - uses: prepor/action-aws-iam-authenticator@master + - run: aws-iam-authenticator version + + - name: Install Kubectl + run: | + #$(curl -Ls https://dl.k8s.io/release/stable.txt) + version=v1.23.6 + echo "using kubectl@$version" + curl -sLO "https://dl.k8s.io/release/$version/bin/linux/amd64/kubectl" -o kubectl + chmod +x kubectl + mv kubectl /usr/local/bin + mkdir $HOME/.kube + sudo apt-get update + sudo apt-get install less + echo ${{ secrets.KUBE_CONFIG_DATA }} | base64 --decode > $HOME/.kube/config + aws sts get-caller-identity + + - name: Deploy to EKS (DEV) + # uses: steebchen/kubectl@v2.0.0 + if: github.ref == 'refs/heads/develop' + run: | # defaults to latest kubectl binary version + kubectl apply -f desci-server/kubernetes/deployment_dev.yaml + kubectl set image deployment/desci-server-dev desci-server-dev=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} --record + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} + docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest + docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-dev:latest + + - name: Deploy to EKS (PROD) + if: github.ref == 'refs/heads/main' + run: | # defaults to latest kubectl binary version + kubectl apply -f desci-server/kubernetes/deployment_prod.yaml + kubectl set image deployment/desci-server desci-server=$AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} --record + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + docker pull $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} + docker tag $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:${{ github.sha }} $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:latest + docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$CONTAINER_IMAGE-prod:latest + + - name: Verify EKS Deployment (DEV) + if: github.ref == 'refs/heads/develop' + run: | + kubectl rollout status deployment/automating-metadata-dev + + - name: Verify EKS Deployment (PROD) + if: github.ref == 'refs/heads/main' + run: | + kubectl rollout status deployment/automating-metadata-prod + \ No newline at end of file diff --git a/DOCKERFILE b/DOCKERFILE deleted file mode 100755 index c17b114..0000000 --- a/DOCKERFILE +++ /dev/null @@ -1,20 +0,0 @@ -# Use a base image with Python installed -FROM python:3.9 - -# Set the working directory in the container -WORKDIR /app - -# Copy the script and requirements file into the container -COPY ./app ./ - -# Install dependencies -RUN pip install -r requirements.txt - -#run service - Expose (what is the request response model) -EXPOSE 5001 -EXPOSE 5005 - -ENV FLASK_APP=server.py - -# Define the command to run when the container starts -CMD ["flask", "run", "--host=0.0.0.0", "--port=5001"] \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..5d01b96 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# Use a base image with Python installed +FROM python:3.9 + +# Set the working directory in the container +WORKDIR / + +# we put this before the COPY app step just to cache the installation of the requirements, so if we make code changes without requirements updates builds are faster +COPY ./app/requirements.txt ./app/requirements.txt +RUN pip install -r ./app/requirements.txt + +# Copy the script and requirements file into the container +COPY ./app ./app + +# Install dependencies +RUN pip install gunicorn + +#run service - Expose (what is the request response model) +EXPOSE 5001 +EXPOSE 5005 + +ENV FLASK_APP=server.py + +# Define the command to run when the container starts +# CMD ["flask", "run", "--host=0.0.0.0", "--port=5001"] +# gunicorn is a production ready web server for flask, with ability to handle multiple requests +CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5001", "app:server"] \ No newline at end of file diff --git a/app/requirements.txt b/app/requirements.txt index 4e099db..c89807d 100755 Binary files a/app/requirements.txt and b/app/requirements.txt differ diff --git a/app/server.py b/app/server.py index c720d24..8617a21 100644 --- a/app/server.py +++ b/app/server.py @@ -27,5 +27,9 @@ def invoke_script(): output = run_langchain(pdf, cr_mailto, pyalex_email) return jsonify({'output': output}) +@app.route('/', methods=['GET']) +def index(): + return jsonify({'message': 'metadata server healthy'}) + if __name__ == '__main__': app.run(host='0.0.0.0', port=5001) \ No newline at end of file diff --git a/kubernetes/deployment_dev.yaml b/kubernetes/deployment_dev.yaml new file mode 100644 index 0000000..5c21cc6 --- /dev/null +++ b/kubernetes/deployment_dev.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: Service +metadata: + name: automating-metadata-dev +spec: + selector: + App: AutomatingMetadataDev + ports: + - port: 80 + targetPort: 5001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: automating-metadata-dev + labels: + App: AutomatingMetadataDev +spec: + replicas: 2 + revisionHistoryLimit: 8 + selector: + matchLabels: + App: AutomatingMetadataDev + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: 'true' + vault.hashicorp.com/agent-inject-status: 'update' + vault.hashicorp.com/role: app-vault-reader + vault.hashicorp.com/agent-inject-secret-config: secrets/desci-server/staging/db + vault.hashicorp.com/agent-inject-template-config: | + {{- with secret "secrets/automating-metadata/dev" -}} + export OPENAI_API_KEY={{ .Data.OPENAI_API_KEY }} + export crmailto={{ .Data.crmailto }} + export pyalexemail={{ .Data.pyalexemail }} + export AM_API_KEY={{ .Data.AM_API_KEY }} + export IPFS_GATEWAY_URL={{ .Data.IPFS_GATEWAY_URL }} + + echo "dbset"; + {{- end -}} + labels: + App: AutomatingMetadataDev + spec: + containers: + - image: 523044037273.dkr.ecr.us-east-2.amazonaws.com/automating-metadata-dev:latest + name: automating-metadata-dev + command: ['/bin/bash', '-c'] + args: + - echo "SOURCING ENV"; source /vault/secrets/config; flask run --host=0.0.0.0 --port=5001 + ports: + - containerPort: 5001 + name: server-api + resources: + limits: + cpu: '1.0' + memory: 5Gi + requests: + cpu: '0.8' + memory: 5Gi + # restart pod after failureThreshold*periodSeconds total seconds + livenessProbe: + httpGet: + path: / + port: server-api + failureThreshold: 80 + periodSeconds: 3 + # temporarily stop sending traffic to pod after failureThreshold*periodSeconds total seconds + readinessProbe: + httpGet: + path: / + port: server-api + failureThreshold: 3 + periodSeconds: 1 + # wait for pod to start for failureThreshold*periodSeconds total seconds + startupProbe: + httpGet: + path: / + port: server-api + failureThreshold: 200 + periodSeconds: 1 + serviceAccountName: 'vault-auth' diff --git a/kubernetes/deployment_prod.yaml b/kubernetes/deployment_prod.yaml new file mode 100644 index 0000000..37c9d79 --- /dev/null +++ b/kubernetes/deployment_prod.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: Service +metadata: + name: automating-metadata-prod +spec: + selector: + App: AutomatingMetadataProd + ports: + - port: 80 + targetPort: 5001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: automating-metadata-prod + labels: + App: AutomatingMetadataProd +spec: + replicas: 2 + revisionHistoryLimit: 8 + selector: + matchLabels: + App: AutomatingMetadataProd + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: 'true' + vault.hashicorp.com/agent-inject-status: 'update' + vault.hashicorp.com/role: app-vault-reader + vault.hashicorp.com/agent-inject-secret-config: secrets/desci-server/staging/db + vault.hashicorp.com/agent-inject-template-config: | + {{- with secret "secrets/automating-metadata/prod" -}} + export OPENAI_API_KEY={{ .Data.OPENAI_API_KEY }} + export crmailto={{ .Data.crmailto }} + export pyalexemail={{ .Data.pyalexemail }} + export AM_API_KEY={{ .Data.AM_API_KEY }} + export IPFS_GATEWAY_URL={{ .Data.IPFS_GATEWAY_URL }} + + echo "dbset"; + {{- end -}} + labels: + App: AutomatingMetadataProd + spec: + containers: + - image: 523044037273.dkr.ecr.us-east-2.amazonaws.com/automating-metadata-prod:latest + name: automating-metadata-prod + command: ['/bin/bash', '-c'] + args: + - echo "SOURCING ENV"; source /vault/secrets/config; flask run --host=0.0.0.0 --port=5001 + ports: + - containerPort: 5001 + name: server-api + resources: + limits: + cpu: '1.0' + memory: 5Gi + requests: + cpu: '0.8' + memory: 5Gi + # restart pod after failureThreshold*periodSeconds total seconds + livenessProbe: + httpGet: + path: / + port: server-api + failureThreshold: 80 + periodSeconds: 3 + # temporarily stop sending traffic to pod after failureThreshold*periodSeconds total seconds + readinessProbe: + httpGet: + path: / + port: server-api + failureThreshold: 3 + periodSeconds: 1 + # wait for pod to start for failureThreshold*periodSeconds total seconds + startupProbe: + httpGet: + path: / + port: server-api + failureThreshold: 200 + periodSeconds: 1 + serviceAccountName: 'vault-auth'