diff --git a/CHANGELOG.md b/CHANGELOG.md index 95071a1..aa4f6e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is base on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.05.00.000] - 2024-12-01 +### Added +- Project now supports Apache Flink 1.20.0. + ## [0.04.00.000] - 2024-11-16 ### Added - Issue [#28](https://github.com/j3-signalroom/supercharge_streamlit-apache_flink/issues/28). diff --git a/linux-Dockerfile b/linux-Dockerfile index cc27caa..43679c7 100644 --- a/linux-Dockerfile +++ b/linux-Dockerfile @@ -1,5 +1,5 @@ # Base image from https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/deployment/resource-providers/standalone/docker/ -FROM flink:1.19.1-scala_2.12-java17 +FROM flink:1.20.0-scala_2.12-java17 # Build argument(s) ARG FLINK_LANGUAGE @@ -18,14 +18,14 @@ ENV AWS_S3_BUCKET=${AWS_S3_BUCKET} ENV AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} # Create/set version number container environment variable(s) -ENV APACHE_FLINK_VERSION=1.19 -ENV APACHE_FLINK_PATCH=.1 +ENV APACHE_FLINK_VERSION=1.20 +ENV APACHE_FLINK_PATCH=.0 ENV APACHE_FLINK_VERSION_WITH_PATCH=${APACHE_FLINK_VERSION}${APACHE_FLINK_PATCH} ENV PYTHON_VERSION=3.11 ENV PYTHON_PATCH=.9 ENV PYTHON_VERSION_WITH_PATCH=${PYTHON_VERSION}${PYTHON_PATCH} -ENV HADOOP_VERSION=3.4.0 -ENV ICEBERG_VERSION=1.6.1 +ENV HADOOP_VERSION=3.4.1 +ENV ICEBERG_VERSION=1.7.0 # Create/set folder contianer environment variable(s) ENV FLINK_CONF_DIR=/opt/flink/conf @@ -72,11 +72,13 @@ RUN curl -L "${MAVEN_ROOT_URL}flink/flink-s3-fs-hadoop/${APACHE_FLINK_VERSION_WI curl -L "${MAVEN_ROOT_URL}flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar" -o "${FLINK_LIB_DIR}flink-shaded-hadoop-2-uber-2.8.3-10.0.jar" && \ curl -L "${MAVEN_ROOT_URL}iceberg/iceberg-flink-runtime-${APACHE_FLINK_VERSION}/${ICEBERG_VERSION}/iceberg-flink-runtime-${APACHE_FLINK_VERSION}-${ICEBERG_VERSION}.jar" -o "${FLINK_LIB_DIR}iceberg-flink-runtime-${APACHE_FLINK_VERSION}-${ICEBERG_VERSION}.jar" && \ curl -L "${MAVEN_ROOT_URL}iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar" -o "${FLINK_LIB_DIR}iceberg-aws-bundle-${ICEBERG_VERSION}.jar" && \ - curl -L "${MAVEN_ROOT_URL}hadoop/hadoop-hdfs-client/3.4.0/hadoop-hdfs-client-3.4.0.jar" -o "${FLINK_LIB_DIR}hadoop-hdfs-client-${HADOOP_VERSION}.jar" && \ - curl -L "${MAVEN_ROOT_URL}flink/flink-sql-connector-kafka/3.2.0-${APACHE_FLINK_VERSION}/flink-sql-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-sql-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" && \ - curl -L "${MAVEN_ROOT_URL}flink/flink-connector-kafka/3.2.0-${APACHE_FLINK_VERSION}/flink-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" && \ + curl -L "${MAVEN_ROOT_URL}hadoop/hadoop-hdfs-client/${HADOOP_VERSION}/hadoop-hdfs-client-${HADOOP_VERSION}.jar" -o "${FLINK_LIB_DIR}hadoop-hdfs-client-${HADOOP_VERSION}.jar" && \ + curl -L "${MAVEN_ROOT_URL}flink/flink-sql-connector-kafka/3.3.0-${APACHE_FLINK_VERSION}/flink-sql-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-sql-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" && \ + curl -L "${MAVEN_ROOT_URL}flink/flink-connector-kafka/3.3.0-${APACHE_FLINK_VERSION}/flink-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" && \ curl -L "${MAVEN_ROOT_URL}flink/flink-json/${APACHE_FLINK_VERSION_WITH_PATCH}/flink-json-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" -o "${FLINK_LIB_DIR}flink-json-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" && \ - curl -L "${MAVEN_ROOT_URL}kafka/kafka-clients/3.7.0/kafka-clients-3.7.0.jar" -o "${FLINK_LIB_DIR}kafka-clients-3.7.0.jar"; + curl -L "${MAVEN_ROOT_URL}flink/flink-avro/${APACHE_FLINK_VERSION_WITH_PATCH}/flink-avro-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" -o "${FLINK_LIB_DIR}flink-avro-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" && \ + curl -L "${MAVEN_ROOT_URL}avro/avro/1.12.0/avro-1.12.0.jar" -o "${FLINK_LIB_DIR}avro-1.12.0.jar" && \ + curl -L "${MAVEN_ROOT_URL}kafka/kafka-clients/3.8.1/kafka-clients-3.8.1.jar" -o "${FLINK_LIB_DIR}kafka-clients-3.8.1.jar"; # Install Python from source RUN cd /usr/local && \ diff --git a/mac-Dockerfile b/mac-Dockerfile index 64275be..44d88df 100644 --- a/mac-Dockerfile +++ b/mac-Dockerfile @@ -1,5 +1,5 @@ # Base image from https://nightlies.apache.org/flink/flink-docs-release-1.19/docs/deployment/resource-providers/standalone/docker/ -FROM arm64v8/flink:1.19.1-scala_2.12-java17 +FROM arm64v8/flink:1.20.0-scala_2.12-java17 # Build argument(s) ARG FLINK_LANGUAGE @@ -18,14 +18,14 @@ ENV AWS_S3_BUCKET=${AWS_S3_BUCKET} ENV AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN} # Create/set version number container environment variable(s) -ENV APACHE_FLINK_VERSION=1.19 -ENV APACHE_FLINK_PATCH=.1 +ENV APACHE_FLINK_VERSION=1.20 +ENV APACHE_FLINK_PATCH=.0 ENV APACHE_FLINK_VERSION_WITH_PATCH=${APACHE_FLINK_VERSION}${APACHE_FLINK_PATCH} ENV PYTHON_VERSION=3.11 ENV PYTHON_PATCH=.9 ENV PYTHON_VERSION_WITH_PATCH=${PYTHON_VERSION}${PYTHON_PATCH} -ENV HADOOP_VERSION=3.4.0 -ENV ICEBERG_VERSION=1.6.1 +ENV HADOOP_VERSION=3.4.1 +ENV ICEBERG_VERSION=1.7.0 # Create/set folder contianer environment variable(s) ENV FLINK_CONF_DIR=/opt/flink/conf @@ -72,11 +72,13 @@ RUN curl -L "${MAVEN_ROOT_URL}flink/flink-s3-fs-hadoop/${APACHE_FLINK_VERSION_WI curl -L "${MAVEN_ROOT_URL}flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar" -o "${FLINK_LIB_DIR}flink-shaded-hadoop-2-uber-2.8.3-10.0.jar" && \ curl -L "${MAVEN_ROOT_URL}iceberg/iceberg-flink-runtime-${APACHE_FLINK_VERSION}/${ICEBERG_VERSION}/iceberg-flink-runtime-${APACHE_FLINK_VERSION}-${ICEBERG_VERSION}.jar" -o "${FLINK_LIB_DIR}iceberg-flink-runtime-${APACHE_FLINK_VERSION}-${ICEBERG_VERSION}.jar" && \ curl -L "${MAVEN_ROOT_URL}iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar" -o "${FLINK_LIB_DIR}iceberg-aws-bundle-${ICEBERG_VERSION}.jar" && \ - curl -L "${MAVEN_ROOT_URL}hadoop/hadoop-hdfs-client/3.4.0/hadoop-hdfs-client-3.4.0.jar" -o "${FLINK_LIB_DIR}hadoop-hdfs-client-${HADOOP_VERSION}.jar" && \ - curl -L "${MAVEN_ROOT_URL}flink/flink-sql-connector-kafka/3.2.0-${APACHE_FLINK_VERSION}/flink-sql-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-sql-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" && \ - curl -L "${MAVEN_ROOT_URL}flink/flink-connector-kafka/3.2.0-${APACHE_FLINK_VERSION}/flink-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-connector-kafka-3.2.0-${APACHE_FLINK_VERSION}.jar" && \ + curl -L "${MAVEN_ROOT_URL}hadoop/hadoop-hdfs-client/${HADOOP_VERSION}/hadoop-hdfs-client-${HADOOP_VERSION}.jar" -o "${FLINK_LIB_DIR}hadoop-hdfs-client-${HADOOP_VERSION}.jar" && \ + curl -L "${MAVEN_ROOT_URL}flink/flink-sql-connector-kafka/3.3.0-${APACHE_FLINK_VERSION}/flink-sql-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-sql-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" && \ + curl -L "${MAVEN_ROOT_URL}flink/flink-connector-kafka/3.3.0-${APACHE_FLINK_VERSION}/flink-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" -o "${FLINK_LIB_DIR}flink-connector-kafka-3.3.0-${APACHE_FLINK_VERSION}.jar" && \ curl -L "${MAVEN_ROOT_URL}flink/flink-json/${APACHE_FLINK_VERSION_WITH_PATCH}/flink-json-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" -o "${FLINK_LIB_DIR}flink-json-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" && \ - curl -L "${MAVEN_ROOT_URL}kafka/kafka-clients/3.7.0/kafka-clients-3.7.0.jar" -o "${FLINK_LIB_DIR}kafka-clients-3.7.0.jar"; + curl -L "${MAVEN_ROOT_URL}flink/flink-avro/${APACHE_FLINK_VERSION_WITH_PATCH}/flink-avro-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" -o "${FLINK_LIB_DIR}flink-avro-${APACHE_FLINK_VERSION_WITH_PATCH}.jar" && \ + curl -L "${MAVEN_ROOT_URL}avro/avro/1.12.0/avro-1.12.0.jar" -o "${FLINK_LIB_DIR}avro-1.12.0.jar" && \ + curl -L "${MAVEN_ROOT_URL}kafka/kafka-clients/3.8.1/kafka-clients-3.8.1.jar" -o "${FLINK_LIB_DIR}kafka-clients-3.8.1.jar"; # Install Python from source RUN cd /usr/local && \ diff --git a/pyproject.toml b/pyproject.toml index 20568aa..01a51d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [project] name = "supercharge_streamlit" -version = "0.01.00.000" +version = "0.05.00.000" description = "Supercharge Streamlit" readme = "README.md" requires-python = "~=3.11.9" dependencies = [ - "apache-flink==1.19.1", + "apache-flink==1.20.0", "boto3>=1.35.36", "botocore>=1.35.36", "confluent-kafka==2.6.0", diff --git a/uv.lock b/uv.lock index 00796a8..6388e72 100644 --- a/uv.lock +++ b/uv.lock @@ -58,7 +58,7 @@ wheels = [ [[package]] name = "apache-flink" -version = "1.19.1" +version = "1.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-beam" }, @@ -78,18 +78,18 @@ dependencies = [ { name = "requests" }, { name = "ruamel-yaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/34/39cd723c7de2c8e4278fc8809f2a4f6c64787aab9e8ad0358fff1e91de0c/apache-flink-1.19.1.tar.gz", hash = "sha256:f999e3ede4868b3a06e8056edb27b9f2df67b42a8a27170aa55a03e520083386", size = 1517034 } +sdist = { url = "https://files.pythonhosted.org/packages/37/85/830c518b8a80095c371d3af5143c90d51a21958f1d0c4095ce799445bcfe/apache-flink-1.20.0.tar.gz", hash = "sha256:2f93ff09b911c484fda4c4cf0f175a609d73017ba63d7cb21bd59e53c56fd64b", size = 1519192 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/bc/c74e397fbfb49072e07e70ae8bcb5dbcee8c97e07e550ff2042891296dbb/apache_flink-1.19.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:452390143e68b7d2e36359b5d7991802a8caadb2063246ac630d64c2393c7a4a", size = 2562801 }, - { url = "https://files.pythonhosted.org/packages/bf/02/3a94cd86b940c1c294adf7f45891ed25a0e2d802dfac412121f91d91fafb/apache_flink-1.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c430aca50a1d63b74ec1a079b3c259fcbcb1da3fd09b04cca300d40711481e32", size = 2472670 }, - { url = "https://files.pythonhosted.org/packages/1a/bc/29f68caebe932b1b1dd1176741da236d559b1912d5fd591e1dfe63b4fdb0/apache_flink-1.19.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:f3b2c4eacae90e6ef148fea0b5df65ad839498581f0c8282480200504c402490", size = 2634199 }, + { url = "https://files.pythonhosted.org/packages/72/84/c55be6b247d953dab7b50d9639294965469174c01ef8b46a59a7a1b0d50b/apache_flink-1.20.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:05bd6fee6292d5f4d84bda6ebedf954ca434787cdb14035609a56c2e57850606", size = 2565134 }, + { url = "https://files.pythonhosted.org/packages/c3/7d/569ae9990c9824b17e477e541e4aa816cbb2e3e691d9e070d61979f36c2c/apache_flink-1.20.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3d013c0ffcfa5cf4491c233d1cee392ae4c7d9262851e01249bad0d6cb2d232b", size = 2474999 }, + { url = "https://files.pythonhosted.org/packages/0e/83/52fe4520f102d2743527c39fd7e5cdd40ef6c97481eea34d82ce95754780/apache_flink-1.20.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:cb1bbe4a0a50d0bccc9c452695c4d10b4f1540667b5a17bff6963173a2f5261e", size = 2636527 }, ] [[package]] name = "apache-flink-libraries" -version = "1.19.1" +version = "1.20.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/69/b75c152878d45e2727d29d50178eb4b23ccbb0844a60620ce630045dc0ef/apache-flink-libraries-1.19.1.tar.gz", hash = "sha256:f5e6a896e8e931120be74ae0425b604743f95baddd50406277fc3a600153484d", size = 230834904 } +sdist = { url = "https://files.pythonhosted.org/packages/1f/31/80cfa34b6a53a5c19d66faf3e7d29cb0689a69c128a381e584863eb9669a/apache-flink-libraries-1.20.0.tar.gz", hash = "sha256:808b381151c51d988c8fe5b5ae59d944d6e61d8193ee6832064f7ca14c88a583", size = 231542248 } [[package]] name = "attrs" @@ -1010,7 +1010,7 @@ wheels = [ [[package]] name = "supercharge-streamlit" -version = "0.1.0.0" +version = "0.5.0.0" source = { virtual = "." } dependencies = [ { name = "apache-flink" }, @@ -1029,7 +1029,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "apache-flink", specifier = "==1.19.1" }, + { name = "apache-flink", specifier = "==1.20.0" }, { name = "boto3", specifier = ">=1.35.36" }, { name = "botocore", specifier = ">=1.35.36" }, { name = "confluent-kafka", specifier = "==2.6.0" },