From 523f03d1a6acd947a0e023aa0b35fec7dd1f2a69 Mon Sep 17 00:00:00 2001 From: Ken Ellinwood Date: Wed, 19 Nov 2025 11:13:28 -1000 Subject: [PATCH 1/2] JET-2338: Add support for file scheme in dist URL --- Dockerfile-ubuntu.template | 11 +++++++++-- README.md | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Dockerfile-ubuntu.template b/Dockerfile-ubuntu.template index 8d364e0b..212c112c 100644 --- a/Dockerfile-ubuntu.template +++ b/Dockerfile-ubuntu.template @@ -57,8 +57,15 @@ RUN groupadd --system --gid=9999 flink && \ WORKDIR $FLINK_HOME # Install Flink -RUN set -ex; \ - wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ +RUN --mount=type=bind,source=.,target=/build-ctx set -ex; \ + case $FLINK_TGZ_URL in \ + file://*) \ + cp $(echo $FLINK_TGZ_URL | cut -c8-) flink.tgz; \ + ;; \ + *) \ + wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ + ;; \ + esac; \ \ if [ "$CHECK_GPG" = "true" ]; then \ wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \ diff --git a/README.md b/README.md index 121c38a9..d93f852b 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,17 @@ in the same folder and version 1.11. Please substitute your folder structure and 5. Generate docker image (in `flink-docker/dev/flink-1.11-debian`): `docker build -t flink:1.11-SN .` 6. Run custom Flink docker image: `docker run -it flink:1.11-SN jobmanager` +### Build without running a containerized web server (skip step 3, above) + +4. Generate the `Dockerfile` via: `./add-custom.sh -u file:///build-ctx/flink-1.11.tgz -n flink-1.11`. + Note that the URL must begin with `file:////build-ctx/` + Next, copy the tgz file into the build context directory created for your custom build: `cp flink-1.11.tgz -n dev/flink-1.11-debian/.` + Proceed with step 5, above. + +#### Build with Podman and without running a containerized web server (skip step 3, above) + +Podman doesn't complain about `RUN --mount ...` but then triggers permission denied when accessing files in the mounted volume. Here's the workaround... + +4. Generate the `Dockerfile` via: `./add-custom.sh -u file:///build-vol/flink-1.11.tgz -n flink-1.11`. + Next, copy the tgz file into the build dir created for your custom build: `cp flink-1.11.tgz -n dev/flink-1.11-debian/.` +5. Generate docker image (in `flink-docker/dev/flink-1.11-debian`): `docker build -v $PWD:/build-vol -t flink:1.11-SN .` From 2daa7cfdc044e6b339a6e9dd6c22b1fd8f52b574 Mon Sep 17 00:00:00 2001 From: Ken Ellinwood Date: Wed, 26 Nov 2025 15:46:56 -1000 Subject: [PATCH 2/2] JET-2338 Better local file support for distro TGZ file --- Dockerfile-ubuntu.template | 13 +++++++------ README.md | 17 ++++++----------- add-custom.sh | 16 +++++++++++++--- generator.sh | 2 ++ 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/Dockerfile-ubuntu.template b/Dockerfile-ubuntu.template index 212c112c..2aa7502b 100644 --- a/Dockerfile-ubuntu.template +++ b/Dockerfile-ubuntu.template @@ -57,14 +57,15 @@ RUN groupadd --system --gid=9999 flink && \ WORKDIR $FLINK_HOME # Install Flink -RUN --mount=type=bind,source=.,target=/build-ctx set -ex; \ - case $FLINK_TGZ_URL in \ - file://*) \ - cp $(echo $FLINK_TGZ_URL | cut -c8-) flink.tgz; \ - ;; \ - *) \ +%%COPY_LOCAL_TGZ%% +RUN set -ex; \ + case "$FLINK_TGZ_URL" in \ + http*) \ wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \ ;; \ + *) \ + # Use the locally copied TGZ file \ + ;; \ esac; \ \ if [ "$CHECK_GPG" = "true" ]; then \ diff --git a/README.md b/README.md index d93f852b..59c1e6dc 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,10 @@ in the same folder and version 1.11. Please substitute your folder structure and ### Build without running a containerized web server (skip step 3, above) -4. Generate the `Dockerfile` via: `./add-custom.sh -u file:///build-ctx/flink-1.11.tgz -n flink-1.11`. - Note that the URL must begin with `file:////build-ctx/` - Next, copy the tgz file into the build context directory created for your custom build: `cp flink-1.11.tgz -n dev/flink-1.11-debian/.` - Proceed with step 5, above. +4. `./add-custom.sh -l -u flink-1.11.tgz -n flink-1.11` + The important changes from above in this step are to include the `-l` option (local TGZ file), and using a plain file as the argument for `-u` + 4.a Copy the TGZ file into the Docker context dir (dev/flink-1.11) and make sure the copied filename in the context dir has the same name + as given in the `-u` argument. -#### Build with Podman and without running a containerized web server (skip step 3, above) - -Podman doesn't complain about `RUN --mount ...` but then triggers permission denied when accessing files in the mounted volume. Here's the workaround... - -4. Generate the `Dockerfile` via: `./add-custom.sh -u file:///build-vol/flink-1.11.tgz -n flink-1.11`. - Next, copy the tgz file into the build dir created for your custom build: `cp flink-1.11.tgz -n dev/flink-1.11-debian/.` -5. Generate docker image (in `flink-docker/dev/flink-1.11-debian`): `docker build -v $PWD:/build-vol -t flink:1.11-SN .` +Proceed with step 5, above. + diff --git a/add-custom.sh b/add-custom.sh index 8b9dc59b..a5b3f367 100755 --- a/add-custom.sh +++ b/add-custom.sh @@ -7,18 +7,23 @@ source "$(dirname "$0")"/generator.sh function usage() { - echo >&2 "usage: $0 -u binary-download-url [-n name] [-j java_version]" + echo >&2 "usage: $0 -u binary-download-url [-l] [-n name] [-j java_version]" } +local_tgz=false +copy_local_tgz= binary_download_url= name=custom java_version=${DEFAULT_JAVA} -while getopts u:n:j:h arg; do +while getopts u:n:j:lh arg; do case "$arg" in u) binary_download_url=$OPTARG ;; + l) + local_tgz=true + ;; n) name=$OPTARG ;; @@ -41,6 +46,11 @@ if [ -z "${binary_download_url}" ]; then exit 1 fi +if [ "${local_tgz}" = "true" ]; then + # Use local tgz file + copy_local_tgz="COPY $binary_download_url /opt/flink/flink.tgz" + binary_download_url="" +fi mkdir -p "dev" echo -n >&2 "Generating Dockerfiles..." @@ -48,6 +58,6 @@ for source_variant in "${SOURCE_VARIANTS[@]}"; do dir="dev/${name}-${source_variant}" rm -rf "${dir}" mkdir "$dir" - generateDockerfile "${dir}" "${binary_download_url}" "" "" false ${java_version} ${source_variant} + generateDockerfile "${dir}" "${binary_download_url}" "" "" false ${java_version} "${source_variant}" "${copy_local_tgz}" done echo >&2 " done." diff --git a/generator.sh b/generator.sh index 758812a9..b3856147 100644 --- a/generator.sh +++ b/generator.sh @@ -14,6 +14,7 @@ function generateDockerfile { check_gpg=$5 java_version=$6 source_variant=$7 + copy_local_tgz=$8 from_docker_image="eclipse-temurin:${java_version}-jre-jammy" @@ -29,6 +30,7 @@ function generateDockerfile { -e "s/%%GPG_KEY%%/$gpg_key/" \ -e "s/%%CHECK_GPG%%/${check_gpg}/" \ -e "s/%%FROM_IMAGE%%/${from_docker_image}/" \ + -e "s!%%COPY_LOCAL_TGZ%%!${copy_local_tgz}!" \ "Dockerfile-$source_variant.template" > "$dir/Dockerfile" }