From 43fdbd55b669a74a437172374667196912e8e037 Mon Sep 17 00:00:00 2001 From: Israel Roldan Date: Thu, 30 Nov 2023 01:32:54 -0600 Subject: [PATCH] Updated the documentation; Added verbose and free FTP settings. --- .idea/runConfigurations/Dockerfile.xml | 4 - README.md | 81 +++++++------- init.sh | 141 ++++++++++++++++++++----- 3 files changed, 156 insertions(+), 70 deletions(-) diff --git a/.idea/runConfigurations/Dockerfile.xml b/.idea/runConfigurations/Dockerfile.xml index 8e96d17..b42fd77 100644 --- a/.idea/runConfigurations/Dockerfile.xml +++ b/.idea/runConfigurations/Dockerfile.xml @@ -3,11 +3,7 @@ diff --git a/README.md b/README.md index 4366de9..2b5e16f 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,24 @@ -# FTP Deployment: Github Action - -This GitHub action copy the files via FTP from your Git project to your server in a specific path. +# FTP Deployment: GitHub Action +This GitHub action copies the files via FTP from your Git project to your server in a specific path. ## Usage Example -Add this code in `.github/workflows/your_action.yml`. +Add this code in `./.github/workflows/your_action.yml`. -More about Github "secrets" in this article: +More about GitHub "secrets" in this article: [Creating and storing encrypted secrets][1]. ```yaml name: CI -> Deploy to My website on: push: - branches: [ master, Development ] + branches: [ main, development ] jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Here is the deployment action - name: Upload from public_html via FTP uses: airvzxf/ftp-deployment-action@latest @@ -28,37 +27,37 @@ jobs: user: ${{ secrets.FTP_USERNAME }} password: ${{ secrets.FTP_PASSWORD }} local_dir: "./public_html" - delete: "false" ``` -Optional, you can get the live version which has the last commits using the `master` branch like this:
-`uses: airvzxf/ftp-deployment-action@master` - +Optionally, you can get the live version which has the last commits using the `master` branch like this: +`uses: airvzxf/ftp-deployment-action@master`. ## Settings -Usually the 0 (zero) values means unlimited or infinite. - -Option | Description | Required | Default | Example ---- | --- | --- | --- | --- -server | FTP Server. | Yes | N/A | rovisoft.net -user | FTP Username. | Yes | N/A | myself@rovisoft.net -password | FTP Password. | Yes | N/A | ExampleOnlyAlphabets -local_dir | Local directory. | No | "./" | "./public_html" -remote_dir | Remote directory. | No | "./" | "/www/user/home" -delete | Delete all the files inside of the remote directory before the upload process. | No | false | N/A -max_retries | Times that the `lftp` will be executed if an error occurred. | No | 10 | N/A -no_symlinks | Do not create symbolic links. | No | true | N/A -ftp_ssl_allow | FTP - Allow SSL encryption | No | false | N/A -ftp_use_feat | FTP - FEAT: Determining what extended features the FTP server supports. | No | false | N/A -ftp_nop_interval | FTP - Delay in seconds between NOOP commands when downloading tail of a file. | No | 2 | N/A -net_max_retries | NET - Maximum number of operation without success.
0 unlimited.
1 no retries. | No | 1 | N/A -net_persist_retries | NET - Ignore hard errors.
When reply 5xx errors or there is too many users. | No | 5 | N/A -net_timeout | NET - Sets the network protocol timeout. | No | 15s | N/A -dns_max_retries | DNS - 0 no limit trying to lookup an address otherwise try only this number of times. | No | 8 | N/A -dns_fatal_timeout | DNS - Time for DNS queries.
Set to "never" to disable. | No | 10s | N/A - -More information in the official site for [lftp - Manual pages][2] +Usually the zero values mean unlimited or infinite. This table is based on the default values on `lftp-4.9.2`. + +| Option | Description | Required | Default | Example | +|---------------------|---------------------------------------------------------------------------------------|----------|---------|---------------------------------------------------------------------------------------------------| +| server | FTP Server. | Yes | N/A | rovisoft.net | +| user | FTP Username. | Yes | N/A | myself@rovisoft.net | +| password | FTP Password. | Yes | N/A | ExampleOnlyAlphabets | +| local_dir | Local directory. | No | "./" | "./public_html" | +| remote_dir | Remote directory. | No | "./" | "/www/user/home" | +| max_retries | Times that the `lftp` command will be executed if an error occurred. | No | 10 | N/A | +| delete | Delete all the files inside of the remote directory before the upload process. | No | false | N/A | +| no_symlinks | Do not create symbolic links. | No | false | N/A | +| mirror_verbose | Mirror verbosity level. | No | 1 | N/A | +| ftp_ssl_allow | FTP - Allow SSL encryption | No | true | N/A | +| ftp_use_feat | FTP - FEAT: Determining what extended features the FTP server supports. | No | true | N/A | +| ftp_nop_interval | FTP - Delay in seconds between NOOP commands when downloading tail of a file. | No | 2 | N/A | +| net_max_retries | NET - Maximum number of operation without success.
0 unlimited.
1 no retries. | No | 1 | N/A | +| net_persist_retries | NET - Ignore hard errors.
When reply 5xx errors or there is too many users. | No | 5 | N/A | +| net_timeout | NET - Sets the network protocol timeout. | No | 15s | N/A | +| dns_max_retries | DNS - 0 no limit trying to lookup an address otherwise try only this number of times. | No | 8 | N/A | +| dns_fatal_timeout | DNS - Time for DNS queries.
Set to "never" to disable. | No | 10s | N/A | +| lftp_settings | Any other settings that you find in the MAN pages for the LFTP package. | No | "" | "set cache:cache-empty-listings true; set cmd:status-interval 1s; set http:user-agent 'firefox';" | + +More information on the official site for [lftp - Manual pages][2]. Example with NO DEFAULT settings: @@ -66,12 +65,12 @@ Example with NO DEFAULT settings: name: CI -> Deploy to My website on: push: - branches: [ master, Development ] + branches: [ main, development ] jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Here is the deployment action - name: Upload from public_html via FTP uses: airvzxf/ftp-deployment-action@latest @@ -92,19 +91,23 @@ jobs: net_timeout: "13s" dns_max_retries: "17" dns_fatal_timeout: "never" + lftp_settings: "set cache:cache-empty-listings true; set cmd:status-interval 1s; set http:user-agent 'firefox';" ``` ## NOTES Main features: -- Copy all the files inside of the specific folder from your Github repository to the specific folder in your server. + +- Copy all the files inside the specific folder from your GitHub repository to the specific folder in your server. - Option to delete all the files in the specific remote folder before the upload. -- Use Alpine container means small size and faster creation of the container. +- Using Alpine container means small size and faster creation of the container. - Show messages in the console logs for every executed command. TODOs: + - Add options for exclude delete files. -- Take all the logs from the Linux container then attach all into the Workflow Artifacts, to review unknown errors. +- Take all the logs from the Linux container then attach all into the Workflow Artifacts, to review unknown errors. [1]: https://docs.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets -[2]: http://lftp.tech/lftp-man.html \ No newline at end of file + +[2]: https://lftp.yar.ru/lftp-man.html diff --git a/init.sh b/init.sh index bbb2051..6398485 100644 --- a/init.sh +++ b/init.sh @@ -1,14 +1,18 @@ #!/bin/sh -e +# ------------------------------------------------------------------------------ +# Display environment variables. +# ------------------------------------------------------------------------------ echo "=== Environment variables ===" echo "INPUT_SERVER: ${INPUT_SERVER}" echo "INPUT_USER: ${INPUT_USER}" echo "INPUT_PASSWORD: ${INPUT_PASSWORD}" echo "INPUT_LOCAL_DIR: ${INPUT_LOCAL_DIR}" echo "INPUT_REMOTE_DIR: ${INPUT_REMOTE_DIR}" -echo "INPUT_DELETE: ${INPUT_DELETE}" echo "INPUT_MAX_RETRIES: ${INPUT_MAX_RETRIES}" +echo "INPUT_DELETE: ${INPUT_DELETE}" echo "INPUT_NO_SYMLINKS: ${INPUT_NO_SYMLINKS}" +echo "INPUT_MIRROR_VERBOSE: ${INPUT_MIRROR_VERBOSE}" echo "INPUT_FTP_SSL_ALLOW: ${INPUT_FTP_SSL_ALLOW}" echo "INPUT_FTP_USE_FEAT: ${INPUT_FTP_USE_FEAT}" echo "INPUT_FTP_NOP_INTERVAL: ${INPUT_FTP_NOP_INTERVAL}" @@ -17,60 +21,134 @@ echo "INPUT_NET_PERSIST_RETRIES: ${INPUT_NET_PERSIST_RETRIES}" echo "INPUT_NET_TIMEOUT: ${INPUT_NET_TIMEOUT}" echo "INPUT_DNS_MAX_RETRIES: ${INPUT_DNS_MAX_RETRIES}" echo "INPUT_DNS_FATAL_TIMEOUT: ${INPUT_DNS_FATAL_TIMEOUT}" +echo "INPUT_LFTP_SETTINGS: ${INPUT_LFTP_SETTINGS}" echo "" echo "=== Current location ===" pwd -ls -lha echo "" -FTP_SETTINGS="set ftp:ssl-allow ${INPUT_FTP_SSL_ALLOW};" -FTP_SETTINGS="${FTP_SETTINGS} set ftp:use-feat ${INPUT_FTP_USE_FEAT};" -FTP_SETTINGS="${FTP_SETTINGS} set ftp:nop-interval ${INPUT_FTP_NOP_INTERVAL};" -FTP_SETTINGS="${FTP_SETTINGS} set net:max-retries ${INPUT_NET_MAX_RETRIES};" -FTP_SETTINGS="${FTP_SETTINGS} set net:persist-retries ${INPUT_NET_PERSIST_RETRIES};" -FTP_SETTINGS="${FTP_SETTINGS} set net:timeout ${INPUT_NET_TIMEOUT};" -FTP_SETTINGS="${FTP_SETTINGS} set dns:max-retries ${INPUT_DNS_MAX_RETRIES};" -FTP_SETTINGS="${FTP_SETTINGS} set dns:fatal-timeout ${INPUT_DNS_FATAL_TIMEOUT};" +# ------------------------------------------------------------------------------ +# Set the LFTP setting. +# ------------------------------------------------------------------------------ +FTP_SETTINGS="" -MIRROR_COMMAND="mirror --continue --reverse" +# ftp:ssl-allow +if [ -n "${INPUT_FTP_SSL_ALLOW}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set ftp:ssl-allow ${INPUT_FTP_SSL_ALLOW};" +else + FTP_SETTINGS="${FTP_SETTINGS} set ftp:ssl-allow true;" +fi + +# ftp:use-feat +if [ -n "${INPUT_FTP_USE_FEAT}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set ftp:use-feat ${INPUT_FTP_USE_FEAT};" +else + FTP_SETTINGS="${FTP_SETTINGS} set ftp:use-feat true;" +fi + +# ftp:nop-interval +if [ -n "${INPUT_FTP_NOP_INTERVAL}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set ftp:nop-interval ${INPUT_FTP_NOP_INTERVAL};" +else + FTP_SETTINGS="${FTP_SETTINGS} set ftp:nop-interval 2;" +fi + +# net:max-retries +if [ -n "${INPUT_NET_MAX_RETRIES}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set net:max-retries ${INPUT_NET_MAX_RETRIES};" +else + FTP_SETTINGS="${FTP_SETTINGS} set net:max-retries 1;" +fi + +# net:persist-retries +if [ -n "${INPUT_NET_PERSIST_RETRIES}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set net:persist-retries ${INPUT_NET_PERSIST_RETRIES};" +else + FTP_SETTINGS="${FTP_SETTINGS} set net:persist-retries 5;" +fi + +# net:timeout +if [ -n "${INPUT_NET_TIMEOUT}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set net:timeout ${INPUT_NET_TIMEOUT};" +else + FTP_SETTINGS="${FTP_SETTINGS} set net:timeout 15s;" +fi + +# dns:max-retries +if [ -n "${INPUT_DNS_MAX_RETRIES}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set dns:max-retries ${INPUT_DNS_MAX_RETRIES};" +else + FTP_SETTINGS="${FTP_SETTINGS} set dns:max-retries 8;" +fi + +# dns:fatal-timeout +if [ -n "${INPUT_DNS_FATAL_TIMEOUT}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} set dns:fatal-timeout ${INPUT_DNS_FATAL_TIMEOUT};" +else + FTP_SETTINGS="${FTP_SETTINGS} set dns:fatal-timeout 10s;" +fi + +# Any manual settings +if [ -n "${INPUT_LFTP_SETTINGS}" ]; then + FTP_SETTINGS="${FTP_SETTINGS} ${INPUT_LFTP_SETTINGS};" +fi +# Remove first space in settings variable +if [ -n "${FTP_SETTINGS}" ]; then + FTP_SETTINGS="${FTP_SETTINGS#"${FTP_SETTINGS%%[![:space:]]*}"}" +fi + +# Max number of retries if [ -z "${INPUT_MAX_RETRIES}" ]; then INPUT_MAX_RETRIES="10" fi +# Local path to get the directories if [ -z "${INPUT_LOCAL_DIR}" ]; then INPUT_LOCAL_DIR="./" else - if [ "${INPUT_LOCAL_DIR}" != "./" ]; then - INPUT_LOCAL_DIR="${INPUT_LOCAL_DIR}/" - fi + INPUT_LOCAL_DIR="${INPUT_LOCAL_DIR}/" fi +# Remote path to put the directories if [ -z "${INPUT_REMOTE_DIR}" ]; then INPUT_REMOTE_DIR="./" else - if [ "${INPUT_REMOTE_DIR}" != "./" ]; then - INPUT_REMOTE_DIR="${INPUT_REMOTE_DIR}/" - fi + INPUT_REMOTE_DIR="${INPUT_REMOTE_DIR}/" +fi + +# Reverse mirror which uploads or updates a directory tree on server +MIRROR_COMMAND="mirror --continue --reverse" + +# Mirror verbosity level +if [ -n "${INPUT_MIRROR_VERBOSE}" ]; then + MIRROR_COMMAND="${MIRROR_COMMAND} --verbose=${INPUT_MIRROR_VERBOSE}" +else + MIRROR_COMMAND="${MIRROR_COMMAND} --verbose=1" fi +# Don't create symbolic links if [ "${INPUT_NO_SYMLINKS}" = "true" ]; then MIRROR_COMMAND="${MIRROR_COMMAND} --no-symlinks" fi +# Delete files not present at the source if [ "${INPUT_DELETE}" = "true" ]; then MIRROR_COMMAND="${MIRROR_COMMAND} --delete" fi +# ------------------------------------------------------------------------------ +# Display LFTP settings. +# ------------------------------------------------------------------------------ echo "=== Directories ===" echo "INPUT_LOCAL_DIR: ${INPUT_LOCAL_DIR}" echo "INPUT_REMOTE_DIR: ${INPUT_REMOTE_DIR}" echo "" -echo "=== List local directory | ${INPUT_LOCAL_DIR} ===" +echo "=== List local directory ===" +echo "${INPUT_LOCAL_DIR}" ls -lha "${INPUT_LOCAL_DIR}" echo "" -echo "=== LFTP ===" -echo "lftp" +echo "=== LFTP Settings ===" echo " FTP_SETTINGS -> ${FTP_SETTINGS}" echo " MIRROR_COMMAND -> ${MIRROR_COMMAND}" echo " INPUT_LOCAL_DIR -> ${INPUT_LOCAL_DIR}" @@ -79,18 +157,20 @@ echo " INPUT_MAX_RETRIES -> ${INPUT_MAX_RETRIES}" echo "" echo "=== * NOTE * ===" echo "The upload should be fast depends how many files and what size they have." -echo "If the process take for several minutes, please stop the job and run it again." -echo "" +echo "If the process take for several minutes or hours, please stop the job and run it again." +# ------------------------------------------------------------------------------ +# Execute the LFTP actions. +# ------------------------------------------------------------------------------ COUNTER=1 SUCCESS="" -until [ ${COUNTER} -gt ${INPUT_MAX_RETRIES} ]; do +while true; do echo "" - echo "Try #: ${COUNTER}" - echo "# ---------------------------------------------" + echo "Try #${COUNTER}" + echo "-------" + lftp \ - --debug \ -u "${INPUT_USER}","${INPUT_PASSWORD}" \ "${INPUT_SERVER}" \ -e "${FTP_SETTINGS} ${MIRROR_COMMAND} ${INPUT_LOCAL_DIR} ${INPUT_REMOTE_DIR}; quit;" && @@ -100,10 +180,17 @@ until [ ${COUNTER} -gt ${INPUT_MAX_RETRIES} ]; do break fi - sleep 1m COUNTER=$((COUNTER + 1)) + if [ ${COUNTER} -gt ${INPUT_MAX_RETRIES} ]; then + break + fi + + sleep 1m done +# ------------------------------------------------------------------------------ +# Display the status of the LFTP actions. +# ------------------------------------------------------------------------------ if [ -z "${SUCCESS}" ]; then echo "" echo "=============================="