Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Miscellaneous tiny shell script tweaks #7080

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions check/all
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,19 @@ thisdir="$(dirname "${BASH_SOURCE[0]}")" || exit $?
topdir="$(git -C "${thisdir}" rev-parse --show-toplevel)" || exit $?
cd "${topdir}" || exit $?

errors=()
declare -a errors

# Parse arguments.
apply_arg=( )
only_changed=0
rev=( )
declare -a apply_arg
declare -a rev
declare only_changed=0
for arg in "$@"; do
if [[ "${arg}" == "--only-changed-files" ]]; then
only_changed=1
elif [[ "${arg}" == "--apply-format-changes" ]]; then
apply_arg=( "--apply" )
elif [[ "${#rev[@]}" == 0 ]]; then
if [ "$(git cat-file -t "${arg}" 2> /dev/null)" != "commit" ]; then
if [[ "$(git cat-file -t "${arg}" 2> /dev/null)" != "commit" ]]; then
echo -e "\033[31mNo revision '${arg}'.\033[0m" >&2
exit 1
fi
Expand All @@ -61,7 +61,7 @@ done
echo "Running misc"
check/misc || errors+=( "check/misc failed" )

if [ ${only_changed} -ne 0 ]; then
if [[ ${only_changed} -ne 0 ]]; then
echo "Running incremental pylint"
check/pylint-changed-files || errors+=( "check/pylint-changed-files failed" )
else
Expand All @@ -76,7 +76,7 @@ echo "Running incremental format"
check/format-incremental "${rev[@]}" "${apply_arg[@]}" ||
errors+=( "check/format-incremental failed" )

if [ ${only_changed} -ne 0 ]; then
if [[ ${only_changed} -ne 0 ]]; then
echo "Running pytest and incremental coverage on changed files"
check/pytest-changed-files-and-incremental-coverage "${rev[@]}" ||
errors+=( "check/pytest-changed-files-and-incremental-coverage failed" )
Expand All @@ -89,6 +89,7 @@ fi
echo "Running doctest"
check/doctest || errors+=( "check/doctest failed" )

declare -i result
echo
if [[ "${#errors[@]}" == 0 ]]; then
echo "All checks passed."
Expand Down
24 changes: 12 additions & 12 deletions check/format-incremental
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,16 @@ thisdir="$(dirname "${BASH_SOURCE[0]}")" || exit $?
topdir="$(git -C "${thisdir}" rev-parse --show-toplevel)" || exit $?
cd "${topdir}" || exit $?


# Parse arguments.
only_print=1
only_changed=1
rev=""
declare -i only_print=1
declare -i only_changed=1
declare rev=""
for arg in "$@"; do
if [[ "${arg}" == "--apply" ]]; then
only_print=0
elif [[ "${arg}" == "--all" ]]; then
only_changed=0
elif [ -z "${rev}" ]; then
elif [[ -z "${rev}" ]]; then
if ! git rev-parse --verify --quiet --no-revs "${arg}^{commit}"; then
echo -e "\033[31mNo revision '${arg}'.\033[0m" >&2
exit 1
Expand All @@ -57,23 +56,23 @@ for arg in "$@"; do
fi
done

typeset -a format_files
declare -a format_files
if (( only_changed == 1 )); then
# Figure out which branch to compare against.
if [ -z "${rev}" ]; then
if [ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]; then
if [[ -z "${rev}" ]]; then
if [[ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]]; then
rev=upstream/main
elif [ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]]; then
rev=origin/main
elif [ "$(git cat-file -t main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t main 2> /dev/null)" == "commit" ]]; then
rev=main
else
echo -e "\033[31mNo default revision found to compare against. Argument #1 must be what to diff against (e.g. 'origin/main' or 'HEAD~1').\033[0m" >&2
exit 1
fi
fi
base="$(git merge-base "${rev}" HEAD)"
if [ "$(git rev-parse "${rev}")" == "${base}" ]; then
if [[ "$(git rev-parse "${rev}")" == "${base}" ]]; then
echo -e "Comparing against revision '${rev}'." >&2
else
echo -e "Comparing against revision '${rev}' (merge base ${base})." >&2
Expand All @@ -94,11 +93,12 @@ if (( ${#format_files[@]} == 0 )); then
exit 0
fi

declare BLACKVERSION
BLACKVERSION="$(black --version)"

echo "Running the black formatter... (version: $BLACKVERSION)"

args=("--color")
declare -a args=("--color")
if (( only_print == 1 )); then
args+=("--check" "--diff")
fi
Expand Down
2 changes: 1 addition & 1 deletion check/misc
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ cd "${topdir}" || exit $?
# Check for non-contrib references to contrib.
results=$(grep -Rl "\bcirq\.contrib\b" cirq-core | grep -v "cirq/contrib" | grep -v "__")
RESULT=$?
if [ $RESULT -eq 0 ]; then
if [[ $RESULT -eq 0 ]]; then
echo -e "\033[31m'cirq.contrib' mentioned in non-contrib files:\033[0m"
echo "${results}"
echo
Expand Down
2 changes: 1 addition & 1 deletion check/mypy
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ read -r -a CIRQ_PACKAGES < \

echo -e -n "\033[31m"
mypy --config-file=dev_tools/conf/$CONFIG_FILE "$@" "${CIRQ_PACKAGES[@]}" dev_tools examples
result=$?
declare -i result=$?
echo -e -n "\033[0m"

exit ${result}
4 changes: 2 additions & 2 deletions check/nbformat
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
#
################################################################################

only_print=1
declare -i only_print=1
for arg in "$@"; do
if [[ "${arg}" == "--apply" ]]; then
only_print=0
Expand All @@ -39,7 +39,7 @@ my_nbfmt() {
python3 -m tensorflow_docs.tools.nbfmt --indent=1 "$@"
}

exit_code=0
declare -i exit_code=0

# Check if notebooks are already formatted and store output for later use
if output=$(my_nbfmt --test docs 2>&1); then
Expand Down
17 changes: 9 additions & 8 deletions check/pylint-changed-files
Original file line number Diff line number Diff line change
Expand Up @@ -30,41 +30,42 @@ topdir="$(git -C "${thisdir}" rev-parse --show-toplevel)" || exit $?
cd "${topdir}" || exit $?

# Figure out which revision to compare against.
if [ -n "$1" ] && [[ $1 != -* ]]; then
if [[ -n "$1" && $1 != -* ]]; then
if ! git rev-parse --verify --quiet --no-revs "$1^{commit}"; then
echo -e "\033[31mNo revision '$1'.\033[0m" >&2
exit 1
fi
rev=$1
elif [ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]]; then
rev=upstream/main
elif [ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]]; then
rev=origin/main
elif [ "$(git cat-file -t main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t main 2> /dev/null)" == "commit" ]]; then
rev=main
else
echo -e "\033[31mNo default revision found to compare against. Argument #1 must be what to diff against (e.g. 'origin/main' or 'HEAD~1').\033[0m" >&2
exit 1
fi
declare base
base="$(git merge-base "${rev}" HEAD)"
if [ "$(git rev-parse "${rev}")" == "${base}" ]; then
if [[ "$(git rev-parse "${rev}")" == "${base}" ]]; then
echo -e "Comparing against revision '${rev}'." >&2
else
echo -e "Comparing against revision '${rev}' (merge base ${base})." >&2
rev="${base}"
fi

typeset -a changed
declare -a changed
IFS=$'\n' read -r -d '' -a changed < \
<(git diff --name-only "${rev}" -- '*.py' ':(exclude)cirq-google/cirq_google/cloud/*' ':(exclude)*_pb2.py' \
| grep -E "^(cirq|dev_tools|examples).*.py$"
)

num_changed=${#changed[@]}
declare -i num_changed=${#changed[@]}

# Run it.
echo "Found ${num_changed} lintable files associated with changes." >&2
if [ "${num_changed}" -eq 0 ]; then
if [[ "${num_changed}" -eq 0 ]]; then
exit 0
fi
env PYTHONPATH=dev_tools pylint --jobs=0 --rcfile=dev_tools/conf/.pylintrc "${changed[@]}"
6 changes: 3 additions & 3 deletions check/pytest
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ cd "${topdir}" || exit $?

# Run in parallel by default. Pass the `-n0` option for a single-process run.
# (the last `-n` option wins)
PYTEST_ARGS=( "-n=auto" "$@" )
declare -a PYTEST_ARGS=( "-n=auto" "$@" )

source dev_tools/pypath || exit $?

pytest "${PYTEST_ARGS[@]}"
RESULT=$?
declare -i RESULT=$?

if [ "$RESULT" = 5 ]; then
if [[ "$RESULT" = 5 ]]; then
echo "[exit 5] No tests collected, but ignored."
exit 0
fi
Expand Down
39 changes: 20 additions & 19 deletions check/pytest-and-incremental-coverage
Original file line number Diff line number Diff line change
Expand Up @@ -26,42 +26,44 @@
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
cd "$(git rev-parse --show-toplevel)" || exit 1

PYTEST_ARGS=()
ANALYZE_COV=1
BASEREV=""
declare -a pytest_args
declare -i analyze_cov=1
declare baserev=""

if [[ "$1" == [^-]* ]]; then
BASEREV="$1"
baserev="$1"
shift
fi

for arg in "$@"; do
if [[ "${arg}" == "--no-analyze" ]]; then
ANALYZE_COV=0
analyze_cov=0
else
PYTEST_ARGS+=("${arg}")
pytest_args+=("${arg}")
fi
done

# Figure out which revision to compare against.
if [ -n "${BASEREV}" ]; then
if ! git rev-parse --verify --quiet --no-revs "${BASEREV}^{commit}"; then
echo -e "\033[31mNo revision '${BASEREV}'.\033[0m" >&2
declare rev=""
if [[ -n "${baserev}" ]]; then
if ! git rev-parse --verify --quiet --no-revs "${baserev}^{commit}"; then
echo -e "\033[31mNo revision '${baserev}'.\033[0m" >&2
exit 1
fi
rev="${BASEREV}"
elif [ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]; then
rev="${baserev}"
elif [[ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]]; then
rev=upstream/main
elif [ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]]; then
rev=origin/main
elif [ "$(git cat-file -t main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t main 2> /dev/null)" == "commit" ]]; then
rev=main
else
echo -e "\033[31mNo default revision found to compare against. Argument #1 must be what to diff against (e.g. 'origin/main' or 'HEAD~1').\033[0m" >&2
exit 1
fi
declare base
base="$(git merge-base "${rev}" HEAD)"
if [ "$(git rev-parse "${rev}")" == "${base}" ]; then
if [[ "$(git rev-parse "${rev}")" == "${base}" ]]; then
echo -e "Comparing against revision '${rev}'." >&2
else
echo -e "Comparing against revision '${rev}' (merge base ${base})." >&2
Expand All @@ -71,13 +73,12 @@ fi
source dev_tools/pypath || exit $?

# Run tests while producing coverage files.
check/pytest --cov "${PYTEST_ARGS[@]}"
pytest_result=$?

check/pytest --cov "${pytest_args[@]}"
declare -i pytest_result=$?

# assume successful cover_result in case coverage is not run
cover_result=0
if (( ANALYZE_COV )); then
declare -i cover_result=0
if (( analyze_cov )); then
# Convert to .py,cover files.
coverage annotate

Expand Down
17 changes: 9 additions & 8 deletions check/pytest-changed-files
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,20 @@ topdir="$(git -C "${thisdir}" rev-parse --show-toplevel)" || exit $?
cd "${topdir}" || exit $?

# Figure out which branch to compare against.
rest=( "$@" )
if [ -n "$1" ] && [[ $1 != -* ]]; then
declare -a rest=( "$@" )
declare rev=""
if [[ -n "$1" && $1 != -* ]]; then
if ! git rev-parse --verify --quiet --no-revs "$1^{commit}"; then
echo -e "\033[31mNo revision '$1'.\033[0m" >&2
exit 1
fi
rev=$1
rest=( "${@:2}" )
elif [ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t upstream/main 2> /dev/null)" == "commit" ]]; then
rev=upstream/main
elif [ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t origin/main 2> /dev/null)" == "commit" ]]; then
rev=origin/main
elif [ "$(git cat-file -t main 2> /dev/null)" == "commit" ]; then
elif [[ "$(git cat-file -t main 2> /dev/null)" == "commit" ]]; then
rev=main
else
echo -e "\033[31mNo default revision found to compare against. Argument #1 must be what to diff against (e.g. 'origin/main' or 'HEAD~1').\033[0m" >&2
Expand All @@ -54,7 +55,7 @@ fi
echo "Comparing against revision '${rev}'." >&2

# Get the _test version of changed python files.
typeset -a changed
declare -a changed
IFS=$'\n' read -r -d '' -a changed < \
<(git diff --name-only "${rev}" -- \
| grep "\.py$" \
Expand All @@ -68,11 +69,11 @@ if git diff --name-only "${rev}" -- | grep "__init__\.py$" > /dev/null; then
# Include global API tests when an __init__ file is touched.
changed+=('cirq-core/cirq/protocols/json_serialization_test.py')
fi
num_changed=${#changed[@]}
declare -i num_changed=${#changed[@]}

# Run it.
echo "Found ${num_changed} test files associated with changes." >&2
if [ "${num_changed}" -eq 0 ]; then
if [[ "${num_changed}" -eq 0 ]]; then
exit 0
fi

Expand Down
Loading