#!/usr/bin/env bash usage () { cat >&2 <] [FILES] [--ignore FILES] Where can be: * --update-ocamlformat: update all the \`.ocamlformat\` files and git-commit (requires clean repo). * --check-ocamlformat: check the above does nothing. * --check-gitlab-ci-yml: check .gitlab-ci.yml has been updated. * --check-scripts: check the .sh files * --check-redirects: check docs/_build/_redirects. * --check-coq-attributes: check the presence of coq attributes. * --check-rust-toolchain: check the contents of rust-toolchain files * --check-licenses-git-new: check license headers of added OCaml .ml(i) files. * --help: display this and return 0. EOF } ## Testing for dependencies if ! type find > /dev/null 2>&-; then echo "find is required but could not be found. Aborting." exit 1 fi set -e say () { echo "$*" >&2 } declare -a source_directories source_directories=(src docs/doc_gen tezt devtools contrib etherlink) # Set of newline-separated basic regular expressions to exclude from --check-licenses-git-new. license_check_exclude=$( cat <<'EOF' EOF ) update_all_dot_ocamlformats () { if ! type ocamlformat > /dev/null 2>&-; then echo "ocamlformat is required but could not be found. Aborting." exit 1 fi if git diff --name-only HEAD --exit-code then say "Repository clean :thumbsup:" else say "Repository not clean, which is required by this script." exit 2 fi find "${source_directories[@]}" -name ".ocamlformat" -exec git rm {} \; # ocamlformat uses [.git], [.hg], and [dune-project] witness files # to determine the project root and will not cross that boundary # when computing its config. This means that we need a # '.ocamlformat' config file next to each dune-project in order to # cover all source code in the tree. interesting_directories=$(find "${source_directories[@]}" -name "dune-project" -type f -print | sed 's:/[^/]*$::' | LC_COLLATE=C sort -u) for d in $interesting_directories ; do ofmt=$d/.ocamlformat cp .ocamlformat "$ofmt" git add "$ofmt" done # we don't want to reformat protocols (but alpha) because it would alter its hash protocols=$(find src/ -maxdepth 1 -name "proto_*" -not -name "proto_alpha") for d in $protocols ; do ( cd "$d/lib_protocol" && (find ./ -maxdepth 1 -name "*.ml*" | sed 's:^./::' | LC_COLLATE=C sort > ".ocamlformat-ignore" ) ) git add "$d/lib_protocol/.ocamlformat-ignore" done } function shellcheck_script () { shellcheck --external-sources "$1" } check_scripts () { # Gather scripts scripts=$(find "${source_directories[@]}" scripts/ docs/ -name "*.sh" -type f -print) exit_code=0 # Check scripts do not contain the tab character tab="$(printf '%b' '\t')" for f in $scripts ; do if grep -q "$tab" "$f"; then say "$f has tab character(s)" exit_code=1 fi done # Execute shellcheck ./scripts/shellcheck_version.sh || return 1 # Check shellcheck's version shellcheck_skips="" while read -r shellcheck_skip; do shellcheck_skips+=" $shellcheck_skip" done < "scripts/shellcheck_skips" for script in ${scripts}; do if [[ "${shellcheck_skips}" == *"${script}"* ]]; then # check whether the skipped script, in reality, is warning-free if shellcheck_script "${script}" > /dev/null; then say "$script shellcheck marked as SKIPPED but actually pass: update shellcheck_skips ❌️" exit_code=1 else # script is skipped, we leave a log however, to incite # devs to enhance the scripts say "$script shellcheck SKIPPED ⚠️" fi else # script is not skipped, let's shellcheck it if shellcheck_script "${script}"; then say "$script shellcheck PASSED ✅" else say "$script shellcheck FAILED ❌" exit_code=1 fi fi done # Check that shellcheck_skips doesn't contain a deprecated value for shellcheck_skip in ${shellcheck_skips}; do if [[ ! -e "${shellcheck_skip}" ]]; then say "$shellcheck_skip is mentioned in shellcheck_skips, but doesn't exist anymore" say "please delete it from shellcheck_skips" exit_code=1 fi done # Done executing shellcheck exit $exit_code } check_redirects () { if [[ ! -f docs/_build/_redirects ]]; then say "check-redirects should be run after building the full documentation," say "i.e. by running 'make all && make -C docs all'" exit 1 fi exit_code=0 while read -r old new code; do re='^[0-9]+$' if ! [[ $code =~ $re && $code -ge 300 ]] ; then say "in docs/_redirects: redirect $old -> $new has erroneous status code \"$code\"" exit_code=1 fi dest_local=docs/_build${new} if [[ ! -f $dest_local ]]; then say "in docs/_redirects: redirect $old -> $new, $dest_local does not exist" exit_code=1 fi done < docs/_build/_redirects exit $exit_code } check_rust_toolchain_files () { authorized_version=("1.66.0" "1.71.1" "1.73.0") declare -a rust_toolchain_files mapfile -t rust_toolchain_files <<< "$(find src/ -name rust-toolchain)" for file in "${rust_toolchain_files[@]}"; do if [[ ! "${authorized_version[*]}" =~ $(cat "${file}") ]]; then say "in ${file}: version $(cat "${file}") is not authorized" exit 1 fi done } check_gitlab_ci_yml () { # Check that a rule is not defined twice, which would result in the first # one being ignored. Gitlab linter doesn't warn for it. find .gitlab-ci.yml .gitlab/ci/ -iname \*.yml | \ while read -r filename; do repeated=$(grep '^[^ #-]' "$filename" \ | sort \ | grep -v include \ | uniq --repeated) if [ -n "$repeated" ]; then echo "$filename contains repeated rules:" echo "$repeated" touch /tmp/repeated fi done if [ -f /tmp/repeated ]; then rm /tmp/repeated exit 1 fi } check_licenses_git_new () { if [ -z "${CHECK_LICENSES_DIFF_BASE:-}" ]; then echo 'Action --check-licenses-git-new requires that CHECK_LICENSES_DIFF_BASE is set in the environment.' echo 'The value of CHECK_LICENSES_DIFF_BASE should be a commit.' echo 'It is used to discover files that have been added and whose license header should be checked.' echo echo "Typically, it should point to the merge base of the current branch, as given by \`git merge-base\`:" echo echo " CHECK_LICENSES_DIFF_BASE=\$(git merge-base HEAD origin/master) $0 --check-licenses-git-new" return 1 elif ! git cat-file -t "${CHECK_LICENSES_DIFF_BASE:-}" > /dev/null 2>&1; then echo "The commit specified in CHECK_LICENSES_DIFF_BASE ('$CHECK_LICENSES_DIFF_BASE') could not be found." echo 'Consider running:' echo echo " git fetch origin $CHECK_LICENSES_DIFF_BASE" return 1 fi diff=$(mktemp) git diff-tree --no-commit-id --name-only -r --diff-filter=A \ "${CHECK_LICENSES_DIFF_BASE:-}" HEAD -- "${source_directories[@]}" > "$diff" if [ -n "$license_check_exclude" ]; then diff2=$(mktemp) grep -v "$license_check_exclude" "$diff" > "$diff2" mv "$diff2" "$diff" fi # Check that new ml(i) files have a valid license header. if ! grep '\.mli\?$' "$diff" | xargs --no-run-if-empty ocaml scripts/check_license/main.ml --verbose; then echo "/!\\ Some files .ml(i) does not have a correct license header /!\\" ; echo "/!\\ See https://tezos.gitlab.io/developer/guidelines.html#license /!\\" ; res=1 else echo "OCaml file license headers OK!" res=0 fi rm -f "$diff" return $res } if [ $# -eq 0 ] || [[ "$1" != --* ]]; then say "provide one action (see --help)" exit 1 else action="$1" shift fi check_clean=false commit= on_files=false case "$action" in "--update-ocamlformat" ) action=update_all_dot_ocamlformats commit="Update .ocamlformat files" ;; "--check-ocamlformat" ) action=update_all_dot_ocamlformats check_clean=true ;; "--check-gitlab-ci-yml" ) action=check_gitlab_ci_yml ;; "--check-scripts" ) action=check_scripts ;; "--check-redirects" ) action=check_redirects ;; "--check-rust-toolchain" ) action=check_rust_toolchain_files ;; "--check-licenses-git-new" ) action=check_licenses_git_new ;; "help" | "-help" | "--help" | "-h" ) usage exit 0 ;; * ) say "Error no action (arg 1 = '$action') provided" usage exit 2 ;; esac if $on_files; then declare -a input_files files ignored_files input_files=() while [ $# -gt 0 ]; do if [ "$1" = "--ignore" ]; then shift break fi input_files+=("$1") shift done if [ ${#input_files[@]} -eq 0 ]; then mapfile -t input_files <<< "$(find "${source_directories[@]}" \( -name "*.ml" -o -name "*.mli" -o -name "*.mlt" \) -type f -print)" fi ignored_files=("$@") # $input_files may contain `*.pp.ml{i}` files which can't be linted. They # are filtered by the following loop. # # Note: another option would be to filter them before calling the script # but it was more convenient to do it here. files=() for file in "${input_files[@]}"; do if [[ "$file" == *.pp.ml?(i) ]]; then continue; fi for ignored_file in "${ignored_files[@]}"; do if [[ "$file" =~ ^(.*/)?"$ignored_file"$ ]] ; then continue 2; fi done files+=("$file") done $action "${files[@]}" else if [ $# -gt 0 ]; then usage; exit 1; fi $action fi if [ -n "$commit" ]; then git commit -m "$commit" fi if $check_clean; then echo "Files that differ but that shouldn't:" git diff --name-only HEAD --exit-code echo "(none, everything looks good)" fi