summaryrefslogtreecommitdiff
path: root/test/script/run-vader-tests
blob: 15a4a91eb6b93a9bcc911338327fc5c0b99e691d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
#!/usr/bin/env bash

set -e
set -u

docker_flags=(--rm -v "$PWD:/testplugin" -v "$PWD/test:/home" -w /testplugin "$DOCKER_RUN_IMAGE")
red='\033[0;31m'
green='\033[0;32m'
nc='\033[0m'
verbose=0
quiet=0

while [ $# -ne 0 ]; do
    case $1 in
    -v)
        verbose=1
        shift
    ;;
    -q)
        quiet=1
        shift
    ;;
    --)
        shift
        break
    ;;
    -?*)
        echo "Invalid argument: $1" 1>&2
        exit 1
    ;;
    *)
        break
    ;;
    esac
done

vim="$1"
tests="$2"

echo "$vim"

case $vim in
    neovim-v0.2*)
        headless=''
    ;;
    # Neovim 0.6+ requires headless argument to load Vader tests.
    neovim*)
        headless='--headless'
    ;;
    *)
        headless=''
    ;;
esac

# This file will be used to track if tests ran or not.
# We can't use a variable, because we need to set a value in a sub-shell.
run_file="$(mktemp -t tests_ran.XXXXXXXX)"

function filter-vader-output() {
    local hit_first_vader_line=0
    # When verbose mode is off, suppress output until Vader starts.
    local start_output="$verbose"
    local filtered_data=''

    while read -r; do
        # Search for the first Vader output line.
        # We can try starting tests again if they don't start.
        if ((!hit_first_vader_line)); then
            if [[ "$REPLY" = *'Starting Vader:'* ]]; then
                hit_first_vader_line=1
            fi
        fi

        if ((!start_output)); then
            if ((hit_first_vader_line)); then
                start_output=1
            else
                continue
            fi
        fi

        if ((quiet)); then
            if [[ "$REPLY" = *'Starting Vader:'* ]]; then
                filtered_data="$REPLY"
            elif [[ "$REPLY" = *'Success/Total'* ]]; then
                success="$(echo -n "$REPLY" | grep -o '[0-9]\+/' | head -n1 | cut -d/ -f1)"
                total="$(echo -n "$REPLY" | grep -o '/[0-9]\+' | head -n1 | cut -d/ -f2)"

                if [ "$success" -lt "$total" ]; then
                    echo "$filtered_data"
                    echo "$REPLY"
                fi

                filtered_data=''
            else
                filtered_data="$filtered_data"$'\n'"$REPLY"
            fi
        else
            echo "$REPLY"
        fi
    done

    # Note that we managed to get the Vader tests started if we did.
    if ((hit_first_vader_line)); then
        echo 1 > "$run_file"
    fi
}

function color-vader-output() {
    while read -r; do
        if [[ "$REPLY" = *'[EXECUTE] (X)'* ]]; then
            echo -en "$red"
        elif [[ "$REPLY" = *'[EXECUTE]'* ]] || [[ "$REPLY" = *'[  GIVEN]'* ]]; then
            echo -en "$nc"
        fi

        if [[ "$REPLY" = *'Success/Total'* ]]; then
            success="$(echo -n "$REPLY" | grep -o '[0-9]\+/' | head -n1 | cut -d/ -f1)"
            total="$(echo -n "$REPLY" | grep -o '/[0-9]\+' | head -n1 | cut -d/ -f2)"

            if [ "$success" -lt "$total" ]; then
                echo -en "$red"
            else
                echo -en "$green"
            fi

            echo "$REPLY"
            echo -en "$nc"
        else
            echo "$REPLY"
        fi
    done

    echo -en "$nc"
}

echo
echo '========================================'
echo "Running tests for $vim"
echo '========================================'
echo

tries=0

while [ "$tries" -lt 5 ]; do
    tries=$((tries + 1))

    exit_code=0
    set -o pipefail
    docker run -a stderr -e VADER_OUTPUT_FILE=/dev/stderr "${docker_flags[@]}" \
        "/vim-build/bin/$vim" -u test/vimrc ${headless} \
        "+Vader! $tests" 2>&1 | filter-vader-output | color-vader-output || exit_code=$?
    set +o pipefail

    if [ -s "$run_file" ]; then
        break
    fi
done

if [ "$tries" -gt 1 ]; then
    echo
    echo "Tried to run tests $tries times"
fi

rm "$run_file"

exit "$exit_code"