#2132 - lint and fix with ale#command#Run

A new function is added here which will later be modified for public use
in linter and fixer callbacks. All linting and fixing now goes through
this new function, to prove that it works in all cases.

18 files changed, 179 insertions, 52 deletions

diff --git a/test/fix/test_ale_fix.vader b/test/fix/test_ale_fix.vader
index 1b1891ce..500eb71c 100644
--- a/test/fix/test_ale_fix.vader
+++ b/test/fix/test_ale_fix.vader
@@ -11,6 +11,7 @@ Before:
   let g:ale_enabled = 0
   let g:ale_echo_cursor = 0
   let g:ale_run_synchronously = 1
+  unlet! g:ale_run_synchronously_callbacks
   let g:ale_set_lists_synchronously = 1
   let g:ale_fix_buffer_data = {}
   let g:ale_fixers = {
@@ -59,6 +60,10 @@ Before:
     return {'command': 'echo x > %t', 'read_temporary_file': 1}
   endfunction
 
+  function CatWithTempFile(buffer, done, lines) abort
+    return {'command': 'cat %t <(echo d)'}
+  endfunction
+
   function RemoveLastLine(buffer, done, lines) abort
     return ['a', 'b']
   endfunction
@@ -177,6 +182,7 @@ After:
   Restore
   unlet! g:ale_run_synchronously
   unlet! g:ale_set_lists_synchronously
+  unlet! g:ale_run_synchronously_callbacks
   unlet! g:ale_emulate_job_failure
   unlet! b:ale_fixers
   unlet! b:ale_fix_on_save
@@ -187,6 +193,7 @@ After:
   delfunction CatLine
   delfunction CatLineOneArg
   delfunction ReplaceWithTempFile
+  delfunction CatWithTempFile
   delfunction RemoveLastLine
   delfunction RemoveLastLineOneArg
   delfunction TestCallback
@@ -235,11 +242,13 @@ Given testft (A file with three lines):
 
 Execute(ALEFix should complain when there are no functions to call):
   ALEFix
+  call ale#test#FlushJobs()
   AssertEqual 'No fixers have been defined. Try :ALEFixSuggest', GetLastMessage()
 
 Execute(ALEFix should apply simple functions):
   let g:ale_fixers.testft = ['AddCarets']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The first function should be used):
   ^a
@@ -249,6 +258,7 @@ Expect(The first function should be used):
 Execute(ALEFix should apply simple functions in a chain):
   let g:ale_fixers.testft = ['AddCarets', 'AddDollars']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(Both functions should be used):
   $^a
@@ -258,6 +268,7 @@ Expect(Both functions should be used):
 Execute(ALEFix should allow 0 to be returned to skip functions):
   let g:ale_fixers.testft = ['DoNothing', 'AddDollars']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(Only the second function should be applied):
   $a
@@ -268,6 +279,7 @@ Execute(The * fixers shouldn't be used if an empty list is set for fixers):
   let g:ale_fixers.testft = []
   let g:ale_fixers['*'] = ['AddDollars']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(Nothing should be changed):
   a
@@ -277,6 +289,7 @@ Expect(Nothing should be changed):
 Execute(* fixers should be used if no filetype is matched):
   let g:ale_fixers = {'*': ['AddDollars']}
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The file should be changed):
   $a
@@ -290,6 +303,7 @@ Execute(ALEFix should allow commands to be run):
   else
     let g:ale_fixers.testft = ['CatLine']
     ALEFix
+    call ale#test#FlushJobs()
   endif
 
 Expect(An extra line should be added):
@@ -301,6 +315,7 @@ Expect(An extra line should be added):
 Execute(ALEFix should use fixers passed in commandline when provided):
   let g:ale_fixers.testft = ['RemoveLastLine']
   ALEFix AddCarets AddDollars
+  call ale#test#FlushJobs()
 
 Expect(Only fixers passed via command line should be run):
   $^a
@@ -315,11 +330,28 @@ Execute(ALEFix should allow temporary files to be read):
   else
     let g:ale_fixers.testft = ['ReplaceWithTempFile']
     ALEFix
+    call ale#test#FlushJobs()
   endif
 
 Expect(The line we wrote to the temporary file should be used here):
   x
 
+Execute(ALEFix should not read the temporary file when the option is not set):
+  if has('win32')
+    " Just skip this test on Windows, we can't run it.
+    call setline(1, ['a', 'b', 'c', 'd'])
+  else
+    let g:ale_fixers.testft = ['CatWithTempFile']
+    ALEFix
+    call ale#test#FlushJobs()
+  endif
+
+Expect(An extra line should be added):
+  a
+  b
+  c
+  d
+
 Execute(ALEFix should allow jobs and simple functions to be combined):
   if has('win32')
     " Just skip this test on Windows, we can't run it.
@@ -328,6 +360,7 @@ Execute(ALEFix should allow jobs and simple functions to be combined):
   else
     let g:ale_fixers.testft = ['ReplaceWithTempFile', 'AddDollars']
     ALEFix
+    call ale#test#FlushJobs()
   endif
 
 Expect(The lines from the temporary file should be modified):
@@ -340,6 +373,7 @@ Execute(ALEFix should send lines modified by functions to jobs):
   else
     let g:ale_fixers.testft = ['AddDollars', 'CatLine']
     ALEFix
+    call ale#test#FlushJobs()
   endif
 
 Expect(The lines should first be modified by the function, then the job):
@@ -352,6 +386,7 @@ Execute(ALEFix should skip commands when jobs fail to run):
   let g:ale_emulate_job_failure = 1
   let g:ale_fixers.testft = ['CatLine', 'AddDollars']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(Only the second function should be applied):
   $a
@@ -361,6 +396,7 @@ Expect(Only the second function should be applied):
 Execute(ALEFix should handle strings for selecting a single function):
   let g:ale_fixers.testft = 'AddCarets'
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The first function should be used):
   ^a
@@ -371,6 +407,7 @@ Execute(ALEFix should use functions from the registry):
   call ale#fix#registry#Add('add_carets', 'AddCarets', [], 'Add some carets')
   let g:ale_fixers.testft = ['add_carets']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The registry function should be used):
   ^a
@@ -380,6 +417,7 @@ Expect(The registry function should be used):
 Execute(ALEFix should be able to remove the last line for files):
   let g:ale_fixers.testft = ['RemoveLastLine']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(There should be only two lines):
   a
@@ -388,6 +426,7 @@ Expect(There should be only two lines):
 Execute(ALEFix should accept funcrefs):
   let g:ale_fixers.testft = [function('RemoveLastLine')]
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(There should be only two lines):
   a
@@ -401,6 +440,7 @@ Execute(ALEFix should accept lambdas):
   else
     let g:ale_fixers.testft = [{buffer, done, lines -> lines + ['d']}]
     ALEFix
+    call ale#test#FlushJobs()
   endif
 
 Expect(There should be an extra line):
@@ -413,6 +453,7 @@ Execute(ALEFix should user buffer-local fixer settings):
   let g:ale_fixers.testft = ['AddCarets', 'AddDollars']
   let b:ale_fixers = {'testft': ['RemoveLastLine']}
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(There should be only two lines):
   a
@@ -422,6 +463,7 @@ Execute(ALEFix should allow Lists to be used for buffer-local fixer settings):
   let g:ale_fixers.testft = ['AddCarets', 'AddDollars']
   let b:ale_fixers = ['RemoveLastLine']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(There should be only two lines):
   a
@@ -447,6 +489,7 @@ Execute(ALEFix should fix files on the save event):
 
   call SetUpLinters()
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   " We should save the file.
   AssertEqual ['$a', '$b', '$c'], readfile('fix_test_file')
@@ -518,6 +561,7 @@ Execute(ALEFix should still lint with no linters to be applied):
 
   call SetUpLinters()
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   Assert !filereadable('fix_test_file'), 'The file should not have been saved'
 
@@ -552,6 +596,7 @@ Execute(ALEFix should still lint when nothing was fixed on save):
 
   call SetUpLinters()
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   Assert !filereadable('fix_test_file'), 'The file should not have been saved'
 
@@ -597,6 +642,7 @@ Execute(ale#fix#InitBufferData() should set up the correct data):
 Execute(ALEFix simple functions should be able to accept one argument, the buffer):
   let g:ale_fixers.testft = ['RemoveLastLineOneArg']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(There should be only two lines):
   a
@@ -632,6 +678,7 @@ Execute(ALEFix functions returning jobs should be able to accept one argument):
   else
     let g:ale_fixers.testft = ['CatLine']
     ALEFix
+    call ale#test#FlushJobs()
   endif
 
 Expect(An extra line should be added):
@@ -643,22 +690,26 @@ Expect(An extra line should be added):
 Execute(ALE should print a message telling you something isn't a valid fixer when you type some nonsense):
   let g:ale_fixers.testft = ['CatLine', 'invalidname']
   ALEFix
+  call ale#test#FlushJobs()
 
   AssertEqual 'There is no fixer named `invalidname`. Check :ALEFixSuggest', GetLastMessage()
 
 Execute(ALE should complain about invalid fixers with minuses in the name):
   let g:ale_fixers.testft = ['foo-bar']
   ALEFix
+  call ale#test#FlushJobs()
 
   AssertEqual 'There is no fixer named `foo-bar`. Check :ALEFixSuggest', GetLastMessage()
 
 Execute(ALE should tolerate valid fixers with minuses in the name):
   let g:ale_fixers.testft = ['prettier-standard']
   ALEFix
+  call ale#test#FlushJobs()
 
 Execute(Test fixing with chained callbacks):
   let g:ale_fixers.testft = ['FirstChainCallback']
   ALEFix
+  call ale#test#FlushJobs()
 
   " The buffer shouldn't be piped in for earlier commands in the chain.
   AssertEqual
@@ -677,6 +728,7 @@ Expect(The echoed line should be added):
 Execute(Test fixing with chained callback where the first command is skipped):
   let g:ale_fixers.testft = ['FirstChainCallbackSkipped']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The default line should be added):
   a
@@ -687,6 +739,7 @@ Expect(The default line should be added):
 Execute(Test fixing with chained callback where the second command is skipped):
   let g:ale_fixers.testft = ['FirstChainCallbackSecondSkipped']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The default line should be added):
   a
@@ -697,6 +750,7 @@ Expect(The default line should be added):
 Execute(Test fixing with chained callback where the final callback is skipped):
   let g:ale_fixers.testft = ['ChainWhereLastIsSkipped']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The lines should be the same):
   a
@@ -706,6 +760,7 @@ Expect(The lines should be the same):
 Execute(Empty output should be ignored):
   let g:ale_fixers.testft = ['IgnoredEmptyOutput']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The lines should be the same):
   a
@@ -715,6 +770,7 @@ Expect(The lines should be the same):
 Execute(A temporary file shouldn't be piped into the command when disabled):
   let g:ale_fixers.testft = ['EchoLineNoPipe']
   ALEFix
+  call ale#test#FlushJobs()
 
   AssertEqual
   \ string(ale#job#PrepareCommand(bufnr(''), 'echo new line')),
@@ -731,6 +787,7 @@ Expect(The new line should be used):
 Execute(Post-processing should work):
   let g:ale_fixers.testft = ['FixWithJSONPostProcessing']
   ALEFix
+  call ale#test#FlushJobs()
 
 Expect(The lines in the JSON should be used):
   x
@@ -740,5 +797,7 @@ Expect(The lines in the JSON should be used):
 Execute(ALEFix should apply autocmds):
   let g:ale_fixers.testft = ['AddCarets']
   ALEFix
+  call ale#test#FlushJobs()
+
   AssertEqual g:pre_success, 1
   AssertEqual g:post_success, 1
diff --git a/test/lsp/test_reset_lsp.vader b/test/lsp/test_reset_lsp.vader
index 2bec13dc..4a1a155e 100644
--- a/test/lsp/test_reset_lsp.vader
+++ b/test/lsp/test_reset_lsp.vader
@@ -18,6 +18,9 @@ Before:
   endfunction
 
   call ale#engine#InitBufferInfo(bufnr(''))
+  " Call this function first, so we can be sure the module is loaded before we
+  " check if it exists.
+  call ale#lsp_linter#ClearLSPData()
 
   call ale#linter#Define('testft', {
   \   'name': 'lsplinter',
@@ -68,7 +71,10 @@ Execute(ALEStopAllLSPs should clear the loclist):
   \   'linter_name': 'otherlinter',
   \ },
   \]
-  let g:ale_buffer_info[bufnr('')].active_linter_list = ['lsplinter', 'otherlinter']
+  let g:ale_buffer_info[bufnr('')].active_linter_list = [
+  \ {'name': 'lsplinter'},
+  \ {'name': 'otherlinter'},
+  \]
 
   ALEStopAllLSPs
 
@@ -87,4 +93,6 @@ Execute(ALEStopAllLSPs should clear the loclist):
   \]
 
   " The LSP linter should be removed from the active linter list.
-  AssertEqual g:ale_buffer_info[bufnr('')].active_linter_list, ['otherlinter']
+  AssertEqual
+  \ ['otherlinter'],
+  \ map(copy(g:ale_buffer_info[bufnr('')].active_linter_list), 'v:val.name')
diff --git a/test/sign/test_linting_sets_signs.vader b/test/sign/test_linting_sets_signs.vader
index 3ccecf45..c72b0882 100644
--- a/test/sign/test_linting_sets_signs.vader
+++ b/test/sign/test_linting_sets_signs.vader
@@ -63,5 +63,6 @@ After:
 
 Execute(The signs should be updated after linting is done):
   ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual [['1', 'ALEWarningSign'], ['2', 'ALEErrorSign']], CollectSigns()
diff --git a/test/sign/test_sign_placement.vader b/test/sign/test_sign_placement.vader
index e2d95ff0..41bed715 100644
--- a/test/sign/test_sign_placement.vader
+++ b/test/sign/test_sign_placement.vader
@@ -134,6 +134,7 @@ Given testft(A file with warnings/errors):
 
 Execute(The current signs should be set for running a job):
   ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual
   \ [
diff --git a/test/smoke_test.vader b/test/smoke_test.vader
index c87f95b2..53e08a8d 100644
--- a/test/smoke_test.vader
+++ b/test/smoke_test.vader
@@ -66,7 +66,7 @@ Execute(Linters should run with the default options):
   " where tests fail randomly.
   for g:i in range(has('nvim-0.3') || has('win32') ? 5 : 1)
     call ale#Queue(0, '')
-    call ale#engine#WaitForJobs(2000)
+    call ale#test#WaitForJobs(2000)
 
     let g:results = ale#test#GetLoclistWithoutModule()
 
@@ -110,7 +110,7 @@ Execute(Linters should run in PowerShell too):
     \})
 
     call ale#Queue(0, '')
-    call ale#engine#WaitForJobs(4000)
+    call ale#test#WaitForJobs(4000)
 
     AssertEqual [
     \ {
@@ -140,7 +140,7 @@ Execute(Linters should run in PowerShell too):
 
 Execute(Previous errors should be removed when linters change):
   call ale#Queue(0, '')
-  call ale#engine#WaitForJobs(2000)
+  call ale#test#WaitForJobs(2000)
 
   call ale#linter#Reset()
 
@@ -167,7 +167,7 @@ Execute(Previous errors should be removed when linters change):
   " where tests fail randomly.
   for g:i in range(has('nvim-0.3') || has('win32') ? 5 : 1)
     call ale#Queue(0, '')
-    call ale#engine#WaitForJobs(2000)
+    call ale#test#WaitForJobs(2000)
 
     let g:results = ale#test#GetLoclistWithoutModule()
 
diff --git a/test/test_ale_lint_command.vader b/test/test_ale_lint_command.vader
index bc2ebabe..ba7308dd 100644
--- a/test/test_ale_lint_command.vader
+++ b/test/test_ale_lint_command.vader
@@ -1,7 +1,9 @@
 Before:
   Save g:ale_buffer_info
+  Save g:ale_enabled
 
   let g:ale_buffer_info = {}
+  let g:ale_enabled = 1
 
   let g:expected_loclist = [{
   \ 'bufnr': bufnr('%'),
@@ -58,7 +60,7 @@ Execute(ALELint should run the linters):
   " Try to run the linter a few times, as it fails randomly in NeoVim.
   for b:i in range(5)
     ALELint
-    call ale#engine#WaitForJobs(2000)
+    call ale#test#WaitForJobs(2000)
 
     if !has('nvim')
       " Sleep so the delayed list function can run.
diff --git a/test/test_ale_toggle.vader b/test/test_ale_toggle.vader
index db891009..d0bca329 100644
--- a/test/test_ale_toggle.vader
+++ b/test/test_ale_toggle.vader
@@ -10,6 +10,7 @@ Before:
   let g:ale_set_signs = 1
   let g:ale_set_lists_synchronously = 1
   let g:ale_run_synchronously = 1
+  unlet! g:ale_run_synchronously_callbacks
   let g:ale_pattern_options = {}
   let g:ale_pattern_options_enabled = 1
   let g:ale_set_balloons =
@@ -85,6 +86,7 @@ Before:
 After:
   Restore
 
+  unlet! g:ale_run_synchronously_callbacks
   unlet! g:expected_loclist
   unlet! g:expected_groups
   unlet! b:ale_enabled
@@ -113,6 +115,7 @@ Execute(ALEToggle should reset everything and then run again):
   AssertEqual 'foobar', &filetype
 
   ALELint
+  call ale#test#FlushJobs()
 
   " First check that everything is there...
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
@@ -135,6 +138,7 @@ Execute(ALEToggle should reset everything and then run again):
 
   " Toggle ALE on, everything should be set up and run again.
   ALEToggle
+  call ale#test#FlushJobs()
 
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
   AssertEqual [0, [[2, 1000001, 'ALEErrorSign']]], ale#sign#FindCurrentSigns(bufnr('%'))
@@ -157,6 +161,7 @@ Execute(ALEToggle should skip filename keys and preserve them):
   \}
 
   ALELint
+  call ale#test#FlushJobs()
 
   " Now Toggle ALE off.
   ALEToggle
@@ -174,6 +179,7 @@ Execute(ALEToggle should skip filename keys and preserve them):
 
   " Toggle ALE on again.
   ALEToggle
+  call ale#test#FlushJobs()
 
   AssertEqual
   \ {
@@ -188,15 +194,18 @@ Execute(ALEToggle should skip filename keys and preserve them):
 
 Execute(ALEDisable should reset everything and stay disabled):
   ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
 
   ALEDisable
+  call ale#test#FlushJobs()
 
   AssertEqual [], ale#test#GetLoclistWithoutModule()
   AssertEqual 0, g:ale_enabled
 
   ALEDisable
+  call ale#test#FlushJobs()
 
   AssertEqual [], ale#test#GetLoclistWithoutModule()
   AssertEqual 0, g:ale_enabled
@@ -205,6 +214,7 @@ Execute(ALEEnable should enable ALE and lint again):
   let g:ale_enabled = 0
 
   ALEEnable
+  call ale#test#FlushJobs()
 
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
   AssertEqual 1, g:ale_enabled
@@ -213,6 +223,7 @@ Execute(ALEReset should reset everything for a buffer):
   AssertEqual 'foobar', &filetype
 
   ALELint
+  call ale#test#FlushJobs()
 
   " First check that everything is there...
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
@@ -224,6 +235,7 @@ Execute(ALEReset should reset everything for a buffer):
 
   " Now Toggle ALE off.
   ALEReset
+  call ale#test#FlushJobs()
 
   " Everything should be cleared.
   Assert !has_key(g:ale_buffer_info, bufnr('')), 'The g:ale_buffer_info Dictionary was not removed'
@@ -237,6 +249,7 @@ Execute(ALEToggleBuffer should reset everything and then run again):
   AssertEqual 'foobar', &filetype
 
   ALELint
+  call ale#test#FlushJobs()
 
   " First check that everything is there...
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
@@ -257,6 +270,7 @@ Execute(ALEToggleBuffer should reset everything and then run again):
 
   " Toggle ALE on, everything should be set up and run again.
   ALEToggleBuffer
+  call ale#test#FlushJobs()
 
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
   AssertEqual [0, [[2, 1000001, 'ALEErrorSign']]], ale#sign#FindCurrentSigns(bufnr('%'))
@@ -268,10 +282,12 @@ Execute(ALEToggleBuffer should reset everything and then run again):
 
 Execute(ALEDisableBuffer should reset everything and stay disabled):
   ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
 
   ALEDisableBuffer
+  call ale#test#FlushJobs()
 
   AssertEqual [], ale#test#GetLoclistWithoutModule()
   AssertEqual 0, b:ale_enabled
@@ -280,6 +296,7 @@ Execute(ALEEnableBuffer should enable ALE and lint again):
   let b:ale_enabled = 0
 
   ALEEnableBuffer
+  call ale#test#FlushJobs()
 
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
   AssertEqual 1, b:ale_enabled
@@ -303,6 +320,7 @@ Execute(ALEResetBuffer should reset everything for a buffer):
   AssertEqual 'foobar', &filetype
 
   ALELint
+  call ale#test#FlushJobs()
 
   " First check that everything is there...
   AssertEqual g:expected_loclist, ale#test#GetLoclistWithoutModule()
@@ -314,6 +332,7 @@ Execute(ALEResetBuffer should reset everything for a buffer):
 
   " Now Toggle ALE off.
   ALEResetBuffer
+  call ale#test#FlushJobs()
 
   " Everything should be cleared.
   Assert !has_key(g:ale_buffer_info, bufnr('')), 'The g:ale_buffer_info Dictionary was not removed'
diff --git a/test/test_command_chain.vader b/test/test_command_chain.vader
index 591f6f40..329ebc97 100644
--- a/test/test_command_chain.vader
+++ b/test/test_command_chain.vader
@@ -1,6 +1,7 @@
 Before:
   Save &shell, g:ale_run_synchronously
   let g:ale_run_synchronously = 1
+  unlet! g:ale_run_synchronously_callbacks
 
   if !has('win32')
     set shell=/bin/sh
@@ -47,6 +48,7 @@ Before:
 
 After:
   Restore
+  unlet! g:ale_run_synchronously_callbacks
   unlet! g:first_echo_called
   unlet! g:second_echo_called
   unlet! g:final_callback_called
@@ -63,6 +65,7 @@ Given foobar (Some imaginary filetype):
 Execute(Check the results of running the chain):
   AssertEqual 'foobar', &filetype
   call ale#Queue(0)
+  call ale#test#FlushJobs()
 
   Assert g:first_echo_called, 'The first chain item was not called'
   Assert g:second_echo_called, 'The second chain item was not called'
diff --git a/test/test_errors_removed_after_filetype_changed.vader b/test/test_errors_removed_after_filetype_changed.vader
index 651a74f2..a47dbcd1 100644
--- a/test/test_errors_removed_after_filetype_changed.vader
+++ b/test/test_errors_removed_after_filetype_changed.vader
@@ -3,7 +3,6 @@ Before:
   Save g:ale_buffer_info
   Save g:ale_echo_cursor
   Save g:ale_run_synchronously
-  Save g:ale_run_synchronously
   Save g:ale_set_highlights
   Save g:ale_set_loclist
   Save g:ale_set_quickfix
@@ -17,6 +16,7 @@ Before:
   let g:ale_echo_cursor = 0
 
   let g:ale_run_synchronously = 1
+  unlet! g:ale_run_synchronously_callbacks
   call setloclist(0, [])
 
   noautocmd let &filetype = 'foobar'
@@ -44,6 +44,8 @@ Before:
 
 After:
   Restore
+
+  unlet! g:ale_run_synchronously_callbacks
   delfunction TestCallback
 
   call ale#linter#Reset()
@@ -51,6 +53,7 @@ After:
 
 Execute(Error should be removed when the filetype changes to something else we cannot check):
   call ale#Queue(0)
+  call ale#test#FlushJobs()
   sleep 1ms
 
   AssertEqual 1, len(ale#test#GetLoclistWithoutModule())
@@ -58,6 +61,7 @@ Execute(Error should be removed when the filetype changes to something else we c
   noautocmd let &filetype = 'foobar2'
 
   call ale#Queue(0)
+  call ale#test#FlushJobs()
   sleep 1ms
 
   " We should get some items from the second filetype.
@@ -66,6 +70,7 @@ Execute(Error should be removed when the filetype changes to something else we c
   noautocmd let &filetype = 'xxx'
 
   call ale#Queue(0)
+  call ale#test#FlushJobs()
   sleep 1ms
 
   AssertEqual 0, len(ale#test#GetLoclistWithoutModule())
diff --git a/test/test_format_temporary_file_creation.vader b/test/test_format_temporary_file_creation.vader
index 385af908..10409400 100644
--- a/test/test_format_temporary_file_creation.vader
+++ b/test/test_format_temporary_file_creation.vader
@@ -13,6 +13,7 @@ Before:
   let g:ale_echo_cursor = 0
   let g:ale_enabled = 1
   let g:ale_run_synchronously = 1
+  unlet! g:ale_run_synchronously_callbacks
   let g:ale_set_highlights = 0
   let g:ale_set_loclist = 0
   let g:ale_set_quickfix = 0
@@ -41,6 +42,7 @@ Before:
 After:
   Restore
 
+  unlet! g:ale_run_synchronously_callbacks
   unlet! g:output
   delfunction TestCallback
 
@@ -56,5 +58,6 @@ Execute(ALE should be able to read the %t file):
   AssertEqual 'foobar', &filetype
 
   ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual ['foo', 'bar', 'baz'], g:output
diff --git a/test/test_highlight_placement.vader b/test/test_highlight_placement.vader
index 53dcea06..619a964e 100644
--- a/test/test_highlight_placement.vader
+++ b/test/test_highlight_placement.vader
@@ -9,6 +9,7 @@ Before:
   Save g:ale_set_signs
 
   let g:ale_run_synchronously = 1
+  unlet! g:ale_run_synchronously_callbacks
   let g:ale_set_highlights = 1
   let g:ale_set_signs = 1
   let g:ale_buffer_info = {}
@@ -64,6 +65,7 @@ Before:
 After:
   Restore
 
+  unlet! g:ale_run_synchronously_callbacks
   unlet! g:items
   unlet! b:ale_enabled
 
@@ -81,6 +83,7 @@ Given testft(A Javscript file with warnings/errors):
 
 Execute(Highlights should be set when a linter runs):
   ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual
   \ [
diff --git a/test/test_history_saving.vader b/test/test_history_saving.vader
index d7a307b5..18b64db5 100644
--- a/test/test_history_saving.vader
+++ b/test/test_history_saving.vader
@@ -2,6 +2,10 @@ Before:
   Save g:ale_max_buffer_history_size
   Save g:ale_history_log_output
   Save g:ale_run_synchronously
+  Save g:ale_enabled
+
+  let g:ale_enabled = 1
+  let g:ale_run_synchronously = 1
 
   unlet! b:ale_fixers
   unlet! b:ale_enabled
@@ -68,27 +72,19 @@ Given foobar (Some imaginary filetype):
 Execute(History should be set when commands are run):
   AssertEqual 'foobar', &filetype
 
-  let g:expected_results = ['command', 'exit_code', 'job_id', 'status']
-
-  " Retry this test until it works. This one can randomly fail.
-  for g:i in range(has('nvim-0.3') || has('win32') ? 5 : 1)
-    let b:ale_history = []
-    call ale#Queue(0)
-    call ale#engine#WaitForJobs(2000)
+  let b:ale_history = []
+  ALELint
+  call ale#test#FlushJobs()
 
-    let g:history = filter(
-    \ copy(ale#history#Get(bufnr(''))),
-    \ 'v:val.job_id isnot# ''executable''',
-    \)
+  let g:history = filter(
+  \ copy(ale#history#Get(bufnr(''))),
+  \ 'v:val.job_id isnot# ''executable''',
+  \)
 
-    AssertEqual 1, len(g:history)
-
-    if sort(keys(g:history[0])) == g:expected_results
-      break
-    endif
-  endfor
-
-  AssertEqual g:expected_results, sort(keys(g:history[0]))
+  AssertEqual 1, len(g:history)
+  AssertEqual
+  \ ['command', 'exit_code', 'job_id', 'status'],
+  \ sort(keys(g:history[0]))
 
   if has('win32')
     AssertEqual 'cmd /s/c "echo command history test"', g:history[0].command
@@ -106,8 +102,8 @@ Execute(History should be not set when disabled):
 
   let g:ale_history_enabled = 0
 
-  call ale#Queue(0)
-  call ale#engine#WaitForJobs(2000)
+  ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual [], ale#history#Get(bufnr(''))
 
@@ -115,24 +111,21 @@ Execute(History should include command output if logging is enabled):
   AssertEqual 'foobar', &filetype
 
   let g:ale_history_log_output = 1
-  let g:expected_results = ['command history test']
 
   " Retry this test until it works. This one can randomly fail.
-  for g:i in range(has('nvim-0.3') || has('win32') ? 5 : 1)
-    let b:ale_history = []
-    call ale#Queue(0)
-    call ale#engine#WaitForJobs(2000)
+  let b:ale_history = []
+  ALELint
+  call ale#test#FlushJobs()
 
-    let g:history = ale#history#Get(bufnr(''))
+  let g:history = ale#history#Get(bufnr(''))
 
-    AssertEqual 1, len(g:history)
-
-    if get(g:history[0], 'output', []) == g:expected_results
-      break
-    endif
-  endfor
-
-  AssertEqual g:expected_results, get(g:history[0], 'output', [])
+  AssertEqual 1, len(g:history)
+  AssertEqual
+  \ ['command history test'],
+  \ map(
+  \   copy(get(g:history[0], 'output', [])),
+  \   'substitute(v:val, ''[\r ]*$'', '''', ''g'')'
+  \ )
 
 Execute(History items should be popped after going over the max):
   let b:ale_history = map(range(20), '{''status'': ''started'', ''job_id'': v:val, ''command'': ''foobar''}')
@@ -169,10 +162,13 @@ Execute(The history should be updated when fixers are run):
 
   let b:ale_fixers = {'foobar': ['TestFixer']}
   let b:ale_enabled = 0
-  let g:ale_run_synchronously = 1
 
   ALEFix
 
+  AssertEqual ['started'], map(copy(b:ale_history), 'v:val.status')
+
+  call ale#test#FlushJobs()
+
   AssertEqual ['finished'], map(copy(b:ale_history), 'v:val.status')
 
   if has('win32')
diff --git a/test/test_lint_file_linters.vader b/test/test_lint_file_linters.vader
index f67fad44..d16f4aa1 100644
--- a/test/test_lint_file_linters.vader
+++ b/test/test_lint_file_linters.vader
@@ -8,6 +8,7 @@ Before:
 
   let g:ale_buffer_info = {}
   let g:ale_run_synchronously = 1
+  unlet! g:ale_run_synchronously_callbacks
   let g:ale_set_lists_synchronously = 1
   let b:ale_save_event_fired = 0
 
@@ -89,6 +90,7 @@ After:
 
   Restore
 
+  unlet! g:ale_run_synchronously_callbacks
   unlet! b:ale_save_event_fired
   unlet! b:ale_enabled
   unlet g:buffer_result
@@ -111,6 +113,7 @@ Given foobar (Some imaginary filetype):
 
 Execute(Running linters without 'lint_file' should run only buffer linters):
   call ale#Queue(0)
+  call ale#test#FlushJobs()
 
   AssertEqual [
   \ {
@@ -131,6 +134,7 @@ Execute(Running linters with 'lint_file' should run all linters):
   Assert filereadable(expand('%:p')), 'The file was not readable'
 
   call ale#Queue(0, 'lint_file')
+  call ale#test#FlushJobs()
 
   AssertEqual [
   \ {
@@ -163,6 +167,7 @@ Execute(Linter errors from files should be kept):
   Assert filereadable(expand('%:p')), 'The file was not readable'
 
   call ale#Queue(0, 'lint_file')
+  call ale#test#FlushJobs()
 
   " Change the results for the buffer callback.
   let g:buffer_result = [
@@ -175,6 +180,7 @@ Execute(Linter errors from files should be kept):
   \]
 
   call ale#Queue(0)
+  call ale#test#FlushJobs()
 
   AssertEqual [
   \ {
@@ -202,6 +208,7 @@ Execute(Linter errors from files should be kept when no other linters are run):
   Assert filereadable(expand('%:p')), 'The file was not readable'
 
   call ale#Queue(0, 'lint_file')
+  call ale#test#FlushJobs()
 
   AssertEqual [
   \ {
@@ -240,11 +247,13 @@ Execute(The Save event should respect the buffer number):
   Assert filereadable(expand('%:p')), 'The file was not readable'
 
   call ale#events#SaveEvent(bufnr('') + 1)
+  call ale#test#FlushJobs()
 
   " We shouldn't get any prblems yet.
   AssertEqual [], GetSimplerLoclist()
 
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   " We should get them now we used the right buffer number.
   AssertEqual [
@@ -268,6 +277,7 @@ Execute(The Save event should set b:ale_save_event_fired to 1):
 
   call ale#linter#Reset()
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   " This flag needs to be set so windows can be opened, etc.
   AssertEqual 1, b:ale_save_event_fired
@@ -276,6 +286,7 @@ Execute(b:ale_save_event_fired should be set to 0 when results are set):
   let b:ale_save_event_fired = 1
 
   call ale#engine#SetResults(bufnr(''), [])
+  call ale#test#FlushJobs()
 
   AssertEqual 0, b:ale_save_event_fired
 
@@ -289,15 +300,18 @@ Execute(lint_file linters should stay running after checking without them):
   " The lint_file linter should still be running.
   AssertEqual
   \ ['lint_file_linter', 'buffer_linter'],
-  \ g:ale_buffer_info[bufnr('')].active_linter_list
+  \ map(copy(g:ale_buffer_info[bufnr('')].active_linter_list), 'v:val.name')
   " We should have 1 job for each linter.
-  AssertEqual 2, len(g:ale_buffer_info[bufnr('')].job_list)
+  AssertEqual
+  \ 2,
+  \ len(keys(get(get(ale#command#GetData(), bufnr(''), {}), 'jobs', {})))
 
-  call ale#engine#WaitForJobs(2000)
+  call ale#test#WaitForJobs(2000)
 
 Execute(The save event should not lint the buffer when ALE is disabled):
   let g:ale_enabled = 0
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual [], GetSimplerLoclist()
   AssertEqual 0, b:ale_save_event_fired
diff --git a/test/test_lint_on_enter_when_file_changed.vader b/test/test_lint_on_enter_when_file_changed.vader
index 67f43c17..88493005 100644
--- a/test/test_lint_on_enter_when_file_changed.vader
+++ b/test/test_lint_on_enter_when_file_changed.vader
@@ -50,6 +50,7 @@ Execute(The file changed event function should set b:ale_file_changed):
 Execute(The file changed event function should lint the current buffer when it has changed):
   set filetype=foobar
   call ale#events#FileChangedEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual [{
   \   'bufnr': bufnr(''),
@@ -68,6 +69,7 @@ Execute(The buffer should be checked after entering it after the file has change
 
   set filetype=foobar
   call ale#events#ReadOrEnterEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual [{
   \   'bufnr': bufnr(''),
diff --git a/test/test_linting_blacklist.vader b/test/test_linting_blacklist.vader
index 73190b7f..2bcc9576 100644
--- a/test/test_linting_blacklist.vader
+++ b/test/test_linting_blacklist.vader
@@ -11,6 +11,6 @@ Given unite (A Unite.vim file):
 
 Execute(Running ALE on a blacklisted file shouldn't change anything):
   call ale#Queue(0)
-  call ale#engine#WaitForJobs(2000)
+  call ale#test#WaitForJobs(2000)
 
   AssertEqual {}, g:ale_buffer_info
diff --git a/test/test_linting_updates_loclist.vader b/test/test_linting_updates_loclist.vader
index 921cdb08..8a162703 100644
--- a/test/test_linting_updates_loclist.vader
+++ b/test/test_linting_updates_loclist.vader
@@ -64,6 +64,7 @@ Given foobar (Some JavaScript with problems):
 
 Execute(The loclist should be updated after linting is done):
   ALELint
+  call ale#test#FlushJobs()
 
   AssertEqual
   \ [
diff --git a/test/test_no_linting_on_write_quit.vader b/test/test_no_linting_on_write_quit.vader
index 75de06a4..7ad08d08 100644
--- a/test/test_no_linting_on_write_quit.vader
+++ b/test/test_no_linting_on_write_quit.vader
@@ -58,6 +58,7 @@ Execute(No linting should be done on :wq or :x):
 
   " First try just the SaveEvent, to be sure that we set errors in the test.
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual 1, len(ale#test#GetLoclistWithoutModule())
 
@@ -65,6 +66,7 @@ Execute(No linting should be done on :wq or :x):
   call setloclist(0, [])
   call ale#events#QuitEvent(bufnr(''))
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual [], ale#test#GetLoclistWithoutModule()
 
@@ -73,11 +75,13 @@ Execute(No linting should be for :w after :q fails):
   let g:ale_fix_on_save = 0
 
   call ale#events#QuitEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   " Simulate 2 seconds passing.
   let b:ale_quitting -= 1000
 
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual 1, len(ale#test#GetLoclistWithoutModule())
 
@@ -86,6 +90,7 @@ Execute(No linting should be done on :wq or :x after fixing files):
   let g:ale_fix_on_save = 1
 
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual 1, len(ale#test#GetLoclistWithoutModule())
 
@@ -93,6 +98,7 @@ Execute(No linting should be done on :wq or :x after fixing files):
   call setloclist(0, [])
   call ale#events#QuitEvent(bufnr(''))
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual [], ale#test#GetLoclistWithoutModule()
 
@@ -101,10 +107,12 @@ Execute(Linting should be done after :q fails and fixing files):
   let g:ale_fix_on_save = 1
 
   call ale#events#QuitEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   " Simulate 2 seconds passing.
   let b:ale_quitting -= 1000
 
   call ale#events#SaveEvent(bufnr(''))
+  call ale#test#FlushJobs()
 
   AssertEqual 1, len(ale#test#GetLoclistWithoutModule())
diff --git a/test/test_temporary_file_management.vader b/test/test_temporary_file_management.vader
index b25da7ac..9fff1ace 100644
--- a/test/test_temporary_file_management.vader
+++ b/test/test_temporary_file_management.vader
@@ -70,7 +70,7 @@ Execute(ALE should delete managed files/directories appropriately after linting)
   AssertEqual 'foobar', &filetype
 
   call ale#Queue(0)
-  call ale#engine#WaitForJobs(2000)
+  call ale#test#FlushJobs()
 
   Assert !filereadable(g:filename), 'The temporary file was not deleted'
   Assert !isdirectory(g:directory), 'The temporary directory was not deleted'
@@ -82,7 +82,7 @@ Execute(ALE should delete managed files even if no command is run):
   let g:command = ''
 
   call ale#Queue(0)
-  call ale#engine#WaitForJobs(2000)
+  call ale#test#WaitForJobs(2000)
 
   Assert !filereadable(g:filename), 'The temporary file was not deleted'
   Assert !isdirectory(g:directory), 'The temporary directory was not deleted'
@@ -119,24 +119,26 @@ Execute(ALE should create and delete directories for ale#command#CreateDirectory
   Assert !isdirectory(b:dir), 'The directory was not deleted'
   Assert !isdirectory(b:dir2), 'The second directory was not deleted'
 
-Execute(ale#command#ManageFile should add the file even if the buffer info hasn't be set yet):
+Execute(ale#command#ManageFile should add the file even if the buffer info hasn't been set yet):
   call ale#command#ManageFile(bufnr(''), '/foo/bar')
 
   AssertEqual
   \ {
   \   bufnr(''): {
+  \     'jobs': {},
   \     'file_list': ['/foo/bar'],
   \     'directory_list': [],
   \   },
   \ },
   \ ale#command#GetData()
 
-Execute(ale#command#ManageDirectory should add the directory even if the buffer info hasn't be set yet):
+Execute(ale#command#ManageDirectory should add the directory even if the buffer info hasn't been set yet):
   call ale#command#ManageDirectory(bufnr(''), '/foo/bar')
 
   AssertEqual
   \ {
   \   bufnr(''): {
+  \     'jobs': {},
   \     'file_list': [],
   \     'directory_list': ['/foo/bar'],
   \   },