Fix #363 - Detect virtualenv executables and fix import paths for mypy. Use lint_file for mypy

5 files changed, 164 insertions, 31 deletions

diff --git a/ale_linters/python/mypy.vim b/ale_linters/python/mypy.vim
index fff306ae..7e4270a2 100644
--- a/ale_linters/python/mypy.vim
+++ b/ale_linters/python/mypy.vim
@@ -1,18 +1,41 @@
 " Author: Keith Smiley <k@keith.so>, w0rp <devw0rp@gmail.com>
 " Description: mypy support for optional python typechecking
 
+let g:ale_python_mypy_executable =
+\   get(g:, 'ale_python_mypy_executable', 'mypy')
 let g:ale_python_mypy_options = get(g:, 'ale_python_mypy_options', '')
+let g:ale_python_mypy_use_global = get(g:, 'ale_python_mypy_use_global', 0)
+
+" (cd /home/w0rp/git/wazoku/wazoku-spotlight &&  /home/w0rp/git/wazoku/wazoku-spotlight/ve-py3/bin/mypy --show-column-numbers /home/w0rp/git/wazoku/wazoku-spotlight/central/models/__init__.py) | grep ^central/models/__init__.p
+
+function! ale_linters#python#mypy#GetExecutable(buffer) abort
+    if !ale#Var(a:buffer, 'python_mypy_use_global')
+        let l:virtualenv = ale#python#FindVirtualenv(a:buffer)
+
+        if !empty(l:virtualenv)
+            let l:ve_mypy = l:virtualenv . '/bin/mypy'
+
+            if executable(l:ve_mypy)
+                return l:ve_mypy
+            endif
+        endif
+    endif
+
+    return ale#Var(a:buffer, 'python_mypy_executable')
+endfunction
 
 function! ale_linters#python#mypy#GetCommand(buffer) abort
-    let l:automatic_stubs_dir = ale#path#FindNearestDirectory(a:buffer, 'stubs')
-    " TODO: Add Windows support
-    let l:automatic_stubs_command = (has('unix') && !empty(l:automatic_stubs_dir))
-    \   ?  'MYPYPATH=' . l:automatic_stubs_dir . ' '
+    let l:project_root = ale#python#FindProjectRoot(a:buffer)
+    let l:cd_command = !empty(l:project_root)
+    \   ? ale#path#CdString(l:project_root)
     \   : ''
+    let l:executable = ale_linters#python#mypy#GetExecutable(a:buffer)
 
-    return 'mypy --show-column-numbers '
+    return l:cd_command
+    \   . fnameescape(l:executable)
+    \   . ' --show-column-numbers '
     \   . ale#Var(a:buffer, 'python_mypy_options')
-    \   . ' %t'
+    \   . ' %s'
 endfunction
 
 function! ale_linters#python#mypy#Handle(buffer, lines) abort
@@ -23,21 +46,20 @@ function! ale_linters#python#mypy#Handle(buffer, lines) abort
     " Lines like these should be ignored below:
     "
     " file.py:4: note: (Stub files are from https://github.com/python/typeshed)
-    let l:pattern = '\v^[a-zA-Z]?:?[^:]+:(\d+):?(\d+)?: ([^:]+): (.+)$'
+    let l:pattern = '\v^([a-zA-Z]?:?[^:]+):(\d+):?(\d+)?: (error|warning): (.+)$'
     let l:output = []
+    let l:buffer_filename = expand('#' . a:buffer . ':p')
 
     for l:match in ale#util#GetMatches(a:lines, l:pattern)
-        if l:match[4] =~# 'Stub files are from'
-            " The lines telling us where to get stub files from make it so
-            " we can't read the actual errors, so exclude them.
+        if l:buffer_filename[-len(l:match[1]):] !=# l:match[1]
             continue
         endif
 
         call add(l:output, {
-        \   'lnum': l:match[1] + 0,
-        \   'col': l:match[2] + 0,
-        \   'text': l:match[4],
-        \   'type': l:match[3] =~# 'error' ? 'E' : 'W',
+        \   'lnum': l:match[2] + 0,
+        \   'col': l:match[3] + 0,
+        \   'type': l:match[4] =~# 'error' ? 'E' : 'W',
+        \   'text': l:match[5],
         \})
     endfor
 
@@ -46,7 +68,8 @@ endfunction
 
 call ale#linter#Define('python', {
 \   'name': 'mypy',
-\   'executable': 'mypy',
+\   'executable_callback': 'ale_linters#python#mypy#GetExecutable',
 \   'command_callback': 'ale_linters#python#mypy#GetCommand',
 \   'callback': 'ale_linters#python#mypy#Handle',
+\   'lint_file': 1,
 \})
diff --git a/doc/ale-python.txt b/doc/ale-python.txt
index 6b4a4ea4..00aa9b38 100644
--- a/doc/ale-python.txt
+++ b/doc/ale-python.txt
@@ -34,6 +34,14 @@ g:ale_python_flake8_options                       *g:ale_python_flake8_options*
 -------------------------------------------------------------------------------
 mypy                                                          *ale-python-mypy*
 
+g:ale_python_mypy_executable                     *g:ale_python_mypy_executable*
+                                                 *b:ale_python_mypy_executable*
+  Type: |String|
+  Default: `'mypy'`
+
+  This variable can be changed to modify the executable used for mypy.
+
+
 g:ale_python_mypy_options                           *g:ale_python_mypy_options*
                                                     *b:ale_python_mypy_options*
   Type: |String|
@@ -43,6 +51,18 @@ g:ale_python_mypy_options                           *g:ale_python_mypy_options*
   invocation.
 
 
+g:ale_python_mypy_use_global                     *g:ale_python_mypy_use_global*
+                                                 *b:ale_python_mypy_use_global*
+  Type: |Number|
+  Default: `0`
+
+  This variable controls whether or not ALE will search for mypy in a
+  virtualenv directory first. If this variable is set to `1`, then ALE will
+  always use |g:ale_python_mypy_executable| for the executable path.
+
+  Both variables can be set with `b:` buffer variables instead.
+
+
 -------------------------------------------------------------------------------
 pylint                                                      *ale-python-pylint*
 
@@ -67,6 +87,8 @@ g:ale_python_pylint_options                       *g:ale_python_pylint_options*
 
   let g:ale_python_pylint_executable = 'python3'   " or 'python' for Python 2
   let g:ale_python_pylint_options = '-rcfile /path/to/pylint.rc'
+  " The virtualenv detection needs to be disabled.
+  let g:ale_python_pylint_use_global = 0
 
   after making sure it's installed for the appropriate Python versions (e.g.
   `python3 -m pip install --user pylint`).
@@ -77,9 +99,9 @@ g:ale_python_pylint_use_global                 *g:ale_python_pylint_use_global*
   Type: |Number|
   Default: `0`
 
-  This variable controls whether or not ALE will search pylint in a virtualenv
-  directory first. If this variable is set to `1`, then ALE will always use
-  |g:ale_python_pylint_executable| for the executable path.
+  This variable controls whether or not ALE will search for pylint in a
+  virtualenv directory first. If this variable is set to `1`, then ALE will
+  always use |g:ale_python_pylint_executable| for the executable path.
 
   Both variables can be set with `b:` buffer variables instead.
 
diff --git a/test/command_callback/python_paths/with_virtualenv/env/bin/mypy b/test/command_callback/python_paths/with_virtualenv/env/bin/mypy
new file mode 100755
index 00000000..e69de29b
--- /dev/null
+++ b/test/command_callback/python_paths/with_virtualenv/env/bin/mypy
diff --git a/test/command_callback/test_mypy_command_callback.vader b/test/command_callback/test_mypy_command_callback.vader
new file mode 100644
index 00000000..ec82c879
--- /dev/null
+++ b/test/command_callback/test_mypy_command_callback.vader
@@ -0,0 +1,73 @@
+Before:
+  runtime ale_linters/python/mypy.vim
+  silent! execute 'cd /testplugin/test/command_callback'
+  let g:dir = getcwd()
+
+After:
+  silent execute 'cd ' . fnameescape(g:dir)
+  " Set the file to something else,
+  " or we'll cause issues when running other tests
+  silent file 'dummy.py'
+  unlet! g:dir
+
+  call ale#linter#Reset()
+  let g:ale_python_mypy_executable = 'mypy'
+  let g:ale_python_mypy_options = ''
+  let g:ale_python_mypy_use_global = 0
+
+Execute(The mypy callbacks should return the correct default values):
+  AssertEqual
+  \ 'mypy',
+  \ ale_linters#python#mypy#GetExecutable(bufnr(''))
+  AssertEqual
+  \ 'cd ' . g:dir . ' && mypy --show-column-numbers  %s',
+  \ ale_linters#python#mypy#GetCommand(bufnr(''))
+
+Execute(The mypy executable should be configurable, and escaped properly):
+  let g:ale_python_mypy_executable = 'executable with spaces'
+
+  AssertEqual
+  \ 'executable with spaces',
+  \ ale_linters#python#mypy#GetExecutable(bufnr(''))
+  AssertEqual
+  \ 'cd ' . g:dir . ' && executable\ with\ spaces --show-column-numbers  %s',
+  \ ale_linters#python#mypy#GetCommand(bufnr(''))
+
+Execute(The mypy command callback should let you set options):
+  let g:ale_python_mypy_options = '--some-option'
+
+  AssertEqual
+  \ 'cd ' . g:dir . ' && mypy --show-column-numbers --some-option %s',
+  \ ale_linters#python#mypy#GetCommand(bufnr(''))
+
+Execute(The mypy command should switch directories to the detected project root):
+  silent execute 'file ' . fnameescape(g:dir . '/python_paths/no_virtualenv/subdir/foo/bar.py')
+
+  AssertEqual
+  \ 'mypy',
+  \ ale_linters#python#mypy#GetExecutable(bufnr(''))
+  AssertEqual
+  \ 'cd ' . g:dir . '/python_paths/no_virtualenv/subdir && mypy --show-column-numbers  %s',
+  \ ale_linters#python#mypy#GetCommand(bufnr(''))
+
+Execute(The mypy callbacks should detect virtualenv directories and switch to the project root):
+  silent execute 'file ' . fnameescape(g:dir . '/python_paths/with_virtualenv/subdir/foo/bar.py')
+
+  AssertEqual
+  \ g:dir . '/python_paths/with_virtualenv/env/bin/mypy',
+  \ ale_linters#python#mypy#GetExecutable(bufnr(''))
+  AssertEqual
+  \   'cd ' . g:dir . '/python_paths/with_virtualenv/subdir && '
+  \   . g:dir . '/python_paths/with_virtualenv/env/bin/mypy --show-column-numbers  %s',
+  \ ale_linters#python#mypy#GetCommand(bufnr(''))
+
+Execute(You should able able to use the global mypy instead):
+  silent execute 'file ' . fnameescape(g:dir . '/python_paths/with_virtualenv/subdir/foo/bar.py')
+  let g:ale_python_mypy_use_global = 1
+
+  AssertEqual
+  \ 'mypy',
+  \ ale_linters#python#mypy#GetExecutable(bufnr(''))
+  AssertEqual
+  \ 'cd ' . g:dir . '/python_paths/with_virtualenv/subdir && mypy --show-column-numbers  %s',
+  \ ale_linters#python#mypy#GetCommand(bufnr(''))
diff --git a/test/handler/test_mypy_handler.vader b/test/handler/test_mypy_handler.vader
index e1b1562d..c949b1aa 100644
--- a/test/handler/test_mypy_handler.vader
+++ b/test/handler/test_mypy_handler.vader
@@ -3,30 +3,45 @@ Before:
 
 After:
   call ale#linter#Reset()
+  silent file something_else.py
 
 Execute(The mypy handler should parse lines correctly):
+  silent file foo/bar/__init__.py
+
   AssertEqual
   \ [
   \   {
-  \     'lnum': 4,
-  \     'col': 0,
-  \     'text': "No library stub file for module 'django.db'",
+  \     'lnum': 15,
+  \     'col': 3,
+  \     'text': 'Argument 1 to "somefunc" has incompatible type "int"; expected "str"',
   \     'type': 'E',
   \   },
   \   {
-  \     'lnum': 40,
-  \     'col': 5,
-  \     'text': "Some other problem",
-  \     'type': 'E',
+  \     'lnum': 72,
+  \     'col': 1,
+  \     'text': 'Some warning',
+  \     'type': 'W',
   \   },
   \ ],
-  \ ale_linters#python#mypy#Handle(347, [
-  \   "file.py:4: error: No library stub file for module 'django.db'",
-  \   'file.py:4: note: (Stub files are from https://github.com/python/typeshed)',
-  \   "file.py:40:5: error: Some other problem",
+  \ ale_linters#python#mypy#Handle(bufnr(''), [
+  \ 'foo/bar/baz.py: note: In class "SomeClass":',
+  \ 'foo/bar/baz.py:768:38: error: Cannot determine type of ''SOME_SYMBOL''',
+  \ 'foo/bar/baz.py: note: In class "AnotherClass":',
+  \ 'foo/bar/baz.py:821:38: error: Cannot determine type of ''SOME_SYMBOL''',
+  \ 'foo/bar/__init__.py:92: note: In module imported here:',
+  \ 'foo/bar/other.py: note: In class "YetAnotherClass":',
+  \ 'foo/bar/other.py:38:44: error: Cannot determine type of ''ANOTHER_SYMBOL''',
+  \ 'foo/bar/__init__.py: note: At top level:',
+  \ 'foo/bar/__init__.py:15:3: error: Argument 1 to "somefunc" has incompatible type "int"; expected "str"',
+  \ 'another_module/bar.py:14: note: In module imported here,',
+  \ 'another_module/__init__.py:16: note: ... from here,',
+  \ 'foo/bar/__init__.py:72:1: warning: Some warning',
   \ ])
 
 Execute(The mypy handler should handle Windows names with spaces):
+  " This test works on Unix, where this is seen as a single filename
+  silent file C:\\something\\with\ spaces.py
+
   AssertEqual
   \ [
   \   {
@@ -36,6 +51,6 @@ Execute(The mypy handler should handle Windows names with spaces):
   \     'type': 'E',
   \   },
   \ ],
-  \ ale_linters#python#mypy#Handle(347, [
-  \   "C:\something\with spaces.py:4: error: No library stub file for module 'django.db'",
+  \ ale_linters#python#mypy#Handle(bufnr(''), [
+  \   'C:\something\with spaces.py:4: error: No library stub file for module ''django.db''',
   \ ])