diff --git a/autoload/ale/highlight.vim b/autoload/ale/highlight.vim
index 3ce6bff4..cb7911e1 100644
--- a/autoload/ale/highlight.vim
+++ b/autoload/ale/highlight.vim
@@ -52,7 +52,7 @@ endfunction
 
 function! ale#highlight#RemoveHighlights() abort
     for l:match in getmatches()
-        if l:match.group =~# '^ALE'
+        if l:match.group =~? '\v^ALE(Style)?(Error|Warning|Info)(Line)?$'
             call matchdelete(l:match.id)
         endif
     endfor
diff --git a/test/test_highlight_placement.vader b/test/test_highlight_placement.vader
index 619a964e..87ac6073 100644
--- a/test/test_highlight_placement.vader
+++ b/test/test_highlight_placement.vader
@@ -279,6 +279,10 @@ Execute(Highlights should be cleared when ALE is disabled):
   AssertEqual [], GetMatchesWithoutIDs()
 
 Execute(Line highlights should be set when signs are disabled):
+  " This will mess with your settings, but it needs to be tested.
+  " We need to match highlights case-insenstive when removing them.
+  hi link aleerrorline spellbad
+
   let g:ale_set_signs = 0
 
   call ale#highlight#SetHighlights(bufnr(''), [
@@ -292,8 +296,12 @@ Execute(Line highlights should be set when signs are disabled):
   \   {'group': 'ALEError', 'priority': 10, 'pos1': [1, 1, 1]},
   \   {'group': 'ALEWarning', 'priority': 10, 'pos1': [2, 1, 1]},
   \   {'group': 'ALEInfo', 'priority': 10, 'pos1': [3, 1, 1]},
-  \   {'group': 'ALEErrorLine', 'priority': 10, 'pos1': [1]},
+  \   {'group': 'aleerrorline', 'priority': 10, 'pos1': [1]},
   \   {'group': 'ALEWarningLine', 'priority': 10, 'pos1': [2]},
   \   {'group': 'ALEInfoLine', 'priority': 10, 'pos1': [3]},
   \ ],
   \ GetMatchesWithoutIDs()
+
+  " All of the highlights should be removed.
+  call ale#highlight#RemoveHighlights()
+  AssertEqual [], GetMatchesWithoutIDs()
diff --git a/test/vimrc b/test/vimrc
index 018b8f67..3f80c636 100644
--- a/test/vimrc
+++ b/test/vimrc
@@ -3,6 +3,9 @@
 " Make most tests just set lists synchronously when run in Docker, etc.
 let g:ale_set_lists_synchronously = 1
 
+" This lowercase highlight definition is needed for highlight tests.
+hi link aleerrorline spellbad
+
 " Load builtin plugins
 " We need this because run_vim.sh sets -i NONE
 if has('win32')