Moved highlighter code from DocView to Doc

* Only one highlighter state is kept per-document as opposed
  to one per-docview
* Fixes a bug with retaining older highlighter state as a
  DocView wasn't able to detect lines changing above it's viewport
* Renames `highlighter` module to more descriptive `tokenizer`

data/core/docview.lua

@@ -5,7 +5,6 @@ local style = require "core.style"
 local syntax = require "core.syntax"
 local translate = require "core.doc.translate"
 local View = require "core.view"
-local highlighter = require "core.highlighter"
 
 
 local DocView = View:extend()
@@ -51,15 +50,6 @@ DocView.translate = {
 local blink_period = 0.8
 
 
-local function reset_syntax(self)
-  local syn = syntax.get(self.doc.filename or "")
-  if self.syntax ~= syn then
-    self.syntax = syn
-    self.cache = { last_valid = 1 }
-  end
-end
-
-
 function DocView:new(doc)
   DocView.super.new(self)
   self.cursor = "ibeam"
@@ -68,32 +58,6 @@ function DocView:new(doc)
   self.font = "code_font"
   self.last_x_offset = {}
   self.blink_timer = 0
-  reset_syntax(self)
-
-  -- init thread for incremental highlighting
-  self.updated_highlighting = false
-  core.add_thread(function()
-    while true do
-      local _, max = self:get_visible_line_range()
-
-      if self.cache.last_valid > max then
-        coroutine.yield(1 / config.fps)
-
-      else
-        max = math.min(self.cache.last_valid + 20, max)
-        for i = self.cache.last_valid, max do
-          local state = (i > 1) and self.cache[i - 1].state
-          local cl = self.cache[i]
-          if not (cl and cl.init_state == state) then
-            self.cache[i] = self:tokenize_line(i, state)
-          end
-        end
-        self.cache.last_valid = max + 1
-        self.updated_highlighting = true
-        coroutine.yield()
-      end
-    end
-  end, self)
 end
 
 
@@ -131,27 +95,6 @@ function DocView:get_scrollable_size()
 end
 
 
-function DocView:tokenize_line(idx, state)
-  local cl = {}
-  cl.init_state = state
-  cl.text = self.doc.lines[idx]
-  cl.tokens, cl.state = highlighter.tokenize(self.syntax, cl.text, state)
-  return cl
-end
-
-
-function DocView:get_cached_line(idx)
-  local cl = self.cache[idx]
-  if not cl or cl.text ~= self.doc.lines[idx] then
-    local prev = self.cache[idx-1]
-    cl = self:tokenize_line(idx, prev and prev.state)
-    self.cache[idx] = cl
-    self.cache.last_valid = math.min(self.cache.last_valid, idx)
-  end
-  return cl
-end
-
-
 function DocView:get_font()
   return style[self.font]
 end
@@ -308,16 +251,6 @@ function DocView:update()
     self.last_line, self.last_col = line, col
   end
 
-  if self.updated_highlighting then
-    self.updated_highlighting = false
-    core.redraw = true
-  end
-
-  if self.doc.filename ~= self.last_filename then
-    reset_syntax(self)
-    self.last_filename = self.doc.filename
-  end
-
   -- update blink timer
   if self == core.active_view and not self.mouse_selecting then
     local n = blink_period / 2
@@ -339,10 +272,9 @@ end
 
 
 function DocView:draw_line_text(idx, x, y)
-  local cl = self:get_cached_line(idx)
   local tx, ty = x, y + self:get_line_text_y_offset()
   local font = self:get_font()
-  for _, type, text in highlighter.each_token(cl.tokens) do
+  for _, type, text in self.doc.highlighter:each_token(idx) do
     local color = style.syntax[type]
     tx = renderer.draw_text(font, text, tx, ty, color)
   end
@@ -355,9 +287,9 @@ function DocView:draw_line_body(idx, x, y)
   -- draw selection if it overlaps this line
   local line1, col1, line2, col2 = self.doc:get_selection(true)
   if idx >= line1 and idx <= line2 then
-    local cl = self:get_cached_line(idx)
+    local text = self.doc.lines[idx]
     if line1 ~= idx then col1 = 1 end
-    if line2 ~= idx then col2 = #cl.text + 1 end
+    if line2 ~= idx then col2 = #text + 1 end
     local x1 = x + self:get_col_x_offset(idx, col1)
     local x2 = x + self:get_col_x_offset(idx, col2)
     local lh = self:get_line_height()