Moved highlighter code from DocView to Doc

* Only one highlighter state is kept per-document as opposed
  to one per-docview
* Fixes a bug with retaining older highlighter state as a
  DocView wasn't able to detect lines changing above it's viewport
* Renames `highlighter` module to more descriptive `tokenizer`

data/core/doc/highlighter.lua

@@ -0,0 +1,85 @@
+local core = require "core"
+local syntax = require "core.syntax"
+local config = require "core.config"
+local tokenizer = require "core.tokenizer"
+local Object = require "core.object"
+
+
+local Highlighter = Object:extend()
+
+
+function Highlighter:new(doc)
+  self.doc = doc
+  self:reset_syntax()
+
+  -- init incremental syntax highlighting
+  core.add_thread(function()
+    while true do
+      if self.last_valid_line > self.max_wanted_line then
+        self.max_wanted_line = 0
+        coroutine.yield(1 / config.fps)
+
+      else
+        local max = math.min(self.last_valid_line + 40, self.max_wanted_line)
+
+        for i = self.last_valid_line, max do
+          local state = (i > 1) and self.lines[i - 1].state
+          local line = self.lines[i]
+          if not (line and line.init_state == state) then
+            self.lines[i] = self:tokenize_line(i, state)
+          end
+        end
+
+        self.last_valid_line = max + 1
+        core.redraw = true
+        coroutine.yield()
+      end
+    end
+  end, self)
+end
+
+
+function Highlighter:reset_syntax()
+  local syn = syntax.get(self.doc.filename or "")
+  if self.syntax ~= syn then
+    self.syntax = syn
+    self.lines = {}
+    self.last_valid_line = 1
+    self.max_wanted_line = 0
+  end
+end
+
+
+function Highlighter:invalidate(idx)
+  self.last_valid_line = idx
+end
+
+
+function Highlighter:tokenize_line(idx, state)
+  local line = {}
+  line.init_state = state
+  line.text = self.doc.lines[idx]
+  line.tokens, line.state = tokenizer.tokenize(self.syntax, line.text, state)
+  return line
+end
+
+
+function Highlighter:get_line(idx)
+  local line = self.lines[idx]
+  if not line or line.text ~= self.doc.lines[idx] then
+    local prev = self.lines[idx - 1]
+    line = self:tokenize_line(idx, prev and prev.state)
+    self.lines[idx] = line
+    self.last_valid_line = math.min(self.last_valid_line, idx)
+  end
+  self.max_wanted_line = math.max(self.max_wanted_line, idx)
+  return line
+end
+
+
+function Highlighter:each_token(idx)
+  return tokenizer.each_token(self:get_line(idx).tokens)
+end
+
+
+return Highlighter

data/core/doc/init.lua

@@ -1,4 +1,5 @@
 local Object = require "core.object"
+local Highlighter = require "core.doc.highlighter"
 local config = require "core.config"
 local common = require "core.common"
 
@@ -19,7 +20,6 @@ local function splice(t, at, remove, insert)
   insert = insert or {}
   local offset = #insert - remove
   local old_len = #t
-  local new_len = old_len + offset
   if offset < 0 then
     for i = at - offset, old_len - offset do
       t[i + offset] = t[i]
@@ -49,6 +49,7 @@ function Doc:reset()
   self.undo_stack = { idx = 1 }
   self.redo_stack = { idx = 1 }
   self.clean_change_id = 1
+  self.highlighter = Highlighter(self)
 end
 
 
@@ -68,6 +69,7 @@ function Doc:load(filename)
     table.insert(self.lines, "\n")
   end
   fp:close()
+  self.highlighter:reset_syntax()
 end
 
 
@@ -80,6 +82,7 @@ function Doc:save(filename)
   end
   fp:close()
   self.filename = filename or self.filename
+  self.highlighter:reset_syntax()
   self:clean()
 end
 
@@ -233,6 +236,9 @@ local function insert(self, undo_stack, time, line, col, text)
   local line2, col2 = self:position_offset(line, col, #text)
   push_undo(self, undo_stack, time, "selection", self:get_selection())
   push_undo(self, undo_stack, time, "remove", line, col, line2, col2)
+
+  -- update highlighter
+  self.highlighter:invalidate(line)
 end
 
 
@@ -252,6 +258,9 @@ local function remove(self, undo_stack, time, line1, col1, line2, col2)
 
   -- splice line into line array
   splice(self.lines, line1, line2 - line1 + 1, { before .. after })
+
+  -- update highlighter
+  self.highlighter:invalidate(line1)
 end