tests/extensions/fake-lexers/lexer.lua


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

-- A miniature stand-in for the Scintillua lexer module, just enough for the
-- syntax-highlight checks to drive every path through the filter without the
-- real collection installed. detect() answers for the suffixes it knows,
-- matched by case as Scintillua's own does, or fake for a shebang naming it,
-- and load() hands out the toy lexer of that name. Besides fake there is one
-- that stops short, one that raises and one that returns no tokens at all,
-- and python, which detect() knows but load() cannot find, as a collection
-- missing a lexer would answer.

local M = {}

-- Tokens come back the way Scintillua returns them, one flat list of a tag
-- name and the position just past the token it names. Letter runs are
-- keywords, or types when capitalised, digit runs are numbers, the rest of a
-- line after two dashes is a comment, a single quoted run carries a dotted
-- tag so the dotted first component lookup gets exercised, a run after a
-- colon or a dollar sign takes one of the tags only the css lexer uses, and
-- any other byte is whitespace, which maps to no css class.
local patterns = {
	{ "^:%a+", "pseudoclass" },
	{ "^%$%a+", "property" },
	{ "^%-%-[^\n]*", "comment" },
	{ "^%u%a*", "type" },
	{ "^%a+", "keyword" },
	{ "^%d+", "number" },
	{ "^'[^']*'", "string.double" },
}

local function lex_words(self, text)
	local tokens = {}
	local pos = 1
	while pos <= #text do
		local tag, stop = "whitespace", pos
		for _, pattern in ipairs(patterns) do
			local start, finish = text:find(pattern[1], pos)
			if start then
				tag, stop = pattern[2], finish
				break
			end
		end
		tokens[#tokens + 1] = tag
		tokens[#tokens + 1] = stop + 1
		pos = stop + 1
	end
	return tokens
end

-- Tags only the first four bytes and stops, so the filter has to append the
-- untagged tail itself or the check for it fails.
local function lex_short(self, text)
	if #text < 5 then
		return {}
	end
	return { "keyword", 5 }
end

local function lex_raise(self, text)
	error("this lexer always raises")
end

local function lex_nothing(self, text)
	return nil
end

local lexers = {
	fake = { lex = lex_words },
	short = { lex = lex_short },
	badlex = { lex = lex_raise },
	nothing = { lex = lex_nothing },
}

function M.load(name)
	local lexer = lexers[name]
	if lexer == nil then
		error("no fake lexer named " .. tostring(name))
	end
	return lexer
end

local suffixes = {
	fake = "fake", short = "short", badlex = "badlex", nothing = "nothing", py = "python",
}

function M.detect(filename, line)
	if line ~= nil and line:match("^#!.*fake") then
		return "fake"
	end
	return suffixes[filename:match("%.([^.]+)$") or ""]
end

return M