Change project layout and use go.mod

This commit is contained in:
Zachary Yedidia
2019-02-03 23:17:24 -05:00
parent c7f2c9c704
commit 0612af1590
103 changed files with 154 additions and 194 deletions

18
pkg/highlight/ftdetect.go Normal file
View File

@@ -0,0 +1,18 @@
package highlight
import "regexp"
// MatchFiletype will use the list of syntax definitions provided and the filename and first line of the file
// to determine the filetype of the file
// It will return the corresponding syntax definition for the filetype
func MatchFiletype(ftdetect [2]*regexp.Regexp, filename string, firstLine []byte) bool {
if ftdetect[0].MatchString(filename) {
return true
}
if ftdetect[1] != nil {
return ftdetect[1].Match(firstLine)
}
return false
}

View File

@@ -0,0 +1,406 @@
package highlight
import (
"regexp"
"strings"
"unicode/utf8"
)
func sliceStart(slc []byte, index int) []byte {
len := len(slc)
i := 0
totalSize := 0
for totalSize < len {
if i >= index {
return slc[totalSize:]
}
_, size := utf8.DecodeRune(slc[totalSize:])
totalSize += size
i++
}
return slc[totalSize:]
}
func sliceEnd(slc []byte, index int) []byte {
len := len(slc)
i := 0
totalSize := 0
for totalSize < len {
if i >= index {
return slc[:totalSize]
}
_, size := utf8.DecodeRune(slc[totalSize:])
totalSize += size
i++
}
return slc[:totalSize]
}
// RunePos returns the rune index of a given byte index
// This could cause problems if the byte index is between code points
func runePos(p int, str []byte) int {
if p < 0 {
return 0
}
if p >= len(str) {
return utf8.RuneCount(str)
}
return utf8.RuneCount(str[:p])
}
func combineLineMatch(src, dst LineMatch) LineMatch {
for k, v := range src {
if g, ok := dst[k]; ok {
if g == 0 {
dst[k] = v
}
} else {
dst[k] = v
}
}
return dst
}
// A State represents the region at the end of a line
type State *region
// LineStates is an interface for a buffer-like object which can also store the states and matches for every line
type LineStates interface {
LineBytes(n int) []byte
LinesNum() int
State(lineN int) State
SetState(lineN int, s State)
SetMatch(lineN int, m LineMatch)
}
// A Highlighter contains the information needed to highlight a string
type Highlighter struct {
lastRegion *region
Def *Def
}
// NewHighlighter returns a new highlighter from the given syntax definition
func NewHighlighter(def *Def) *Highlighter {
h := new(Highlighter)
h.Def = def
return h
}
// LineMatch represents the syntax highlighting matches for one line. Each index where the coloring is changed is marked with that
// color's group (represented as one byte)
type LineMatch map[int]Group
func findIndex(regex *regexp.Regexp, skip *regexp.Regexp, str []byte, canMatchStart, canMatchEnd bool) []int {
regexStr := regex.String()
if strings.Contains(regexStr, "^") {
if !canMatchStart {
return nil
}
}
if strings.Contains(regexStr, "$") {
if !canMatchEnd {
return nil
}
}
var strbytes []byte
if skip != nil {
strbytes = skip.ReplaceAllFunc(str, func(match []byte) []byte {
res := make([]byte, utf8.RuneCount(match))
return res
})
} else {
strbytes = str
}
match := regex.FindIndex(strbytes)
if match == nil {
return nil
}
// return []int{match.Index, match.Index + match.Length}
return []int{runePos(match[0], str), runePos(match[1], str)}
}
func findAllIndex(regex *regexp.Regexp, str []byte, canMatchStart, canMatchEnd bool) [][]int {
regexStr := regex.String()
if strings.Contains(regexStr, "^") {
if !canMatchStart {
return nil
}
}
if strings.Contains(regexStr, "$") {
if !canMatchEnd {
return nil
}
}
matches := regex.FindAllIndex(str, -1)
for i, m := range matches {
matches[i][0] = runePos(m[0], str)
matches[i][1] = runePos(m[1], str)
}
return matches
}
func (h *Highlighter) highlightRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, curRegion *region, statesOnly bool) LineMatch {
lineLen := utf8.RuneCount(line)
if start == 0 {
if !statesOnly {
if _, ok := highlights[0]; !ok {
highlights[0] = curRegion.group
}
}
}
loc := findIndex(curRegion.end, curRegion.skip, line, start == 0, canMatchEnd)
if loc != nil {
if !statesOnly {
highlights[start+loc[0]] = curRegion.limitGroup
}
if curRegion.parent == nil {
if !statesOnly {
highlights[start+loc[1]] = 0
h.highlightRegion(highlights, start, false, lineNum, sliceEnd(line, loc[0]), curRegion, statesOnly)
}
h.highlightEmptyRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), statesOnly)
return highlights
}
if !statesOnly {
highlights[start+loc[1]] = curRegion.parent.group
h.highlightRegion(highlights, start, false, lineNum, sliceEnd(line, loc[0]), curRegion, statesOnly)
}
h.highlightRegion(highlights, start+loc[1], canMatchEnd, lineNum, sliceStart(line, loc[1]), curRegion.parent, statesOnly)
return highlights
}
if lineLen == 0 || statesOnly {
if canMatchEnd {
h.lastRegion = curRegion
}
return highlights
}
firstLoc := []int{lineLen, 0}
var firstRegion *region
for _, r := range curRegion.rules.regions {
loc := findIndex(r.start, nil, line, start == 0, canMatchEnd)
if loc != nil {
if loc[0] < firstLoc[0] {
firstLoc = loc
firstRegion = r
}
}
}
if firstLoc[0] != lineLen {
highlights[start+firstLoc[0]] = firstRegion.limitGroup
h.highlightRegion(highlights, start, false, lineNum, sliceEnd(line, firstLoc[0]), curRegion, statesOnly)
h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly)
return highlights
}
fullHighlights := make([]Group, lineLen)
for i := 0; i < len(fullHighlights); i++ {
fullHighlights[i] = curRegion.group
}
for _, p := range curRegion.rules.patterns {
matches := findAllIndex(p.regex, line, start == 0, canMatchEnd)
for _, m := range matches {
for i := m[0]; i < m[1]; i++ {
fullHighlights[i] = p.group
}
}
}
for i, h := range fullHighlights {
if i == 0 || h != fullHighlights[i-1] {
highlights[start+i] = h
}
}
if canMatchEnd {
h.lastRegion = curRegion
}
return highlights
}
func (h *Highlighter) highlightEmptyRegion(highlights LineMatch, start int, canMatchEnd bool, lineNum int, line []byte, statesOnly bool) LineMatch {
lineLen := utf8.RuneCount(line)
if lineLen == 0 {
if canMatchEnd {
h.lastRegion = nil
}
return highlights
}
firstLoc := []int{lineLen, 0}
var firstRegion *region
for _, r := range h.Def.rules.regions {
loc := findIndex(r.start, nil, line, start == 0, canMatchEnd)
if loc != nil {
if loc[0] < firstLoc[0] {
firstLoc = loc
firstRegion = r
}
}
}
if firstLoc[0] != lineLen {
if !statesOnly {
highlights[start+firstLoc[0]] = firstRegion.limitGroup
}
h.highlightEmptyRegion(highlights, start, false, lineNum, sliceEnd(line, firstLoc[0]), statesOnly)
h.highlightRegion(highlights, start+firstLoc[1], canMatchEnd, lineNum, sliceStart(line, firstLoc[1]), firstRegion, statesOnly)
return highlights
}
if statesOnly {
if canMatchEnd {
h.lastRegion = nil
}
return highlights
}
fullHighlights := make([]Group, len(line))
for _, p := range h.Def.rules.patterns {
matches := findAllIndex(p.regex, line, start == 0, canMatchEnd)
for _, m := range matches {
for i := m[0]; i < m[1]; i++ {
fullHighlights[i] = p.group
}
}
}
for i, h := range fullHighlights {
if i == 0 || h != fullHighlights[i-1] {
// if _, ok := highlights[start+i]; !ok {
highlights[start+i] = h
// }
}
}
if canMatchEnd {
h.lastRegion = nil
}
return highlights
}
// HighlightString syntax highlights a string
// Use this function for simple syntax highlighting and use the other functions for
// more advanced syntax highlighting. They are optimized for quick rehighlighting of the same
// text with minor changes made
func (h *Highlighter) HighlightString(input string) []LineMatch {
lines := strings.Split(input, "\n")
var lineMatches []LineMatch
for i := 0; i < len(lines); i++ {
line := []byte(lines[i])
highlights := make(LineMatch)
if i == 0 || h.lastRegion == nil {
lineMatches = append(lineMatches, h.highlightEmptyRegion(highlights, 0, true, i, line, false))
} else {
lineMatches = append(lineMatches, h.highlightRegion(highlights, 0, true, i, line, h.lastRegion, false))
}
}
return lineMatches
}
// HighlightStates correctly sets all states for the buffer
func (h *Highlighter) HighlightStates(input LineStates) {
for i := 0; i < input.LinesNum(); i++ {
line := input.LineBytes(i)
// highlights := make(LineMatch)
if i == 0 || h.lastRegion == nil {
h.highlightEmptyRegion(nil, 0, true, i, line, true)
} else {
h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true)
}
curState := h.lastRegion
input.SetState(i, curState)
}
}
// HighlightMatches sets the matches for each line in between startline and endline
// It sets all other matches in the buffer to nil to conserve memory
// This assumes that all the states are set correctly
func (h *Highlighter) HighlightMatches(input LineStates, startline, endline int) {
for i := startline; i < endline; i++ {
if i >= input.LinesNum() {
break
}
line := input.LineBytes(i)
highlights := make(LineMatch)
var match LineMatch
if i == 0 || input.State(i-1) == nil {
match = h.highlightEmptyRegion(highlights, 0, true, i, line, false)
} else {
match = h.highlightRegion(highlights, 0, true, i, line, input.State(i-1), false)
}
input.SetMatch(i, match)
}
}
// ReHighlightStates will scan down from `startline` and set the appropriate end of line state
// for each line until it comes across the same state in two consecutive lines
func (h *Highlighter) ReHighlightStates(input LineStates, startline int) {
// lines := input.LineData()
h.lastRegion = nil
if startline > 0 {
h.lastRegion = input.State(startline - 1)
}
for i := startline; i < input.LinesNum(); i++ {
line := input.LineBytes(i)
// highlights := make(LineMatch)
// var match LineMatch
if i == 0 || h.lastRegion == nil {
h.highlightEmptyRegion(nil, 0, true, i, line, true)
} else {
h.highlightRegion(nil, 0, true, i, line, h.lastRegion, true)
}
curState := h.lastRegion
lastState := input.State(i)
input.SetState(i, curState)
if curState == lastState {
break
}
}
}
// ReHighlightLine will rehighlight the state and match for a single line
func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) {
line := input.LineBytes(lineN)
highlights := make(LineMatch)
h.lastRegion = nil
if lineN > 0 {
h.lastRegion = input.State(lineN - 1)
}
var match LineMatch
if lineN == 0 || h.lastRegion == nil {
match = h.highlightEmptyRegion(highlights, 0, true, lineN, line, false)
} else {
match = h.highlightRegion(highlights, 0, true, lineN, line, h.lastRegion, false)
}
curState := h.lastRegion
input.SetMatch(lineN, match)
input.SetState(lineN, curState)
}

354
pkg/highlight/parser.go Normal file
View File

@@ -0,0 +1,354 @@
package highlight
import (
"errors"
"fmt"
"regexp"
"gopkg.in/yaml.v2"
)
// A Group represents a syntax group
type Group uint8
// Groups contains all of the groups that are defined
// You can access them in the map via their string name
var Groups map[string]Group
var numGroups Group
// String returns the group name attached to the specific group
func (g Group) String() string {
for k, v := range Groups {
if v == g {
return k
}
}
return ""
}
// A Def is a full syntax definition for a language
// It has a filetype, information about how to detect the filetype based
// on filename or header (the first line of the file)
// Then it has the rules which define how to highlight the file
type Def struct {
*Header
rules *rules
}
type Header struct {
FileType string
FtDetect [2]*regexp.Regexp
}
type File struct {
FileType string
yamlSrc map[interface{}]interface{}
}
// A Pattern is one simple syntax rule
// It has a group that the rule belongs to, as well as
// the regular expression to match the pattern
type pattern struct {
group Group
regex *regexp.Regexp
}
// rules defines which patterns and regions can be used to highlight
// a filetype
type rules struct {
regions []*region
patterns []*pattern
includes []string
}
// A region is a highlighted region (such as a multiline comment, or a string)
// It belongs to a group, and has start and end regular expressions
// A region also has rules of its own that only apply when matching inside the
// region and also rules from the above region do not match inside this region
// Note that a region may contain more regions
type region struct {
group Group
limitGroup Group
parent *region
start *regexp.Regexp
end *regexp.Regexp
skip *regexp.Regexp
rules *rules
}
func init() {
Groups = make(map[string]Group)
}
func ParseFtDetect(file *File) (r [2]*regexp.Regexp, err error) {
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("pkg: %v", r)
}
}
}()
rules := file.yamlSrc
loaded := 0
for k, v := range rules {
if k == "detect" {
ftdetect := v.(map[interface{}]interface{})
if len(ftdetect) >= 1 {
syntax, err := regexp.Compile(ftdetect["filename"].(string))
if err != nil {
return r, err
}
r[0] = syntax
}
if len(ftdetect) >= 2 {
header, err := regexp.Compile(ftdetect["header"].(string))
if err != nil {
return r, err
}
r[1] = header
}
loaded++
}
if loaded >= 2 {
break
}
}
if loaded == 0 {
return r, errors.New("No detect regexes found")
}
return r, err
}
func ParseFile(input []byte) (f *File, err error) {
// This is just so if we have an error, we can exit cleanly and return the parse error to the user
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("pkg: %v", r)
}
}
}()
var rules map[interface{}]interface{}
if err = yaml.Unmarshal(input, &rules); err != nil {
return nil, err
}
f = new(File)
f.yamlSrc = rules
for k, v := range rules {
if k == "filetype" {
filetype := v.(string)
f.FileType = filetype
break
}
}
return f, err
}
// ParseDef parses an input syntax file into a highlight Def
func ParseDef(f *File, header *Header) (s *Def, err error) {
// This is just so if we have an error, we can exit cleanly and return the parse error to the user
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("pkg: %v", r)
}
}
}()
rules := f.yamlSrc
s = new(Def)
s.Header = header
for k, v := range rules {
if k == "rules" {
inputRules := v.([]interface{})
rules, err := parseRules(inputRules, nil)
if err != nil {
return nil, err
}
s.rules = rules
}
}
return s, err
}
// ResolveIncludes will sort out the rules for including other filetypes
// You should call this after parsing all the Defs
func ResolveIncludes(def *Def, files []*File) {
resolveIncludesInDef(files, def)
}
func resolveIncludesInDef(files []*File, d *Def) {
for _, lang := range d.rules.includes {
for _, searchFile := range files {
if lang == searchFile.FileType {
searchDef, _ := ParseDef(searchFile, nil)
d.rules.patterns = append(d.rules.patterns, searchDef.rules.patterns...)
d.rules.regions = append(d.rules.regions, searchDef.rules.regions...)
}
}
}
for _, r := range d.rules.regions {
resolveIncludesInRegion(files, r)
r.parent = nil
}
}
func resolveIncludesInRegion(files []*File, region *region) {
for _, lang := range region.rules.includes {
for _, searchFile := range files {
if lang == searchFile.FileType {
searchDef, _ := ParseDef(searchFile, nil)
region.rules.patterns = append(region.rules.patterns, searchDef.rules.patterns...)
region.rules.regions = append(region.rules.regions, searchDef.rules.regions...)
}
}
}
for _, r := range region.rules.regions {
resolveIncludesInRegion(files, r)
r.parent = region
}
}
func parseRules(input []interface{}, curRegion *region) (ru *rules, err error) {
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("pkg: %v", r)
}
}
}()
ru = new(rules)
for _, v := range input {
rule := v.(map[interface{}]interface{})
for k, val := range rule {
group := k
switch object := val.(type) {
case string:
if k == "include" {
ru.includes = append(ru.includes, object)
} else {
// Pattern
r, err := regexp.Compile(object)
if err != nil {
return nil, err
}
groupStr := group.(string)
if _, ok := Groups[groupStr]; !ok {
numGroups++
Groups[groupStr] = numGroups
}
groupNum := Groups[groupStr]
ru.patterns = append(ru.patterns, &pattern{groupNum, r})
}
case map[interface{}]interface{}:
// region
region, err := parseRegion(group.(string), object, curRegion)
if err != nil {
return nil, err
}
ru.regions = append(ru.regions, region)
default:
return nil, fmt.Errorf("Bad type %T", object)
}
}
}
return ru, nil
}
func parseRegion(group string, regionInfo map[interface{}]interface{}, prevRegion *region) (r *region, err error) {
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("pkg: %v", r)
}
}
}()
r = new(region)
if _, ok := Groups[group]; !ok {
numGroups++
Groups[group] = numGroups
}
groupNum := Groups[group]
r.group = groupNum
r.parent = prevRegion
r.start, err = regexp.Compile(regionInfo["start"].(string))
if err != nil {
return nil, err
}
r.end, err = regexp.Compile(regionInfo["end"].(string))
if err != nil {
return nil, err
}
// skip is optional
if _, ok := regionInfo["skip"]; ok {
r.skip, err = regexp.Compile(regionInfo["skip"].(string))
if err != nil {
return nil, err
}
}
// limit-color is optional
if _, ok := regionInfo["limit-group"]; ok {
groupStr := regionInfo["limit-group"].(string)
if _, ok := Groups[groupStr]; !ok {
numGroups++
Groups[groupStr] = numGroups
}
groupNum := Groups[groupStr]
r.limitGroup = groupNum
if err != nil {
return nil, err
}
} else {
r.limitGroup = r.group
}
r.rules, err = parseRules(regionInfo["rules"].([]interface{}), r)
if err != nil {
return nil, err
}
return r, nil
}