Use less memory when opening very large files

cmd/micro/actions.go

@@ -1483,7 +1483,7 @@ func (v *View) AddTab(usePlugin bool) bool {
 		return false
 	}
 
-	tab := NewTabFromView(NewView(NewBuffer(strings.NewReader(""), "")))
+	tab := NewTabFromView(NewView(NewBufferFromString("", "")))
 	tab.SetNum(len(tabs))
 	tabs = append(tabs, tab)
 	curTab = len(tabs) - 1
@@ -1543,7 +1543,7 @@ func (v *View) VSplitBinding(usePlugin bool) bool {
 		return false
 	}
 
-	v.VSplit(NewBuffer(strings.NewReader(""), ""))
+	v.VSplit(NewBufferFromString("", ""))
 
 	if usePlugin {
 		return PostActionCall("VSplit", v)
@@ -1557,7 +1557,7 @@ func (v *View) HSplitBinding(usePlugin bool) bool {
 		return false
 	}
 
-	v.HSplit(NewBuffer(strings.NewReader(""), ""))
+	v.HSplit(NewBufferFromString("", ""))
 
 	if usePlugin {
 		return PostActionCall("HSplit", v)

cmd/micro/buffer.go

@@ -61,11 +61,11 @@ type SerializedBuffer struct {
 }
 
 func NewBufferFromString(text, path string) *Buffer {
-	return NewBuffer(strings.NewReader(text), path)
+	return NewBuffer(strings.NewReader(text), int64(len(text)), path)
 }
 
 // NewBuffer creates a new buffer from a given reader with a given path
-func NewBuffer(reader io.Reader, path string) *Buffer {
+func NewBuffer(reader io.Reader, size int64, path string) *Buffer {
 	if path != "" {
 		for _, tab := range tabs {
 			for _, view := range tab.views {
@@ -77,7 +77,7 @@ func NewBuffer(reader io.Reader, path string) *Buffer {
 	}
 
 	b := new(Buffer)
-	b.LineArray = NewLineArray(reader)
+	b.LineArray = NewLineArray(size, reader)
 
 	b.Settings = DefaultLocalSettings()
 	for k, v := range globalSettings {

cmd/micro/command.go

@@ -320,7 +320,7 @@ func Help(args []string) {
 // If no file is given, it opens an empty buffer in a new split
 func VSplit(args []string) {
 	if len(args) == 0 {
-		CurView().VSplit(NewBuffer(strings.NewReader(""), ""))
+		CurView().VSplit(NewBufferFromString("", ""))
 	} else {
 		filename := args[0]
 		home, _ := homedir.Dir()
@@ -338,9 +338,9 @@ func VSplit(args []string) {
 		var buf *Buffer
 		if err != nil {
 			// File does not exist -- create an empty buffer with that name
-			buf = NewBuffer(strings.NewReader(""), filename)
+			buf = NewBufferFromString("", filename)
 		} else {
-			buf = NewBuffer(file, filename)
+			buf = NewBuffer(file, FSize(file), filename)
 		}
 		CurView().VSplit(buf)
 	}
@@ -350,7 +350,7 @@ func VSplit(args []string) {
 // If no file is given, it opens an empty buffer in a new split
 func HSplit(args []string) {
 	if len(args) == 0 {
-		CurView().HSplit(NewBuffer(strings.NewReader(""), ""))
+		CurView().HSplit(NewBufferFromString("", ""))
 	} else {
 		filename := args[0]
 		home, _ := homedir.Dir()
@@ -368,9 +368,9 @@ func HSplit(args []string) {
 		var buf *Buffer
 		if err != nil {
 			// File does not exist -- create an empty buffer with that name
-			buf = NewBuffer(strings.NewReader(""), filename)
+			buf = NewBufferFromString("", filename)
 		} else {
-			buf = NewBuffer(file, filename)
+			buf = NewBuffer(file, FSize(file), filename)
 		}
 		CurView().HSplit(buf)
 	}
@@ -408,9 +408,9 @@ func NewTab(args []string) {
 
 		var buf *Buffer
 		if err != nil {
-			buf = NewBuffer(strings.NewReader(""), filename)
+			buf = NewBufferFromString("", filename)
 		} else {
-			buf = NewBuffer(file, filename)
+			buf = NewBuffer(file, FSize(file), filename)
 		}
 
 		tab := NewTabFromView(NewView(buf))

cmd/micro/lineArray.go

@@ -63,32 +63,60 @@ type LineArray struct {
 	lines []Line
 }
 
+func Append(slice []Line, data ...Line) []Line {
+	l := len(slice)
+	if l+len(data) > cap(slice) { // reallocate
+		// Allocate double what's needed, for future growth.
+		newSlice := make([]Line, (l+len(data))+10000)
+		// The copy function is predeclared and works for any slice type.
+		copy(newSlice, slice)
+		slice = newSlice
+	}
+	slice = slice[0 : l+len(data)]
+	for i, c := range data {
+		slice[l+i] = c
+	}
+	return slice
+}
+
 // NewLineArray returns a new line array from an array of bytes
-func NewLineArray(reader io.Reader) *LineArray {
+func NewLineArray(size int64, reader io.Reader) *LineArray {
 	la := new(LineArray)
 
-	var buf bytes.Buffer
-	tee := io.TeeReader(reader, &buf)
-	numlines, _ := lineCounter(tee)
-	numlines++
+	la.lines = make([]Line, 0, 1000)
 
-	la.lines = make([]Line, numlines)
+	br := bufio.NewReader(reader)
+	var loaded int
 
-	br := bufio.NewReader(&buf)
-
-	for i := 0; i < numlines; i++ {
+	n := 0
+	for {
 		data, err := br.ReadBytes('\n')
+
+		if n >= 1000 && loaded >= 0 {
+			totalLinesNum := int(float64(size) * (float64(n) / float64(loaded)))
+			newSlice := make([]Line, len(la.lines), totalLinesNum+10000)
+			// The copy function is predeclared and works for any slice type.
+			copy(newSlice, la.lines)
+			la.lines = newSlice
+			loaded = -1
+		}
+
+		if loaded >= 0 {
+			loaded += len(data)
+		}
+
 		if err != nil {
 			if err == io.EOF {
-				// la.lines[i] = Line{data[:len(data)], nil, nil, false}
-				la.lines[i].data = data
+				la.lines = Append(la.lines, Line{data[:len(data)], nil, nil, false})
+				// la.lines = Append(la.lines, Line{data[:len(data)]})
 			}
 			// Last line was read
 			break
 		} else {
-			la.lines[i].data = data[:len(data)-1]
-			// la.lines[i] = Line{data[:len(data)-1], nil, nil, false}
+			// la.lines = Append(la.lines, Line{data[:len(data)-1]})
+			la.lines = Append(la.lines, Line{data[:len(data)-1], nil, nil, false})
 		}
+		n++
 	}
 
 	return la

cmd/micro/messenger.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"strconv"
-	"strings"
 
 	"github.com/zyedidia/clipboard"
 	"github.com/zyedidia/tcell"
@@ -79,7 +78,7 @@ func (m *Messenger) AddLog(msg string) {
 
 func (m *Messenger) getBuffer() *Buffer {
 	if m.log == nil {
-		m.log = NewBuffer(strings.NewReader(""), "")
+		m.log = NewBufferFromString("", "")
 		m.log.name = "Log"
 	}
 	return m.log

cmd/micro/micro.go

@@ -110,9 +110,9 @@ func LoadInput() []*Buffer {
 			}
 			// If the file didn't exist, input will be empty, and we'll open an empty buffer
 			if input != nil {
-				buffers = append(buffers, NewBuffer(input, filename))
+				buffers = append(buffers, NewBuffer(input, FSize(input), filename))
 			} else {
-				buffers = append(buffers, NewBuffer(strings.NewReader(""), filename))
+				buffers = append(buffers, NewBufferFromString("", filename))
 			}
 		}
 	} else if !isatty.IsTerminal(os.Stdin.Fd()) {
@@ -124,10 +124,10 @@ func LoadInput() []*Buffer {
 			TermMessage("Error reading from stdin: ", err)
 			input = []byte{}
 		}
-		buffers = append(buffers, NewBuffer(strings.NewReader(string(input)), filename))
+		buffers = append(buffers, NewBufferFromString(string(input), filename))
 	} else {
 		// Option 3, just open an empty buffer
-		buffers = append(buffers, NewBuffer(strings.NewReader(string(input)), filename))
+		buffers = append(buffers, NewBufferFromString(string(input), filename))
 	}
 
 	return buffers

cmd/micro/util.go

@@ -59,6 +59,12 @@ func Max(a, b int) int {
 	return b
 }
 
+func FSize(f *os.File) int64 {
+	fi, _ := f.Stat()
+	// get the size
+	return fi.Size()
+}
+
 // IsWordChar returns whether or not the string is a 'word character'
 // If it is a unicode character, then it does not match
 // Word characters are defined as [A-Za-z0-9_]

cmd/micro/view.go

@@ -258,9 +258,9 @@ func (v *View) Open(filename string) {
 	if err != nil {
 		messenger.Message(err.Error())
 		// File does not exist -- create an empty buffer with that name
-		buf = NewBuffer(strings.NewReader(""), filename)
+		buf = NewBufferFromString("", filename)
 	} else {
-		buf = NewBuffer(file, filename)
+		buf = NewBuffer(file, FSize(file), filename)
 	}
 	v.OpenBuffer(buf)
 }
@@ -649,7 +649,7 @@ func (v *View) openHelp(helpPage string) {
 	if data, err := FindRuntimeFile(RTHelp, helpPage).Data(); err != nil {
 		TermMessage("Unable to load help text", helpPage, "\n", err)
 	} else {
-		helpBuffer := NewBuffer(strings.NewReader(string(data)), helpPage+".md")
+		helpBuffer := NewBufferFromString(string(data), helpPage+".md")
 		helpBuffer.name = "Help"
 
 		if v.Type == vtHelp {