Removed useless div in edit form

Otherwise, questionmarks in page names aren't handled correctly.

Makefile

@@ -25,4 +25,4 @@ test:
 upload:
 	go build
 	rsync --itemize-changes --archive oddmu oddmu.service *.html README.md sibirocobombus.root:/home/oddmu/
-	ssh sibirocobombus.root "systemctl restart oddmu"
+	ssh sibirocobombus.root "systemctl restart oddmu; systemctl restart alex"

README.md

@@ -29,11 +29,15 @@ Oddmu. 🙃
 
 ## Markdown
 
-This wiki uses Markdown. There is no additional wiki markup, most
-importantly double square brackets are not a link. If you're used to
+This wiki uses a [Markdown
+library](https://github.com/gomarkdown/markdown) to generate the web
+pages from Markdown. There is no additional wiki markup. Most
+importantly, double square brackets are not a link. If you're used to
 that, it'll be strange as you need to repeat the name: `[like
-this](like this)`. The Markdown processor comes with a few extensions,
-some of which are enable by default:
+this](like this)`.
+
+The Markdown processor comes with a few extensions, some of which are
+enable by default:
 
 * emphasis markers inside words are ignored
 * tables are supported
@@ -46,10 +50,6 @@ some of which are enable by default:
 * definition lists are supported
 * MathJax is supported (but needs a separte setup)
 
-See the section on
-[extensions](https://github.com/russross/blackfriday#extensions) in
-the Blackfriday library for information on the various extensions.
-
 A table with footers and a columnspan:
 
 ```text
@@ -71,28 +71,63 @@ Internet
 : Vector of transmission for pictures of cats
 ```
 
+There is another extension made: hashtags link to searches for the
+hashtag. Hashtags are separate from titles because there is no space
+after the hash. Use the underscore to use hashtags consisting of
+multiple words.
+
+```
+# Title
+
+Text
+
+#Tag #Another_Tag
+```
+
 ## Templates
 
-Feel free to change the templates `view.html` and `edit.html` and
-restart the server. Modifying the styles in the templates would be a
-good start to get a feel for it.
+The template files are the HTML files in the working directory:
+`add.html`, `edit.html`, `search.html` and `view.html`. Feel free to
+change the templates and restart the server. The first change you
+should make is to replace the email address in `view.html`. 😄
 
-The first change you should make is to replace the email address in
-`view.html`. 😄
+See [Structuring the web
+with HTML](https://developer.mozilla.org/en-US/docs/Learn/HTML) to
+learn more about HTML.
+
+Modifying the styles in the templates would be another good start to
+get a feel for it. See [Learn to style HTML using
+CSS](https://developer.mozilla.org/en-US/docs/Learn/CSS) to learn more
+about style sheets.
 
 The templates can refer to the following properties of a page:
 
 `{{.Title}}` is the page title. If the page doesn't provide its own
 title, the page name is used.
 
-`{{.Name}}` is the page name. The page name doesn't include the `.md`
-extension.
+`{{.Name}}` is the page name, escaped for use in URLs. More
+specifically, it is URI escaped except for the slashes. The page name
+doesn't include the `.md` extension.
 
 `{{.Html}}` is the rendered Markdown, as HTML.
 
 `{{printf "%s" .Body}}` is the Markdown, as a string (the data itself
 is a byte array and that's why we need to call `printf`).
 
+For the `search.html` template only:
+
+`{{.Results}}` indicates if there were any search results.
+
+`{{.Items}}` is an array of pages, each containing a search result. A
+search result is a page (with the properties seen above). Thus, to
+refer to them, you need to use a `{{range .Items}}` … `{{end}}`
+construct.
+
+For search results, `{{.Html}}` is the rendered Markdown of a page
+summary, as HTML.
+
+`{{.Score}}` is a numerical score for search results.
+
 When calling the `save` action, the page name is take from the URL and
 the page content is taken from the `body` form parameter. To
 illustrate, here's how to edit a page using `curl`:
@@ -102,6 +137,11 @@ curl --form body="Did you bring a towel?" \
   http://localhost:8080/save/welcome
 ```
 
+The wiki uses the standard
+[html/template](https://pkg.go.dev/html/template) library to do this.
+There's more information on writing templates in the documentation for
+the [text/template](https://pkg.go.dev/text/template) library.
+
 ## Non-English hyphenation
 
 Automatic hyphenation by the browser requires two things: The style
@@ -110,12 +150,11 @@ and that element must have a `lang` set (usually a two letter language
 code such as `de` for German). This happens in the template files,
 such as `view.html` and `search.html`.
 
-If have pages in different languages, the problem is that they all use
-the same template and that's not good. In such cases, it might be
-better to not specificy the `lang` attribute in the template. This
-also disables hyphenation by the browser, unfortunately. It might
-still be better than using English hyphenation patterns for
-non-English languages.
+Oddmu uses the [lingua](github.com/pemistahl/lingua-go) library to
+detect languages. If you know that you're only going to use a small
+number of languages – or just a single language! – you can set the
+environment variable ODDMU_LANGUAGES to a comma-separated list of ISO
+639-1 codes, e.g. "en" or "en,de,fr,pt".
 
 ## Building
 
@@ -142,6 +181,9 @@ If you ran it in the source directory, try
 http://localhost:8080/view/README – this serves the README file you're
 currently reading.
 
+You can change the port by setting the ODDMU_PORT environment
+variable.
+
 ## Deploying it using systemd
 
 As root, on your server:
@@ -160,6 +202,7 @@ likely have to take care of:
 ExecStart=/home/oddmu/oddmu
 WorkingDirectory=/home/oddmu
 Environment="ODDMU_PORT=8080"
+Environment="ODDMU_LANGUAGES=en,de"
 ```
 
 Install the service file and enable it:
@@ -207,28 +250,21 @@ MDCertificateAgreement accepted
     ServerAdmin alex@alexschroeder.ch
     ServerName transjovian.org
     SSLEngine on
-
-    RewriteEngine on
-    RewriteRule ^/$ http://%{HTTP_HOST}:8080/view/index [redirect]
-    RewriteRule ^/(view|edit|save|search)/(.*) http://%{HTTP_HOST}:8080/$1/$2 [proxy]
+    ProxyPassMatch ^/(search|(view|edit|save|add|append)/(.*))?$ http://localhost:8080/$1
 </VirtualHost>
 ```
 
 First, it manages the domain, getting the necessary certificates. It
 redirects regular HTTP traffic from port 80 to port 443. It turns on
-the SSL engine for port 443. It redirects `/` to `/view/index` and any
-path that starts with `/view/`, `/edit/`, `/save/`, `/add/`,
-`/append/` or `/search/` is proxied to port 8080 where the Oddmu
-program can handle it.
+the SSL engine for port 443. It proxies the requests for the wiki to
+port 8080.
 
 Thus, this is what happens:
 
-* The user tells the browser to visit `http://transjovian.org` (on port 80)
-* Apache redirects this to `http://transjovian.org/` by default (still on port 80)
-* Our first virtual host redirects this to `https://transjovian.org/` (encrypted, on port 443)
-* Our second virtual host redirects this to `https://transjovian.org/wiki/view/index` (still on port 443)
-* This is proxied to `http://transjovian.org:8080/view/index` (no on port 8080, without encryption)
-* The wiki converts `index.md` to HTML, adds it to the template, and serves it.
+* The user tells the browser to visit `transjovian.org`
+* The browser sends a request for `http://transjovian.org` (on port 80)
+* Apache redirects this to `https://transjovian.org/` by default (now on port 443)
+* This is proxied to `http://transjovian.org:8080/` (no encryption, on port 8080)
 
 Restart the server, gracefully:
 
@@ -236,6 +272,11 @@ Restart the server, gracefully:
 apachectl graceful
 ```
 
+To serve both HTTP and HTTPS, don't redirect from the first virtual
+host to the second – instead just proxy to the wiki like you did for
+the second virtual host: use a copy of the `ProxyPassMatch` directive
+instead of `RewriteEngine on` and `RewriteRule`.
+
 ## Access
 
 Access control is not part of the wiki. By default, the wiki is
@@ -343,11 +384,14 @@ that matches everything:
 </Location>
 ```
 
-## Customization (with recompilation)
+## Virtual hosting
 
-The Markdown parser can be customized and
-[extensions](https://github.com/russross/blackfriday#extensions) can
-be added. You'll need to make changes to `renderHtml` yourself.
+Virtual hosting in this context means that the program serves two
+different sites for two different domains from the same machine. Oddmu
+doesn't support that, but your webserver does. Therefore, start an
+Oddmu instance for every domain name, each listening on a different
+port. Then set up your web server such that ever domain acts as a
+reverse proxy to a different Oddmu instance.
 
 ## Understanding search
 
@@ -393,6 +437,8 @@ The pages are indexed as the server starts and the index is kept in
 memory. If you have a ton of pages, this surely wastes a lot of
 memory.
 
+Files may not end with a tilde (`~`) – these are backup files.
+
 ## Bugs
 
 If you spot any, [contact](https://alexschroeder.ch/wiki/Contact) me.

TODO.md

@@ -3,10 +3,3 @@ Upload files.
 Automatically scale or process files.
 
 Post by Delta Chat? That is, allow certain encrypted emails to post.
-
-Convert the existing wiki.
-
-Investigate how to run a multi-linugual wiki where an appropriate
-version of a page is served based on language preferences of the user.
-This is a low priority issue since it's probably only of interest for
-corporate or governmental sites.

concurrency_test.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+// Use go test -race to see whether this is a race condition.
+func TestLoadAndSearch(t *testing.T) {
+	go loadIndex()
+	q := "Oddµ"
+	pages := search(q)
+	assert.Zero(t, len(pages))
+}

edit.html

@@ -13,11 +13,11 @@ form, textarea { width: 100%; }
   <body>
     <h1>Editing {{.Title}}</h1>
     <form action="/save/{{.Name}}" method="POST">
-      <div><textarea name="body" rows="20" cols="80" placeholder="# Title
+      <textarea name="body" rows="20" cols="80" placeholder="# Title
 
-Text" autofocus>{{printf "%s" .Body}}</textarea></div>
+Text" autofocus>{{printf "%s" .Body}}</textarea>
       <p><input type="submit" value="Save">
-        <a href="/view/{{.Name}}"><button>Cancel</button></a></p>
+        <a href="/view/{{.Name}}"><button type="button">Cancel</button></a></p>
     </form>
   </body>
 </html>

go.mod

@@ -4,9 +4,10 @@ go 1.21.0
 
 require (
 	github.com/dgryski/go-trigram v0.0.0-20160407183937-79ec494e1ad0
+	github.com/gomarkdown/markdown v0.0.0-20230912175223-14b07df9d538
 	github.com/microcosm-cc/bluemonday v1.0.25
 	github.com/pemistahl/lingua-go v1.4.0
-	github.com/russross/blackfriday/v2 v2.1.0
+	github.com/stretchr/testify v1.8.4
 )
 
 require (
@@ -15,7 +16,6 @@ require (
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
 	golang.org/x/exp v0.0.0-20221106115401-f9659909a136 // indirect
 	golang.org/x/net v0.12.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect

go.sum

@@ -5,6 +5,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dgryski/go-trigram v0.0.0-20160407183937-79ec494e1ad0 h1:b+7JSiBM+hnLQjP/lXztks5hnLt1PS46hktG9VOJgzo=
 github.com/dgryski/go-trigram v0.0.0-20160407183937-79ec494e1ad0/go.mod h1:qzKC/DpcxK67zaSHdCmIv3L9WJViHVinYXN2S7l3RM8=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12 h1:uK3X/2mt4tbSGoHvbLBHUny7CKiuwUip3MArtukol4E=
+github.com/gomarkdown/markdown v0.0.0-20230716120725-531d2d74bc12/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+github.com/gomarkdown/markdown v0.0.0-20230912175223-14b07df9d538 h1:ePDpFu7l0QUV46/9A7icfL2wvIOzTJLCWh4RO2NECzE=
+github.com/gomarkdown/markdown v0.0.0-20230912175223-14b07df9d538/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -16,8 +20,6 @@ github.com/pemistahl/lingua-go v1.4.0 h1:ifYhthrlW7iO4icdubwlduYnmwU37V1sbNrwhKB
 github.com/pemistahl/lingua-go v1.4.0/go.mod h1:ECuM1Hp/3hvyh7k8aWSqNCPlTxLemFZsRjocUf3KgME=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
@@ -30,6 +32,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

languages.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"errors"
+	"github.com/pemistahl/lingua-go"
+	"os"
+	"strings"
+)
+
+// getLangauges returns the environment variable ODDMU_LANGUAGES or
+// all languages.
+func getLanguages() ([]lingua.Language, error) {
+	v := os.Getenv("ODDMU_LANGUAGES")
+	if v == "" {
+		return lingua.AllLanguages(), nil
+	}
+	codes := strings.Split(v, ",")
+	if len(codes) == 1 {
+		return nil, errors.New("detection unnecessary")
+	}
+
+	var langs []lingua.Language
+	for _, lang := range codes {
+		langs = append(langs, lingua.GetLanguageFromIsoCode639_1(lingua.GetIsoCode639_1FromValue(lang)))
+	}
+	return langs, nil
+}
+
+// detector is the LanguageDetector initialized at startup by loadLanguages.
+var detector lingua.LanguageDetector
+
+// loadLanguages initializes the detector using the languages returned
+// by getLanguages and returns the number of languages loaded.
+func loadLanguages() int {
+	langs, err := getLanguages()
+	if err == nil {
+		detector = lingua.NewLanguageDetectorBuilder().
+			FromLanguages(langs...).
+			WithPreloadedLanguageModels().
+			WithLowAccuracyMode().
+			Build()
+	} else {
+		detector = nil
+	}
+	return len(langs)
+}
+
+// language returns the language used for a string, as a lower case
+// ISO 639-1 string, e.g. "en" or "de".
+func language(s string) string {
+	if detector == nil {
+		return os.Getenv("ODDMU_LANGUAGES")
+	}
+	if language, ok := detector.DetectLanguageOf(s); ok {
+		return strings.ToLower(language.IsoCode639_1().String())
+	}
+	return ""
+}

languages_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestAllLanguage(t *testing.T) {
+	os.Unsetenv("ODDMU_LANGUAGES")
+	loadLanguages()
+	l := language(`
+My back hurts at night
+My shoulders won't budge today
+Winter bones I say`)
+	assert.Equal(t, "en", l)
+}
+
+func TestSomeLanguages(t *testing.T) {
+	os.Setenv("ODDMU_LANGUAGES", "en,de")
+	loadLanguages()
+	l := language(`
+Kühle Morgenluft
+Keine Amsel singt heute
+Mensch im Dämmerlicht
+`)
+	assert.Equal(t, "de", l)
+}
+
+func TestOneLanguages(t *testing.T) {
+	os.Setenv("ODDMU_LANGUAGES", "en")
+	loadLanguages()
+	l := language(`
+Schwer wiegt die Luft hier
+Atme ein, ermahn' ich mich
+Erinnerungen
+`)
+	assert.Equal(t, "en", l)
+}
+
+func TestWrongLanguages(t *testing.T) {
+	os.Setenv("ODDMU_LANGUAGES", "de,fr")
+	loadLanguages()
+	l := language(`
+Something drifts down there
+Head submerged oh god a man
+Drowning as we stare
+`)
+	assert.NotEqual(t, "en", l)
+}

page.go

@@ -3,30 +3,17 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/ast"
+	"github.com/gomarkdown/markdown/parser"
 	"github.com/microcosm-cc/bluemonday"
-	"github.com/pemistahl/lingua-go"
-	"github.com/russross/blackfriday/v2"
 	"html/template"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 )
 
-// languages is the list of languages the wiki understands. This is
-// passed along to the template so that it can be added to the
-// template which allows browsers to (maybe) do hyphenation correctly.
-var languages = []lingua.Language{
-	lingua.English,
-	lingua.German,
-}
-
-// detector is built once based on the list languages.
-var detector = lingua.NewLanguageDetectorBuilder().
-	FromLanguages(languages...).
-	WithPreloadedLanguageModels().
-	WithLowAccuracyMode().
-	Build()
-
 // Page is a struct containing information about a single page. Title
 // is the title extracted from the page content using titleRegexp.
 // Name is the filename without extension (so a filename of "foo.md"
@@ -42,14 +29,26 @@ type Page struct {
 	Score    int
 }
 
+// santize uses bluemonday to sanitize the HTML.
 func sanitize(s string) template.HTML {
 	return template.HTML(bluemonday.UGCPolicy().Sanitize(s))
 }
 
+// santizeBytes uses bluemonday to sanitize the HTML.
 func sanitizeBytes(bytes []byte) template.HTML {
 	return template.HTML(bluemonday.UGCPolicy().SanitizeBytes(bytes))
 }
 
+// nameEscape returns the page name safe for use in URLs. That is,
+// percent escaping is used except for the slashes.
+func nameEscape(s string) string {
+	parts := strings.Split(s, "/")
+	for i, part := range parts {
+		parts[i] = url.PathEscape(part)
+	}
+	return strings.Join(parts, "/")
+}
+
 // save saves a Page. The filename is based on the Page.Name and gets
 // the ".md" extension. Page.Body is saved, without any carriage
 // return characters ("\r"). The file permissions used are readable
@@ -58,17 +57,20 @@ func sanitizeBytes(bytes []byte) template.HTML {
 func (p *Page) save() error {
 	filename := p.Name + ".md"
 	s := bytes.ReplaceAll(p.Body, []byte{'\r'}, []byte{})
+	if len(s) == 0 {
+		return os.Remove(filename)
+	}
 	p.Body = s
 	p.updateIndex()
 	d := filepath.Dir(filename)
 	if d != "." {
-		err := os.MkdirAll(d, 0700)
+		err := os.MkdirAll(d, 0755)
 		if err != nil {
 			fmt.Printf("Creating directory %s failed", d)
 			return err
 		}
 	}
-	return os.WriteFile(filename, s, 0600)
+	return os.WriteFile(filename, s, 0644)
 }
 
 // loadPage loads a Page given a name. The filename loaded is that
@@ -99,9 +101,31 @@ func (p *Page) handleTitle(replace bool) {
 	}
 }
 
+func hashtag(p *parser.Parser, data []byte, offset int) (int, ast.Node) {
+	data = data[offset:]
+	i := 0
+	n := len(data)
+	for i < n && !parser.IsSpace(data[i]) {
+		i++
+	}
+	if i == 0 {
+		return 0, nil
+	}
+	link := &ast.Link{
+		Destination: append([]byte("/search?q=%23"), data[1:i]...),
+		Title:       data[0:i],
+	}
+	text := bytes.ReplaceAll(data[0:i], []byte("_"), []byte(" "))
+	ast.AppendChild(link, &ast.Text{Leaf: ast.Leaf{Literal: text}})
+	return i, link
+}
+
 // renderHtml renders the Page.Body to HTML and sets Page.Html.
 func (p *Page) renderHtml() {
-	maybeUnsafeHTML := blackfriday.Run(p.Body)
+	parser := parser.New()
+	parser.RegisterInline('#', hashtag)
+	maybeUnsafeHTML := markdown.ToHTML(p.Body, parser, nil)
+	p.Name = nameEscape(p.Name)
 	p.Html = sanitizeBytes(maybeUnsafeHTML)
 	p.Language = language(p.plainText())
 }
@@ -110,30 +134,26 @@ func (p *Page) renderHtml() {
 // ignoring all the Markdown and all the newlines. The result is one
 // long single line of text.
 func (p *Page) plainText() string {
-	optList := []blackfriday.Option{blackfriday.WithExtensions(blackfriday.CommonExtensions)}
-	parser := blackfriday.New(optList...)
-	ast := parser.Parse(p.Body)
+	parser := parser.New()
+	doc := markdown.Parse(p.Body, parser)
 	text := []byte("")
-	ast.Walk(func(node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
-		if entering && node.Type == blackfriday.Text {
-			s := node.Literal
-			if len(s) == 0 {
-				return blackfriday.GoToNext
-			}
-			// Some Markdown still contains newlines
-			for i, c := range s {
-				if c == '\n' {
-					s[i] = ' '
-				}
-			}
-			if len(text) > 0 && text[len(text)-1] != ' ' && s[0] != ' ' {
-				text = append(text, ' ')
-			}
-
-			text = append(text, s...)
+	ast.WalkFunc(doc, func(node ast.Node, entering bool) ast.WalkStatus {
+		if entering && node.AsLeaf() != nil {
+			text = append(text, node.AsLeaf().Literal...)
+			text = append(text, []byte(" ")...)
 		}
-		return blackfriday.GoToNext
+		return ast.GoToNext
 	})
+	// Some Markdown still contains newlines
+	for i, c := range text {
+		if c == '\n' {
+			text[i] = ' '
+		}
+	}
+	// Remove trailing space
+	for len(text) > 0 && text[len(text)-1] == ' ' {
+		text = text[0 : len(text)-1]
+	}
 	return string(text)
 }
 
@@ -145,10 +165,3 @@ func (p *Page) summarize(q string) {
 	p.Html = sanitize(snippets(q, t))
 	p.Language = language(t)
 }
-
-func language(s string) string {
-	if language, ok := detector.DetectLanguageOf(s); ok {
-		return strings.ToLower(language.IsoCode639_1().String())
-	}
-	return ""
-}

page_test.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"github.com/stretchr/testify/assert"
 	"os"
 	"regexp"
 	"testing"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestPageTitle(t *testing.T) {
@@ -43,6 +43,25 @@ A cruel sun stares down</p>
 	assert.Equal(t, r, string(p.Html))
 }
 
+func TestPageHtmlHashtag(t *testing.T) {
+	p := &Page{Body: []byte(`# Comet
+Stars flicker above
+Too faint to focus, so far
+I am cold, alone
+
+#Haiku #Cold_Poets`)}
+	p.renderHtml()
+	r := `<h1>Comet</h1>
+
+<p>Stars flicker above
+Too faint to focus, so far
+I am cold, alone</p>
+
+<p><a href="/search?q=%23Haiku" rel="nofollow">#Haiku</a> <a href="/search?q=%23Cold_Poets" rel="nofollow">#Cold Poets</a></p>
+`
+	assert.Equal(t, r, string(p.Html))
+}
+
 func TestPageDir(t *testing.T) {
 	_ = os.RemoveAll("testdata")
 	loadIndex()
@@ -54,15 +73,13 @@ Moonlight floods the aisle`)}
 	o, err := loadPage("testdata/moon")
 	assert.NoError(t, err, "load page")
 	assert.Equal(t, p.Body, o.Body)
+	assert.FileExists(t, "testdata/moon.md")
+
+	// Saving an empty page deletes it.
+	p = &Page{Name: "testdata/moon", Body: []byte("")}
+	p.save()
+	assert.NoFileExists(t, "testdata/moon.md")
 	t.Cleanup(func() {
 		_ = os.RemoveAll("testdata")
 	})
 }
-
-func TestLanguage(t *testing.T) {
-	l := language(`
-My back hurts at night
-My shoulders won't budge today
-Winter bones I say`)
-	assert.Equal(t, "en", l)
-}

score.go

@@ -18,10 +18,7 @@ func score(q string, s string) int {
 			score += len(m)
 		}
 	}
-	for _, v := range strings.Split(q, " ") {
-		if len(v) == 0 {
-			continue
-		}
+	for _, v := range strings.Fields(q) {
 		re, err := regexp.Compile(`(?is)(\pL?)(` + v + `)(\pL?)`)
 		if err != nil {
 			continue

search.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"sync"
 	"unicode"
 	"unicode/utf8"
 )
@@ -21,15 +22,23 @@ type Search struct {
 	Results bool
 }
 
-// index is a struct containing the trigram index for search. It is
-// generated at startup and updated after every page edit. The index
-// is case-insensitive.
-var index trigram.Index
+// idx contains the two maps used for search. Make sure to lock and
+// unlock as appropriate.
+var idx = struct {
+	sync.RWMutex
 
-// documents is a map, mapping document ids of the index to page
-// names.
-var documents map[trigram.DocID]string
+	// index is a struct containing the trigram index for search. It is
+	// generated at startup and updated after every page edit. The index
+	// is case-insensitive.
+	index trigram.Index
 
+	// documents is a map, mapping document ids of the index to page
+	// names.
+	documents map[trigram.DocID]string
+}{}
+
+// indexAdd reads a file and adds it to the index. This must happen
+// while the idx is locked, which is true when called from loadIndex.
 func indexAdd(path string, info fs.FileInfo, err error) error {
 	if err != nil {
 		return err
@@ -43,58 +52,91 @@ func indexAdd(path string, info fs.FileInfo, err error) error {
 	if err != nil {
 		return err
 	}
-	id := index.Add(strings.ToLower(string(p.Body)))
-	documents[id] = p.Name
+	id := idx.index.Add(strings.ToLower(string(p.Body)))
+	idx.documents[id] = p.Name
 	return nil
 }
 
-func loadIndex() error {
-	index = make(trigram.Index)
-	documents = make(map[trigram.DocID]string)
+// loadIndex loads all the pages and indexes them. This takes a while.
+// It returns the number of pages indexed.
+func loadIndex() (int, error) {
+	idx.Lock()
+	defer idx.Unlock()
+	idx.index = make(trigram.Index)
+	idx.documents = make(map[trigram.DocID]string)
 	err := filepath.Walk(".", indexAdd)
 	if err != nil {
-		fmt.Println("Indexing failed")
-		index = nil
-		documents = nil
+		idx.index = nil
+		idx.documents = nil
+		return 0, err
 	}
-	return err
+	n := len(idx.documents)
+	return n, nil
 }
 
+// updateIndex updates the index for a single page. The old text is
+// loaded from the disk and removed from the index first, if it
+// exists.
 func (p *Page) updateIndex() {
+	idx.Lock()
+	defer idx.Unlock()
 	var id trigram.DocID
-	for docId, name := range documents {
+	// This function does not rely on files actually existing, so
+	// let's quickly find the document id.
+	for docId, name := range idx.documents {
 		if name == p.Name {
 			id = docId
 			break
 		}
 	}
 	if id == 0 {
-		id = index.Add(strings.ToLower(string(p.Body)))
-		documents[id] = p.Name
+		id = idx.index.Add(strings.ToLower(string(p.Body)))
+		idx.documents[id] = p.Name
 	} else {
 		o, err := loadPage(p.Name)
 		if err == nil {
-			index.Delete(strings.ToLower(string(o.Body)), id)
+			idx.index.Delete(strings.ToLower(string(o.Body)), id)
 		}
-		index.Insert(strings.ToLower(string(p.Body)), id)
+		idx.index.Insert(strings.ToLower(string(p.Body)), id)
 	}
 }
 
-// search returns a sorted []Page where each page contains an extract
-// of the actual Page.Body in its Page.Html.
-func search(q string) []Page {
-	if len(q) == 0 {
-		return make([]Page, 0)
+func sortItems(a, b Page) int {
+	// Sort by score
+	if a.Score < b.Score {
+		return 1
+	} else if a.Score > b.Score {
+		return -1
 	}
-	words := strings.Split(strings.ToLower(q), " ")
-	var trigrams []trigram.T
-	for _, word := range words {
-		trigrams = trigram.Extract(word, trigrams)
+	// If the score is the same and both page names start
+	// with a number (like an ISO date), sort descending.
+	ra, _ := utf8.DecodeRuneInString(a.Title)
+	rb, _ := utf8.DecodeRuneInString(b.Title)
+	if unicode.IsNumber(ra) && unicode.IsNumber(rb) {
+		if a.Title < b.Title {
+			return 1
+		} else if a.Title > b.Title {
+			return -1
+		} else {
+			return 0
+		}
 	}
-	ids := index.QueryTrigrams(trigrams)
-	items := make([]Page, len(ids))
-	for i, id := range ids {
-		name := documents[id]
+	// Otherwise sort ascending.
+	if a.Title < b.Title {
+		return -1
+	} else if a.Title > b.Title {
+		return 1
+	} else {
+		return 0
+	}
+}
+
+// loadAndSummarize loads the pages named and summarizes them for the
+// query give.
+func loadAndSummarize(names []string, q string) []Page {
+	// Load and summarize the items.
+	items := make([]Page, len(names))
+	for i, name := range names {
 		p, err := loadPage(name)
 		if err != nil {
 			fmt.Printf("Error loading %s\n", name)
@@ -103,35 +145,30 @@ func search(q string) []Page {
 			items[i] = *p
 		}
 	}
-	fn := func(a, b Page) int {
-		// Sort by score
-		if a.Score < b.Score {
-			return 1
-		} else if a.Score > b.Score {
-			return -1
-		}
-		// If the score is the same and both page names start
-		// with a number (like an ISO date), sort descending.
-		ra, _ := utf8.DecodeRuneInString(a.Title)
-		rb, _ := utf8.DecodeRuneInString(b.Title)
-		if unicode.IsNumber(ra) && unicode.IsNumber(rb) {
-			if a.Title < b.Title {
-				return 1
-			} else if a.Title > b.Title {
-				return -1
-			} else {
-				return 0
-			}
-		}
-		// Otherwise sort ascending.
-		if a.Title < b.Title {
-			return -1
-		} else if a.Title > b.Title {
-			return 1
-		} else {
-			return 0
-		}
-	}
-	slices.SortFunc(items, fn)
+	return items
+}
+
+// search returns a sorted []Page where each page contains an extract
+// of the actual Page.Body in its Page.Html.
+func search(q string) []Page {
+	if len(q) == 0 {
+		return make([]Page, 0)
+	}
+	words := strings.Fields(strings.ToLower(q))
+	var trigrams []trigram.T
+	for _, word := range words {
+		trigrams = trigram.Extract(word, trigrams)
+	}
+	// Keep the read lock for a short as possible. Make a list of
+	// the names we need to load and summarize.
+	idx.RLock()
+	ids := idx.index.QueryTrigrams(trigrams)
+	names := make([]string, len(ids))
+	for i, id := range ids {
+		names[i] = idx.documents[id]
+	}
+	idx.RUnlock()
+	items := loadAndSummarize(names, q)
+	slices.SortFunc(items, sortItems)
 	return items
 }

search_test.go

@@ -1,40 +1,41 @@
 package main
 
 import (
+	"github.com/stretchr/testify/assert"
 	"os"
 	"strings"
 	"testing"
 )
 
-var name string = "test"
-
 // TestIndex relies on README.md being indexed
 func TestIndex(t *testing.T) {
-	_ = os.Remove(name + ".md")
 	loadIndex()
 	q := "Oddµ"
 	pages := search(q)
-	if len(pages) == 0 {
-		t.Log("Search found no result")
-		t.Fail()
-	}
+	assert.NotZero(t, len(pages))
 	for _, p := range pages {
-		if strings.Contains(p.Title, "<b>") {
-			t.Logf("Page %s contains <b>", p.Name)
-			t.Fail()
-		}
-		if !strings.Contains(string(p.Body), q) && !strings.Contains(string(p.Title), q) {
-			t.Logf("Page %s does not contain %s", p.Name, q)
-			t.Fail()
-		}
-		if p.Score == 0 {
-			t.Logf("Page %s has no score", p.Name)
-			t.Fail()
-		}
+		assert.NotContains(t, p.Title, "<b>")
+		assert.True(t, strings.Contains(string(p.Body), q) || strings.Contains(string(p.Title), q))
+		assert.NotZero(t, p.Score)
 	}
+}
+
+func TestSearchHashtag(t *testing.T) {
+	loadIndex()
+	q := "#Another_Tag"
+	pages := search(q)
+	assert.NotZero(t, len(pages))
+}
+
+func TestIndexUpdates(t *testing.T) {
+	name := "test"
+	_ = os.Remove(name + ".md")
+	loadIndex()
 	p := &Page{Name: name, Body: []byte("This is a test.")}
 	p.save()
-	pages = search("This is a test")
+
+	// Find the phrase
+	pages := search("This is a test")
 	found := false
 	for _, p := range pages {
 		if p.Name == name {
@@ -42,10 +43,9 @@ func TestIndex(t *testing.T) {
 			break
 		}
 	}
-	if !found {
-		t.Logf("Page '%s' was not found", name)
-		t.Fail()
-	}
+	assert.True(t, found)
+
+	// Find the phrase, case insensitive
 	pages = search("this is a test")
 	found = false
 	for _, p := range pages {
@@ -54,10 +54,9 @@ func TestIndex(t *testing.T) {
 			break
 		}
 	}
-	if !found {
-		t.Logf("Page '%s' was not found using the lower case text", name)
-		t.Fail()
-	}
+	assert.True(t, found)
+
+	// Find some words
 	pages = search("this test")
 	found = false
 	for _, p := range pages {
@@ -66,10 +65,9 @@ func TestIndex(t *testing.T) {
 			break
 		}
 	}
-	if !found {
-		t.Logf("Page '%s' was not found using a query missing some words", name)
-		t.Fail()
-	}
+	assert.True(t, found)
+
+	// Update the page and no longer find it with the old phrase
 	p = &Page{Name: name, Body: []byte("Guvf vf n grfg.")}
 	p.save()
 	pages = search("This is a test")
@@ -80,10 +78,9 @@ func TestIndex(t *testing.T) {
 			break
 		}
 	}
-	if found {
-		t.Logf("Page '%s' was still found using the old content: %s", name, p.Body)
-		t.Fail()
-	}
+	assert.False(t, found)
+
+	// Find page using a new word
 	pages = search("Guvf")
 	found = false
 	for _, p := range pages {
@@ -92,10 +89,8 @@ func TestIndex(t *testing.T) {
 			break
 		}
 	}
-	if !found {
-		t.Logf("Page '%s' not found using the new content: %s", name, p.Body)
-		t.Fail()
-	}
+	assert.True(t, found)
+
 	t.Cleanup(func() {
 		_ = os.Remove(name + ".md")
 	})

wiki.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"os"
 	"regexp"
-	"strings"
 )
 
 // Templates are parsed at startup.
@@ -14,10 +13,10 @@ var templates = template.Must(
 	template.ParseFiles("edit.html", "add.html", "view.html", "search.html"))
 
 // validPath is a regular expression where the second group matches a
-// page, so when the handler for "/edit/" is called, a URL path of
-// "/edit/foo" results in the editHandler being called with title
-// "foo". The regular expression doesn't define the handlers (this
-// happens in the main function).
+// page, so when the editHandler is called, a URL path of "/edit/foo"
+// results in the editHandler being called with title "foo". The
+// regular expression doesn't define the handlers (this happens in the
+// main function).
 var validPath = regexp.MustCompile("^/([^/]+)/(.+)$")
 
 // titleRegexp is a regular expression matching a level 1 header line
@@ -40,28 +39,25 @@ func rootHandler(w http.ResponseWriter, r *http.Request) {
 	http.Redirect(w, r, "/view/index", http.StatusFound)
 }
 
-// viewHandler renders a text file, if the name ends in ".txt" and
-// such a file exists. Otherwise, it loads the page. If this didn't
-// work, the browser is redirected to an edit page. Otherwise, the
-// "view.html" template is used to show the rendered HTML.
+// viewHandler serves existing files (including markdown files with
+// the .md extension). If the requested file does not exist, a page
+// with the same name is loaded. This means adding the .md extension
+// and using the "view.html" template to render the HTML. Both
+// attempts fail, the browser is redirected to an edit page.
 func viewHandler(w http.ResponseWriter, r *http.Request, name string) {
-	// Short cut for text files
-	if strings.HasSuffix(name, ".txt") {
-		body, err := os.ReadFile(name)
-		if err == nil {
-			w.Write(body)
-			return
-		}
-	}
-	// Attempt to load Markdown page; edit it if this fails
-	p, err := loadPage(name)
-	if err != nil {
-		http.Redirect(w, r, "/edit/"+name, http.StatusFound)
+	body, err := os.ReadFile(name)
+	if err == nil {
+		w.Write(body)
 		return
 	}
-	p.handleTitle(true)
-	p.renderHtml()
-	renderTemplate(w, "view", p)
+	p, err := loadPage(name)
+	if err == nil {
+		p.handleTitle(true)
+		p.renderHtml()
+		renderTemplate(w, "view", p)
+		return
+	}
+	http.Redirect(w, r, "/edit/"+name, http.StatusFound)
 }
 
 // editHandler uses the "edit.html" template to present an edit page.
@@ -159,6 +155,28 @@ func getPort() string {
 	return port
 }
 
+// scheduleLoadIndex calls loadIndex and prints some messages before
+// and after. For testing, call loadIndex directly and skip the
+// messages.
+func scheduleLoadIndex() {
+	fmt.Print("Indexing pages\n")
+	n, err := loadIndex()
+	if err == nil {
+		fmt.Printf("Indexed %d pages\n", n)
+	} else {
+		fmt.Println("Indexing failed")
+	}
+}
+
+// scheduleLoadLanguages calls loadLanguages and prints some messages before
+// and after. For testing, call loadLanguages directly and skip the
+// messages.
+func scheduleLoadLanguages() {
+	fmt.Print("Loading languages\n")
+	n := loadLanguages()
+	fmt.Printf("Loaded %d languages\n", n)
+}
+
 func main() {
 	http.HandleFunc("/", rootHandler)
 	http.HandleFunc("/view/", makeHandler(viewHandler))
@@ -167,8 +185,8 @@ func main() {
 	http.HandleFunc("/add/", makeHandler(addHandler))
 	http.HandleFunc("/append/", makeHandler(appendHandler))
 	http.HandleFunc("/search", searchHandler)
-	fmt.Print("Indexing all pages\n")
-	loadIndex()
+	go scheduleLoadIndex()
+	go scheduleLoadLanguages()
 	port := getPort()
 	fmt.Printf("Serving a wiki on port %s\n", port)
 	http.ListenAndServe(":"+port, nil)

wiki_test.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// HTTPHeaders is a helper that returns HTTP headers of the response. It returns
+// nil if building a new request fails.
+func HTTPHeaders(handler http.HandlerFunc, method, url string, values url.Values, header string) []string {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(method, url+"?"+values.Encode(), nil)
+	if err != nil {
+		return nil
+	}
+	handler(w, req)
+	return w.Result().Header[header]
+}
+
+// HTTPRedirectTo checks that the request results in a redirect and it
+// checks the destination of the redirect. It returns whether the
+// request did in fact result in a redirect. Note: This method assumes
+// that POST requests ignore the query part of the URL which is often
+// true but not mandated by the standards.
+func HTTPRedirectTo(t *testing.T, handler http.HandlerFunc, method, url string, values url.Values, destination string) bool {
+	w := httptest.NewRecorder()
+	var req *http.Request
+	var err error
+	if method == http.MethodPost {
+		body := strings.NewReader(values.Encode())
+		req, err = http.NewRequest(method, url, body)
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	} else {
+		req, err = http.NewRequest(method, url+"?"+values.Encode(), nil)
+	}
+	if err != nil {
+		assert.Fail(t, fmt.Sprintf("Failed to build test request, got error: %s", err))
+	}
+	handler(w, req)
+	code := w.Code
+	isRedirectCode := code >= http.StatusMultipleChoices && code <= http.StatusTemporaryRedirect
+	if !isRedirectCode {
+		assert.Fail(t, fmt.Sprintf("Expected HTTP redirect status code for %q but received %d", url+"?"+values.Encode(), code))
+	}
+	headers := w.Result().Header["Location"]
+	if len(headers) != 1 || headers[0] != destination {
+		assert.Fail(t, fmt.Sprintf("Expected HTTP redirect location %s for %q but received %v", destination, url+"?"+values.Encode(), headers))
+	}
+	return isRedirectCode
+}
+
+func TestRootHandler(t *testing.T) {
+	HTTPRedirectTo(t, rootHandler, "GET", "/", nil, "/view/index")
+}
+
+// relies on index.md in the current directory!
+func TestViewHandler(t *testing.T) {
+	assert.Regexp(t, regexp.MustCompile("Welcome to Oddµ"),
+		assert.HTTPBody(makeHandler(viewHandler), "GET", "/view/index", nil))
+}
+
+// wipes testdata
+func TestEditSave(t *testing.T) {
+	_ = os.RemoveAll("testdata")
+
+	data := url.Values{}
+	data.Set("body", "Hallo!")
+
+	HTTPRedirectTo(t, makeHandler(viewHandler), "GET", "/view/testdata/alex", nil, "/edit/testdata/alex")
+	assert.HTTPStatusCode(t, makeHandler(editHandler), "GET", "/edit/testdata/alex", nil, 200)
+	HTTPRedirectTo(t, makeHandler(saveHandler), "POST", "/save/testdata/alex", data, "/view/testdata/alex")
+	assert.Regexp(t, regexp.MustCompile("Hallo!"),
+		assert.HTTPBody(makeHandler(viewHandler), "GET", "/view/testdata/alex", nil))
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll("testdata")
+	})
+}
+
+// wipes testdata
+func TestAddAppend(t *testing.T) {
+	_ = os.RemoveAll("testdata")
+
+	p := &Page{Name: "testdata/fire", Body: []byte(`# Fire
+Orange sky above
+Reflects a distant fire
+It's not `)}
+	p.save()
+
+	data := url.Values{}
+	data.Set("body", "barbecue")
+
+	assert.Regexp(t, regexp.MustCompile("a distant fire"),
+		assert.HTTPBody(makeHandler(viewHandler), "GET", "/view/testdata/fire", nil))
+	assert.NotRegexp(t, regexp.MustCompile("a distant fire"),
+		assert.HTTPBody(makeHandler(addHandler), "GET", "/add/testdata/fire", nil))
+	HTTPRedirectTo(t, makeHandler(appendHandler), "POST", "/append/testdata/fire", data, "/view/testdata/fire")
+	assert.Regexp(t, regexp.MustCompile("It’s not barbecue"),
+		assert.HTTPBody(makeHandler(viewHandler), "GET", "/view/testdata/fire", nil))
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll("testdata")
+	})
+}
+
+// wipes testdata
+func TestPageTitleWithAmp(t *testing.T) {
+	_ = os.RemoveAll("testdata")
+
+	p := &Page{Name: "testdata/Rock & Roll", Body: []byte("Dancing")}
+	p.save()
+
+	assert.Regexp(t, regexp.MustCompile("Rock &amp; Roll"),
+		assert.HTTPBody(makeHandler(viewHandler), "GET", "/view/testdata/Rock%20%26%20Roll", nil))
+
+	p = &Page{Name: "testdata/Rock & Roll", Body: []byte("# Sex & Drugs & Rock'n'Roll\nOh no!")}
+	p.save()
+
+	assert.Regexp(t, regexp.MustCompile("Sex &amp; Drugs"),
+		assert.HTTPBody(makeHandler(viewHandler), "GET", "/view/testdata/Rock%20%26%20Roll", nil))
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll("testdata")
+	})
+}
+
+func TestPageTitleWithQuestionMark(t *testing.T) {
+	_ = os.RemoveAll("testdata")
+
+	p := &Page{Name: "testdata/How about no?", Body: []byte("No means no")}
+	p.save()
+
+	body := assert.HTTPBody(makeHandler(viewHandler), "GET", "/view/testdata/How%20about%20no%3F", nil)
+	assert.Contains(t, body, "No means no")
+	assert.Contains(t, body, "<a href=\"/edit/testdata/How%20about%20no%3F\">Edit</a>")
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll("testdata")
+	})
+}