Fixed GetHiddenValue encoding errors.

GetHiddenValue was returning strings that had lost their UTF-8 flag or something like that. In order to fix this, I needed to replace the call to $q->hidden with a call to $q->input with -type=>"hidden".
git.pl: run "git add ." as part of maintenance.
2012-08-15 18:47:10 +02:00 · 2012-08-14 21:41:19 +02:00 · 2012-08-14 09:06:16 -04:00 · 2012-08-14 09:05:22 -04:00 · 2012-08-13 16:00:47 +02:00 · 2012-08-07 17:26:12 +02:00
88 changed files with 1234 additions and 532 deletions
--- a/.cvsignore
+++ b/.cvsignore
@@ -1,8 +0,0 @@
-.DS_Store
-oddmuse-1.*
-oddmuse-inkscape-1.*
-*.patch
-*.patch.gz
-*.diff
-*.diff.gz
-test-data
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,7 @@
 *~
-/current.pl
+/build/
 \#*\#
 /test-data
+/Mac/pkg/
+*.dmg
+*.pkg
--- a/Mac/.cvsignore
+++ b/Mac/.cvsignore
@@ -1 +0,0 @@
-pkg
--- a/54
+++ b/54
@@ -2,28 +2,55 @@
 # Make sure the CVS keywords for the sed command on the next line are not expanded.

 VERSION_NO=$(shell git describe --tags)
-VERSION=oddmuse-$(VERSION_NO)
-UPLOADVERSION=oddmuse-inkscape-$(VERSION_NO)
 TRANSLATIONS=$(wildcard modules/translations/[a-z]*-utf8.pl$)
 MODULES=$(wildcard modules/*.pl)
+BUILD=build/wiki.pl $(foreach file, $(notdir $(MODULES)) $(notdir $(TRANSLATIONS)), build/$(file))
+
+# PREPARE/BUILD: this creates copies of wiki.pl and all the modules
+# and translations in the build subdirectory. These copies all contain
+# a reference to the revision they were created from (git describe
+# --tags).
+
+prepare: build $(BUILD)
+
+build:
+	mkdir -p build
+
+build/wiki.pl: wiki.pl
+	sed "s/\\\$$q->a({-href=>'http:\/\/www.oddmuse.org\/'}, 'Oddmuse')/\\\$$q->a({-href=>'http:\/\/git.savannah.gnu.org\/cgit\/oddmuse.git\/tag\/?id=$(VERSION_NO)'}, 'wiki.pl') . ' ($(VERSION_NO)), see ' . &/" < $< > $@
+
+build/%-utf8.pl: modules/translations/%-utf8.pl
+	sed "s/<a href=\"http:\/\/git.savannah.gnu.org\/cgit\/oddmuse.git\/tree\/modules\/translations\/\\(.*\\).pl\">\\(.*\\).pl<\/a>/<a href=\"http:\/\/git.savannah.gnu.org\/cgit\/oddmuse.git\/tree\/modules\/translations\/\\1.pl?id=$(VERSION_NO)\">\\1.pl<\/a> (for $(VERSION_NO))/" < $< > $@
+
+# Currently oddtrans introduces encoding errors!
+
+# %-utf8.pl: wiki.pl $(MODULES)
+# 	perl oddtrans -l $@ $^ > $@-new && mv $@-new $@
+
+# from: http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/namespaces.pl
+#   to: http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/namespaces.pl?id=2.1-11-gd4f1e27
+
+build/%.pl: modules/%.pl
+	sed "s/<a href=\"http:\/\/git.savannah.gnu.org\/cgit\/oddmuse.git\/tree\/modules\/\\(.*\\).pl\">\\(.*\\).pl<\/a>/<a href=\"http:\/\/git.savannah.gnu.org\/cgit\/oddmuse.git\/tree\/modules\/\\1.pl?id=$(VERSION_NO)\">\\1.pl<\/a> (for $(VERSION_NO))/" < $< > $@
+
+
+
+
+# UNTESTED/OBSOLETE: these targets have not been tested in a long time
+# and are potentially obsolete.
+VERSION=oddmuse-$(VERSION_NO)
+UPLOADVERSION=oddmuse-inkscape-$(VERSION_NO)
 INKSCAPE=GPL $(wildcard inkscape/*.py inkscape/*.inx inkscape/*.sh)
 PACKAGEMAKER=/Developer/Applications/Utilities/PackageMaker.app/Contents/MacOS/PackageMaker
-PWD=$(shell pwd)
 DIST=$(VERSION).tar.gz $(VERSION).tar.gz.sig \
 	contrib/simple-install/$(VERSION)-simple.tar.gz \
 	contrib/simple-install/$(VERSION)-simple.tar.gz.sig
-
-# These targets no longer work are have not been verified in a long time.
 OLDDIST=$(VERSION).dmg $(VERSION).dmg.sig \
-	$(VERSION).tar.gz $(VERSION).tar.gz.sig \
-	$(VERSION).tgz $(VERSION).tgz.sig \
-	$(UPLOADVERSION).tar.gz $(UPLOADVERSION).tar.gz.sig
+	$(VERSION).tgz $(VERSION).tgz.sig
+PWD=$(shell pwd)

 dist: $(DIST)

-current.pl: wiki.pl
-	sed "s/\\\$$q->a({-href=>'http:\/\/www.oddmuse.org\/'}, 'Oddmuse')/\\\$$q->a({-href=>'http:\/\/git.savannah.gnu.org\/cgit\/oddmuse.git\/tag\/?id=$(VERSION_NO)'}, 'wiki.pl') . ', see ' . &/" < $< > $@
-
 upload: $(DIST)
 	for f in $^; do \
 		scp $$f as@dl.sv.nongnu.org:/releases/oddmuse/; \
@@ -107,9 +134,6 @@ $(VERSION).tgz: wiki.pl modules/creole.pl Mac/config Mac/wiki
 	sudo chmod 775 Slack/var/www/cgi-bin/wiki
 	cd Slack && tar czf ../$@ var install

-%-utf8.pl: wiki.pl $(MODULES)
-	perl oddtrans -l $@ $^ > $@-new && mv $@-new $@
-
 update-translations: $(TRANSLATIONS)

 upload-translations: always
@@ -119,8 +143,6 @@ upload-translations: always
 		"http://www.oddmuse.org/cgi-bin/oddmuse/raw/$$f" < $$f; \
 	done

-.PHONY: always
-
 deb:
 	equivs-build control

--- a/Slack/.cvsignore
+++ b/Slack/.cvsignore
@@ -1,2 +0,0 @@
-install
-var
--- a/contrib/simple-install/.cvsignore
+++ b/contrib/simple-install/.cvsignore
@@ -1,5 +0,0 @@
-current.pl
-FDL
-GPL
-*.tar.gz
-*.tar.gz.sig
--- a/css/alex-2012.css
+++ b/css/alex-2012.css
@@ -336,6 +336,7 @@ div.image span.caption {

 .left { float:left; margin-right: 1em; }
 .right { float:right; margin-left: 1em; }
+.half a img { height: 50%; width: 50%; }
 div.left .left, div.right .right {
    float:none;
 }
--- a/modules/.cvsignore
+++ b/modules/.cvsignore
@@ -1 +0,0 @@
-.DS_Store
--- a/modules/atom.pl
+++ b/modules/atom.pl
@@ -55,7 +55,7 @@ sub DoAtomIntrospection {
  push(@types, @UploadTypes) if $UploadAllowed;
  my $upload = '<accept>' . join(', ', @types) . '</accept>';
  print <<EOT;
-<?xml version="1.0" encoding='$HttpCharset'?>
+<?xml version="1.0" encoding='UTF-8'?>
 <service xmlns="http://purl.org/atom/app#">
 <workspace title="Wiki" >
 <collection title="$SiteName" href="$ScriptName/atom/wiki">
@@ -79,7 +79,7 @@ sub GetRcAtom {
  my $historyPrefix = QuoteHtml($ScriptName) . "?action=history;id=";
  my $limit = GetParam("rsslimit", 15); # Only take the first 15 entries
  my $count = 0;
-  my $feed = qq{<?xml version="1.0" encoding="$HttpCharset"?>\n};
+  my $feed = qq{<?xml version="1.0" encoding="UTF-8"?>\n};
  if ($RssStyleSheet =~ /\.(xslt?|xml)$/) {
    $feed .= qq{<?xml-stylesheet type="text/xml" href="$RssStyleSheet" ?>\n};
  } elsif ($RssStyleSheet) {
--- a/modules/config.pl
+++ b/modules/config.pl
@@ -29,25 +29,21 @@ sub DoConfig {
 \$EditPass = "";
 };
  my $source = GetRaw('http://www.emacswiki.org/scripts/current');
-  foreach my $var qw($HomePage $MaxPost $HttpCharset $StyleSheet
-		     $StyleSheetPage $NotFoundPg $NewText $NewComment
-		     $EditAllowed $BannedHosts $BannedCanRead
-		     $BannedContent $WikiLinks $FreeLinks $BracketText
-		     $BracketWiki $NetworkFile $AllNetworkFiles
-		     $PermanentAnchors $InterMap $NearMap
-		     $RssInterwikiTranslate $SurgeProtection
-		     $SurgeProtectionTime $SurgeProtectionViews
+  foreach my $var qw($HomePage $MaxPost $StyleSheet $StyleSheetPage $NotFoundPg
+		     $NewText $NewComment $EditAllowed $BannedHosts
+		     $BannedCanRead $BannedContent $WikiLinks $FreeLinks
+		     $BracketText $BracketWiki $NetworkFile $AllNetworkFiles
+		     $PermanentAnchors $InterMap $NearMap $RssInterwikiTranslate
+		     $SurgeProtection $SurgeProtectionTime $SurgeProtectionViews
 		     $DeletedPage $RCName @RcDays $RcDefault $KeepDays
-		     $KeepMajor $SummaryHours $SummaryDefaultLength
-		     $ShowEdits $UseLookup $RecentTop $RecentLink
-		     $PageCluster $InterWikiMoniker $SiteDescription
-		     $RssImageUrl $RssRights $RssExclude
-		     $RssCacheHours $RssStyleSheet $UploadAllowed
-		     @UploadTypes $EmbedWiki $FooterNote $EditNote
-		     $TopLinkBar @UserGotoBarPages $UserGotoBar
-		     $ValidatorLink $CommentsPrefix $HtmlHeaders
-		     $IndentLimit $LanguageLimit $JournalLimit
-		     $SisterSiteLogoUrl %SpecialDays %Smilies
+		     $KeepMajor $SummaryHours $SummaryDefaultLength $ShowEdits
+		     $UseLookup $RecentTop $RecentLink $PageCluster
+		     $InterWikiMoniker $SiteDescription $RssImageUrl $RssRights
+		     $RssExclude $RssCacheHours $RssStyleSheet $UploadAllowed
+		     @UploadTypes $EmbedWiki $FooterNote $EditNote $TopLinkBar
+		     @UserGotoBarPages $UserGotoBar $ValidatorLink
+		     $CommentsPrefix $HtmlHeaders $IndentLimit $LanguageLimit
+		     $JournalLimit $SisterSiteLogoUrl %SpecialDays %Smilies
 		     %Languages) {
    my $default = undef;
    my $re = quotemeta($var);
--- a/modules/creole.pl
+++ b/modules/creole.pl
@@ -156,7 +156,7 @@ sub CreoleInit {
  # $FullUrlPattern = "((?:$UrlProtocols:|/)$UrlChars+)";

  # Permit page authors to link to other pages having semicolons in their names.
-  # my $LinkCharsSansZero = "-;,.()' _1-9A-Za-z\x80-\xff";
+  # my $LinkCharsSansZero = "-;,.()' _1-9A-Za-z\x{0080}-\x{fffd}";
  # my $LinkChars = $LinkCharsSansZero.'0';
  # $FreeLinkPattern = "([$LinkCharsSansZero]|[$LinkChars][$LinkChars]+)";
 }
@@ -623,8 +623,8 @@ sub CreoleRuleRecursive {
    elsif (m/\G\s+/cg) {
      $html .= ' ';
    }
-    elsif (   m/\G([A-Za-z\x80-\xff]+([ \t]+[a-z\x80-\xff]+)*[ \t]+)/cg
-           or m/\G([A-Za-z\x80-\xff]+)/cg
+    elsif (   m/\G([A-Za-z\x{0080}-\x{fffd}]+([ \t]+[a-z\x{0080}-\x{fffd}]+)*[ \t]+)/cg
+           or m/\G([A-Za-z\x{0080}-\x{fffd}]+)/cg
           or m/\G(\S)/cg) {
      $html .= $1;  # multiple words but do not match http://foo
    }
--- a/modules/drafts.pl
+++ b/modules/drafts.pl
@@ -1,21 +1,18 @@
-# Copyright (C) 2006  Alex Schroeder <alex@emacswiki.org>
+# Copyright (C) 2006, 2012  Alex Schroeder <alex@gnu.org>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the
-#    Free Software Foundation, Inc.
-#    59 Temple Place, Suite 330
-#    Boston, MA 02111-1307 USA
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.

+use File::Glob ':glob';
 use vars qw($DraftDir);

 $DraftDir = $DataDir."/draft"; # directory for drafts
@@ -70,15 +67,23 @@ sub DraftNewGetEditForm {

 push(@MyMaintenance, \&DraftCleanup);

+sub DraftFiles {
+  return map {
+    $_ = substr($_, length($DraftDir) + 1);
+    utf8::decode($_);
+    $_;
+  } bsd_glob("$DraftDir/*"), bsd_glob("$DraftDir/.*");
+}
+
 sub DraftCleanup {
  print '<p>' . T('Draft Cleanup');
-  foreach my $draft (glob("$DraftDir/* $DraftDir/.*")) {
-    next if $draft =~ m!/\.\.?$!;
-    my $ts = (stat($draft))[9];
+  foreach my $draft (DraftFiles()) {
+    next if $draft eq '.' or $draft eq '..';
+    my $ts = (stat("$DraftDir/$draft"))[9];
    if ($Now - $ts < 1209600) { # 14*24*60*60
      print $q->br(), Tss("%1 was last modified %2 and was kept",
 		$draft, CalcTimeSince($Now - $ts));
-    } elsif (unlink($draft)) {
+    } elsif (unlink("$DraftDir/$draft")) {
      print $q->br(), Tss("%1 was last modified %2 and was deleted",
 		$draft, CalcTimeSince($Now - $ts));
    } else {
--- a/modules/dynamic-comments.pl
+++ b/modules/dynamic-comments.pl
@@ -36,29 +36,7 @@ function togglecomments (id) {
 } unless $HtmlHeaders =~ /commenthidden/; # mod_perl?
 }

-sub SafeId {
-  my $id = shift;
-  my $regexp = "";
-  $regexp = "|\xc3[\x80-\x96\x98-\xb6\xb8-\xff]|[\xc4-\xca].|\xcb[\x00-\xbf]"
-          . "|\xcd[\xb0-\xbd\xbf-\xff]|[\xce-\xDF].|\xe0..|\xe1[\x00-\xbe]."
-          . "|\xe1\xbf[\x00-\xbf]|\xe2\x80[\x8c\x8d]"
-      if $HttpCharset eq 'UTF-8';
-  # Unicode Codepoint   UTF-8 encoding
-  # [#xC0-#xD6]              c3 80 - c3 96
-  # [#xD8-#xF6]              c3 98 - c3 b6
-  # [#xF8-#x2FF]             c3 b8 - cb bf
-  # [#x370-#x37D]            cd b0 - cd bd
-  # [#x37F-#x1FFF]           cd bf - e1 bf bf
-  # [#x200C-#x200D]       e2 80 8c - e2 80 8d
-  # [#x2070-#x218F]       e2 81 b0 - e2 86 8f    -- FIXME \
-  # [#x2C00-#x2FEF]       e2 b0 80 - e2 bf af    -- FIXME |
-  # [#x3001-#xD7FF]       e3 80 81 - ed 9f bf    -- FIXME | these are missing
-  # [#xF900-#xFDCF]       ef a4 80 - ef b7 8f    -- FIXME | in the regexp above
-  # [#xFDF0-#xFFFD]       ef b7 b0 - ef bf bd    -- FIXME |
-  # [#x10000-#xEFFFF]  f0 90 80 80 - f3 af bf bf -- FIXME /
-  $id = ":$id" unless $id =~ /^[:_A-Za-z]$regexp/;
-  return join('', $id =~ m/([-.:_A-Za-z0-9]$regexp)/g);
-}
+my $num = 0;

 *DynamicCommentsOldGetPageLink = *GetPageLink;
 *GetPageLink = *DynamicCommentsNewGetPageLink;
@@ -70,10 +48,10 @@ sub DynamicCommentsNewGetPageLink {
    $title =~ s/_/ /g;
    my $page = PageHtml($id);
    if ($page) {
-      my $safe = SafeId($id);
-      return qq{<a href="javascript:togglecomments('$safe')">$title</a>}
+      my $anchor = "id" . $num++;
+      return qq{<a href="javascript:togglecomments('$anchor')">$title</a>}
        . '</p>' # close p before opening div
-        . $q->div({-class=>commenthidden, -id=>$safe},
+        . $q->div({-class=>commenthidden, -id=>$anchor},
                  $page,
                  $q->p(DynamicCommentsOldGetPageLink($id, T('Add Comment'))))
        . '<p>'; # open an empty p that will be closed in PrintAllPages
--- a/modules/git.pl
+++ b/modules/git.pl
@@ -147,8 +147,11 @@ sub GitCleanup {
      OpenPage($id);
      WriteStringToFile("$GitRepo/$id", $Page{text});
    }
-    # commit the new state
+    # run git!
    chdir($GitRepo); # important for all the git commands that follow!
+    # add any new files
+    GitRun('add', '.');
+    # commit the new state
    GitRun('commit', '--quiet', '-a', '-m', 'maintenance job',
 	   "--author=Oddmuse <$GitMail>");
  }
--- a/modules/latex.pl
+++ b/modules/latex.pl
@@ -32,6 +32,7 @@
 # img.InlineMath
 # img.DisplayMath

+use File::Glob ':glob';
 use vars qw($LatexDir $LatexLinkDir $LatexExtendPath $LatexSingleDollars);

 # One of the following options must be set correctly to the full path of
@@ -172,7 +173,7 @@ sub MakeLaTeX {
      #setup rendering directory
      my $dir = "$LatexDir/$hash";
      if (-d $dir) {
-	  unlink (glob('$dir/*'));
+	  unlink (bsd_glob('$dir/*'));
      } else {
 	  mkdir($dir) or return "[Unable to create $dir]";
      }
@@ -220,5 +221,3 @@ sub MakeLaTeX {
  rmdir ($dir);
  return $result;
 }
-
-
--- a/modules/link-all.pl
+++ b/modules/link-all.pl
@@ -22,7 +22,7 @@ push(@MyRules, \&LinkAllRule);
 $RuleOrder{\&LinkAllRule} = 1000;

 sub LinkAllRule {
-  if (/\G([A-Za-z\x80-\xff]+)/gc) {
+  if (/\G([A-Za-z\x{0080}-\x{fffd}]+)/gc) {
    my $oldpos = pos;
    Dirty($1);
    # print the word, or the link to the word
--- a/modules/mac.pl
+++ b/modules/mac.pl
@@ -18,24 +18,79 @@

 $ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/mac.pl">mac.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Mac">Mac</a></p>';

-use Encode;
 use Unicode::Normalize;

-*OldAllPagesList = *AllPagesList;
-*AllPagesList = *NewAllPagesList;
+*OldMacAllPagesList = *AllPagesList;
+*AllPagesList = *NewMacAllPagesList;

-sub NewAllPagesList {
+sub NewMacAllPagesList {
  $refresh = GetParam('refresh', 0);
  if ($IndexInit && !$refresh) {
    return @IndexList;
  }
-  OldAllPagesList(@_);
+  OldMacAllPagesList(@_);
  my @new = ();
  %IndexHash = ();
  foreach my $id (@IndexList) {
-    $id = encode_utf8(NFC(decode_utf8($id)));
+    $id = NFC($id);
    push(@new, $id);
    $IndexHash{$id} = 1;
  }
+  @IndexList = @new;
  return @new;
 }
+
+*OldMacGrepFiltered = *GrepFiltered;
+*GrepFiltered = *NewMacGrepFiltered;
+
+sub NewMacGrepFiltered {
+  my @pages = OldMacGrepFiltered(@_);
+  foreach my $id (@pages) {
+    $id = NFC($id);
+  }
+  return @pages;
+}
+
+push(@MyInitVariables, \&MacFixEncoding);
+
+sub MacFixEncoding {
+  # disable grep if searching for non-ascii stuff:
+
+  # $ mkdir /tmp/dir
+  # $ echo schroeder > /tmp/dir/schroeder
+  # $ echo schröder > /tmp/dir/schröder
+  # $ echo SCHRÖDER > /tmp/dir/SCHRÖDER-UP # don't use SCHRÖDER because of HFS
+  # $ grep -rli schröder /tmp/dir
+  # /tmp/dir/schröder
+  # $ grep -rli SCHRÖDER /tmp/dir
+  # /tmp/dir/schröder
+  #
+  # Why is grep not finding the upper case variant in the SCHRÖDER-UP
+  # file?
+
+  $UseGrep = 0 if GetParam('search', '') =~ /[x{0080}-\x{fffd}]/;
+
+  # the rest is only necessary if using namespaces.pl
+  return unless defined %Namespaces;
+  while (my ($key, $value) = each %Namespaces) {
+    delete $Namespaces{$key};
+    utf8::decode($key);
+    $key = NFC($key);
+    $Namespaces{$key} = $NamespaceRoot . '/' . $key . '/';
+  }
+  while (my ($key, $value) = each %InterSite) {
+    delete $InterSite{$key};
+    utf8::decode($key);
+    $key = NFC($key);
+    $InterSite{$key} = $Namespaces{$key} if $Namespaces{$key};
+  }
+}
+
+# for drafts.pl
+
+*OldMacDraftFiles = *DraftFiles;
+*DraftFiles = *NewMacDraftFiles;
+
+sub NewMacDraftFiles {
+  return map { NFC($_) } OldMacDraftFiles(@_);
+}
--- a/modules/markdown.pl
+++ b/modules/markdown.pl
@@ -152,7 +152,7 @@ sub MarkdownQuoteHtml {
 	$html =~ s/&/&amp;/g;
 	$html =~ s/</&lt;/g;
 #	$html =~ s/>/&gt;/g;
-	$html =~ s/[\x00-\x08\x0b\x0c\x0e-\x1f]/ /g; # legal xml: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
+	$html =~ s/[\x00-\x08\x0b\x0c\x0e-\x1f]/ /g; # legal xml: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFD]

 	return $html;
 }
@@ -198,8 +198,8 @@ sub MarkdownNearInit {
 sub DoWikiWords {
 	
 	my $text = shift;
-	my $WikiWord = '[A-Z]+[a-z\x80-\xff]+[A-Z][A-Za-z\x80-\xff]*';
-	my $FreeLinkPattern = "([-,.()' _0-9A-Za-z\x80-\xff]+)";
+	my $WikiWord = '[A-Z]+[a-z\x{0080}-\x{fffd}]+[A-Z][A-Za-z\x{0080}-\x{fffd}]*';
+	my $FreeLinkPattern = "([-,.()' _0-9A-Za-z\x{0080}-\x{fffd}]+)";

 	if ($FreeLinks) {
 		# FreeLinks
@@ -299,7 +299,7 @@ sub CreateWikiLink {

 sub UnescapeWikiWords {
 	my $text = shift;
-	my $WikiWord = '[A-Z]+[a-z\x80-\xff]+[A-Z][A-Za-z\x80-\xff]*';
+	my $WikiWord = '[A-Z]+[a-z\x{0080}-\x{fffd}]+[A-Z][A-Za-z\x{0080}-\x{fffd}]*';
 	
 	# Unescape escaped WikiWords
 	$text =~ s/\\($WikiWord)/$1/g;
@@ -379,7 +379,7 @@ sub NewEncodeCode {
 	$text = OldEncodeCode($text);
 	
 	# Protect Wiki Words
-	my $WikiWord = '[A-Z]+[a-z\x80-\xff]+[A-Z][A-Za-z\x80-\xff]*';
+	my $WikiWord = '[A-Z]+[a-z\x{0080}-\x{fffd}]+[A-Z][A-Za-z\x{0080}-\x{fffd}]*';
 	$text =~ s!($WikiWord)!\\$1!gx;

 	return $text;
@@ -471,7 +471,7 @@ sub MarkdownAddComment {

 sub NewDoAnchors {
 	my $text = shift;
-	my $WikiWord = '[A-Z]+[a-z\x80-\xff]+[A-Z][A-Za-z\x80-\xff]*';
+	my $WikiWord = '[A-Z]+[a-z\x{0080}-\x{fffd}]+[A-Z][A-Za-z\x{0080}-\x{fffd}]*';
 		
 	return Markdown::_DoAnchors($text);
 }
--- a/modules/markup.pl
+++ b/modules/markup.pl
@@ -80,9 +80,9 @@ $RuleOrder{\&MarkupRule} = 150;
 %MarkupLines = ('>' => 'pre',
 	       );

-my $words = '([A-Za-z\x80-\xff][-%.,:;\'"!?0-9 A-Za-z\x80-\xff]*?)';
+my $words = '([A-Za-z\x{0080}-\x{fffd}][-%.,:;\'"!?0-9 A-Za-z\x{0080}-\x{fffd}]*?)';
 # zero-width look-ahead assertion to prevent km/h from counting
-my $noword = '(?=[^-0-9A-Za-z\x80-\xff]|$)';
+my $noword = '(?=[^-0-9A-Za-z\x{0080}-\x{fffd}]|$)';

 my $markup_pairs_re = '';
 my $markup_forced_pairs_re = '';
@@ -174,7 +174,7 @@ sub MarkupRule {
    return $MarkupSingles{UnquoteHtml($1)};
  } elsif ($MarkupPairs{'/'} and m|\G~/|gc) {
    return '~/'; # fix ~/elisp/ example
-  } elsif ($MarkupPairs{'/'} and m|\G(/[-A-Za-z0-9\x80-\xff/]+/$words/)|gc) {
+  } elsif ($MarkupPairs{'/'} and m|\G(/[-A-Za-z0-9\x{0080}-\x{fffd}/]+/$words/)|gc) {
    return $1; # fix /usr/share/lib/! example
  }
  # "foo
--- a/modules/namespaces.pl
+++ b/modules/namespaces.pl
@@ -1,17 +1,16 @@
-# Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2012  Alex Schroeder <alex@gnu.org>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.

 =head1 Namespaces Extension

@@ -38,6 +37,7 @@ be changed using the C<$NamespacesSelf> option.

 $ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/namespaces.pl">namespaces.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Namespaces_Extension">Namespaces Extension</a></p>';

+use File::Glob ':glob';
 use vars qw($NamespacesMain $NamespacesSelf $NamespaceCurrent
 	    $NamespaceRoot $NamespaceSlashing @NamespaceParameters
 	    %Namespaces);
@@ -80,7 +80,8 @@ sub NamespacesInitVariables {
  # Do this before changing the $DataDir and $ScriptName
  if (!$Monolithic and $UsePathInfo) {
    $Namespaces{$NamespacesMain} = $ScriptName . '/';
-    foreach my $name (glob("$DataDir/*")) {
+    foreach my $name (bsd_glob("$DataDir/*")) {
+      utf8::decode($name);
      if (-d $name
 	  and $name =~ m|/($InterSitePattern)$|
 	  and $name ne $NamespacesMain
@@ -92,19 +93,21 @@ sub NamespacesInitVariables {
  $NamespaceRoot = $ScriptName; # $ScriptName may be changed below
  $NamespaceCurrent = '';
  my $ns = GetParam('ns', '');
+  if (not $ns and $UsePathInfo) {
+    my $path_info = $q->path_info();
+    utf8::decode($path_info);
+    # make sure ordinary page names are not matched!
+    if ($path_info =~ m|^/($InterSitePattern)(/.*)?|
+	and ($2 or $q->keywords or NamespaceRequiredByParameter())) {
+      $ns = $1;
+    }
+  }
  ReportError(Ts('%s is not a legal name for a namespace', $ns))
    if $ns and $ns !~ m/^($InterSitePattern)$/;
-  if (($UsePathInfo
-       # make sure ordinary page names are not matched!
-       and $q->path_info() =~ m|^/($InterSitePattern)(/.*)?|
-       and ($1 ne $NamespacesMain)
-       and ($1 ne $NamespacesSelf)
-       and ($2 or $q->keywords or NamespaceRequiredByParameter()))
-      or
-      ($ns =~ m/^($InterSitePattern)$/
-       and ($1 ne $NamespacesMain)
-       and ($1 ne $NamespacesSelf))) {
-    $NamespaceCurrent = $1;
+  if ($ns
+      and $ns ne $NamespacesMain
+      and $ns ne $NamespacesSelf) {
+    $NamespaceCurrent = $ns;
    # Change some stuff from the original InitVariables call:
    $SiteName   .= ' ' . $NamespaceCurrent;
    $InterWikiMoniker = $NamespaceCurrent;
@@ -115,7 +118,7 @@ sub NamespacesInitVariables {
    $TempDir     = "$DataDir/temp";
    $LockDir     = "$TempDir/lock";
    $NoEditFile  = "$DataDir/noedit";
-    $RcFile = "$DataDir/rc.log";
+    $RcFile      = "$DataDir/rc.log";
    $RcOldFile   = "$DataDir/oldrc.log";
    $IndexFile   = "$DataDir/pageidx";
    $VisitorFile = "$DataDir/visitors.log";
@@ -133,8 +136,8 @@ sub NamespacesInitVariables {
    my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, $atime,$mtime,$ctime,$blksize,$blocks)
      = stat($IndexFile);
    $LastUpdate = $mtime;
-    CreateDir($DataDir); # Create directory if it doesn't exist
-    ReportError(Ts('Could not create %s', $DataDir) . ": $!", '500 INTERNAL SERVER ERROR')
+    CreateDir($DataDir);	# Create directory if it doesn't exist
+    ReportError(Ts('Cannot create %s', $DataDir) . ": $!", '500 INTERNAL SERVER ERROR')
      unless -d $DataDir;
  }
  $Namespaces{$NamespacesSelf} = $ScriptName . '?';
@@ -203,8 +206,8 @@ sub NewNamespaceGetRcLines { # starttime, hash of seen pages to use as a second
    # directory. This reduces the chances of getting different
    # results.
    foreach my $site (keys %InterSite) {
-      if ($InterSite{$site} =~ m|^$ScriptName/([^/]*)|) {
-	my $ns = $1 or next;
+      if (substr($InterSite{$site}, 0, length($ScriptName)) eq $ScriptName) {
+	my $ns = $site;
 	my $file = "$DataDir/$ns/rc.log";
 	push(@rcfiles, $file);
 	$namespaces{$file} = $ns;
@@ -217,7 +220,7 @@ sub NewNamespaceGetRcLines { # starttime, hash of seen pages to use as a second
  # starttime. If any rcfile exists with no timestamp before the
  # starttime, we need to open its rcoldfile.
  foreach my $file (@rcfiles) {
-    open(F, $file);
+    open(F, '<:encoding(UTF-8)', $file);
    my $line = <F>;
    my ($ts) = split(/$FS/o, $line); # the first timestamp in the regular rc file
    my @new;
@@ -249,7 +252,7 @@ sub NewNamespaceGetRcLines { # starttime, hash of seen pages to use as a second
  return LatestChanges(@result);
 }

-=head RSS feed
+=head2 RSS feed

 When retrieving the RSS feed with the parameter full=1, one would
 expect the various items to contain the fully rendered HTML.
--- a/modules/national-days-de.pl
+++ b/modules/national-days-de.pl
@@ -24,6 +24,8 @@
 # If you are running a multilingual site, then you should explicitly
 # load this file from your language-specific config file.

+use utf8;
+
 $ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/national-days-de.pl">national-days-de.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Special_Days">Special Days</a></p>';

 %SpecialDays = (
--- a/modules/national-days.pl
+++ b/modules/national-days.pl
@@ -16,6 +16,8 @@
 #    59 Temple Place, Suite 330
 #    Boston, MA 02111-1307 USA

+use utf8;
+
 $ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/national-days.pl">national-days.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Special_Days">Special Days</a></p>';

 %SpecialDays = (
--- a/modules/not-found-handler.pl
+++ b/modules/not-found-handler.pl
@@ -18,6 +18,7 @@

 $ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/not-found-handler.pl">not-found-handler.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/404_Handler_Extension">404 Handler Extension</a></p>';

+use File::Glob ':glob';
 use vars qw($NotFoundHandlerDir, $LinkFile, %LinkDb, $LinkDbInit);

 $NotFoundHandlerDir = '/tmp/oddmuse/cache';
@@ -29,7 +30,7 @@ $Action{clearcache} = \&DoClearCache;

 sub DoClearCache {
  print GetHeader('', QuoteHtml(T('Clearing Cache')), '');
-  unlink(glob("$NotFoundHandlerDir/*"));
+  unlink(bsd_glob("$NotFoundHandlerDir/*"));
  print $q->p(T('Done.'));
  PrintFooter();
 }
@@ -103,7 +104,7 @@ sub NewNotFoundHandlerSave {
    NotFoundHandlerCacheUpdate($id);
  } else {
    # unlink PageName, PageName.en, PageName.de, etc.
-    unlink("$NotFoundHandlerDir/$id", glob("$NotFoundHandlerDir/$id.[a-z][a-z]"));
+    unlink("$NotFoundHandlerDir/$id", bsd_glob("$NotFoundHandlerDir/$id.[a-z][a-z]"));
  }
 }

@@ -128,7 +129,7 @@ sub NotFoundHandlerCacheUpdate {
  foreach my $source (keys %LinkDb) {
    warn "Examining $source\n";
    if (grep(/$target/, @{$LinkDb{$source}})) {
-      unlink("$NotFoundHandlerDir/$source", glob("$NotFoundHandlerDir/$source.[a-z][a-z]"));
+      unlink("$NotFoundHandlerDir/$source", bsd_glob("$NotFoundHandlerDir/$source.[a-z][a-z]"));
      warn "Unlinking $source\n";
    }
  }
--- a/modules/questionasker.pl
+++ b/modules/questionasker.pl
@@ -77,7 +77,7 @@ sub NewQuestionaskerDoPost {
 	  or QuestionaskerException($id)) {
    print GetHeader('', T('Edit Denied'), undef, undef, '403 FORBIDDEN');
    print $q->p(T('You did not answer correctly.'));
-    print $q->start_form, QuestionaskerGetQuestion(1),
+    print GetFormStart(), QuestionaskerGetQuestion(1),
      (map { $q->hidden($_, '') }
       qw(title text oldtime summary recent_edit aftertext)), $q->end_form;
    PrintFooter();
--- a/modules/recaptcha.pl
+++ b/modules/recaptcha.pl
@@ -230,7 +230,7 @@ sub NewReCaptchaDoPost {
    print GetHeader('', T('Edit Denied'), undef, undef, '403 FORBIDDEN');
    print $q->start_div({-class=>'error'});
    print $q->p(T('You did not answer correctly.'));
-    print $q->start_form, ReCaptchaGetQuestion(1),
+    print GetFormStart(), ReCaptchaGetQuestion(1),
      (map { $q->hidden($_, '') }
       qw(title text oldtime summary recent_edit aftertext)), $q->end_form;
    print $q->end_div();
--- a/modules/search-freetext.pl
+++ b/modules/search-freetext.pl
@@ -33,7 +33,7 @@ package OddMuse::Tokenize;

 use vars qw($regexp);

-$regexp = qr'[A-Za-z0-9_\x80-\xff]+';
+$regexp = qr'[A-Za-z0-9_\x{0080}-\x{fffd}]+';

 sub new {
    my ($classname, @args) = @_;
--- a/modules/setext.pl
+++ b/modules/setext.pl
@@ -33,7 +33,7 @@ push(@MyRules, \&SeTextRule);
 # If the length does not match, pos is reset and zero is returned so
 # that the remaining rules will be tested instead.

-my $word = '([-A-Za-z\x80-\xff]+)';
+my $word = '([-A-Za-z\x{0080}-\x{fffd}]+)';
 sub SeTextRule {
  my $oldpos = pos;
  if ($bol && ((m/\G((.+?)[ \t]*\n(-+|=+)[ \t]*\n)/gc
--- a/modules/simple-rules-test.pl
+++ b/modules/simple-rules-test.pl
@@ -5,7 +5,7 @@ use vars qw($FS $FreeLinkPattern $UrlProtocols $UrlChars $EndChars
 $UrlPattern $q);

 $FS = "\x1e";
-$FreeLinkPattern = "([-,.()' _0-9A-Za-z\x80-\xff]+)";
+$FreeLinkPattern = "([-,.()' _0-9A-Za-z\x{0080}-\x{fffd}]+)";
 $UrlProtocols = 'http|https|ftp|afs|news|nntp|mid|cid|mailto|wais|prospero|telnet|gopher|irc';
 $UrlChars = '[-a-zA-Z0-9/@=+$_~*.,;:?!\'"()&#%]'; # see RFC 2396
 $EndChars = '[-a-zA-Z0-9/@=+$_~*]'; # no punctuation at the end of the url.
--- a/modules/smiles.pl
+++ b/modules/smiles.pl
@@ -16,7 +16,7 @@
 #    59 Temple Place, Suite 330
 #    Boston, MA 02111-1307 USA

-$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/smiles.pl">smiles.pl</a></p>';
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/smiles.pl">smiles.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Logo_And_Icons#Smilies">Logo And Icons</a></p>';

 %Smilies = (
     ':-?D'      => 'http://www.emacswiki.org/pics/grin.png',
--- a/modules/static-copy.pl
+++ b/modules/static-copy.pl
@@ -158,7 +158,7 @@ sub StaticHtml {
 <title>$SiteName: $id</title>
 <link type="text/css" rel="stylesheet" href="static.css" />
 <meta http-equiv="refresh" content="0; url=$target">
-<meta http-equiv="content-type" content="text/html; charset=$HttpCharset">
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
 </head>
 <body>
 <p>Redirected to <a href="$target">$1</a>.</p>
@@ -173,7 +173,7 @@ EOT
 <head>
 <title>$SiteName: $id</title>
 <link type="text/css" rel="stylesheet" href="static.css" />
-<meta http-equiv="content-type" content="text/html; charset=$HttpCharset">
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
 </head>
 <body>
 EOT
--- a/modules/static-hybrid.pl
+++ b/modules/static-hybrid.pl
@@ -173,7 +173,7 @@ sub StaticHtml {
 	# Process the page
 	local $Message = "";
 	# encoding is left off, so fix it:
-	print qq!<?xml version="1.0" encoding="$HttpCharset" ?>!;
+	print qq!<?xml version="1.0" encoding="UTF-8" ?>!;
 	print GetHeader($id, QuoteHtml($id), undef, "");
 	print $q->start_div({-class=> 'content browse'});
 	print PageHtml($id);
--- a/modules/tables-long.pl
+++ b/modules/tables-long.pl
@@ -33,7 +33,7 @@ sub TablesLongRule {
  # a new row is started when a cell is repeated
  # if cells are missing, column spans are created (the first row
  # could use row spans...)
-  if ($bol && m|\G\s*\n*\&lt;table(/[A-Za-z\x80-\xff/]+)? +([A-Za-z\x80-\xff,;\/ ]+)\&gt; *\n|cg) {
+  if ($bol && m|\G\s*\n*\&lt;table(/[A-Za-z\x{0080}-\x{fffd}/]+)? +([A-Za-z\x{0080}-\x{fffd},;\/ ]+)\&gt; *\n|cg) {
    my $class = join(' ', split(m|/|, $1)); # leading / in $1 will make sure we have leading space
    Clean(CloseHtmlEnvironments() . "<table class=\"user long$class\">");
    # labels and their default class
@@ -60,7 +60,7 @@ sub TablesLongRule {
    my $rowspan = '';
    my $first = 1;
    for my $line (@lines) {
-      if ($line =~ m|^($regexp)/?([0-9]+)?/?([A-Za-z\x80-\xff/]+)?[:=] *(.*)|) { # regexp changes for other tables
+      if ($line =~ m|^($regexp)/?([0-9]+)?/?([A-Za-z\x{0080}-\x{fffd}/]+)?[:=] *(.*)|) { # regexp changes for other tables
 	$label = $1;
 	$rowspan = $2;
 	$class = join(' ', split(m|/|, $3)); # no leading / therefore no leading space
--- a/modules/tags.pl
+++ b/modules/tags.pl
@@ -218,6 +218,7 @@ sub TagFind {
  my %page;
  foreach my $tag (@tags) {
    foreach my $id (split(/$FS/, $h{lc($tag)})) {
+      utf8::decode($id);
      $page{$id} = 1;
    }
  }
@@ -248,9 +249,7 @@ sub NewTagGrepFiltered { # called within a lock!
  }
  # filter out the tags from the search string
  $string = join(' ', grep(!/^-?tag:/, $string =~ /\"([^\"]+)\"|(\S+)/g));
-  # if no query terms remain, just return the pages we found
-  # return sort keys %page if $string eq '';
-  # otherwise run grep
+  # run the old code for any remaining search terms
  return OldTagGrepFiltered($string, sort keys %page);
 }

@@ -353,6 +352,11 @@ sub DoTagsReindex {
       $Page{text} =~ m/\[\[tag:$FreeLinkPattern\|([^]|]+)\]\]/g);
    next unless %tag;

+    utf8::encode($id); # back to bytes for the following installation:
+    # This is perl, v5.10.1 (*) built for i486-linux-gnu-thread-multi
+    # (with 56 registered patches, see perl -V for more detail)
+    # (FIXME: get rid of this call, or explain why no UTF-8 can be stored in %h)
+
    # For each tag we list the files tagged. Add the current file for
    # all tags.
    foreach my $tag (keys %tag) {
--- a/modules/tex.pl
+++ b/modules/tex.pl
@@ -19,6 +19,7 @@
 $ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/tex.pl">tex.pl</a></p>';

 use vars qw($TeXInit);
+use utf8;

 my %h = qw( !` ¡ {\pounds} £ \pounds £ {\S} § \S § \"{} ¨ {\copyright} ©
 \copyright © $^a$ ª \={} ¯ $\pm$ ± \pm ± $^2$ ² $^3$ ³ \'{} ´ {\P} ¶
--- a/modules/throttle.pl
+++ b/modules/throttle.pl
@@ -23,6 +23,7 @@

 $ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/throttle.pl">throttle.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Limit_Number_Of_Instances_Running">Limit Number Of Instances Running</a></p>';

+use File::Glob ':glob';
 use vars qw($InstanceThrottleDir $InstanceThrottleLimit);

 $InstanceThrottleDir = $DataDir."/pids"; # directory for pid files
@@ -47,7 +48,7 @@ sub NewDoBrowseRequest {

 # limit the script to a maximum of $InstanceThrottleLimit instances
 sub DoInstanceThrottle {
-  my @pids = glob($InstanceThrottleDir."/*");
+  my @pids = bsd_glob($InstanceThrottleDir."/*");
  # Go over all pids: validate each pid by sending signal 0, unlink
  # pidfile if pid does not exist and return 0. Count the number of
  # zeros (= removed files = zombies) with grep.
--- a/modules/thumbs.pl
+++ b/modules/thumbs.pl
@@ -0,0 +1,256 @@
+# Copyright (C) 2004, 2012 Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2005 Rob Neild
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Thumbnail (and improved image handling) module for OddMuse wiki
+# Conflicts with the "Image extension module"
+
+require MIME::Base64;
+
+use File::Path;
+
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/thumbs.pl">thumbs.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Image_Thumbnails">Image Thumbnails</a></p>';
+
+# Tempoary directory to create thumbnails in
+$ThumbnailTempDir = '/tmp';
+
+# Path and name of external program to use to create thumbnails. Only
+# ImageMagick 'convert' can be used. You may have to set the MAGICK_HOME
+# environment variable in your config file if you set it to
+# /usr/local/bin/convert and get the following error:
+#   convert: no decode delegate for this image format
+# For your config file:
+#   $ENV{MAGICK_HOME} = '/usr/local';
+$ThumbnailConvert = '/usr/bin/convert';
+
+# Max size for a thumbnail. If larger size is specified just shows
+# regular image
+$ThumbnailMaxSize  = 500;
+
+# Default thumbnail size if non is specified
+$ThumbnailDefaultSize = 100;
+
+# MIME types to create thumbnail for, all allowed if empty list
+@ThumbnailTypes =  @UploadTypes;
+
+# As well as using ALT, use TITLE. This enables comments to popup when
+# hovering mouse over thumbnail
+$ThumbnailImageUseTitle = 0;
+
+$ThumbnailCacheDir = "oddmuse_thumbnail_cache";
+$ThumbnailCacheUrl = "/oddmuse_thumbnail_cache";
+
+# Define new formatting rule "thumb" that inserts an auto generated thumbnail
+# Syntax is [[thumb:page name | etc. ]]
+
+push(@MyRules, \&ThumbNailSupportRule);
+
+sub ThumbNailSupportRule {
+   my $result;
+   my $RawMatch;
+
+  if (m!\G(\[\[thumb:$FreeLinkPattern(\|.*?)?\]\])!gc)
+  {
+
+       $RawMatch = $1;
+
+       # Try and extract out all the options. They can be in any order, apart from comment at end
+
+       my $name = $2;
+
+       my $size="$ThumbnailDefaultSize";              # default size for thumbnail
+       my $frame;
+       my $comment;                                          # default alignment for a non framed picture
+       my $alignment_framed = 'tright';              # default alignment for a framed picture
+       my $alignment;
+
+       my $params = $3 . '|';
+
+       if($params =~ s/\|([0-9]+)px\|/\|/)  { $size = $1; }
+
+       if($params =~ s/\|thumb\|/\|/) { $frame = 'yes' ;}
+       if($params =~ s/\|frame\|/\|/) { $frame = 'yes'; }
+
+       if ($params =~ s/\|none\|/\|/)  { $alignment_framed= 'tnone'; }
+       if ($params =~ s/\|right\|/\|/)  { $alignment_framed= 'tright'; $alignment='floatright';}
+       if ($params =~ s/\|left\|/\|/)  { $alignment_framed= 'tleft'; $alignment='floatleft'; }
+
+       if ($params =~ m/\|(.+)\|$/) { $comment = $1; }
+
+      my $id = FreeToNormal($name);
+      AllPagesList();
+
+      # if the page does exists
+    
+      if ($IndexHash{$id})
+      {
+
+
+            if (! -e "$ThumbnailCacheDir/$id/$size")
+           { 
+                   GenerateThumbNail ($id, $size); 
+           }
+
+    
+         my %img_attribs;
+
+          my $action = "$ThumbnailCacheUrl/" . UrlEncode($id) . "/$size"; 
+          
+         $img_attribs{'-src'} = $action;
+
+         if (defined $comment)  { 
+                     $img_attribs{'-alt'} ="$comment";
+                     $img_attribs{'-title'} = "$comment" if $ThumbnailImageUseTitle==1;
+        } 
+           else { $img_attribs{'-alt'} = "$name"; }
+
+
+        $img_attribs{'-class'} = 'upload';
+
+         $result = $q->img(\%img_attribs);
+         $result = ScriptLink(UrlEncode($id) , $result, 'image');
+
+         if (defined $frame) {
+              if (defined $comment)  { $result = $result . $q->div({-class=>'thumbcaption'}, "$comment"); }
+              
+              if ($size>0) {
+                   $result = $q->div({-style=>"width:" .  ($size+2) . "px"}, $result); 
+                   $result = $q->div({-class=>"thumb " .  $alignment_framed}, $result);
+              }
+         }
+         else
+         {
+              if (defined $alignment) { $result = $q->div({-class=>"$alignment" }, $result); }
+         }
+     }
+     else
+     {
+           # if the image does not exist
+           $result = '[' . T('thumb') . ':' . $name . GetEditLink($id, '?', 1) . ']';
+     }
+
+   }
+
+    if (defined $result) 
+    { 
+           Dirty($RawMatch); 
+           print $result;
+          
+           $result = '';
+     }
+
+    return $result;
+    
+}
+
+
+
+
+# define new action "thumbnail" that actually does the on fly generation of the image
+# thumbnails are put into the file so they only need be generated once
+# we also store the size of thumbnail so that can be used in the markup
+
+# if we get passed a size of zero then all we need to do is check whether we have the image size stored in thumbnail_0
+# this enbles markup for non-thumbnail images better
+
+
+sub GenerateThumbNail {
+   my ($id, $size) = (@_); 
+
+   ValidIdOrDie($id);
+
+    AllPagesList();
+    
+     if (not $IndexHash{$id}) { ReportError(Ts('Error creating thumbnail from non existant page %s.' , $id), '500 INTERNAL SERVER ERROR'); }   # Page Doesn't exist,
+
+
+     my $openpage = $OpenPageName;       # remember  the current page we are on
+
+
+    RequestLockOrError();  
+    OpenPage($id);
+
+     # Parse out some data
+     #   Check MIME type supported
+     #   Check is a file
+
+     my ($text, $revision) = GetTextRevision(GetParam('revision', '')); # maybe revision reset!
+     my ($type) = TextIsFile($text); # MIME type if an uploaded file
+     my $data = substr($text, index($text, "\n") + 1);
+
+     if ($type)
+     {
+           my $regexp = quotemeta($type);
+
+            if (@ThumbnailTypes and not grep(/^$regexp$/, @ThumbnailTypes)) {
+                ReportError(Ts('Can not create thumbnail for file type %s.' , $type), '415 UNSUPPORTED MEDIA TYPE');
+           }
+      }
+      else
+      {
+             ReportError(T('Can not create thumbnail for a text document'), '500 INTERNAL SERVER ERROR');
+      }
+
+
+     my $filename = $ThumbnailTempDir . "/odd" . $id . "_" . $size;
+
+     # Decode the original image to a temp file
+
+     open(FD, "> $filename") or ReportError(Ts("Could not open %s for writing whilst trying to save image before creating thumbnail. Check write permissions.",$filename), '500 INTERNAL SERVER ERROR');  
+     binmode(FD);
+     print FD MIME::Base64::decode($data);
+     close(FD);
+
+     eval { mkpath("$ThumbnailCacheDir/$id") };
+     if ($@) {
+         ReportError(Ts('Can not create path for thumbnail - %s', $@), '500 INTERNAL SERVER ERROR');
+     }
+
+    # create the thumbnail
+
+     my $command = "$ThumbnailConvert '$filename' -verbose -resize ${size}x '$ThumbnailCacheDir/$id/$size' 2>&1";
+     open (MESSAGE, '-|', $command)
+       or ReportError(Tss("Failed to run %1 to create thumbnail: %2", $ThumbnailConvert, $!),
+		      '500 INTERNAL SERVER ERROR');
+
+      my $convert = <MESSAGE>;
+      close(MESSAGE);
+
+      my $scaled_size_x;
+      my $scaled_size_y;
+
+	my $thumbnail_data= '';
+
+        if($?) {
+                 ReportError(Ts("%s ran into an error", $ThumbnailConvert), '500 INTERNAL SERVER ERROR', undef,
+			     $q->pre($command . "\n" . $convert));
+	} elsif($convert =~ m/=>(\d+)x(\d+)/) {
+                 $scaled_size_x = $1;
+                 $scaled_size_y = $2;
+	} elsif (!$convert) {
+                 ReportError(Ts("%s produced no output", $ThumbnailConvert), '500 INTERNAL SERVER ERROR');
+        } else {
+                 ReportError(Ts("Failed to parse %s.", $convert), '500 INTERNAL SERVER ERROR');
+        }
+
+        unlink($filename);
+
+        # save tag to page
+        #$Page{'thumbnail_' . $size} = '#FILE ' . $type . ' created=' . $Now . ' revision=' . $Page{'revision'} . ' size=' . $scaled_size_x . 'x' . $scaled_size_y . "\n" . $thumbnail_data;
+        #SavePage();
+        
+        ReleaseLock();
+
+        OpenPage($openpage);      # restore original open page
+}
--- a/modules/toc.pl
+++ b/modules/toc.pl
@@ -25,7 +25,6 @@ use vars qw($TocHeaderText
            $TocClass
            $TocAutomatic
            $TocAnchorPrefix
-
            $TocIsApplyingAutomaticRules);

 =head2 $TocHeaderText
@@ -140,7 +139,7 @@ that table. This is optional. If not specified, it defaults to "toc".
 sub TocRule {
  # <toc...> markup. This explicitly displays a table of contents at this point.
  if ($bol and
-      m~\G&lt;toc(/([A-Za-z\x80-\xff/]+))?    # $1
+      m~\G&lt;toc(/([A-Za-z\x{0080}-\x{fffd}/]+))?    # $1
        (\s+(?:header_text\s*=\s*)?"(.+?)")?  # $3
        (\s+(?:class\s*=\s*)?"(.+?)")?        # $5
        &gt;[ \t]*(\n|$)~cgx) {               # $7
@@ -229,8 +228,11 @@ sub NewTocApplyRules {
  {
    local *STDOUT;
    open(  STDOUT, '>', \$html) or die "Can't open memory file: $!";
+    binmode STDOUT, ":utf8";
    ($blocks, $flags) = OldTocApplyRules(@_);
    close  STDOUT;
+    utf8::decode($blocks);
+    utf8::decode($html);
  }
  # If there are at least two HTML headers on this page, insert a table of
  # contents.
--- a/modules/translation-links.pl
+++ b/modules/translation-links.pl
@@ -1,17 +1,17 @@
-# Copyright (C) 2008  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2008, 2012  Alex Schroeder <alex@gnu.org>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.

 =head1 Translation Links

@@ -76,7 +76,6 @@ sub TranslationLinkInit {
  $TranslationLinkPattern .= '|' if $FreeLinks or $WikiLinks;
  $TranslationLinkPattern .= $LinkPattern if $WikiLinks;
  $TranslationLinkPattern .= ')\]\]';
-  my $text = GetPageContent(FreeToNormal(GetId()));
  %TranslationLinkData = ();
 }

--- a/modules/translations/.cvsignore
+++ b/modules/translations/.cvsignore
@@ -1 +0,0 @@
-*.wiki
--- a/modules/translations/brazilian-portuguese-utf8.pl
+++ b/modules/translations/brazilian-portuguese-utf8.pl
@@ -1,24 +1,3 @@
-# UTF-8 encoded Brazilian Portuguese language file for use with Oddmuse
-#
-# Copyright (c) 2003  Marcelo Toledo <rw@locked.org>.
-# Copyright (c) 2006, 2007  Hélio Nunes <dedalu@dedalu.art.br>.
-#
-# Permission is granted to copy, distribute and/or modify this
-# document under the terms of the GNU Free Documentation License,
-# Version 1.2 or any later version published by the Free Software
-# Foundation; with no Invariant Sections, no Front-Cover Texts, and no
-# Back-Cover Texts.  A copy of the license could be found at:
-# http://www.gnu.org/licenses/fdl.txt.
-#
-# Installation:
-# =============
-#
-# Create a modules subdirectory in your data directory, and put the
-# file in there. It will be loaded automatically.
-#
-# This translation was last checked for Oddmuse version 1.753.
-#
-$ModulesDescription .= '<p>$Id: brazilian-portuguese-utf8.pl,v 1.14 2009/06/07 19:30:37 as Exp $</p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Incluir páginas normais
@@ -92,6 +71,8 @@ Please go on to %s.
 Por favor, vá para %s.
 Updates since %s
 Atualizações desde %s
+up to %s
+
 Updates in the last %s days
 Atualizações nos últimos %s dias
 Updates in the last %s day
@@ -176,6 +157,8 @@ The two revisions are the same.
 As duas versões são idênticas.
 Editing not allowed for %s.
 Edição não permitida para %s.
+Rollback of %s would restore banned content.
+
 Rollback to %s
 Desfazer para %s
 %s rolled back
@@ -196,8 +179,6 @@ Unlock site
 Desbloquear site
 Lock site
 Bloquear site
-Install CSS
-Instalar CSS
 Unlock %s
 Desbloquear %s
 Lock %s
@@ -320,6 +301,8 @@ Could not get %s lock
 Não foi possível conseguir o bloqueio %s
 The lock was created %s.
 O bloqueio foi criado %s.
+Maybe the user running this script is no longer allowed to remove the lock directory?
+
 This operation may take several seconds...
 Essa operação pode demorar vários segundos...
 Forced unlock of %s lock.
@@ -410,6 +393,8 @@ Filtro:
 (para %s)
 %s pages found.
 %s páginas encontradas.
+Malformed regular expression in %s
+
 Replaced: %s
 Substituído: %s
 Search for: %s
@@ -492,16 +477,12 @@ Displaying Wiki Version
 Mostrando a versão do Wiki
 Debugging Information

-Inter links:
-Links inter-:
 Too many connections by %s
 Muitas conexões de %s
 Please do not fetch more than %1 pages in %2 seconds.
 Por favor, abra mais do que %1 páginas em %2 segundos.
 Check whether the web server can create the directory %s and whether it can create files in it.
 Verifique se o servidor web pode criar o diretório %s e se pode criar arquivos nele.
-Copy one of the following stylesheets to %s:
-Copie uma das seguintes folhas de estilo para %s:
 Deleting %s
 Excluindo %s
 Deleted %s
@@ -618,6 +599,20 @@ Compilation for %s
 Compilação para %s
 Compilation tag is missing a regular expression.
 Falta expressão regular na etiqueta de compilação.
+Install CSS
+Instalar CSS
+Copy one of the following stylesheets to %s:
+Copie uma das seguintes folhas de estilo para %s:
+Reset
+
+Extract all dates from the database
+
+Dates
+
+No dates found.
+
+Inter links:
+Links inter-:
 List spammed pages
 Listar páginas com spam
 Despamming pages
@@ -658,6 +653,14 @@ ordinary changes
 alterações normais
 Matching page names:
 Coincidindo com os nomes de página:
+no summary available
+
+page was marked for deletion
+
+Oddmuse
+
+Email: 
+
 Could not find %1.html template in %2
 Não foi possível encontrar o modelo %1.html em %2
 Only Editors are allowed to see this hidden page.
@@ -686,6 +689,14 @@ O modelo %s ou está vazio ou não existe.
 -- definido em %s
 Local names defined on %1: %2
 Nomes locais definidos em %1: %2
+Name: 
+
+URL: 
+
+Define Local Names
+
+Define external redirect: 
+
 Locked Pages

 Register for %s
@@ -788,8 +799,6 @@ unsubscribe

 subscribe

-Email: 
-
 %s appears to be an invalid mail address

 Your mail subscriptions
@@ -826,6 +835,8 @@ You linked more than %s times to the same domain. It would seem that only a spam
 %s não é um nome legal para um espaço de nomes
 Namespaces

+ (create locally)
+ (criar localmente)
 Getting page index file for %s.
 Obtendo arquivo de índice de página para %s.
 Near links:
@@ -842,8 +853,6 @@ EditNearLinks
 EditarLinksPróximos
 The same page on other sites:
 A mesma página em outros sites:
- (create locally)
- (criar localmente)
 image
 imagem
 download
@@ -858,6 +867,12 @@ Generating Link Database
 Gerando Banco de Dados de Link
 The 404 handler extension requires the link data extension (links.pl).
 A extensão 404 handler requer a extensão link data (links.pl)
+Make available offline
+
+Offline
+
+You are currently offline and what you requested is not part of the offline application. You need to be online to do this.
+
 LocalMap
 MapaLocal
 No page id for action localmap
@@ -872,20 +887,6 @@ Self-ban by %s
 Auto-banimento por %s
 You have banned your own IP.
 Você baniu seu próprio IP.
-OpenID Login
-
-Your identity is saved in a cookie, if you have cookies enabled. Cookies may get lost if you connect from another machine, from another account, or using another software.
-
-Your homepage is set to %s.
-
-You have no homepage set.
-
-Homepage:
-
-Homepage is missing
-
-OpenID error %s
-
 Orphan List
 Lista de Órfãs
 Trail: 
@@ -918,6 +919,8 @@ No target wiki was specified in the config file.
 Não foi especificado um wiki alvo no arquivo de configuração.
 The target wiki was misconfigured.
 O wiki alvo está mal configurado.
+Upload is limited to %s bytes
+
 You did not answer correctly.
 Você não respondeu corretamente à pergunta.
 To save this page you must answer this question:
@@ -970,6 +973,10 @@ Static Copy
 Cópia Estática
 Back to %s
 Voltar para %s
+Edit image in the browser
+
+Summary of your changes: 
+
 Copy to %1 succeeded: %2.
 Sucesso na cópia de %1: %2.
 Copy to %1 failed: %2.
@@ -1008,12 +1015,40 @@ Too many instances.  Only %s allowed.
 Muitas instâncias. Apenas %s permitidas.
 Please try again later. Perhaps somebody is running maintenance or doing a long search. Unfortunately the site has limited resources, and so we must ask you for a bit of patience.
 Por favor, tente novamente mais tarde. Talvez alguém esteja executando manutenção ou uma longa busca. Infelizmente o site tem recursos limitados e, por isso, pedimos um pouco de paciência.
+thumb
+
+Error creating thumbnail from non existant page %s.
+
+Can not create thumbnail for file type %s.
+
+Can not create thumbnail for a text document
+
+Could not open %s for writing whilst trying to save image before creating thumbnail. Check write permissions.
+
+Can not create path for thumbnail - %s
+
+Failed to run %1 to create thumbnail: %2
+
+%s ran into an error
+
+%s produced no output
+
+Failed to parse %s.
+
+Timezone
+
+Pick your timezone:
+
+Set
+
 Contents
 Conteúdo
 Create a new page for today

 Add Translation

+Please provide a different page name for the translation.
+
 Added translation: %1 (%2)

 Translate %s
@@ -1056,6 +1091,8 @@ Páginas Procuradas
 %s páginas
 %s, referenced from:
 %s, referenciada por:
+Web application for offline browsing
+
 Upload of %s file
 Envio de arquivos %s
 Blog
--- a/modules/translations/bulgarian-utf8.pl
+++ b/modules/translations/bulgarian-utf8.pl
@@ -1,24 +1,3 @@
-# UTF-8 encoded Bulgarian translation file for use with Oddmuse
-#
-# Copyright (c) 2004 Stanislav Traykov <stanislav@tortoises.org>
-# Copyright (c) 2003, 2004  Alex Schröder <alex@emacswiki.org>
-#
-# Permission is granted to copy, distribute and/or modify this
-# document under the terms of the GNU Free Documentation License,
-# Version 1.2 or any later version published by the Free Software
-# Foundation; with no Invariant Sections, no Front-Cover Texts, and no
-# Back-Cover Texts.  A copy of the license could be found at:
-# http://www.gnu.org/licenses/fdl.txt .
-#
-# Installation:
-# =============
-#
-# Create a modules subdirectory in your data directory, and put the
-# file in there. It will be loaded automatically.
-#
-# This translation was updated for Oddmuse 1.354.
-#
-$ModulesDescription .= '<p>$Id: bulgarian-utf8.pl,v 1.11 2009/06/07 19:30:37 as Exp $</p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

@@ -92,6 +71,8 @@ Please go on to %s.
 Моля продължи на %s.
 Updates since %s
 Промени от %s насам
+up to %s
+
 Updates in the last %s days
 Промени през последните %s дни
 Updates in the last %s day
@@ -176,6 +157,8 @@ The two revisions are the same.

 Editing not allowed for %s.
 Редакция на %s не е разрешена.
+Rollback of %s would restore banned content.
+
 Rollback to %s
 Връщане до %s
 %s rolled back
@@ -196,8 +179,6 @@ Unlock site

 Lock site

-Install CSS
-
 Unlock %s

 Lock %s
@@ -320,6 +301,8 @@ Could not get %s lock
 Не може да се резервират изключителни права върху %s.
 The lock was created %s.

+Maybe the user running this script is no longer allowed to remove the lock directory?
+
 This operation may take several seconds...
 Тази операция може да потрае малко...
 Forced unlock of %s lock.
@@ -410,6 +393,8 @@ Filter:
 (за %s)
 %s pages found.
 %s намерени страници.
+Malformed regular expression in %s
+
 Replaced: %s
 Заместено: %s
 Search for: %s
@@ -492,16 +477,12 @@ Displaying Wiki Version

 Debugging Information

-Inter links:
-Интер-линкове:
 Too many connections by %s
 Прекалено много връзки с %s
 Please do not fetch more than %1 pages in %2 seconds.

 Check whether the web server can create the directory %s and whether it can create files in it.

-Copy one of the following stylesheets to %s:
-
 Deleting %s

 Deleted %s
@@ -618,6 +599,20 @@ Compilation for %s

 Compilation tag is missing a regular expression.

+Install CSS
+
+Copy one of the following stylesheets to %s:
+
+Reset
+
+Extract all dates from the database
+
+Dates
+
+No dates found.
+
+Inter links:
+Интер-линкове:
 List spammed pages

 Despamming pages
@@ -658,6 +653,14 @@ ordinary changes

 Matching page names:

+no summary available
+
+page was marked for deletion
+
+Oddmuse
+
+Email: 
+
 Could not find %1.html template in %2

 Only Editors are allowed to see this hidden page.
@@ -686,6 +689,14 @@ The template %s is either empty or does not exist.

 Local names defined on %1: %2

+Name: 
+
+URL: 
+
+Define Local Names
+
+Define external redirect: 
+
 Locked Pages

 Register for %s
@@ -788,8 +799,6 @@ unsubscribe

 subscribe

-Email: 
-
 %s appears to be an invalid mail address

 Your mail subscriptions
@@ -826,6 +835,8 @@ You linked more than %s times to the same domain. It would seem that only a spam

 Namespaces

+ (create locally)
+
 Getting page index file for %s.
 Получаване на индекс за %s.
 Near links:
@@ -842,8 +853,6 @@ EditNearLinks
 Редакция на близки линкове
 The same page on other sites:
 Същата страница на други места:
- (create locally)
-
 image

 download
@@ -858,6 +867,12 @@ Generating Link Database

 The 404 handler extension requires the link data extension (links.pl).

+Make available offline
+
+Offline
+
+You are currently offline and what you requested is not part of the offline application. You need to be online to do this.
+
 LocalMap

 No page id for action localmap
@@ -872,20 +887,6 @@ Self-ban by %s

 You have banned your own IP.

-OpenID Login
-
-Your identity is saved in a cookie, if you have cookies enabled. Cookies may get lost if you connect from another machine, from another account, or using another software.
-
-Your homepage is set to %s.
-
-You have no homepage set.
-
-Homepage:
-
-Homepage is missing
-
-OpenID error %s
-
 Orphan List

 Trail: 
@@ -918,6 +919,8 @@ No target wiki was specified in the config file.

 The target wiki was misconfigured.

+Upload is limited to %s bytes
+
 You did not answer correctly.

 To save this page you must answer this question:
@@ -970,6 +973,10 @@ Static Copy

 Back to %s
 Обратно към %s
+Edit image in the browser
+
+Summary of your changes: 
+
 Copy to %1 succeeded: %2.

 Copy to %1 failed: %2.
@@ -1008,12 +1015,40 @@ Too many instances.  Only %s allowed.

 Please try again later. Perhaps somebody is running maintenance or doing a long search. Unfortunately the site has limited resources, and so we must ask you for a bit of patience.

+thumb
+
+Error creating thumbnail from non existant page %s.
+
+Can not create thumbnail for file type %s.
+
+Can not create thumbnail for a text document
+
+Could not open %s for writing whilst trying to save image before creating thumbnail. Check write permissions.
+
+Can not create path for thumbnail - %s
+
+Failed to run %1 to create thumbnail: %2
+
+%s ran into an error
+
+%s produced no output
+
+Failed to parse %s.
+
+Timezone
+
+Pick your timezone:
+
+Set
+
 Contents

 Create a new page for today

 Add Translation

+Please provide a different page name for the translation.
+
 Added translation: %1 (%2)

 Translate %s
@@ -1056,6 +1091,8 @@ Wanted Pages

 %s, referenced from:

+Web application for offline browsing
+
 Upload of %s file

 Blog
--- a/modules/translations/chinese-utf8.pl
+++ b/modules/translations/chinese-utf8.pl
@@ -1,24 +1,3 @@
-# UTF-8 encoded Traditional Chinese language file for use with Oddmuse
-#
-# Copyright (c) 2003, 2004  wctang <wctang@csie.nctu.edu.tw>
-# Copyright (c) 2007  Wei Ren Wang <weithenn@gmail.com>
-#
-# Permission is granted to copy, distribute and/or modify this
-# document under the terms of the GNU Free Documentation License,
-# Version 1.2 or any later version published by the Free Software
-# Foundation; with no Invariant Sections, no Front-Cover Texts, and no
-# Back-Cover Texts.  A copy of the license could be found at:
-# http://www.gnu.org/licenses/fdl.txt.
-#
-# Installation:
-# =============
-#
-# Create a modules subdirectory in your data directory, and put the
-# file in there. It will be loaded automatically.
-#
-# This translation was last checked for Oddmuse version 1.504.
-#
-$ModulesDescription .= '<p>$Id: chinese-utf8.pl,v 1.12 2009/06/07 19:30:37 as Exp $</p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 包含正常頁面
@@ -92,6 +71,8 @@ Please go on to %s.
 請繼續前住 %s 。
 Updates since %s
 自 %s 以來的修改
+up to %s
+
 Updates in the last %s days
 在 %s 天之內的更動
 Updates in the last %s day
@@ -176,6 +157,8 @@ The two revisions are the same.
 二個版本相同
 Editing not allowed for %s.
 不允許編輯 %s 。
+Rollback of %s would restore banned content.
+
 Rollback to %s
 回復至 %s
 %s rolled back
@@ -196,8 +179,6 @@ Unlock site
 網站解鎖
 Lock site
 網站鎖定
-Install CSS
-安裝 CSS
 Unlock %s
 解鎖 %s
 Lock %s
@@ -320,6 +301,8 @@ Could not get %s lock
 無法取得 %s 鎖定
 The lock was created %s.
 建立鎖定 %s
+Maybe the user running this script is no longer allowed to remove the lock directory?
+
 This operation may take several seconds...
 這個動作可能要花幾秒…
 Forced unlock of %s lock.
@@ -410,6 +393,8 @@ Filter:
 (列出 %s )
 %s pages found.
 找到 %s 個頁面。
+Malformed regular expression in %s
+
 Replaced: %s
 取代：%s
 Search for: %s
@@ -492,16 +477,12 @@ Displaying Wiki Version
 顯示 Wiki 主機相關套件版本
 Debugging Information

-Inter links:
-內部連結：
 Too many connections by %s
 太多來自 %s 的連線
 Please do not fetch more than %1 pages in %2 seconds.
 請不要在 %2 秒內抓取超過 %1 頁的資料。
 Check whether the web server can create the directory %s and whether it can create files in it.
 請確認網站伺服器是否可建立 %s 目錄，並且在其中建立檔案。
-Copy one of the following stylesheets to %s:
-複製以下 CSS 模版至 %s
 Deleting %s
 正在刪除 %s
 Deleted %s
@@ -618,6 +599,20 @@ Compilation for %s
 %s 的彙整
 Compilation tag is missing a regular expression.
 匯編的標記缺少一個正規表示式
+Install CSS
+安裝 CSS
+Copy one of the following stylesheets to %s:
+複製以下 CSS 模版至 %s
+Reset
+
+Extract all dates from the database
+
+Dates
+
+No dates found.
+
+Inter links:
+內部連結：
 List spammed pages
 列出 SPAM 頁面
 Despamming pages
@@ -658,6 +653,14 @@ ordinary changes
 普通變更
 Matching page names:
 匹配頁面的名稱:
+no summary available
+
+page was marked for deletion
+
+Oddmuse
+
+Email: 
+
 Could not find %1.html template in %2
 無法在 %2 找到 %1.html 的範本
 Only Editors are allowed to see this hidden page.
@@ -686,6 +689,14 @@ The template %s is either empty or does not exist.
 -- 在 %s 中定義
 Local names defined on %1: %2
 定義本地名稱在 %1: %2
+Name: 
+
+URL: 
+
+Define Local Names
+
+Define external redirect: 
+
 Locked Pages

 Register for %s
@@ -788,8 +799,6 @@ unsubscribe

 subscribe

-Email: 
-
 %s appears to be an invalid mail address

 Your mail subscriptions
@@ -826,6 +835,8 @@ You linked more than %s times to the same domain. It would seem that only a spam
 %s 不是一個正常的命名空間
 Namespaces

+ (create locally)
+ (本地建立)
 Getting page index file for %s.
 自 %s 取得頁面索引資料。
 Near links:
@@ -842,8 +853,6 @@ EditNearLinks
 編輯接近連結
 The same page on other sites:
 其他網站的相同頁面
- (create locally)
- (本地建立)
 image
 圖像
 download
@@ -858,6 +867,12 @@ Generating Link Database
 產生連結資料庫
 The 404 handler extension requires the link data extension (links.pl).
 404 訊息，您需要安裝 (links.pl) 擴充模組
+Make available offline
+
+Offline
+
+You are currently offline and what you requested is not part of the offline application. You need to be online to do this.
+
 LocalMap
 本地地圖
 No page id for action localmap
@@ -872,20 +887,6 @@ Self-ban by %s
 被 %s 禁止
 You have banned your own IP.
 您禁止了自已的 IP Address
-OpenID Login
-
-Your identity is saved in a cookie, if you have cookies enabled. Cookies may get lost if you connect from another machine, from another account, or using another software.
-
-Your homepage is set to %s.
-
-You have no homepage set.
-
-Homepage:
-
-Homepage is missing
-
-OpenID error %s
-
 Orphan List
 孤立頁面列表
 Trail: 
@@ -918,6 +919,8 @@ No target wiki was specified in the config file.
 設定檔案中沒有設定目標(Target) Wiki
 The target wiki was misconfigured.
 目標(Target) Wiki 設定錯誤
+Upload is limited to %s bytes
+
 You did not answer correctly.
 您沒有回答正確的答案
 To save this page you must answer this question:
@@ -970,6 +973,10 @@ Static Copy
 靜態頁面備份
 Back to %s
 返回 %s
+Edit image in the browser
+
+Summary of your changes: 
+
 Copy to %1 succeeded: %2.
 從 %2 複製到 %1 成功
 Copy to %1 failed: %2.
@@ -1008,12 +1015,40 @@ Too many instances.  Only %s allowed.
 太多請求，只允許 %s
 Please try again later. Perhaps somebody is running maintenance or doing a long search. Unfortunately the site has limited resources, and so we must ask you for a bit of patience.
 系統忙碌中請稍後在試一次，可能有人正在執行維護動作或長期搜尋
+thumb
+
+Error creating thumbnail from non existant page %s.
+
+Can not create thumbnail for file type %s.
+
+Can not create thumbnail for a text document
+
+Could not open %s for writing whilst trying to save image before creating thumbnail. Check write permissions.
+
+Can not create path for thumbnail - %s
+
+Failed to run %1 to create thumbnail: %2
+
+%s ran into an error
+
+%s produced no output
+
+Failed to parse %s.
+
+Timezone
+
+Pick your timezone:
+
+Set
+
 Contents
 內容
 Create a new page for today
 建立今日頁面
 Add Translation

+Please provide a different page name for the translation.
+
 Added translation: %1 (%2)

 Translate %s
@@ -1056,6 +1091,8 @@ Wanted Pages
 %s 頁面
 %s, referenced from:
 %s 引用自:
+Web application for offline browsing
+
 Upload of %s file
 上傳 %s 檔案
 Blog
--- a/modules/translations/chinese_cn-utf8.pl
+++ b/modules/translations/chinese_cn-utf8.pl
@@ -24,7 +24,8 @@
 #by wctang <wctang@csie.nctu.edu.tw> and using the tool cnmap
 #(http://search.cpan.org/~qjzhou/Encode-CNMap-0.32/bin/cnmap) by Qing-Jie Zhou <qjzhou@hotmail.com>.
 #
-$ModulesDescription .= '<p>$Id: chinese_cn-utf8.pl,v 1.11 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/chinese_cn-utf8.pl">chinese_cn-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Chinese">Chinese</a></p>';
 ##############################################################
 #  for those who want to use Chinese even for special pages
 #  please uncomment the corresponding lines to use translated 
--- a/modules/translations/dutch-utf8.pl
+++ b/modules/translations/dutch-utf8.pl
@@ -24,7 +24,8 @@
 #
 # This translation was last checked for Oddmuse version 1.215.
 #
-$ModulesDescription .= '<p>$Id: dutch-utf8.pl,v 1.12 2011/05/17 13:24:13 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/dutch-utf8.pl">dutch-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Dutch">Dutch</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

--- a/modules/translations/finnish-utf8.pl
+++ b/modules/translations/finnish-utf8.pl
@@ -16,7 +16,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: finnish-utf8.pl,v 1.11 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/finnish-utf8.pl">finnish-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Finnish">Finnish</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

--- a/modules/translations/french-utf8.pl
+++ b/modules/translations/french-utf8.pl
@@ -19,7 +19,8 @@
 #
 # This translation was last checked for Oddmuse version 1.296.
 #
-$ModulesDescription .= '<p>$Id: french-utf8.pl,v 1.20 2011/02/05 12:29:07 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/french-utf8.pl">french-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/French">French</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Comprend les pages normales
--- a/modules/translations/german-utf8.pl
+++ b/modules/translations/german-utf8.pl
@@ -15,7 +15,8 @@
 # Create a modules subdirectory in your data directory, and put the file in
 # there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: german-utf8.pl,v 1.30 2011/02/05 12:40:38 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/german-utf8.pl">german-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/German">German</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Mit normalen Seiten
--- a/modules/translations/greek-utf8.pl
+++ b/modules/translations/greek-utf8.pl
@@ -15,7 +15,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: greek-utf8.pl,v 1.2 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/greek-utf8.pl">greek-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Greek">Greek</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Με τις κανονικές σελίδες.
--- a/modules/translations/hebrew-utf8.pl
+++ b/modules/translations/hebrew-utf8.pl
@@ -17,7 +17,8 @@
 #
 # This translation was last checked for Oddmuse version 1.195.
 #
-$ModulesDescription .= '<p>$Id: hebrew-utf8.pl,v 1.11 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/hebrew-utf8.pl">hebrew-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Hebrew">Hebrew</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

--- a/modules/translations/italian-utf8.pl
+++ b/modules/translations/italian-utf8.pl
@@ -15,7 +15,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: italian-utf8.pl,v 1.11 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/italian-utf8.pl">italian-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Italian">Italian</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Includi le pagine normali
--- a/modules/translations/japanese-utf8.pl
+++ b/modules/translations/japanese-utf8.pl
@@ -17,7 +17,8 @@
 #
 # This translation was last checked for Oddmuse version 1.215.
 #
-$ModulesDescription .= '<p>$Id: japanese-utf8.pl,v 1.11 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/japanese-utf8.pl">japanese-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Japanese">Japanese</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

--- a/modules/translations/korean-utf8.pl
+++ b/modules/translations/korean-utf8.pl
@@ -15,7 +15,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: korean-utf8.pl,v 1.5 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/korean-utf8.pl">korean-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Korean">Korean</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 정상 페이지를 포함하여
--- a/modules/translations/new-utf8.pl
+++ b/modules/translations/new-utf8.pl
@@ -15,7 +15,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: new-utf8.pl,v 1.12 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/fixme-utf8.pl">fixme-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Fixme">Fixme</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

--- a/modules/translations/polish-utf8.pl
+++ b/modules/translations/polish-utf8.pl
@@ -17,7 +17,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: polish-utf8.pl,v 1.7 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/polish-utf8.pl">polish-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Polish">Polish</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Ze zwykłymi stronami
--- a/modules/translations/portuguese-utf8.pl
+++ b/modules/translations/portuguese-utf8.pl
@@ -20,7 +20,8 @@
 #
 # This translation was last checked for Oddmuse version 1.195.
 #
-$ModulesDescription .= '<p>$Id: portuguese-utf8.pl,v 1.15 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/portuguese-utf8.pl">portuguese-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Portuguese">Portuguese</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Incluir páginas normais
--- a/modules/translations/romanian-utf8.pl
+++ b/modules/translations/romanian-utf8.pl
@@ -15,7 +15,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: romanian-utf8.pl,v 1.9 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/romanian-utf8.pl">romanian-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Romanian">Romanian</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

--- a/modules/translations/russian-utf8.pl
+++ b/modules/translations/russian-utf8.pl
@@ -20,7 +20,8 @@
 #
 # This script was last checked for Oddmuse version 1.658.
 #
-$ModulesDescription .= '<p>$Id: russian-utf8.pl,v 1.13 2007/08/19 11:42:08 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/russian-utf8.pl">russian-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Russian">Russian</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Reading not allowed: user, ip, or network is blocked.
 Просмотр недоступен: имя пользователя, IP-адрес или сеть заблокированы.
--- a/modules/translations/serbian-utf8.pl
+++ b/modules/translations/serbian-utf8.pl
@@ -17,7 +17,8 @@
 #
 # This translation was last checked for Oddmuse version 1.195.
 #
-$ModulesDescription .= '<p>$Id: serbian-utf8.pl,v 1.11 2009/06/07 19:30:37 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/serbian-utf8.pl">serbian-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Serbian">Serbian</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages

--- a/modules/translations/spanish-utf8.pl
+++ b/modules/translations/spanish-utf8.pl
@@ -18,7 +18,8 @@
 #
 # This translation was last checked for Oddmuse version 1.195.
 #
-$ModulesDescription .= '<p>$Id: spanish-utf8.pl,v 1.13 2011/07/05 00:30:18 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/spanish-utf8.pl">spanish-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Spanish">Spanish</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Incluir páginas normales
--- a/modules/translations/swedish-utf8.pl
+++ b/modules/translations/swedish-utf8.pl
@@ -20,7 +20,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: swedish-utf8.pl,v 1.18 2009/06/07 19:30:38 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/swedish-utf8.pl">swedish-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Swedish">Swedish</a></p><p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Med vanliga sidor
--- a/modules/translations/ukrainian-utf8.pl
+++ b/modules/translations/ukrainian-utf8.pl
@@ -15,7 +15,8 @@
 # Create a modules subdirectory in your data directory, and put the
 # file in there. It will be loaded automatically.
 #
-$ModulesDescription .= '<p>$Id: ukrainian-utf8.pl,v 1.6 2009/06/07 19:30:38 as Exp $</p>';
+use utf8;
+$ModulesDescription .= '<p><a href="http://git.savannah.gnu.org/cgit/oddmuse.git/tree/modules/translations/ukrainian-utf8.pl">ukrainian-utf8.pl</a>, see <a href="http://www.oddmuse.org/cgi-bin/oddmuse/Ukrainian">Ukrainian</a></p>';
 %Translate = split(/\n/,<<END_OF_TRANSLATION);
 Include normal pages
 Включати звичайні сторінки
--- a/modules/webdav.pl
+++ b/modules/webdav.pl
@@ -109,7 +109,7 @@ sub get {
    } else {
      print $q->header( -cache_control  => 'max-age=10',
 			-etag           => $OddMuse::Page{ts},
-			-type           => "text/plain; charset=$OddMuse::HttpCharset",
+			-type           => "text/plain; charset=UTF-8",
 			-status         => "200 OK",);
      print $OddMuse::Page{text} unless $head;
    }
@@ -373,11 +373,6 @@ sub propfind {
      $resp->addChild($propstat);
    }
  }
-  # The XML Parser handles UTF-8 correctly, but Perl will
-  # automatically convert it to Latin-1 upon printing to STDOUT unless
-  # we use binmode.
-  eval { local $SIG{__DIE__}; binmode(STDOUT, ":utf8"); }
-    if $OddMuse::HttpCharset eq 'UTF-8';
  warn "RESPONSE: 207\n" . $doc->toString(1) . "\n" if $verbose;
  print $doc->toString(1);
 }
--- a/3
+++ b/3
@@ -2,6 +2,9 @@
 # Based on umtrans.pl version 1.0 (April 8, 2001) by Clifford Adams.
 # Extracts translation strings from wiki script and extensions.

+binmode(STDIN, ':utf8');
+binmode(STDOUT, ':utf8');
+
 $help = q{
 NAME
       oddtrans - complement translation tables for Oddmuse
--- a/rc2mail.pl
+++ b/rc2mail.pl
@@ -119,7 +119,6 @@ sub get_rss {
  my $rss = new XML::RSS;
  $rss->parse($response->content);
  print "Found " . @{$rss->{items}} . " items.\n" if $debug;
-  update_timestamp();
  return $rss;
 }

@@ -145,6 +144,7 @@ sub send_file {
  my ($id, $title, $item, @subscribers) = @_;
  return unless @subscribers;
  my $fh = File::Temp->new(SUFFIX => '.html');
+  binmode($fh, ":utf8");
  warn "No content for $title\n" unless $item->{description};
  my $link = $item->{link};
  my $sub = "$root?action=subscriptions";
@@ -207,6 +207,7 @@ sub main {
  my $subscribers = get_subscribers();
  return unless %{$subscribers};
  send_files($rss, $subscribers);
+  update_timestamp();
 }

 main ();
--- a/t/ampersand.t
+++ b/t/ampersand.t
@@ -15,6 +15,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 39;
+use utf8; # tests contain UTF-8 characters and it matters

 clear_pages();

--- a/t/crossbar.t
+++ b/t/crossbar.t
@@ -29,7 +29,10 @@ test_page_negative($page, 'mu');
 AppendStringToFile($ConfigFile, "\$UploadAllowed = 1;\n");
 test_page(update_page('Alex', "#FILE image/png\niVBORw0KGgoAAAA"),
 	  'This page contains an uploaded file:');
-$page = get_page('action=download id=Alex');
+{
+  local $raw = 1;
+  $page = get_page('action=download id=Alex');
+}
 $page =~ s/^.*\r\n\r\n//s; # strip headers
 require MIME::Base64;
 test_page(MIME::Base64::encode($page), '^iVBORw0KGgoAAAA');
--- a/t/dotfiles.t
+++ b/t/dotfiles.t
@@ -0,0 +1,24 @@
+# Copyright (C) 2012  Alex Schroeder <alex@gnu.org>
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 't/test.pl';
+package OddMuse;
+use Test::More tests => 4;
+
+clear_pages();
+
+test_page(update_page('Test', 'some text'), 'some text');
+test_page(update_page('.emacs', 'some code'), 'some code');
+unlink $IndexFile;
+test_page(get_page('action=index raw=1'), 'Test', '\.emacs');
--- a/t/download.t
+++ b/t/download.t
@@ -14,7 +14,7 @@

 require 't/test.pl';
 package OddMuse;
-use Test::More tests => 12;
+use Test::More tests => 13;

 clear_pages();

@@ -25,7 +25,15 @@ AppendStringToFile($ConfigFile, "\$LogoUrl = '/pic/logo.png';\n");
 xpath_test(get_page('HomePage'), '//a[@class="logo"]/img[@class="logo"][@src="/pic/logo.png"][@alt="[Home]"]');
 AppendStringToFile($ConfigFile, "\$LogoUrl = 'Logo';\n");
 xpath_test(get_page('HomePage'), '//a[@class="logo"]/img[@class="logo"][@src="Logo"][@alt="[Home]"]');
+
 update_page('Logo', "#FILE image/png\niVBORw0KGgoAAAA");
+
+# make sure we don't supply "content-type: image/png; charset=utf-8"
+{
+ local $raw = 1;
+ test_page_negative(get_page('action=download id=Logo'), 'charset');
+}
+
 xpath_test(get_page('HomePage'), '//a[@class="logo"]/img[@class="logo"][@src="http://localhost/wiki.pl/download/Logo"][@alt="[Home]"]');
 AppendStringToFile($ConfigFile, "\$UsePathInfo = 0;\n");
 xpath_test(get_page('HomePage'), '//a[@class="logo"]/img[@class="logo"][@src="http://localhost/wiki.pl?action=download;id=Logo"][@alt="[Home]"]');
@@ -46,7 +54,10 @@ test_page(update_page('Trogs', $page), 'contains an uploaded file');

 xpath_test(get_page('Trogs'),
 	   '//p/img[@class="upload"][@src="http://localhost/wiki.pl?action=download;id=Trogs"][@alt="Trogs"]');
-$page = get_page('action=download id=Trogs');
+{
+  local $raw = 1;
+  $page = get_page('action=download id=Trogs');
+}
 test_page($page,
 	  'Content-Type: image/svg\+xml',
 	  'Content-encoding: gzip');
--- a/t/drafts.t
+++ b/t/drafts.t
@@ -1,24 +1,21 @@
-# Copyright (C) 2006  Alex Schroeder <alex@emacswiki.org>
+# Copyright (C) 2006, 2012  Alex Schroeder <alex@gnu.org>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the
-#    Free Software Foundation, Inc.
-#    59 Temple Place, Suite 330
-#    Boston, MA 02111-1307 USA
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.

 require 't/test.pl';
 package OddMuse;
-use Test::More tests => 23;
+use Test::More tests => 26;
+use utf8;

 clear_pages();

@@ -77,3 +74,10 @@ utime($Now-1300000, $Now-1300000, "$DraftDir/Alex");
 # Second maintenance requires admin password and deletes one draft
 test_page(get_page('action=maintain pwd=foo'), 'Alex was last modified [^<>]* ago and was deleted');
 ok(! -f "$DraftDir/.berta", "$DraftDir/Alex is gone");
+
+# Testing UTF-8
+# Saving draft uses 204 No Content status
+test_page(get_page('title=HomePage text=foo username=Schröder Draft=1'),
+	 'Status: 204', 'username%251eSchr%C3%B6der');
+test_page(get_page('action=maintain pwd=foo'),
+	  'Schröder was last modified [^<>]* and was kept');
--- a/t/dynamic-comments.t
+++ b/t/dynamic-comments.t
@@ -14,7 +14,8 @@

 require 't/test.pl';
 package OddMuse;
-use Test::More tests => 9;
+use Test::More tests => 11;
+use utf8; # tests contain UTF-8 characters and it matters

 clear_pages();

@@ -35,12 +36,16 @@ test_page(update_page('Comments_on_2011-07-06', 'Yo'),
 	  'Yo');

 xpath_test(get_page('Hi'),
-	   '//div[@class="journal"]/div[@class="page"]/p[@class="comment"]/a[@href="javascript:togglecomments(\'Comments_on_2011-07-06\')"][text()="Comments on 2011-07-06"]');
+	   '//div[@class="journal"]/div[@class="page"]/p[@class="comment"]/a[@href="javascript:togglecomments(\'id0\')"][text()="Comments on 2011-07-06"]');
+
+# encoding basics
+$page = update_page('2011-07-06_(…)_Dü', 'Hallo Dü');
+test_page($page, 'Hallo Dü');
+xpath_test($page, '//p[contains(text(), "Dü")]');

-update_page('2011-07-06_(…)_Dü', 'Hallo');
 update_page('Comments_on_2011-07-06_(…)_Dü', 'Yo');

 xpath_test(update_page('Hi', '<journal>'),
 	  '//h1/a[text()="2011-07-06 (…) Dü"]',
 	  '//div[@class="journal"]/div[@class="page"]/p[@class="comment"]/a[text()="Comments on 2011-07-06 (…) Dü"]',
-	  '//div[@class="journal"]/div[@class="page"]/p[@class="comment"]/a[@href="javascript:togglecomments(\'Comments_on_2011-07-06__Dü\')"]');
+	  '//div[@class="journal"]/div[@class="page"]/p[@class="comment"]/a[@href="javascript:togglecomments(\'id0\')"]');
--- a/t/encoding.t
+++ b/t/encoding.t
@@ -0,0 +1,116 @@
+# Copyright (C) 2012  Alex Schroeder <alex@gnu.org>
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 't/test.pl';
+package OddMuse;
+use Test::More tests => 41;
+use utf8; # tests contain UTF-8 characters and it matters
+
+clear_pages();
+
+# ASCII basics
+
+$page = update_page('Aal', 'aal');
+test_page($page, '<h1><a .*>Aal</a></h1>', '<p>aal</p>');
+xpath_test($page, '//h1/a[text()="Aal"]', '//p[text()="aal"]');
+
+$page = get_page('Aal');
+test_page($page, '<h1><a .*>Aal</a></h1>', '<p>aal</p>');
+xpath_test($page, '//h1/a[text()="Aal"]', '//p[text()="aal"]');
+
+# non-ASCII
+
+$page = update_page('Öl', 'öl');
+test_page($page, '<h1><a .*>Öl</a></h1>', '<p>öl</p>');
+xpath_test($page, '//h1/a[text()="Öl"]', '//p[text()="öl"]');
+
+$page = get_page('Öl');
+test_page($page, '<h1><a .*>Öl</a></h1>', '<p>öl</p>');
+xpath_test($page, '//h1/a[text()="Öl"]', '//p[text()="öl"]');
+
+$page = get_page('action=index raw=1');
+test_page($page, 'Aal', 'Öl');
+
+test_page(get_page('Aal'), 'aal');
+test_page(get_page('Öl'), 'öl');
+
+# rc
+
+test_page(get_page('action=rc raw=1'),
+	  'title: Öl', 'description: öl');
+
+# diff
+
+update_page('Öl', 'Ähren');
+xpath_test(get_page('action=browse id=Öl diff=1'),
+	   '//div[@class="old"]/p/strong[@class="changes"][text()="öl"]',
+	   '//div[@class="new"]/p/strong[@class="changes"][text()="Ähren"]');
+
+# search
+
+# testing with non-ASCII is important on a Mac
+
+# ASCII
+$page = get_page('search=aal raw=1');
+test_page($page, 'title: Search for: aal', 'title: Aal');
+
+# matching page name does not involve grep working
+$page = get_page('search=öl raw=1');
+test_page($page, 'title: Search for: öl', 'title: Öl');
+
+# this fails with grep on a Mac, thus testing if mac.pl
+# managed to switch of the use of grep
+test_page(get_page('search=ähren raw=1'),
+	  'title: Search for: ähren', 'title: Öl');
+
+# the username keeps getting reported as changed
+test_page(get_page('action=browse id=Möglich username=Schr%C3%B6der'),
+	  'Set-Cookie: Wiki=username%251eSchr%C3%B6der',
+	  'username=Schröder');
+
+# verify that non-ASCII parameters work as intended
+AppendStringToFile($ConfigFile, "use utf8;\n\$CookieParameters{ärger} = 1;\n");
+test_page(get_page('action=browse id=Test %C3%A4rger=hallo'),
+	  'Set-Cookie: Wiki=%C3%A4rger%251ehallo');
+
+# this causes wide character in print somehow? otherwise harmless
+test_page(update_page("Russian", "Русский Hello"),
+	  "Русский");
+
+# checking for errors in the rss feed
+test_page(get_page("action=rss match=Russian full=1"),
+	  "Русский");
+
+# with toc.pl, however, a problem: Русский is corrupted
+add_module('toc.pl');
+test_page(update_page("Russian", "Русский Hello again"),
+	  "Русский");
+
+# and with inclusion, too:
+test_page(update_page("All", qq{<include "Russian">}),
+	  "Русский");
+
+# and checking the cache
+test_page(get_page("All"), "Русский");
+
+# and checking without the cache
+test_page(get_page("action=browse id=All cache=0"), "Русский");
+
+# testing search
+test_page(get_page('search=Русский raw=1'),
+	  qw(Russian));
+
+# testing page editing
+test_page(update_page("Русский", "друзья"),
+	  "друзья");
--- a/t/image.t
+++ b/t/image.t
@@ -15,6 +15,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 33;
+use utf8; # tests contain UTF-8 characters and it matters

 clear_pages();

--- a/t/indexed-search.t
+++ b/t/indexed-search.t
@@ -16,6 +16,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 61;
+use utf8; # tests contain UTF-8 characters and it matters

 SKIP: {
  eval { require Search::FreeText };
--- a/t/languages.t
+++ b/t/languages.t
@@ -19,6 +19,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 5;
+use utf8; # tests contain UTF-8 characters and it matters

 %Languages = ('de' => '\b(der|die|das|und|oder)\b',
 	      'fr' => '\b(et|le|la|pas)\b', );
--- a/t/markup.t
+++ b/t/markup.t
@@ -122,12 +122,10 @@ i"m tired
 i"m tired
 He said, "[w]hen I voice complaints..."
 He said, &#x201c;[w]hen I voice complaints&#x2026;&#x201d;
+[foo]'s problem
+[foo]&#x2019;s problem
 EOT

-xpath_run_tests("[http://foo.org/ foo]'s problem",
-		'//a[@class="url http outside"][@href="http://foo.org/"][text()="foo"]'
-		. '/following-sibling::text()[string()="’s problem"]');
-
 $MarkupQuotes = 0;
 run_tests(q{"Get lost!", they say, and I answer: "I'm not 'lost'!"},
 	  q{"Get lost!", they say, and I answer: "I'm not 'lost'!"});
--- a/t/namespaces.t
+++ b/t/namespaces.t
@@ -1,21 +1,22 @@
-# Copyright (C) 2006, 2007, 2008, 2009  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2006, 2007, 2008, 2009, 2012  Alex Schroeder <alex@gnu.org>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.

 require 't/test.pl';
 package OddMuse;
-use Test::More tests => 71;
+use Test::More tests => 73;
+use utf8; # tests contain UTF-8 characters and it matters
+
 clear_pages();

 add_module('namespaces.pl');
@@ -112,15 +113,20 @@ test_page(get_page('action=rss'),
 	  '<link>http://localhost/wiki.pl/Muu/Mu</link>',
 	  '<wiki:history>http://localhost/wiki.pl/Muu\?action=history;id=Mu</wiki:history>',
 	  '<wiki:diff>http://localhost/wiki.pl/Muu\?action=browse;diff=1;id=Mu</wiki:diff>');
-# Test non-ASCII characters in namespaces
+# Test Unicode characters in namespaces (BLACK HEART SUIT)
+test_page(update_page('Umlaute', 'namespace mit herz',
+		      'wo steckt das ü', undef, undef,
+		      'ns=Zürich♥'), 'namespace mit herz');
+xpath_test(get_page('action=rc'),
+	   # the exact result depends on filesystem encoding!
+	   '//a[@class="local"][@href="http://localhost/wiki.pl/Z%c3%bcrich%e2%99%a5/Umlaute"]');
+# Test potential Latin-1 characters in namespaces (LATIN SMALL LETTER U DIAERESIS)
 test_page(update_page('Umlaute', 'namespace mit umlaut',
 		      'wo steckt das ü', undef, undef,
 		      'ns=Zürich'), 'namespace mit umlaut');
 xpath_test(get_page('action=rc'),
 	   # the exact result depends on filesystem encoding!
-	   '//a[@class="local"][@href="http://localhost/wiki.pl/Zu%cc%88rich/Umlaute"'
-	   . ' or @href="http://localhost/wiki.pl/Zu%fcrich/Umlaute"'
-	   . ' or @href="http://localhost/wiki.pl/Z%c3%bcrich/Umlaute"]');
+	   '//a[@class="local"][@href="http://localhost/wiki.pl/Z%c3%bcrich/Umlaute"]');

 # Test rollbacks
 test_page(get_page('action=browse ns=Muu id=Test'),
--- a/t/permanent-anchors.t
+++ b/t/permanent-anchors.t
@@ -16,6 +16,8 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 25;
+use utf8; # tests contain UTF-8 characters and it matters
+
 clear_pages();

 add_module('permanent-anchors.pl');
@@ -57,7 +59,7 @@ $page = update_page('Keith_Jarret', 'plays unlike [::Thelonius Mönk]');
 like($page, qr(the page (.*?) also exists), 'the page ... also exists');
 $page =~ qr(the page (.*?) also exists);
 $link = $1;
-xpath_test($link, Encode::encode_utf8('//a[@class="local"][@href="http://localhost/wiki.pl?action=browse;anchor=0;id=Thelonius_M%c3%b6nk"][text()="Thelonius Mönk"]'));
+xpath_test($link, '//a[@class="local"][@href="http://localhost/wiki.pl?action=browse;anchor=0;id=Thelonius_M%c3%b6nk"][text()="Thelonius Mönk"]');
 # verify that the redirection works
 test_page(get_page('action=browse id=Thelonius_Mönk'),
 	  'Status: 302',
--- a/t/redirection.t
+++ b/t/redirection.t
@@ -16,6 +16,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 9;
+use utf8; # tests contain UTF-8 characters and it matters

 clear_pages();

--- a/t/referrer-tracking.t
+++ b/t/referrer-tracking.t
@@ -19,6 +19,7 @@ require 't/test.pl';
 package OddMuse;

 use Test::More tests => 10; # update two numbers below!
+use utf8; # tests contain UTF-8 characters and it matters

 SKIP: {

--- a/t/rollback.t
+++ b/t/rollback.t
@@ -16,6 +16,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 63;
+use utf8; # tests contain UTF-8 characters and it matters

 clear_pages();
 WriteStringToFile($RcFile, "1FirstPage1\n");
--- a/t/rss.t
+++ b/t/rss.t
@@ -16,6 +16,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 100;
+use utf8; # tests contain UTF-8 characters and it matters

 clear_pages();
 AppendStringToFile($ConfigFile, "\$CommentsPrefix = 'Comments on ';\n");
--- a/t/search.t
+++ b/t/search.t
@@ -16,6 +16,7 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 38;
+use utf8; # tests contain UTF-8 characters and it matters

 clear_pages();

--- a/t/tags.t
+++ b/t/tags.t
@@ -16,6 +16,8 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 70;
+use utf8;
+
 clear_pages();

 add_module('tags.pl');
@@ -43,7 +45,7 @@ xpath_run_tests(split('\n',<<'EOT'));
 EOT

 update_page('Brilliant', 'Gameologists [[tag:podcast]] [[tag:mag]]');
-update_page('Podgecast', 'Another [[tag:podcast]]');
+update_page('Pödgecäst', 'Another [[tag:podcast]]');
 update_page('Alex', 'Me! [[tag:Old School]]');

 # open the DB file
@@ -53,11 +55,11 @@ tie %h, "DB_File", $TagFile;
 %tag = map {$_=>1} split($FS, $h{"_Brilliant"});
 ok($tag{podcast}, 'Brilliant page tagged podcast');
 ok($tag{mag}, 'Brilliant page tagged mag');
-%tag = map {$_=>1} split($FS, $h{"_Podgecast"});
-ok($tag{podcast}, 'Podgecast page tagged podcast');
+%tag = map {$_=>1} split($FS, $h{"_Pödgecäst"});
+ok($tag{podcast}, 'Pödgecäst page tagged podcast');
 %file = map {$_=>1} split($FS, $h{"podcast"});
 ok($file{Brilliant}, 'Tag podcast applies to page Brilliant');
-ok($file{Podgecast}, 'Tag podcast applies to page Podgecast');
+ok($file{Pödgecäst}, 'Tag podcast applies to page Pödgecäst');
 %file = map {$_=>1} split($FS, $h{"mag"});
 ok($file{Brilliant}, 'Tag mag applies to page Brilliant');
 %file = map {$_=>1} split($FS, $h{"old_school"});
@@ -76,7 +78,7 @@ ok(!$tag{podcast}, 'Brilliant page no longer tagged podcast');
 ok($tag{mag}, 'Brilliant page still tagged mag');
 %file = map {$_=>1} split($FS, $h{"podcast"});
 ok(!$file{Brilliant}, 'Tag podcast no longer applies to page Brilliant');
-ok($file{Podgecast}, 'Tag podcast still applies to page Podgecast');
+ok($file{Pödgecäst}, 'Tag podcast still applies to page Pödgecäst');

 # close the DB file before making changes via the wiki!
 untie %h;
@@ -99,46 +101,46 @@ update_page('Jeff', 'a blog [[tag:Old School]]');

 # ordinary search finds Alex
 $page = get_page('search=podcast raw=1');
-test_page($page, qw(Podgecast Brilliant Sons Alex));
+test_page($page, qw(Pödgecäst Brilliant Sons Alex));

 # tag search skips Alex
 $page = get_page('search=tag:podcast raw=1');
-test_page($page, qw(Podgecast Brilliant Sons));
+test_page($page, qw(Pödgecäst Brilliant Sons));
 test_page_negative($page, qw(Alex));

 # tag search is case insensitive
 $page = get_page('search=tag:PODCAST raw=1');
-test_page($page, qw(Podgecast Brilliant Sons));
+test_page($page, qw(Pödgecäst Brilliant Sons));
 test_page_negative($page, qw(Alex));

 # exclude tag search skips Brilliant
 $page = get_page('search=-tag:mag raw=1');
-test_page($page, qw(Podgecast Sons Alex));
+test_page($page, qw(Pödgecäst Sons Alex));
 test_page_negative($page, qw(Brilliant));

 # combine include and exclude tag search to exclude both Alex and
 # Brilliant
 $page = get_page('search=tag:podcast%20-tag:mag raw=1');
-test_page($page, qw(Podgecast Sons));
+test_page($page, qw(Pödgecäst Sons));
 test_page_negative($page, qw(Brilliant Alex));

 # combine ordinary search with include and exclude tag search to
 # exclude both Alex and Brilliant
 $page = get_page('search=kryos%20tag:podcast%20-tag:mag raw=1');
 test_page($page, qw(Sons));
-test_page_negative($page, qw(Podgecast Brilliant Alex));
+test_page_negative($page, qw(Pödgecäst Brilliant Alex));

 # search for a tag containing spaces
 $page = get_page('search=tag:old_school raw=1');
 test_page($page, qw(Jeff));
-test_page_negative($page, qw(Sons Podgecast Brilliant Alex));
+test_page_negative($page, qw(Sons Pödgecäst Brilliant Alex));

 test_page(get_page('action=reindex pwd=foo'),
-	  qw(Podgecast Brilliant Sons Alex));
+	  qw(Pödgecäst Brilliant Sons Alex));

 # tag search skips Alex -- repeat test after reindexing
 $page = get_page('search=tag:podcast raw=1');
-test_page($page, qw(Podgecast Brilliant Sons));
+test_page($page, qw(Pödgecäst Brilliant Sons));
 test_page_negative($page, qw(Alex));

 add_module('near-links.pl');
@@ -158,7 +160,7 @@ test_page_negative($page, qw(AlexSchroeder Foo));

 # check journal pages
 $page = update_page('Podcasts', '<journal "." search tag:podcast>');
-test_page($page, qw(Podgecast Brilliant Sons));
+test_page($page, qw(Pödgecäst Brilliant Sons));
 test_page_negative($page, qw(Alex Foo));

 # check the tag cloud
@@ -175,4 +177,4 @@ AppendStringToFile($ConfigFile, "\$LocalNamesCollect = 1;\n");
 update_page('LocalNames', 'test');
 update_page('Alex', 'is a [[tag:podcast]] after all');
 $page = get_page('search=tag:podcast raw=1');
-test_page($page, qw(Podgecast Brilliant Sons Alex));
+test_page($page, qw(Pödgecäst Brilliant Sons Alex));
--- a/t/test.pl
+++ b/t/test.pl
@@ -1,32 +1,33 @@
-# Copyright (C) 2004, 2005, 2006, 2008  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2004, 2005, 2006, 2008, 2012  Alex Schroeder <alex@gnu.org>
 #
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.

 package OddMuse;
 use lib '.';
 use XML::LibXML;
-use Encode;
+use utf8;
+use vars qw($raw);

 # Import the functions

+$raw = 0;       # capture utf8 is the default
 $RunCGI = 0;    # don't print HTML on stdout
 $UseConfig = 0; # don't read module files
 $DataDir = 'test-data';
 $ENV{WikiDataDir} = $DataDir;
 require 'wiki.pl';
+$ENV{PATH} = '/usr/local/bin:' . $ENV{PATH}; # location of perl?
 Init();
-
 use vars qw($redirect);

 undef $/;
@@ -35,6 +36,7 @@ $| = 1; # no output buffering
 sub url_encode {
  my $str = shift;
  return '' unless $str;
+  utf8::encode($str); # turn to byte string
  my @letters = split(//, $str);
  my @safe = ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '-', '_', '.'); # shell metachars are unsafe
  foreach my $letter (@letters) {
@@ -46,6 +48,25 @@ sub url_encode {
  return join('', @letters);
 }

+# Run perl in a subprocess and make sure it prints UTF-8 and not Latin-1
+# If you use the download action, the output will be raw bytes. Use
+# something like the following:
+# {
+#   local $raw = 1;
+#   $page = get_page('action=download id=Trogs');
+# }
+sub capture {
+  my $command = shift;
+  if ($raw) {
+    open (CL, '-|', $command) or die "Can't run $command: $!";
+  } else {
+    open (CL, '-|:encoding(utf-8)', $command) or die "Can't run $command: $!";
+  }
+  my $result = <CL>;
+  close CL;
+  return $result;
+}
+
 sub update_page {
  my ($id, $text, $summary, $minor, $admin, @rest) = @_;
  my $pwd = $admin ? 'foo' : 'wrong';
@@ -54,8 +75,8 @@ sub update_page {
  $summary = url_encode($summary);
  $minor = $minor ? 'on' : 'off';
  my $rest = join(' ', @rest);
-  $redirect = `perl wiki.pl 'Save=1' 'title=$page' 'summary=$summary' 'recent_edit=$minor' 'text=$text' 'pwd=$pwd' $rest`;
-  $output = `perl wiki.pl action=browse id=$page $rest`;
+  $redirect = capture("perl wiki.pl 'Save=1' 'title=$page' 'summary=$summary' 'recent_edit=$minor' 'text=$text' 'pwd=$pwd' $rest");
+  $output = capture("perl wiki.pl action=browse id=$page $rest");
  if ($redirect =~ /^Status: 302 /) {
    # just in case a new page got created or NearMap or InterMap
    $IndexHash{$id} = 1;
@@ -66,16 +87,14 @@ sub update_page {
 }

 sub get_page {
-  open(F, "perl wiki.pl @_ |");
-  my $output = <F>;
-  close F;
-  return $output;
+  return capture("perl wiki.pl @_");
 }

 sub name {
  $_ = shift;
  s/\n/\\n/g;
  $_ = '...' . substr($_, -60) if length > 63;
+  utf8::encode($_);
  return $_;
 }

@@ -120,8 +139,8 @@ sub run_macro_tests {
 # one string, many tests
 sub test_page {
  my $page = shift;
-  foreach my $str (@_) {
-    like($page, qr($str), name($str));
+  foreach my $test (@_) {
+    like($page, qr($test), name($test));
  }
 }

@@ -147,10 +166,19 @@ sub xpath_do {
    skip("Cannot parse ".name($page).": $@", $#tests + 1) if $@;
    foreach my $test (@tests) {
      my $nodelist;
-      eval { $nodelist = $doc->findnodes($test) };
+      my $bytes = $test;
+      # utf8::encode: Converts in-place the character sequence to the
+      # corresponding octet sequence in *UTF-X*. The UTF8 flag is
+      # turned off, so that after this operation, the string is a byte
+      # string. (I have no idea why this is necessary, but there you
+      # go. See encoding.t tests and make sure the page file is
+      # encoded correctly.)
+      utf8::encode($bytes);
+      eval { $nodelist = $doc->findnodes($bytes) };
      if ($@) {
 	fail(&$check(1) ? "$test: $@" : "not $test: $@");
-      } elsif (ok(&$check($nodelist->size()), name(&$check(1) ? $test : "not $test"))) {
+      } elsif (ok(&$check($nodelist->size()),
+		  name(&$check(1) ? $test : "not $test"))) {
 	$result .= $nodelist->string_value();
      } else {
 	$page =~ s/^.*?<html/<html/s;
@@ -201,16 +229,18 @@ sub remove_rule {
 }

 sub add_module {
-  my $mod = shift;
+  my ($mod, $subdir) = @_;
+  $subdir .= '/' if $subdir and substr($subdir, -1) ne '/';
+  my $filename = 
  mkdir $ModuleDir unless -d $ModuleDir;
  my $dir = `/bin/pwd`;
  chop($dir);
  if (-l "$ModuleDir/$mod") {
    # do nothing
-  } elsif (eval{ symlink("$dir/modules/$mod", "$ModuleDir/$mod"); 1; }) {
+  } elsif (eval{ symlink("$dir/modules/$subdir$mod", "$ModuleDir/$mod"); 1; }) {
    # do nothing
  } else {
-    system("copy '$dir/modules/$mod' '$ModuleDir/$mod'");
+    system("copy '$dir/modules/$subdir$mod' '$ModuleDir/$mod'");
  }
  die "Cannot symlink $mod: $!" unless -e "$ModuleDir/$mod";
  do "$ModuleDir/$mod";
@@ -231,7 +261,8 @@ sub clear_pages {
  }
  die "Cannot remove $DataDir!\n" if -e $DataDir;
  mkdir $DataDir;
-  open(F,">$DataDir/config");
+  add_module('mac.pl') if $^O eq 'darwin'; # guessing HFS filesystem
+  open(F, '>:encoding(utf-8)', "$DataDir/config");
  print F "\$AdminPass = 'foo';\n";
  # this used to be the default in earlier CGI.pm versions
  print F "\$ScriptName = 'http://localhost/wiki.pl';\n";
@@ -240,6 +271,7 @@ sub clear_pages {
  $ScriptName = 'http://localhost/test.pl'; # different!
  $IndexInit = 0;
  %IndexHash = ();
+  @IndexList = ();
  $InterSiteInit = 0;
  %InterSite = ();
  $NearSiteInit = 0;
--- a/t/tex.t
+++ b/t/tex.t
@@ -19,6 +19,8 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 3;
+use utf8; # tests contain UTF-8 characters and it matters
+
 clear_pages();
 add_module('tex.pl');

--- a/t/translation-links.t
+++ b/t/translation-links.t
@@ -16,11 +16,14 @@
 require 't/test.pl';
 package OddMuse;
 use Test::More tests => 20;
+use utf8; # tests contain UTF-8 characters and it matters
+
 clear_pages();

 add_module('translation-links.pl');

 AppendStringToFile($ConfigFile, q{
+use utf8;
 %Languages = ('de' => '\b(der|die|das|und|oder)\b',
              'en' => '\b(the|he|she|that|this)\b');
 $Translate{de} = 'Deutsch';
--- a/t/translations.t
+++ b/t/translations.t
@@ -0,0 +1,28 @@
+# Copyright (C) 2012  Alex Schroeder <alex@gnu.org>
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 3 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 't/test.pl';
+package OddMuse;
+use Test::More tests => 6;
+use utf8; # tests contain UTF-8 characters and it matters
+
+clear_pages();
+
+test_page(update_page('HomePage', 'tätärätää!'),
+	  'Edit this page', 'Last edited', 'tätärätää!');
+
+add_module('german-utf8.pl', 'translations');
+
+test_page(get_page('HomePage'),
+	  'Diese Seite bearbeiten', 'Zuletzt geändert', 'tätärätää!');
--- a/wiki.pl
+++ b/wiki.pl
@@ -1,8 +1,7 @@
 #! /usr/bin/perl
-# Version       $Id: wiki.pl,v 1.960 2012/03/08 15:28:18 as Exp $
-# Copyleft      2008 Brian Curry <http://www.raiazome.com>
-# Copyright (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
+# Copyright (C) 2001-2012
 #     Alex Schroeder <alex@gnu.org>
+# Copyleft      2008 Brian Curry <http://www.raiazome.com>
 # ... including lots of patches from the UseModWiki site
 # Copyright (C) 2001, 2002  various authors
 # ... which was based on UseModWiki version 0.92 (April 21, 2001)
@@ -29,15 +28,12 @@
 # this program. If not, see <http://www.gnu.org/licenses/>.

 package OddMuse;
-
 use strict;
 use CGI;
 use CGI::Carp qw(fatalsToBrowser);
-use vars qw($VERSION);
+use File::Glob ':glob';
 local $| = 1;  # Do not buffer output (localized for mod_perl)

-$VERSION=(split(/ +/, q{$Revision: 1.960 $}))[1]; # for MakeMaker
-
 # Options:
 use vars qw($RssLicense $RssCacheHours @RcDays $TempDir $LockDir $DataDir
 $KeepDir $PageDir $RcOldFile $IndexFile $BannedContent $NoEditFile $BannedHosts
@@ -47,20 +43,19 @@ $EmbedWiki $BracketText $UseConfig $UseLookup $AdminPass $EditPass $NetworkFile
 $BracketWiki $FreeLinks $WikiLinks $SummaryHours $FreeLinkPattern $RCName
 $RunCGI $ShowEdits $LinkPattern $RssExclude $InterLinkPattern $MaxPost $UseGrep
 $UrlPattern $UrlProtocols $ImageExtensions $InterSitePattern $FS $CookieName
-$SiteBase $StyleSheet $NotFoundPg $FooterNote $NewText $EditNote $HttpCharset
-$UserGotoBar $VisitorFile $RcFile %Smilies %SpecialDays $InterWikiMoniker
-$SiteDescription $RssImageUrl $ReadMe $RssRights $BannedCanRead $SurgeProtection
-$TopLinkBar $LanguageLimit $SurgeProtectionTime $SurgeProtectionViews
-$DeletedPage %Languages $InterMap $ValidatorLink %LockOnCreation
-$RssStyleSheet %CookieParameters @UserGotoBarPages $NewComment $HtmlHeaders
-$StyleSheetPage $ConfigPage $ScriptName $CommentsPrefix @UploadTypes
-$AllNetworkFiles $UsePathInfo $UploadAllowed $LastUpdate $PageCluster
-%PlainTextPages $RssInterwikiTranslate $UseCache $Counter $ModuleDir
-$FullUrlPattern $SummaryDefaultLength $FreeInterLinkPattern
-%InvisibleCookieParameters %AdminPages $UseQuestionmark $JournalLimit
-$LockExpiration $RssStrip %LockExpires @IndexOptions @Debugging $DocumentHeader
-%HtmlEnvironmentContainers @MyAdminCode @MyFooters @MyInitVariables @MyMacros
-@MyMaintenance @MyRules);
+$SiteBase $StyleSheet $NotFoundPg $FooterNote $NewText $EditNote $UserGotoBar
+$VisitorFile $RcFile %Smilies %SpecialDays $InterWikiMoniker $SiteDescription
+$RssImageUrl $ReadMe $RssRights $BannedCanRead $SurgeProtection $TopLinkBar
+$LanguageLimit $SurgeProtectionTime $SurgeProtectionViews $DeletedPage
+%Languages $InterMap $ValidatorLink %LockOnCreation $RssStyleSheet
+%CookieParameters @UserGotoBarPages $NewComment $HtmlHeaders $StyleSheetPage
+$ConfigPage $ScriptName $CommentsPrefix @UploadTypes $AllNetworkFiles
+$UsePathInfo $UploadAllowed $LastUpdate $PageCluster %PlainTextPages
+$RssInterwikiTranslate $UseCache $Counter $ModuleDir $FullUrlPattern
+$SummaryDefaultLength $FreeInterLinkPattern %InvisibleCookieParameters
+%AdminPages $UseQuestionmark $JournalLimit $LockExpiration $RssStrip
+%LockExpires @IndexOptions @Debugging $DocumentHeader %HtmlEnvironmentContainers
+@MyAdminCode @MyFooters @MyInitVariables @MyMacros @MyMaintenance @MyRules);

 # Internal variables:
 use vars qw(%Page %InterSite %IndexHash %Translate %OldCookie $FootnoteNumber
@@ -95,7 +90,6 @@ $CookieName  = 'Wiki';          # Name for this wiki (for multi-wiki sites)

 $SiteBase    = '';              # Full URL for <BASE> header
 $MaxPost     = 1024 * 210;      # Maximum 210K posts (about 200K for pages)
-$HttpCharset = 'UTF-8';         # You are on your own if you change this!
 $StyleSheet  = '';              # URL for CSS stylesheet (like '/wiki.css')
 $StyleSheetPage = 'css';        # Page for CSS sheet
 $LogoUrl     = '';              # URL for site logo ('' for no logo)
@@ -207,7 +201,7 @@ sub DoWikiRequest {

 sub ReportError {   # fatal!
  my ($errmsg, $status, $log, @html) = @_;
-  $q = new CGI unless $q; # make sure we can report errors before InitRequest
+  InitRequest(); # make sure we can report errors before InitRequest
  print GetHttpHeader('text/html', 'nocache', $status), GetHtmlHeader(T('Error')),
    $q->start_div({class=>"error"}), $q->h1(QuoteHtml($errmsg)), @html, $q->end_div,
      $q->end_html, "\n\n"; # newlines for FCGI because of exit()
@@ -217,22 +211,21 @@ sub ReportError {   # fatal!
 }

 sub Init {
+  binmode(STDOUT, ':utf8');
  InitDirConfig();
  $FS  = "\x1e"; # The FS character is the RECORD SEPARATOR control char in ASCII
  $Message = ''; # Warnings and non-fatal errors.
  InitLinkPatterns(); # Link pattern can be changed in config files
  InitModules(); # Modules come first so that users can change module variables in config
  InitConfig(); # Config comes as early as possible; remember $q is not available here
-  InitRequest(); # get $q with $MaxPost and $HttpCharset; set these in the config file
+  InitRequest(); # get $q with $MaxPost; set these in the config file
  InitCookie(); # After InitRequest, because $q is used
  InitVariables(); # After config, to change variables, after InitCookie for GetParam
 }

 sub InitModules {
  if ($UseConfig and $ModuleDir and -d $ModuleDir) {
-    foreach my $lib (glob("$ModuleDir/*.pm $ModuleDir/*.pl")) {
-      next unless ($lib =~ /^($ModuleDir\/[-\w.]+\.p[lm])$/o);
-      $lib = $1;    # untaint
+    foreach my $lib (bsd_glob("$ModuleDir/*.p[ml]")) {
      do $lib unless $MyInc{$lib};
      $MyInc{$lib} = 1;   # Cannot use %INC in mod_perl settings
      $Message .= CGI::p("$lib: $@") if $@; # no $q exists, yet
@@ -245,7 +238,7 @@ sub InitConfig {
    do $ConfigFile; # these options must be set in a wrapper script or via the environment
    $Message .= CGI::p("$ConfigFile: $@") if $@; # remember, no $q exists, yet
  }
-  if ($ConfigPage) { # $FS, $HttpCharset, $MaxPost must be set in config file!
+  if ($ConfigPage) { # $FS and $MaxPost must be set in config file!
    my ($status, $data) = ReadFile(GetPageFile(FreeToNormal($ConfigPage)));
    my %data = ParseData($data); # before InitVariables so GetPageContent won't work
    eval $data{text} if $data{text};
@@ -254,6 +247,7 @@ sub InitConfig {
 }

 sub InitDirConfig {
+  utf8::decode($DataDir); # just in case, eg. "WikiDataDir=/tmp/Zürich♥ perl wiki.pl"
  $PageDir     = "$DataDir/page";  # Stores page data
  $KeepDir     = "$DataDir/keep";  # Stores kept (old) page data
  $TempDir     = "$DataDir/temp";  # Temporary files and locks
@@ -271,11 +265,9 @@ sub InitDirConfig {
  $ModuleDir   = "$DataDir/modules" unless $ModuleDir;
 }

-sub InitRequest {
+sub InitRequest { # set up $q
  $CGI::POST_MAX = $MaxPost;
  $q = new CGI unless $q;
-  $q->charset($HttpCharset) if $HttpCharset;
-  eval { local $SIG{__DIE__}; binmode(STDOUT, ":raw"); }; # we treat input and output as bytes
 }

 sub InitVariables {  # Init global session variables for mod_perl!
@@ -312,7 +304,7 @@ sub InitVariables {  # Init global session variables for mod_perl!
  $LastUpdate = $ts;
  unshift(@MyRules, \&MyRules) if defined(&MyRules) && (not @MyRules or $MyRules[0] != \&MyRules);
  @MyRules = sort {$RuleOrder{$a} <=> $RuleOrder{$b}} @MyRules; # default is 0
-  ReportError(Ts('Could not create %s', $DataDir) . ": $!", '500 INTERNAL SERVER ERROR')
+  ReportError(Ts('Cannot create %s', $DataDir) . ": $!", '500 INTERNAL SERVER ERROR')
    unless -d $DataDir;
  foreach my $sub (@MyInitVariables) {
    my $result = &$sub;
@@ -329,12 +321,10 @@ sub ReInit {   # init everything we need if we want to link to stuff

 sub InitCookie {
  undef $q->{'.cookies'};   # Clear cache if it exists (for SpeedyCGI)
-  if ($q->cookie($CookieName)) {
-    %OldCookie = split(/$FS/o, UrlDecode($q->cookie($CookieName)));
-  } else {
-    %OldCookie = ();
-  }
-  my %provided = map { $_ => 1 } $q->param;
+  my $cookie = $q->cookie($CookieName);
+  utf8::decode($cookie); # make sure it's decoded as UTF-8
+  %OldCookie = split(/$FS/o, UrlDecode($cookie));
+  my %provided = map { utf8::decode($_); $_ => 1 } $q->param;
  for my $key (keys %OldCookie) {
    SetParam($key, $OldCookie{$key}) unless $provided{$key};
  }
@@ -370,8 +360,10 @@ sub CookieRollbackFix {

 sub GetParam {
  my ($name, $default) = @_;
+  utf8::encode($name); # may fail
  my $result = $q->param($name);
  $result = $default unless defined($result);
+  utf8::decode($result); # may fail
  return QuoteHtml($result); # you need to unquote anything that can have <tags>
 }

@@ -383,13 +375,13 @@ sub SetParam {
 sub InitLinkPatterns {
  my ($WikiWord, $QDelim);
  $QDelim = '(?:"")?'; # Optional quote delimiter (removed from the output)
-  $WikiWord = '[A-Z]+[a-z\x80-\xff]+[A-Z][A-Za-z\x80-\xff]*';
+  $WikiWord = '[A-Z]+[a-z\x{0080}-\x{fffd}]+[A-Z][A-Za-z\x{0080}-\x{fffd}]*'; # exclude noncharacters FFFE and FFFF
  $LinkPattern = "($WikiWord)$QDelim";
-  $FreeLinkPattern = "([-,.()'%&?;<> _1-9A-Za-z\x80-\xff]|[-,.()'%&?;<> _0-9A-Za-z\x80-\xff][-,.()'%&?;<> _0-9A-Za-z\x80-\xff]+)"; # disallow "0" and must match HTML and plain text (ie. > and &gt;)
+  $FreeLinkPattern = "([-,.()'%&?;<> _1-9A-Za-z\x{0080}-\x{fffd}]|[-,.()'%&?;<> _0-9A-Za-z\x{0080}-\x{fffd}][-,.()'%&?;<> _0-9A-Za-z\x{0080}-\x{fffd}]+)"; # disallow "0" and must match HTML and plain text (ie. > and &gt;)
  # Intersites must start with uppercase letter to avoid confusion with URLs.
-  $InterSitePattern = '[A-Z\x80-\xff]+[A-Za-z\x80-\xff]+';
-  $InterLinkPattern = "($InterSitePattern:[-a-zA-Z0-9\x80-\xff_=!?#\$\@~`\%&*+\\/:;.,]*[-a-zA-Z0-9\x80-\xff_=#\$\@~`\%&*+\\/])$QDelim";
-  $FreeInterLinkPattern = "($InterSitePattern:[-a-zA-Z0-9\x80-\xff_=!?#\$\@~`\%&*+\\/:;.,()' ]+)"; # plus space and other characters, and no restrictions on the end of the pattern
+  $InterSitePattern = '[A-Z\x{0080}-\x{fffd}]+[A-Za-z\x{0080}-\x{fffd}]+';
+  $InterLinkPattern = "($InterSitePattern:[-a-zA-Z0-9\x{0080}-\x{fffd}_=!?#\$\@~`\%&*+\\/:;.,]*[-a-zA-Z0-9\x{0080}-\x{fffd}_=#\$\@~`\%&*+\\/])$QDelim";
+  $FreeInterLinkPattern = "($InterSitePattern:[-a-zA-Z0-9\x{0080}-\x{fffd}_=!?#\$\@~`\%&*+\\/:;.,()' ]+)"; # plus space and other characters, and no restrictions on the end of the pattern
  $UrlProtocols = 'http|https|ftp|afs|news|nntp|mid|cid|mailto|wais|prospero|telnet|gopher|irc|feed';
  $UrlProtocols .= '|file'  if $NetworkFile;
  my $UrlChars = '[-a-zA-Z0-9/@=+$_~*.,;:?!\'"()&#%]'; # see RFC 2396
@@ -485,9 +477,7 @@ sub ApplyRules {
 	Clean(CloseHtmlEnvironments());
 	Dirty($1);
 	my ($oldpos, $old_) = (pos, $_); # remember these because of the call to RSS()
-	eval { local $SIG{__DIE__}; binmode(STDOUT, ":utf8"); } if $HttpCharset eq 'UTF-8';
 	print RSS($3 ? $3 : 15, split(/\s+/, UnquoteHtml($4)));
-	eval { local $SIG{__DIE__}; binmode(STDOUT, ":raw"); };
 	Clean(AddHtmlEnvironment('p')); # if dirty block is looked at later, this will disappear
 	($_, pos) = ($old_, $oldpos); # restore \G (assignment order matters!)
      } elsif (/\G(&lt;search (.*?)&gt;)/cgis) {
@@ -522,8 +512,8 @@ sub ApplyRules {
 	Clean("&$1;");
      } elsif (m/\G\s+/cg) {
 	Clean(' ');
-      } elsif (m/\G([A-Za-z\x80-\xff]+([ \t]+[a-z\x80-\xff]+)*[ \t]+)/cg
-	       or m/\G([A-Za-z\x80-\xff]+)/cg or m/\G(\S)/cg) {
+      } elsif (m/\G([A-Za-z\x{0080}-\x{fffd}]+([ \t]+[a-z\x{0080}-\x{fffd}]+)*[ \t]+)/cg
+	       or m/\G([A-Za-z\x{0080}-\x{fffd}]+)/cg or m/\G(\S)/cg) {
 	Clean($1);    # multiple words but do not match http://foo
      } else {
 	last;
@@ -670,7 +660,7 @@ sub OpenHtmlEnvironment {  # close the previous $html_tag and open a new one
  @HtmlStack = @stack if $found; # if not starting a new list
  $depth = $IndentLimit if $depth > $IndentLimit; # requested depth 0 makes no sense
  $html_tag_attr = qq/class="$html_tag_attr"/ # backwards-compatibility hack: classically, the third argument to this function was a single CSS class, rather than string of HTML tag attributes as in the second argument to the "AddHtmlEnvironment" function. To allow both sorts, we conditionally change this string to 'class="$html_tag_attr"' when this string is a single CSS class.
-    if $html_tag_attr && $html_tag_attr !~ m/^\s*[:alpha:]+\s*=\s*('|").+\1/;
+    if $html_tag_attr && $html_tag_attr !~ m/^\s*[[:alpha:]]@@+\s*=\s*('|").+\1/;
  splice(@HtmlAttrStack, 0, @HtmlAttrStack - @HtmlStack); # truncate to size of @HtmlStack
  foreach ($found..$depth-1) {
    unshift(@HtmlStack,     $html_tag);
@@ -794,6 +784,7 @@ sub UnquoteHtml {
 sub UrlEncode {
  my $str = shift;
  return '' unless $str;
+  utf8::encode($str); # turn to byte string
  my @letters = split(//, $str);
  my %safe = map {$_ => 1} ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '-', '_', '.', '!', '~', '*', "'", '(', ')', '#');
  foreach my $letter (@letters) {
@@ -912,13 +903,6 @@ sub RSS {
  # translations will be double encoded when printing the result.
  my $tDiff = T('diff');
  my $tHistory = T('history');
-  if ($HttpCharset eq 'UTF-8' and ($tDiff ne 'diff' or $tHistory ne 'history')) {
-    eval { local $SIG{__DIE__};
-	   require Encode;
-	   $tDiff = Encode::decode_utf8($tDiff);
-	   $tHistory = Encode::decode_utf8($tHistory);
-	 }
-  }
  my $wikins = 'http://purl.org/rss/1.0/modules/wiki/';
  my $rdfns = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
  @uris = map { s/^"?(.*?)"?$/$1/; $_; } @uris; # strip quotes of uris
@@ -1030,8 +1014,8 @@ sub GetRss {
  if (GetParam('cache', $UseCache) > 0) {
    foreach my $uri (keys %todo) { # read cached rss files if possible
      if ($Now - (stat($todo{$uri}))[9] < $RssCacheHours * 3600) {
-  $data{$uri} = ReadFile($todo{$uri});
-  delete($todo{$uri});  # no need to fetch them below
+	$data{$uri} = ReadFile($todo{$uri});
+	delete($todo{$uri});  # no need to fetch them below
      }
    }
  }
@@ -1048,8 +1032,8 @@ sub GetRss {
      %todo = (); # because the uris in the response may have changed due to redirects
      my $entries = $pua->wait();
      foreach (keys %$entries) {
-  my $uri = $entries->{$_}->request->uri;
-  $data{$uri} = $entries->{$_}->response->content;
+	my $uri = $entries->{$_}->request->uri;
+	$data{$uri} = $entries->{$_}->response->content;
      }
    }
  }
@@ -1059,7 +1043,10 @@ sub GetRss {
  if (GetParam('cache', $UseCache) > 0) {
    CreateDir($RssDir);
    foreach my $uri (@need_cache) {
-      WriteStringToFile(GetRssFile($uri), $data{$uri});
+      my $data = $data{$uri};
+      # possibly a Latin-1 file without encoding attribute will cause a problem?
+      $data =~ s/encoding="[^"]*"/encoding="UTF-8"/; # content was converted
+      WriteStringToFile(GetRssFile($uri), $data) if $data;
    }
  }
  return $str, %data;
@@ -1277,18 +1264,22 @@ sub PrintPageDiff {   # print diff for open page
  }
 }

-sub PageHtml {  #FIXME: A bit buggy, this. STDOUT should be explicitly closed before returning.
+sub PageHtml {
  my ($id, $limit, $error) = @_;
-  my $result = '';
+  my ($diff, $page);
  local *STDOUT;
  OpenPage($id);
-  open(STDOUT, '>', \$result) or die "Can't open memory file: $!";
+  open(STDOUT, '>', \$diff) or die "Can't open memory file: $!";
+  binmode(STDOUT, ":utf8");
  PrintPageDiff();
-  return $error if $limit and length($result) > $limit;
-  my $diff = $result;
+  utf8::decode($diff);
+  return $error if $limit and length($diff) > $limit;
+  open(STDOUT, '>', \$page) or die "Can't open memory file: $!";
+  binmode(STDOUT, ":utf8");
  PrintPageHtml();
-  return $diff . $q->p($error) if $limit and length($result) > $limit;
-  return $result;
+  utf8::decode($page);
+  return $diff . $q->p($error) if $limit and length($diff . $page) > $limit;
+  return $diff . $page;
 }

 sub T {
@@ -1313,9 +1304,17 @@ sub Tss {

 sub GetId {
  my $id = UnquoteHtml(GetParam('id', GetParam('title', ''))); # id=x or title=x -> x
-  $id = join('_', $q->keywords) unless $id; # script?p+q -> p_q
+  if (not $id) {
+    my @keywords = $q->keywords;
+    foreach my $keyword (@keywords) {
+      utf8::decode($keyword);
+    }
+    $id = join('_', @keywords) unless $id; # script?p+q -> p_q
+  }
  if ($UsePathInfo) {
-    my @path = split(/\//, $q->path_info);
+    my $path = $q->path_info;
+    utf8::decode($path);
+    my @path = split(/\//, $path);
    $id = pop(@path) unless $id; # script/p/q -> q
    foreach my $p (@path) {
      SetParam($p, 1);    # script/p/q -> p=1
@@ -1500,7 +1499,7 @@ sub GetRcLines { # starttime, hash of seen pages to use as a second return value
  my %following = ();
  my @result = ();
  # check the first timestamp in the default file, maybe read old log file
-  open(F, $RcFile);
+  open(F, '<:encoding(UTF-8)', $RcFile);
  my $line = <F>;
  my ($ts) = split(/$FS/o, $line); # the first timestamp in the regular rc file
  if (not $ts or $ts > $starttime) { # we need to read the old rc file, too
@@ -1586,7 +1585,7 @@ sub GetRcLinesFor {
        rcclusteronly rcfilteronly match lang followup);
  # parsing and filtering
  my @result = ();
-  open(F,$file) or return ();
+  open(F, '<:encoding(UTF-8)', $file) or return ();
  while (my $line = <F>) {
    chomp($line);
    my ($ts, $id, $minor, $summary, $host, $username, $revision,
@@ -1872,7 +1871,7 @@ sub GetRcRss {
      }
    }
  }
-  my $rss = qq{<?xml version="1.0" encoding="$HttpCharset"?>\n};
+  my $rss = qq{<?xml version="1.0" encoding="UTF-8"?>\n};
  if ($RssStyleSheet =~ /\.(xslt?|xml)$/) {
    $rss .= qq{<?xml-stylesheet type="text/xml" href="$RssStyleSheet" ?>\n};
  } elsif ($RssStyleSheet) {
@@ -2218,15 +2217,16 @@ sub GetHeader {
  $result .= $q->start_div({-class=>'header'});
  if (not $embed and $LogoUrl) {
    my $url = $IndexHash{$LogoUrl} ? GetDownloadLink($LogoUrl, 2) : $LogoUrl;
-    $result .= ScriptLink(UrlEncode($HomePage), $q->img({-src=>$url, -alt=>$alt, -class=>'logo'}), 'logo');
+    $result .= ScriptLink(UrlEncode($HomePage),
+			  $q->img({-src=>$url, -alt=>$alt, -class=>'logo'}), 'logo');
  }
  if (GetParam('toplinkbar', $TopLinkBar)) {
    $result .= GetGotoBar($id);
    if (%SpecialDays) {
      my ($sec, $min, $hour, $mday, $mon, $year) = gmtime($Now);
      if ($SpecialDays{($mon + 1) . '-' . $mday}) {
-  $result .= $q->br() . $q->span({-class=>'specialdays'},
-               $SpecialDays{($mon + 1) . '-' . $mday});
+	$result .= $q->br() . $q->span({-class=>'specialdays'},
+				       $SpecialDays{($mon + 1) . '-' . $mday});
      }
    }
  }
@@ -2243,11 +2243,11 @@ sub GetHttpHeader {
  return if $PrintedHeader;
  $PrintedHeader = 1;
  my ($type, $ts, $status, $encoding) = @_; # $ts is undef, a ts, or 'nocache'
+  $q->charset($type =~ m!^(text/|application/xml)! ? 'utf-8' : ''); # text/plain, text/html, application/xml: UTF-8
  my %headers = (-cache_control=>($UseCache < 0 ? 'no-cache' : 'max-age=10'));
  $headers{-etag} = $ts || PageEtag() if GetParam('cache', $UseCache) >= 2;
  $headers{'-last-modified'} = TimeToRFC822($ts) if $ts and $ts ne 'nocache'; # RFC 2616 section 13.3.4
  $headers{-type} = GetParam('mime-type', $type);
-  $headers{-type} .= "; charset=$HttpCharset" if $HttpCharset;
  $headers{-status} = $status if $status;
  $headers{-Content_Encoding} = $encoding if $encoding;
  my $cookie = Cookie();
@@ -2263,7 +2263,7 @@ sub CookieData {
  my ($changed, $visible, %params);
  foreach my $key (keys %CookieParameters) {
    my $default = $CookieParameters{$key};
-    my $value = GetParam($key, $default); # values are URL encoded
+    my $value = GetParam($key, $default);
    $params{$key} = $value  if $value ne $default;
    # The cookie is considered to have changed under the following
    # condition: If the value was already set, and the new value is
@@ -2280,6 +2280,7 @@ sub Cookie {
  my ($changed, $visible, %params) = CookieData(); # params are URL encoded
  if ($changed) {
    my $cookie = join(UrlEncode($FS), %params); # no CTL in field values
+    utf8::encode($cookie); # prevent casting to Latin 1
    my $result = $q->cookie(-name=>$CookieName, -value=>$cookie,
 			    -expires=>'+2y');
    $Message .= $q->p(T('Cookie: ') . $CookieName . ', '
@@ -2297,10 +2298,9 @@ sub GetHtmlHeader {   # always HTML!
    . T('Edit this page') . '" href="'
    . ScriptUrl('action=edit;id=' . UrlEncode(GetId())) . '" />' if $id;
  return $DocumentHeader
-      . $q->head($q->title($title) . $base
-     . GetCss() . GetRobots() . GetFeeds() . $HtmlHeaders
-     . '<meta http-equiv="Content-Type" content="text/html; charset='
-     . $HttpCharset . '"/>')
+    . $q->head($q->title($title) . $base
+      . GetCss() . GetRobots() . GetFeeds() . $HtmlHeaders
+      . '<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />')
      . '<body class="' . GetParam('theme', $ScriptName) . '">';
 }

@@ -2319,7 +2319,7 @@ sub GetFeeds {      # default for $HtmlHeaders
  my $id = GetId(); # runs during Init, not during DoBrowseRequest
  $html .= '<link rel="alternate" type="application/rss+xml" title="'
    . QuoteHtml("$SiteName: $id") . '" href="' . $ScriptName
-      . '?action=rss;rcidonly=' . UrlEncode(FreeToNormal($id)) . '" />' if $id;
+    . '?action=rss;rcidonly=' . UrlEncode(FreeToNormal($id)) . '" />' if $id;
  my $username = GetParam('username', '');
  $html .= '<link rel="alternate" type="application/rss+xml" '
    . 'title="Follow-ups for ' . NormalToFree($username) . '" '
@@ -2452,8 +2452,9 @@ sub GetCommentForm {
 sub GetFormStart {
  my ($ignore, $method, $class) = @_;
  $method ||= 'post';
+  $class  ||= 'form';
  return $q->start_multipart_form(-method=>$method, -action=>$FullUrl,
-				  -class=>$class||'form');
+				  -accept_charset=>'utf-8', -class=>$class);
 }

 sub GetSearchForm {
@@ -2537,6 +2538,7 @@ sub DoDiff {      # Actualy call the diff program
  WriteStringToFile($oldName, $_[0]);
  WriteStringToFile($newName, $_[1]);
  my $diff_out = `diff $oldName $newName`;
+  utf8::decode($diff_out); # needs decoding
  $diff_out =~ s/\\ No newline.*\n//g; # Get rid of common complaint.
  ReleaseLockDir('diff');
  # No need to unlink temp files--next diff will just overwrite.
@@ -2663,7 +2665,9 @@ sub OpenPage {      # Sets global variables
    %Page = ();
    $Page{ts} = $Now;
    $Page{revision} = 0;
-    if ($id eq $HomePage and (open(F, $ReadMe) or open(F, 'README'))) {
+    if ($id eq $HomePage
+	and (open(F, '<:encoding(UTF-8)', $ReadMe)
+	     or open(F, '<:encoding(UTF-8)', 'README'))) {
      local $/ = undef;
      $Page{text} = <F>;
      close F;
@@ -2723,7 +2727,7 @@ sub GetPageFile {
 }

 sub GetKeepFile {
-  my ($id, $revision) = @_; die 'No revision' unless $revision; #FIXME
+  my ($id, $revision) = @_; die "No revision for $id" unless $revision; #FIXME
  return $KeepDir . '/' . GetPageDirectory($id) . "/$id/$revision.kp";
 }

@@ -2733,7 +2737,7 @@ sub GetKeepDir {
 }

 sub GetKeepFiles {
-  return glob(GetKeepDir(shift) . '/*.kp'); # files such as 1.kp, 2.kp, etc.
+  return bsd_glob(GetKeepDir(shift) . '/*.kp'); # files such as 1.kp, 2.kp, etc.
 }

 sub GetKeepRevisions {
@@ -2791,8 +2795,9 @@ sub ExpireKeepFiles {   # call with opened page
 }

 sub ReadFile {
-  my $fileName = shift;
-  if (open(IN, "<$fileName")) {
+  my $file = shift;
+  utf8::encode($file); # filenames are bytes!
+  if (open(IN, '<:encoding(UTF-8)', $file)) {
    local $/ = undef;   # Read complete files
    my $data=<IN>;
    close IN;
@@ -2802,18 +2807,19 @@ sub ReadFile {
 }

 sub ReadFileOrDie {
-  my ($fileName) = @_;
+  my ($file) = @_;
  my ($status, $data);
-  ($status, $data) = ReadFile($fileName);
+  ($status, $data) = ReadFile($file);
  if (!$status) {
-    ReportError(Ts('Cannot open %s', $fileName) . ": $!", '500 INTERNAL SERVER ERROR');
+    ReportError(Ts('Cannot open %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
  }
  return $data;
 }

 sub WriteStringToFile {
  my ($file, $string) = @_;
-  open(OUT, ">$file")
+  utf8::encode($file);
+  open(OUT, '>:encoding(UTF-8)', $file)
    or ReportError(Ts('Cannot write %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
  print OUT  $string;
  close(OUT);
@@ -2821,7 +2827,8 @@ sub WriteStringToFile {

 sub AppendStringToFile {
  my ($file, $string) = @_;
-  open(OUT, ">>$file")
+  utf8::encode($file);
+  open(OUT, '>>:encoding(UTF-8)', $file)
    or ReportError(Ts('Cannot write %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
  print OUT  $string;
  close(OUT);
@@ -2829,6 +2836,7 @@ sub AppendStringToFile {

 sub CreateDir {
  my ($newdir) = @_;
+  utf8::encode($newdir);
  return if -d $newdir;
  mkdir($newdir, 0775)
    or ReportError(Ts('Cannot create %s', $newdir) . ": $!", '500 INTERNAL SERVER ERROR');
@@ -2896,7 +2904,7 @@ sub ReleaseLock {
 sub ForceReleaseLock {
  my $pattern = shift;
  my $forced;
-  foreach my $name (glob $pattern) {
+  foreach my $name (bsd_glob $pattern) {
    # First try to obtain lock (in case of normal edit lock)
    $forced = 1 if !RequestLockDir($name, 5, 3, 0);
    ReleaseLockDir($name); # Release the lock, even if we didn't get it.
@@ -2969,7 +2977,7 @@ sub TimeToRFC822 {
 sub GetHiddenValue {
  my ($name, $value) = @_;
  $q->param($name, $value);
-  return $q->hidden($name);
+  return $q->input({-type=>"hidden", -name=>$name, -value=>$value});
 }

 sub GetRemoteHost { # when testing, these variables are undefined.
@@ -3123,6 +3131,7 @@ sub DoDownload {
      if @UploadTypes and not $allowed{$type};
    print GetHttpHeader($type, $ts, undef, $encoding);
    require MIME::Base64;
+    binmode(STDOUT, ":pop:raw"); # need to pop utf8 for Windows users!?
    print MIME::Base64::decode($data);
  } else {
    print GetHttpHeader('text/plain', $ts);
@@ -3258,8 +3267,8 @@ sub DoIndex {
    push(@menu, $q->b(Ts('(for %s)', GetParam('lang', '')))) if GetParam('lang', '');
    print $q->start_div({-class=>'content index'}),
      GetFormStart(undef, 'get', 'index'), GetHiddenValue('action', 'index'),
-  $q->p(join($q->br(), @menu)), $q->end_form(),
-    $q->h2(Ts('%s pages found.', ($#pages + 1))), $q->start_p();
+      $q->p(join($q->br(), @menu)), $q->end_form(),
+      $q->h2(Ts('%s pages found.', ($#pages + 1))), $q->start_p();
  }
  foreach (@pages) {
    PrintPage($_);
@@ -3290,6 +3299,7 @@ sub PrintPage {
 sub AllPagesList {
  my $refresh = GetParam('refresh', 0);
  return @IndexList if @IndexList and not $refresh;
+  SetParam('refresh', 0) if $refresh;
  if (not $refresh and -f $IndexFile) {
    my ($status, $rawIndex) = ReadFile($IndexFile); # not fatal
    if ($status) {
@@ -3301,11 +3311,12 @@ sub AllPagesList {
  }
  @IndexList = ();
  %IndexHash = ();
-  # Try to write out the list for future runs.  If file exists and cannot be changed, error!
+  # If file exists and cannot be changed, error!
  my $locked = RequestLockDir('index', undef, undef, -f $IndexFile);
-  foreach (glob("$PageDir/*/*.pg $PageDir/*/.*.pg")) { # find .dotfiles, too
+  foreach (bsd_glob("$PageDir/*/*.pg"), bsd_glob("$PageDir/*/.*.pg")) {
    next unless m|/.*/(.+)\.pg$|;
    my $id = $1;
+    utf8::decode($id);
    push(@IndexList, $id);
    $IndexHash{$id} = 1;
  }
@@ -3359,7 +3370,9 @@ sub PageIsUploadedFile {
  return undef if $OpenPageName eq $id;
  if ($IndexHash{$id}) {
    my $file = GetPageFile($id);
-    open(FILE, "<$file") or ReportError(Ts('Cannot open %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
+    utf8::encode($file); # filenames are bytes!
+    open(FILE, '<:encoding(UTF-8)', $file)
+      or ReportError(Ts('Cannot open %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
    while (defined($_ = <FILE>) and $_ !~ /^text: /) {
    }          # read lines until we get to the text key
    close FILE;
@@ -3403,7 +3416,7 @@ sub GrepFiltered { # grep is so much faster!!
  # if we know of any remaining grep incompatibilities we should
  # return @pages here!
  $regexp = quotemeta($regexp);
-  open(F,"grep -rli $regexp '$PageDir' 2>/dev/null |");
+  open(F, '-|:encoding(UTF-8)', "grep -rli $regexp '$PageDir' 2>/dev/null");
  while (<F>) {
    push(@result, $1) if m/.*\/(.*)\.pg/ and not $found{$1};
  }
Author	SHA1	Message	Date
Alex Schroeder	0f55e3d59e	Fixed GetHiddenValue encoding errors. GetHiddenValue was returning strings that had lost their UTF-8 flag or something like that. In order to fix this, I needed to replace the call to $q->hidden with a call to $q->input with -type=>"hidden".	2012-08-15 18:47:10 +02:00
Alex Schroeder	60a5a8fcee	git.pl: run "git add ." as part of maintenance.	2012-08-14 21:41:19 +02:00
Alex Schroeder	598b5e06f6	Merge branch 'master' of ssh://as@git.sv.gnu.org/srv/git/oddmuse	2012-08-14 09:06:16 -04:00
Alex Schroeder	f451e62d30	Delay writing of the timestamp.	2012-08-14 09:05:22 -04:00
Alex Schroeder	a41920a5fd	Add another test regarding UTF-8 pagenames. Use binmode(":pop:raw") because this appears to be necessary for Windows users -- Ingo Belka was using ActivePerl versions 5.10.1 and 5.14.2 to test it.	2012-08-13 16:00:47 +02:00
Alex Schroeder	e8683fdb02	Making sure that the charset parameter is only added when necessary. It's only added to the content-type if it's text/plain, text/html or RSS (application/xml).	2012-08-07 17:26:12 +02:00
Alex Schroeder	f08a404bde	clear_page unsets @IndexList Without this, running t/tags.t will alternate between running ok and producing an error: "Cannot open test-data/page/I/InterMap.pg: No such file or directory"	2012-08-05 02:28:39 +02:00
Alex Schroeder	e1f0d909c1	In DoTagsReindex, call utf8::encode on the id before adding it to the index. I'm not sure why this is necessary. My tests in t/test.t work just fine on my Mac with mac.pl installed. Without it, reindexing results in a silent crash of Perl on the machine where emacswiki.org is installed.	2012-08-05 01:59:20 +02:00
Alex Schroeder	3ab606d96a	Fixed UTF-8 support in various automated tests. Issues: - the Mac layer was masking issues because of the NFC/NFD difference and existing compatiblity hacks in mac.pl - drafts.pl was suffering from a double encoding issue - crossbar.t and download.t tests were failing because I had recently fixed DoDownload output to be raw instead of encoded - test.pl now has a way to capture the raw, unencoded output produced by DoDownload - tags.t got some tests to prove that recent changes to wiki.pl actually work	2012-07-30 23:30:27 +02:00
Alex Schroeder	93dc6b415f	Merge branch 'master' of git.sv.gnu.org:/srv/git/oddmuse	2012-07-30 21:12:44 +02:00
Alex Schroeder	6d8314cb39	Fix regular expression using character class.	2012-07-30 21:12:06 +02:00
Alex Schroeder	ce18377109	Use utf8::decode on filenames passed to PageIsUploadedFile Without this, search will break on pagenames with non-ASCII characters. I noted this in conjunction with tags.pl. There, I needed to encode the page names for search to work correctly.	2012-07-30 18:18:29 +02:00
Alex Schroeder	d4edb159ae	Use binmode to set stdout to raw when printing uploaded files. The bug involved uploaded images not being decoded correctly.	2012-07-25 13:43:48 +02:00
Alex Schroeder	16198f8784	Filenames also need to be encoded such that the bytes are used. Otherwise pages containing non-ASCII names cannot be read or written.	2012-07-21 11:55:45 -04:00
Alex Schroeder	03e08464bb	New test to verify that bsd_glob works on dotfiles. The question is whether bsd_glob("$PageDir//.pg") will find page/other/.emacs.pg -- and apparently it does not, so that had to be added back in.	2012-07-21 00:58:27 +02:00
Alex Schroeder	7ca0f6172b	Use bsd_glob instead of glob. Glob will split patterns on whitespace and non-breaking spaces are now treated as whitespace.	2012-07-21 00:40:42 +02:00
Alex Schroeder	276c1c2076	Handle namespaces containing a non-breaking space. Since the non-breaking space is now recognized as a whitespace character, the behaviour of the glob function has changed. Use bsd_glob instead!	2012-07-20 02:14:36 +02:00
Alex Schroeder	9c996cee04	No longer call oddtrans when modules are changed because of encoding errors introduced.	2012-07-19 09:07:59 -04:00
Alex Schroeder	6c813b8297	$q->path_info lacks UTF-8 decoding. Fix other encoding issues with namespaces.	2012-07-19 09:05:07 -04:00
Alex Schroeder	ec4cba6fb3	Move binmode calls to the top of the file.	2012-07-19 08:28:51 -04:00
Alex Schroeder	76c50674f5	Handle encoding and avoid "wide character in print" when running oddtrans."	2012-07-19 08:06:41 -04:00
Alex Schroeder	a73a9e4a96	Merge branch 'master' of git.sv.gnu.org:/srv/git/oddmuse	2012-07-19 14:01:04 +02:00
Alex Schroeder	f27ec6c142	Use utf8::encode when creating a directory. There is no way to provide an encoding layer to directory names. Therefore they need to be raw bytes and not characters. This becomes apparent when creating namespaces containing non-ASCII characters.	2012-07-19 13:58:58 +02:00
Alex Schroeder	37c32783f3	Changed \x{ffff} to \x{fffd} in all regular expressions. Using Perl 5.8.8 resulted in a crash when compiling regular expressions referring to \x{ffff}.	2012-07-19 06:01:30 -04:00
Alex Schroeder	f922c0ae9a	Fixed encoding of brazilian-portuguese-utf8.pl. Some left-over texts for bulgarian-utf8.pl and chinese-utf8.pl.	2012-07-19 05:42:03 -04:00
Alex Schroeder	b46fcb3c90	Merge branch 'master' of ssh://as@git.sv.gnu.org/srv/git/oddmuse	2012-07-19 05:29:19 -04:00
Alex Schroeder	b885287ea2	Use binmode utf8 when sending files.	2012-07-19 10:53:13 +02:00
Alex Schroeder	67d68a370a	Merge branch 'master' of ssh://as@git.sv.gnu.org/srv/git/oddmuse Conflicts: Makefile	2012-07-19 04:49:13 -04:00
Alex Schroeder	ccef879ac1	Added "prepare" target to Makefile to add version info to all the sources. .gitignore added build directory and stuff created by the Mac disk image rules.	2012-07-18 18:34:57 +02:00
Alex Schroeder	e8b9708f40	Set $ModulesDescription to an appropriate link for all translation files.	2012-07-18 18:16:57 +02:00
Alex Schroeder	d4f1e27cae	Add a space to the META HTTP-EQUIV tag. Add an encoding test.	2012-07-18 10:59:58 +02:00
Alex Schroeder	b0a8e49975	Add link to documentation for %Smilies.	2012-07-18 10:58:58 +02:00
Alex Schroeder	9383adcba5	Merge branch 'master' of git.sv.gnu.org:/srv/git/oddmuse	2012-07-06 15:40:36 +02:00
Alex Schroeder	cde2d41dd1	Worked on encoding errors in PageHtml I was getting "wide character in print" warnings when printing the RSS feed. This should fix it.	2012-07-06 15:39:27 +02:00
Alex Schroeder	6e5766f431	On some encoding problems The TOC extension causes problems when including non-ASCII Text.	2012-07-04 21:36:56 +02:00
Alex Schroeder	a2fe639a57	Removed statement that had no effect but caused an error when opening a non-ASCII page The page in question was "Übersicht" on Emacs Wiki.	2012-07-04 17:14:51 +02:00
Alex Schroeder	f452d99e2f	Removed $VERSION	2012-07-04 16:45:12 +02:00
Alex Schroeder	49f0c6d200	duplicate use utf8 removed	2012-07-04 16:43:12 +02:00
Alex Schroeder	f9efba3976	use utf8;	2012-07-04 15:26:08 +02:00
Alex Schroeder	36ef964d0d	Removed $Id$ in wiki.pl because it was no longer being updated.	2012-07-04 11:30:21 +02:00
Alex Schroeder	eb4a4653a2	Support "half" class, eg. [[image/half/center:...]]	2012-06-27 16:52:07 +02:00
Alex Schroeder	741601489f	Fix cookie encoding issues If the username was set to Schröder, the wiki would keep reporting the username as changed. This was caused by encoding errors if cookie values. This change also allows non-ASCII parameters to be stored in the cookie.	2012-06-23 01:08:46 +02:00
Alex Schroeder	217055fab2	Pod fix. Changed =head to =head2 as suggested by anonymous visitor to the wiki.	2012-06-22 18:23:51 +02:00
Alex Schroeder	8f68442db1	Fix UTF-8 decoding issue when using toc.pl The output of ApplyRules is no explicitly decoded as UTF8.	2012-06-22 02:54:29 +02:00
Alex Schroeder	b9d0c60080	Added copyright year.	2012-06-14 10:21:40 +02:00
Alex Schroeder	2b2e45b952	Merge branch 'master' of git.sv.gnu.org:/srv/git/oddmuse	2012-06-14 10:20:17 +02:00
Alex Schroeder	dfc3555184	Merge branch 'add-thumbs'	2012-06-14 10:19:33 +02:00
Alex Schroeder	474798c5cd	Handle the new file format for uploaded files Recently, uploaded files don't just contain #FILE and a MIME type -- the MIME type is followed by a space and optionally more information. I replaced the hand-coded parsing with a call to TextIsFile and added better error checking and fixed the error messages (they used $s instead of %s).	2012-06-14 10:04:54 +02:00
Alex Schroeder	0a54f14a6f	From the wiki Taken the version from the wiki and added the standard $ModulesDescription.	2012-06-14 08:39:25 +02:00
Alex Schroeder	04cdf0be24	Fixed cookie decoding If the username contained a non-ASCII character, eg. Schröder, then the script would keep printing the cookie message, telling the user that the cookie had changed when in fact it had not.	2012-05-25 17:41:01 +02:00
Alex Schroeder	e531f9d569	Encoding the cookie and fixing drafts.pl Drafts are saved using the username as filename. This must also be encoded and decoded correctly. Because of NFC and NFD issues on Mac HFS, an appropriate normalization was added to mac.pl. As the username is also part of the cookie, this showed that the Cookie content wasn't being encoded correctly, so that was fixed, too.	2012-05-25 11:56:46 +02:00
Alex Schroeder	4f6407fd38	Resolved conflict.	2012-05-25 08:29:34 +02:00
Alex Schroeder	3174e184f9	Fix an encoding error in namespaces.pl on Debian The Debian installation uses ext3 and therefore raw bytes for filenames unlike the HFS filesystem of Mac OSX. Copyright years were updated. The maintenance output of for drafts was cleaned up.	2012-05-25 01:00:10 +02:00
Alex Schroeder	8d94a0a50f	Merge branch 'master' of git.sv.gnu.org:/srv/git/oddmuse Conflicts: modules/mac.pl t/encoding.t wiki.pl	2012-05-24 18:31:44 +02:00
Alex Schroeder	67650e3dc8	More UTF-8 fixes All the source files containing non-ASCII characters needed to have utf8 added. This will be necessary for user config files as well! The regular expressions identifying page names had to be changed. UrlEncode translates the string back to bytes before encoding it. Cached RSS files are saved with UTF-8 encoding and therefore need their meta-data changed (using the XML::RSS module to do this correctly didn't work for some of the test files). The CGI object's parameters, keywords and info_path are decoded correctly. File access uses the UTF-8 layer (reading, writing, appending, access to the log of recent changes, running sub processes with grep and diff). The mac compatibility extension will also disable the use of grep if non-ASCII characters are searched for because of an unexplained problem with grep.	2012-05-24 18:08:42 +02:00
Alex Schroeder	700d412a01	More UTF-8 fixes All the source files containing non-ASCII characters needed to have utf8 added. This will be necessary for user config files as well! The regular expressions identifying page names had to be changed. UrlEncode translates the string back to bytes before encoding it. Cached RSS files are saved with UTF-8 encoding and therefore need their meta-data changed (using the XML::RSS module to do this correctly didn't work for some of the test files). The CGI object's parameters, keywords and info_path are decoded correctly. File access uses the UTF-8 layer (reading, writing, appending, access to the log of recent changes, running sub processes with grep and diff). The mac compatibility extension will also disable the use of grep, because I could not get it to work (and I don't think there will be large Oddmuse installations running on Apple's HFS).	2012-05-24 16:58:10 +02:00
Alex Schroeder	cd2b4d624e	Remove $HttpCharset and moved everything to UTF-8 This also required some changes to the tests where explicit UTF-8 encoding had been used in the past.	2012-05-22 11:50:23 +02:00
Alex Schroeder	17dbca2353	current.pl target now depends on always The identifier added changes with every commit. A dependency on wiki.pl alone doesn't work. That's why it now depends on "always" which is phony.	2012-04-25 04:22:38 -04:00