test.pl: move to encode_utf8 as well

Add access function for rename.

modules/admin.pl

@@ -68,13 +68,13 @@ sub AdminPowerRename {
   Unlink($IndexFile);
   # page file
   CreateDir($PageDir); # It might not exist yet
-  rename($fname, $newfname)
+  Rename($fname, $newfname)
     or ReportError(Tss('Cannot rename %1 to %2', $fname, $newfname) . ": $!", '500 INTERNAL SERVER ERROR');
   # keep directory
   my $kdir = GetKeepDir($id);
   my $newkdir = GetKeepDir($new);
   CreateDir($KeepDir); # It might not exist yet (only the parent directory!)
-  rename($kdir, $newkdir)
+  Rename($kdir, $newkdir)
     or ReportError(Tss('Cannot rename %1 to %2', $kdir, $newkdir) . ": $!", '500 INTERNAL SERVER ERROR')
       if IsDir($kdir);
   # refer file
@@ -82,7 +82,7 @@ sub AdminPowerRename {
     my $rdir = GetRefererFile($id);
     my $newrdir = GetRefererFile($new);
     CreateDir($RefererDir); # It might not exist yet
-    rename($rdir, $newrdir)
+    Rename($rdir, $newrdir)
       or ReportError(Tss('Cannot rename %1 to %2', $rdir, $newrdir) . ": $!", '500 INTERNAL SERVER ERROR')
 	if IsDir($rdir);
   }

modules/askpage.pl

@@ -29,8 +29,7 @@ $NewQuestion = 'Write your question here:';
 
 sub IncrementInFile {
   my $filename = shift;
-  utf8::encode($filename);
-  sysopen my $fh, $filename, O_RDWR|O_CREAT or die "can't open $filename: $!";
+  sysopen my $fh, encode_utf8($filename), O_RDWR|O_CREAT or die "can't open $filename: $!";
   flock $fh, LOCK_EX or die "can't flock $filename: $!";
   my $num = <$fh> || 1;
   seek $fh, 0, 0 or die "can't rewind $filename: $!";

modules/backlinkage.pl

@@ -44,10 +44,8 @@ sub BacklinksMenu {
 $Action{buildback} = \&BuildBacklinkDatabase;
 sub BuildBacklinkDatabase {
     print GetHttpHeader('text/plain');
-    my $file = $backfile;
-    utf8::encode($file); # bytes
-    unlink($file); # Remove old database
-    tie my %backhash, 'MLDBM', $file or die "Cannot open file $backfile $!\n";
+    Unlink($backfile); # Remove old database
+    tie my %backhash, 'MLDBM', encode_utf8($backfile) or die "Cannot open file $backfile $!\n";
     log1("Starting Database Store Process ... please wait\n\n");
 
     foreach my $name (AllPagesList()) {
@@ -103,9 +101,7 @@ sub GetBackLink {
 
     our ($BacklinkBanned);
     $BacklinkBanned = "HomePage|ScratchPad" if !$BacklinkBanned;
-    my $file = $backfile;
-    utf8::encode($file);
-    tie my %backhash, 'MLDBM', $file, O_CREAT|O_RDWR, oct(644) or die "Cannot open file $backfile $!\n";
+    tie my %backhash, 'MLDBM', encode_utf8($backfile), O_CREAT|O_RDWR, oct(644) or die "Cannot open file $backfile $!\n";
 
     # Search database for matches
     while ( my ($source, $hashes) = each %backhash ) {

modules/diff.pl

@@ -77,8 +77,7 @@ sub DoUnifiedDiff { # copied from DoDiff
   RequestLockDir('diff') or return '';
   WriteStringToFile($oldName, $_[0]);
   WriteStringToFile($newName, $_[1]);
-  my $diff_out = `diff -U 99999 -- \Q$oldName\E \Q$newName\E | tail -n +7`; # should be +4, but we always add extra line # TODO that workaround is ugly, fix it!
-  utf8::decode($diff_out); # needs decoding
+  my $diff_out = decode_utf8(`diff -U 99999 -- \Q$oldName\E \Q$newName\E | tail -n +7`); # should be +4, but we always add extra line # TODO that workaround is ugly, fix it!
   $diff_out =~ s/\n\K\\ No newline.*\n//g; # Get rid of common complaint.
   ReleaseLockDir('diff');
   # No need to unlink temp files--next diff will just overwrite.

modules/drafts.pl

@@ -76,10 +76,7 @@ push(@MyMaintenance, \&DraftCleanup);
 
 sub DraftFiles {
   return map {
-    my $x = $_;
-    $x = substr($x, length($DraftDir) + 1);
-    utf8::decode($x);
-    $x;
+    substr($_, length($DraftDir) + 1);
   } Glob("$DraftDir/*"), Glob("$DraftDir/.*");
 }
 

modules/fix-encoding.pl

@@ -27,8 +27,7 @@ sub FixEncoding {
   ValidIdOrDie($id);
   RequestLockOrError();
   OpenPage($id);
-  my $text = $Page{text};
-  utf8::decode($text);
+  my $text = decode_utf8($Page{text});
   Save($id, $text, T('Fix character encoding'), 1) if $text ne $Page{text};
   ReleaseLock();
   ReBrowsePage($id);

modules/gd_security_image.pl

@@ -163,9 +163,7 @@ sub GdSecurityImageGenerate {
   my ($imgData) = $img->out(force => 'png');
   my $ticketId = Digest::MD5::md5_hex(rand());
   CreateDir($GdSecurityImageDir);
-  my $file = GdSecurityImageGetImageFile($ticketId);
-  utf8::encode($file);
-  open my $fh, ">:raw", $file
+  open my $fh, ">:raw", encode_utf8(GdSecurityImageGetImageFile($ticketId))
     or ReportError(Ts('Image storing failed. (%s)', $!), '500 INTERNAL SERVER ERROR');
   print $fh $imgData;
   #print $fh $png; ### experimental ###
@@ -188,9 +186,7 @@ sub GdSecurityImageIsValidId {
 }
 
 sub GdSecurityImageReadImageFile {
-  my $file = shift;
-  utf8::encode($file); # filenames are bytes!
-  if (open(my $IN, '<:raw', $file)) {
+  if (open(my $IN, '<:raw', encode_utf8(shift))) {
     local $/ = undef;   # Read complete files
     my $data=<$IN>;
     close $IN;

modules/git.pl

@@ -191,12 +191,9 @@ sub GitCleanup {
     print $q->p('Git cleanup starting');
     AllPagesList();
     # delete all the files including all the files starting with a dot
-    my $dir = $GitRepo;
-    utf8::encode($dir);
-    opendir(DIR, $dir) or ReportError("cannot open directory $GitRepo: $!");
+    opendir(DIR, encode_utf8($GitRepo)) or ReportError("cannot open directory $GitRepo: $!");
     foreach my $file (readdir(DIR)) {
-      my $name = $file;
-      utf8::decode($name); # filenames are bytes
+      my $name = decode_utf8($file);
       next if $file eq '.git' or $file eq '.' or $file eq '..' or $IndexHash{$name};
       print $q->p("Deleting left over file $name");
       Unlink("$GitRepo/$file") or ReportError("cannot delete $GitRepo/$name: $!");

modules/latex.pl

@@ -161,9 +161,7 @@ sub MakeLaTeX {
     # read template and replace <math>
     CreateDir($LatexDir);
     if (not IsFile($LatexDefaultTemplateName)) {
-      my $file = $LatexDefaultTemplateName;
-      utf8::encode($file);
-      open (my $F, '>', $file) or return '[Unable to write template]';
+      open (my $F, '>', encode_utf8($LatexDefaultTemplateName)) or return '[Unable to write template]';
       print $F $LatexDefaultTemplate;
       close $F;
     }

modules/localnames.pl

@@ -64,9 +64,7 @@ You can change this expiry time by setting C<$LnCacheHours>.
 push (@MyMaintenance, \&LnMaintenance);
 
 sub LnMaintenance {
-  my $dir = $RssDir;
-  utf8::encode($dir);
-  if (opendir(DIR, $dir)) { # cleanup if they should expire anyway
+  if (opendir(DIR, encode_utf8($RssDir))) { # cleanup if they should expire anyway
     foreach my $file (readdir(DIR)) {
       unlink("$RssDir/$file") if -M $file > $LnCacheHours * 3600;
     }

modules/login.pl

@@ -328,9 +328,7 @@ sub DoProcessLogout {
 
 sub UserExists {
 	my $username = shift;
-	my $file = $PasswordFile;
-	utf8::encode($file);
-	if (open (my $PASSWD, '<', $file)) {
+	if (open (my $PASSWD, '<', encode_utf8($PasswordFile))) {
 		while ( <$PASSWD> ) {
 			if ($_ =~ /^$username:/) {
 				return 1;
@@ -340,9 +338,7 @@ sub UserExists {
 	}
 
 	if ($RegistrationsMustBeApproved) {
-		$file = $PendingPasswordFile;
-		utf8::encode($file);
-		if (open (my $PASSWD, '<', $file)) {
+		if (open (my $PASSWD, '<', encode_utf8($PendingPasswordFile))) {
 			while ( <$PASSWD> ) {
 				if ($_ =~ /^$username:/) {
 					return 1;
@@ -353,9 +349,7 @@ sub UserExists {
 	}
 
 	if ($ConfirmEmailAddress) {
-		$file = $UnconfirmedPasswordFile;
-		utf8::encode($file);
-		if (open (my $PASSWD, '<', $UnconfirmedPasswordFile)) {
+		if (open (my $PASSWD, '<', encode_utf8($UnconfirmedPasswordFile))) {
 			while ( <$PASSWD> ) {
 				if ($_ =~ /^$username:/) {
 					return 1;
@@ -496,9 +490,7 @@ sub ConfirmUser {
 	my ($username, $key) = @_;
 	my $FileToUse = $RegistrationsMustBeApproved
 		? $PendingPasswordFile : $PasswordFileToUse;
-	my $file = $UnconfirmedPasswordFile;
-	utf8::encode($file);
-	if (open(my $PASSWD, '<', $file)) {
+	if (open(my $PASSWD, '<', encode_utf8($UnconfirmedPasswordFile))) {
 		while (<$PASSWD>) {
 			if ($_ =~ /^$username:(.*):(.*)/) {
 				if (crypt($1,$key) eq $key) {
@@ -522,8 +514,7 @@ sub RemoveUser {
 
 	my %passwords = ();
 	my %emails = ();
-	utf8::encode($FileToUse);
-	if (open (my $PASSWD, '<', $FileToUse)) {
+	if (open (my $PASSWD, '<', encode_utf8($FileToUse))) {
 		while ( <$PASSWD> ) {
 			if ($_ =~ /^(.*):(.*):(.*)$/) {
 				next if ($1 eq $username);
@@ -606,9 +597,7 @@ sub ChangePassword {
 
 	my %passwords = ();
 	my %emails = ();
-	my $file = $PasswordFile;
-	utf8::encode($file);
-	if (open (my $PASSWD, '<', $file)) {
+	if (open (my $PASSWD, '<', encode_utf8($PasswordFile))) {
 		while ( <$PASSWD> ) {
 			if ($_ =~ /^(.*):(.*):(.*)$/) {
 				$passwords{$1}=$2;
@@ -620,7 +609,7 @@ sub ChangePassword {
 
 	$passwords{$user} = $hash;
 
-	open (my $PASSWD, '>', $file);
+	open (my $PASSWD, '>', encode_utf8($PasswordFile));
 	foreach my $key ( sort keys(%passwords)) {
 		print $PASSWD "$key:$passwords{$key}:$emails{$key}\n";
 	}
@@ -727,9 +716,7 @@ sub DoApprovePending {
 		}
 	} else {
 		print '<ul>';
-		my $file = $PendingPasswordFile;
-		utf8::encode($file);
-		if (open(my $PASSWD, '<', $file)) {
+		if (open(my $PASSWD, '<', encode_utf8($PendingPasswordFile))) {
 			while (<$PASSWD>) {
 				if ($_ =~ /^(.*):(.*):(.*)$/) {
 					print '<li>' . ScriptLink("action=approve_pending;user=$1;",$1) . ' - ' . $3 . '</li>';
@@ -750,9 +737,7 @@ sub DoApprovePending {
 
 sub ApproveUser {
 	my ($username) = @_;
-	my $file = $PendingPasswordFile;
-	utf8::encode($file);
-	if (open(my $PASSWD, '<', $file)) {
+	if (open(my $PASSWD, '<', encode_utf8($PendingPasswordFile))) {
 		while (<$PASSWD>) {
 			if ($_ =~ /^$username:(.*):(.*)/) {
 				AddUser($username,$1,$2,$PasswordFile);

modules/mac.pl

@@ -56,14 +56,12 @@ sub MacFixEncoding {
   return unless %Namespaces;
   my %hash = ();
   for my $key (keys %Namespaces) {
-    utf8::decode($key);
     $key = NFC($key);
     $hash{$key} = $NamespaceRoot . '/' . $key . '/';
   }
   %Namespaces = %hash;
   %hash = ();
   for my $key (keys %InterSite) {
-    utf8::decode($key);
     $key = NFC($key);
     $hash{$key} = $Namespaces{$key} if $Namespaces{$key};
   }

modules/mail.pl

@@ -120,8 +120,7 @@ sub MailIsSubscribed {
   return 0 unless $mail;
   # open the DB file
   require DB_File;
-  utf8::encode($MailFile);
-  tie my %h, "DB_File", $MailFile;
+  tie my %h, "DB_File", encode_utf8($MailFile);
   my %subscribers = map {$_=>1} split(/$FS/, UrlDecode($h{UrlEncode($id)}));
   untie %h;
   return $subscribers{$mail};
@@ -198,8 +197,7 @@ sub NewMailDeletePage {
 sub MailDeletePage {
   my $id = shift;
   require DB_File;
-  utf8::encode($MailFile);
-  tie my %h, "DB_File", $MailFile;
+  tie my %h, "DB_File", encode_utf8($MailFile);
   foreach my $mail (split(/$FS/, UrlDecode(delete $h{UrlEncode($id)}))) {
     my %subscriptions = map {$_=>1} split(/$FS/, UrlDecode($h{UrlEncode($mail)}));
     delete $subscriptions{$id};
@@ -276,8 +274,7 @@ sub MailSubscription {
   my $mail = shift;
   return unless $mail;
   require DB_File;
-  utf8::encode($MailFile);
-  tie my %h, "DB_File", $MailFile;
+  tie my %h, "DB_File", encode_utf8($MailFile);
   my @result = split(/$FS/, UrlDecode($h{UrlEncode($mail)}));
   untie %h;
   @result = sort @result;
@@ -306,8 +303,7 @@ sub DoMailSubscriptionList {
       '<ul>';
   }
   require DB_File;
-  utf8::encode($MailFile);
-  tie my %h, "DB_File", $MailFile;
+  tie my %h, "DB_File", encode_utf8($MailFile);
   foreach my $encodedkey (sort keys %h) {
     my @values = sort split(/$FS/, UrlDecode($h{$encodedkey}));
     my $key = UrlDecode($encodedkey);
@@ -386,8 +382,7 @@ sub MailSubscribe {
   return unless $mail and @pages;
   # open the DB file
   require DB_File;
-  utf8::encode($MailFile);
-  tie my %h, "DB_File", $MailFile;
+  tie my %h, "DB_File", encode_utf8($MailFile);
   # add to the mail entry
   my %subscriptions = map {$_=>1} split(/$FS/, UrlDecode($h{UrlEncode($mail)}));
   for my $id (@pages) {
@@ -446,8 +441,7 @@ sub MailUnsubscribe {
   my ($mail, @pages) = @_;
   return unless $mail and @pages;
   require DB_File;
-  utf8::encode($MailFile);
-  tie my %h, "DB_File", $MailFile;
+  tie my %h, "DB_File", encode_utf8($MailFile);
   my %subscriptions = map {$_=>1} split(/$FS/, UrlDecode($h{UrlEncode($mail)}));
   foreach my $id (@pages) {
     delete $subscriptions{$id};
@@ -486,8 +480,7 @@ sub DoMailMigration {
     $q->start_div({-class=>'content mailmigrate'});
 
   require DB_File;
-  utf8::encode($MailFile);
-  tie my %h, "DB_File", $MailFile;
+  tie my %h, "DB_File", encode_utf8($MailFile);
   my $found = 0;
   foreach my $key (keys %h) {
     if (index($key, '@') != -1) {

modules/module-updater.pl

@@ -82,9 +82,7 @@ sub ProcessModule {
     return;
   }
   my $file = "$TempDir/$module";
-  utf8::encode($file);
-  open my $fh, ">", $file or die("Could not open file $TempDir/$module: $!");
-  utf8::encode($moduleData);
+  open my $fh, ">:utf8", encode_utf8($file) or die("Could not open file $TempDir/$module: $!");
   print $fh $moduleData;
   close $fh;
 
@@ -111,7 +109,6 @@ sub ProcessModule {
 }
 
 sub DoModuleDiff {
-  my $diff = `diff -U 3 -- \Q$_[0]\E \Q$_[1]\E`;
-  utf8::decode($diff); # needs decoding
+  my $diff = decode_utf8(`diff -U 3 -- \Q$_[0]\E \Q$_[1]\E`);
   return $diff;
 }

modules/namespaces.pl

@@ -86,7 +86,6 @@ sub NamespacesInitVariables {
   if ($UsePathInfo) {
     $Namespaces{$NamespacesMain} = $ScriptName . '/';
     foreach my $name (Glob("$DataDir/*")) {
-      utf8::decode($name);
       if (IsDir($name)
 	  and $name =~ m|/($InterSitePattern)$|
 	  and $name ne $NamespacesMain
@@ -97,13 +96,9 @@ sub NamespacesInitVariables {
   }
   $NamespaceRoot = $ScriptName; # $ScriptName may be changed below
   $NamespaceCurrent = '';
-  my $ns = GetParam('ns', '');
-  if ($ns) {
-    # GetParam quotes HTML but we don't care
-    utf8::decode($ns); # don't forget non-ASCII
-  } elsif (not $ns and $UsePathInfo) {
-    my $path_info = $q->path_info();
-    utf8::decode($path_info);
+  my $ns = decode_utf8(GetParam('ns', ''));
+  if (not $ns and $UsePathInfo) {
+    my $path_info = decode_utf8($q->path_info());
     # make sure ordinary page names are not matched!
     if ($path_info =~ m|^/($InterSitePattern)(/.*)?|
 	and ($2 or $q->keywords or NamespaceRequiredByParameter())) {
@@ -223,9 +218,7 @@ sub NewNamespaceGetRcLines { # starttime, hash of seen pages to use as a second
   # starttime. If any rcfile exists with no timestamp before the
   # starttime, we need to open its rcoldfile.
   foreach my $rcfile (@rcfiles) {
-    my $file = $rcfile;
-    utf8::encode($file);
-    open(my $F, '<:encoding(UTF-8)', $file);
+    open(my $F, '<:encoding(UTF-8)', encode_utf8($rcfile));
     my $line = <$F>;
     my ($ts) = split(/$FS/, $line); # the first timestamp in the regular rc file
     my @new;
@@ -441,8 +434,6 @@ sub NamespacesNewGetId {
   # In this case GetId() will have set the parameter Test to 1.
   # http://example.org/cgi-bin/wiki.pl/Test?rollback-1234=foo
   # This doesn't set the Test parameter.
-  if ($UsePathInfo and $id eq $NamespaceCurrent and not GetParam($id) and not GetParam('ns')) {
-    $id = undef;
-  }
+  return if $id and $UsePathInfo and $id eq $NamespaceCurrent and not GetParam($id) and not GetParam('ns');
   return $id;
 }

modules/private-wiki.pl

@@ -41,7 +41,7 @@ sub PrivateWikiInit {
   }
 }
 
-sub PadTo16Bytes { # use this only on UTF-X strings (after utf8::encode)
+sub PadTo16Bytes { # use this only on bytes (after encode_utf8)
   my ($data, $minLength) = @_;
   my $endBytes = length($data) % 16;
   $data .= "\0" x (16 - $endBytes) if $endBytes != 0;
@@ -62,8 +62,7 @@ sub NewPrivateWikiReadFile {
 	      $q->p($errorMessage)) if not UserIsEditor();
   PrivateWikiInit();
   my $file = shift;
-  utf8::encode($file); # filenames are bytes!
-  if (open(my $IN, '<', $file)) {
+  if (open(my $IN, '<', encode_utf8($file))) {
     local $/ = undef; # Read complete files
     my $data = <$IN>;
     close $IN;
@@ -72,8 +71,7 @@ sub NewPrivateWikiReadFile {
     $data = $cipher->decrypt(substr $data, 16);
     my $copy = $data; # copying is required, see https://github.com/briandfoy/crypt-rijndael/issues/5
     $copy =~ s/\0+$//;
-    utf8::decode($copy);
-    return (1, $copy);
+    return (1, decode_utf8($copy));
   }
   return (0, '');
 }
@@ -86,13 +84,12 @@ sub NewPrivateWikiWriteStringToFile {
 	      $q->p($errorMessage)) if not UserIsEditor();
   PrivateWikiInit();
   my ($file, $string) = @_;
-  utf8::encode($file);
-  open(my $OUT, '>', $file) or ReportError(Ts('Cannot write %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
-  utf8::encode($string);
+  open(my $OUT, '>', encode_utf8($file))
+      or ReportError(Ts('Cannot write %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
   my $iv = $random->random_bytes(16);
   $cipher->set_iv($iv);
   print $OUT $iv;
-  print $OUT $cipher->encrypt(PadTo16Bytes $string);
+  print $OUT $cipher->encrypt(PadTo16Bytes(encode_utf8($string)));
   close($OUT);
 }
 
@@ -111,9 +108,7 @@ sub AppendStringToFile {
 sub NewPrivateWikiRefreshIndex {
   if (not IsFile($IndexFile)) { # Index file does not exist yet, this is a new wiki
     my $fh;
-    my $file = $IndexFile;
-    utf8::encode($file);
-    open($fh, '>', $file) or die "Unable to open file $IndexFile : $!"; # 'touch' equivalent
+    open($fh, '>', encode_utf8($IndexFile)) or die "Unable to open file $IndexFile : $!"; # 'touch' equivalent
     close($fh) or die "Unable to close file : $IndexFile $!";
     return;
   }
@@ -165,8 +160,7 @@ sub GetPrivatePageFile {
   }
   $cipher->set_iv($iv);
   # We cannot use full byte range because of the filesystem limits
-  utf8::encode($id);
-  my $returnName = unpack "H*", $iv . $cipher->encrypt(PadTo16Bytes $id, 96); # to hex string
+  my $returnName = unpack "H*", $iv . $cipher->encrypt(PadTo16Bytes(encode_utf8($id)), 96); # to hex string
   return $returnName;
 }
 
@@ -221,8 +215,7 @@ sub DoDiff {      # Actualy call the diff program
   $LockCleaners{'diff'} = sub { Unlink($oldName) if IsFile($oldName); Unlink($newName) if IsFile($newName); };
   OldPrivateWikiWriteStringToFile($oldName, $_[0]); # CHANGED Here we use the old sub!
   OldPrivateWikiWriteStringToFile($newName, $_[1]); # CHANGED
-  my $diff_out = `diff -- \Q$oldName\E \Q$newName\E`;
-  utf8::decode($diff_out); # needs decoding
+  my $diff_out = decode_utf8(`diff -- \Q$oldName\E \Q$newName\E`);
   $diff_out =~ s/\n\K\\ No newline.*\n//g; # Get rid of common complaint.
   # CHANGED We have to unlink the files because we don't want to store them in plaintext!
   Unlink($oldName, $newName); # CHANGED
@@ -245,8 +238,7 @@ sub MergeRevisions {   # merge change from file2 to file3 into file1
   OldPrivateWikiWriteStringToFile($name2, $file2); # CHANGED
   OldPrivateWikiWriteStringToFile($name3, $file3); # CHANGED
   my ($you, $ancestor, $other) = (T('you'), T('ancestor'), T('other'));
-  my $output = `diff3 -m -L \Q$you\E -L \Q$ancestor\E -L \Q$other\E -- \Q$name1\E \Q$name2\E \Q$name3\E`;
-  utf8::decode($output); # needs decoding
+  my $output = decode_utf8(`diff3 -m -L \Q$you\E -L \Q$ancestor\E -L \Q$other\E -- \Q$name1\E \Q$name2\E \Q$name3\E`);
   Unlink($name1, $name2, $name3); # CHANGED unlink temp files -- we don't want to store them in plaintext!
   ReleaseLockDir('merge');
   return $output;

modules/pygmentize.pl

@@ -52,11 +52,9 @@ sub DoPygmentize {
 
   RequestLockDir('pygmentize') or return '';
   WriteStringToFile("$TempDir/pygmentize", $contents);
-  my $output = `pygmentize $lexer -f html -O encoding=utf8 $args -- \Q$TempDir/pygmentize\E  2>&1`;
+  my $output = decode_utf8(`pygmentize $lexer -f html -O encoding=utf8 $args -- \Q$TempDir/pygmentize\E  2>&1`);
   ReleaseLockDir('pygmentize');
 
-  utf8::decode($output);
-
   if ($?) {
     $output = $q->p($q->strong($output)) # "sh: pygmentize: command not found"
         . $q->pre($contents);

modules/relation.pl

@@ -32,9 +32,7 @@ my $dummy = RelationRead();
 
 sub RelationRead {
 #   return scalar(@RelationLinking) if (scalar(@RelationLinking));
-   my $file = "$DataDir/$referencefile";
-   utf8::encode($file);
-   open (my $RRR, '<', $file) || return(0);
+   open (my $RRR, '<', encode_utf8("$DataDir/$referencefile")) || return(0);
    while (<$RRR>) {
       chomp;
       my ($a,$b,$c) = split(';');
@@ -174,9 +172,7 @@ $Action{'updaterelates'} = sub {
   else {
         print "no new source<br />\n";
   }
-  my $file = "$DataDir/$referencefile";
-  utf8::encode($file);
-  open (my $RRR, '>', $file);
+  open (my $RRR, '>', encode_utf8("$DataDir/$referencefile"));
   print "<br />\n";
   foreach my $t (@RelationLinking) {
       next unless (defined($t));

modules/smiley-dir.pl

@@ -24,13 +24,13 @@ AddModuleDescription('smiley-dir.pl', 'Smiley Directory Extension');
 our (@MyInitVariables, $ImageExtensions, %Smilies);
 our ($SmileyDir, $SmileyUrlPath);
 
+# $SmileyDir must be bytes! (use encode_utf8 if necessary)
 $SmileyDir = '/mnt/pics'; # directory with all the smileys
 $SmileyUrlPath = '/pics'; # path where all the smileys can be found (URL)
 
 push(@MyInitVariables, \&SmileyDirInit);
 
 sub SmileyDirInit {
-  # $SmileyDir must be bytes! (use utf8::encode if necessary)
   if (opendir(DIR, $SmileyDir)) {
     map {
       if (/^((.*)\.$ImageExtensions$)/ and IsFile("$SmileyDir/$_")) {

modules/static-copy.pl

@@ -135,9 +135,7 @@ sub StaticWriteFile {
   my ($mimetype, $encoding, $data) =
       $Page{text} =~ /^\#FILE ([^ \n]+) ?([^ \n]*)\n(.*)/s;
   my $filename = StaticFileName($id);
-  my $file = "$StaticDir/$filename";
-  utf8::encode($file);
-  open(my $fh, '>', $file)
+  open(my $fh, '>', encode_utf8("$StaticDir/$filename"))
     or ReportError(Ts('Cannot write %s', $filename));
   if ($data) {
     binmode($fh);

modules/static-hybrid.pl

@@ -137,9 +137,7 @@ sub StaticWriteFile {
 	my ($mimetype, $data) = $Page{text} =~ /^\#FILE ([^ \n]+)\n(.*)/s;
 	return unless $html or $data;
 	my $filename = StaticFileName($id);
-	my $file = "$StaticDir/$filename";
-	utf8::encode($file);
-	open(my $F, '>', $file) or ReportError(Ts('Cannot write %s', $filename));
+	open(my $F, '>', encode_utf8("$StaticDir/$filename")) or ReportError(Ts('Cannot write %s', $filename));
 	if ($data) {
 		StaticFile($id, $mimetype, $data, $F);
 	} elsif ($html) {

modules/tags.pl

@@ -84,9 +84,7 @@ sub TagsGetLink {
 
 sub TagReadHash {
   require Storable;
-  my $file = $TagFile;
-  utf8::encode($file);
-  return %{ Storable::retrieve($file) } if -f $file;
+  return %{ Storable::retrieve(encode_utf8($TagFile)) } if IsFile($TagFile);
 }
 
 
@@ -94,9 +92,7 @@ sub TagReadHash {
 sub TagWriteHash {
   my $h = shift;
   require Storable;
-  my $file = $TagFile;
-  utf8::encode($file);
-  return Storable::store($h, $file);
+  return Storable::store($h, encode_utf8($TagFile));
 }
 
 push(@MyRules, \&TagsRule);

modules/thumbs.pl

@@ -209,10 +209,8 @@ sub GenerateThumbNail {
 
 
      my $filename = $ThumbnailTempDir . "/odd" . $id . "_" . $size;
-     my $file = $filename;
-     utf8::encode($file);
      # Decode the original image to a temp file
-     open(my $FD, '>', $file) or ReportError(Ts("Could not open %s for writing whilst trying to save image before creating thumbnail. Check write permissions.",$filename), '500 INTERNAL SERVER ERROR');
+     open(my $FD, '>', encode_utf8($filename)) or ReportError(Ts("Could not open %s for writing whilst trying to save image before creating thumbnail. Check write permissions.",$filename), '500 INTERNAL SERVER ERROR');
      binmode($FD);
      print $FD MIME::Base64::decode($data);
      close($FD);

modules/toc.pl

@@ -234,12 +234,10 @@ sub NewTocApplyRules {
     my $html_unfixed;
     open(  STDOUT, '>', \$html_unfixed) or die "Can't open memory file: $!";
     binmode STDOUT, ":encoding(UTF-8)";
-    ($blocks, $flags) = OldTocApplyRules(@_);
-    close  STDOUT;
-    utf8::decode($blocks);
+    ($blocks, $flags) = map { decode_utf8($_) } OldTocApplyRules(@_);
+    close STDOUT;
     # do not delete!
-    $html = $html_unfixed; # this is a workarond for perl bug
-    utf8::decode($html);   # otherwise UTF8 characters are SOMETIMES not decoded.
+    $html = decode_utf8($html_unfixed);
   }
   # If there are at least two HTML headers on this page, insert a table of
   # contents.

modules/upgrade.pl

@@ -77,17 +77,13 @@ sub DoUpgrade {
       $dir =~ s/^$DataDir/$DataDir\/$ns/ if $ns;
       for my $old (Glob("$dir/*/*"), Glob("$dir/*/.*")) {
 	next if $old =~ /\/\.\.?$/;
-	my $oldname = $old;
-	utf8::decode($oldname);
-	print "<br />\n$oldname";
+	print "<br />\n$old";
 	my $new = $old;
 	$new =~ s!/([A-Z]|other)/!/!;
 	if ($old eq $new) {
 	  print " does not fit the pattern!";
-	} elsif (not rename $old, $new) {
-	  my $newname = $new;
-	  utf8::decode($newname);
-	  print " → $newname failed!";
+	} elsif (not Rename($old, $new)) {
+	  print " → $new failed!";
 	}
       }
       for my $subdir (grep(/\/([A-Z]|other)$/, Glob("$dir/*"), Glob("$dir/.*"))) {
@@ -99,7 +95,7 @@ sub DoUpgrade {
 
   print $q->end_p();
 
-  if (rename "$ModuleDir/upgrade.pl", "$ModuleDir/upgrade.done") {
+  if (Rename("$ModuleDir/upgrade.pl", "$ModuleDir/upgrade.done")) {
     print $q->p(T("Upgrade complete."))
   } else {
     print $q->p(T("Upgrade complete. Please remove $ModuleDir/upgade.pl, now."))

modules/upload.pl

@@ -71,15 +71,12 @@ for (my $i=0; $q->param("fileToUpload$i"); $i++) {
   } else {
     squeak 'Error: Filename contains invalid characters'; # this should not happen
   }
-  utf8::encode($logFile);
-  open(my $LOGFILE, '>>', $logFile) or squeak "$!";
+  open(my $LOGFILE, '>>', encode_utf8($logFile)) or squeak "$!";
   print $LOGFILE $q->param("key") . ' ' . $ENV{REMOTE_ADDR} . ' ' . $curFilename . "\n";
   close $LOGFILE;
 
   my $uploadFileHandle = $q->upload("fileToUpload$i");
-  my $file = "$uploadDir/$curFilename";
-  utf8::encode($file);
-  open(my $UPLOADFILE, '>', $file) or squeak "$!";
+  open(my $UPLOADFILE, '>', encode_utf8("$uploadDir/$curFilename")) or squeak "$!";
   binmode $UPLOADFILE;
   while (<$uploadFileHandle>) {
     print $UPLOADFILE;

t/test.pl

@@ -17,6 +17,7 @@ package OddMuse;
 use lib '.';
 use XML::LibXML;
 use utf8;
+use Encode qw(encode_utf8 decode_utf8);
 use vars qw($raw);
 
 # Test::More explains how to fix wide character in print issues
@@ -56,8 +57,7 @@ $| = 1; # no output buffering
 sub url_encode {
   my $str = shift;
   return '' unless $str;
-  utf8::encode($str); # turn to byte string
-  my @letters = split(//, $str);
+  my @letters = split(//, encode_utf8($str));
   my @safe = ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '-', '_', '.'); # shell metachars are unsafe
   foreach my $letter (@letters) {
     my $pattern = quotemeta($letter);
@@ -209,15 +209,8 @@ sub xpath_do {
     skip("Cannot parse ".name($page).": $@", $#tests + 1) if $@;
     foreach my $test (@tests) {
       my $nodelist;
-      my $bytes = $test;
-      # utf8::encode: Converts in-place the character sequence to the
-      # corresponding octet sequence in *UTF-X*. The UTF8 flag is
-      # turned off, so that after this operation, the string is a byte
-      # string. (I have no idea why this is necessary, but there you
-      # go. See encoding.t tests and make sure the page file is
-      # encoded correctly.)
-      utf8::encode($bytes);
-      eval { $nodelist = $doc->findnodes($bytes) };
+      # libxml2 is not aware of UTF8 flag
+      eval { $nodelist = $doc->findnodes(encode_utf8($test)) };
       if ($@) {
 	fail(&$check(1) ? "$test: $@" : "not $test: $@");
       } elsif (ok(&$check($nodelist->size()),

wiki.pl

@@ -38,6 +38,7 @@ use utf8; # in case anybody ever adds UTF8 characters to the source
 use CGI qw/-utf8/;
 use CGI::Carp qw(fatalsToBrowser);
 use File::Glob ':glob';
+use Encode qw(encode_utf8 decode_utf8);
 use sigtrap 'handler' => \&HandleSignals, 'normal-signals', 'error-signals';
 local $| = 1; # Do not buffer output (localized for mod_perl)
 
@@ -66,7 +67,7 @@ our $UseConfig //= 1;
 
 # Main wiki directory
 our $DataDir;
-$DataDir    ||= $ENV{WikiDataDir} if $UseConfig;
+$DataDir    ||= decode_utf8($ENV{WikiDataDir}) if $UseConfig;
 $DataDir    ||= '/tmp/oddmuse'; # FIXME: /var/opt/oddmuse/wiki ?
 
 our $ConfigFile;
@@ -229,9 +230,12 @@ sub Init {
 sub InitModules {
   if ($UseConfig and $ModuleDir and IsDir($ModuleDir)) {
     foreach my $lib (Glob("$ModuleDir/*.p[ml]")) {
-      do $lib unless $MyInc{$lib};
-      $MyInc{$lib} = 1;   # Cannot use %INC in mod_perl settings
-      $Message .= CGI::p("$lib: $@") if $@; # no $q exists, yet
+      if (not $MyInc{$lib}) {
+	$MyInc{$lib} = 1;   # Cannot use %INC in mod_perl settings
+	my $file = encode_utf8($lib);
+	do $file;
+	$Message .= CGI::p("$lib: $@") if $@; # no $q exists, yet
+      }
     }
   }
 }
@@ -250,7 +254,6 @@ sub InitConfig {
 }
 
 sub InitDirConfig {
-  utf8::decode($DataDir); # just in case, eg. "WikiDataDir=/tmp/Zürich♥ perl wiki.pl"
   $PageDir     = "$DataDir/page";  # Stores page data
   $KeepDir     = "$DataDir/keep";  # Stores kept (old) page data
   $TempDir     = "$DataDir/temp";  # Temporary files and locks
@@ -360,8 +363,7 @@ sub CookieRollbackFix {
 
 sub GetParam {
   my ($name, $default) = @_;
-  utf8::encode($name); # turn to byte string
-  my $result = $q->param($name);
+  my $result = $q->param(encode_utf8($name));
   $result //= $default;
   return QuoteHtml($result); # you need to unquote anything that can have <tags>
 }
@@ -786,8 +788,7 @@ sub UnquoteHtml {
 sub UrlEncode {
   my $str = shift;
   return '' unless $str;
-  utf8::encode($str); # turn to byte string
-  my @letters = split(//, $str);
+  my @letters = split(//, encode_utf8($str));
   my %safe = map {$_ => 1} ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '-', '_', '.', '!', '~', '*', "'", '(', ')', '#');
   foreach my $letter (@letters) {
     $letter = sprintf("%%%02x", ord($letter)) unless $safe{$letter};
@@ -798,8 +799,7 @@ sub UrlEncode {
 sub UrlDecode {
   my $str = shift;
   $str =~ s/%([0-9a-f][0-9a-f])/chr(hex($1))/eg;
-  utf8::decode($str); # make internal string
-  return $str;
+  return decode_utf8($str); # make internal string
 }
 
 sub QuoteRegexp {
@@ -1263,9 +1263,8 @@ sub ToString {
   $sub_ref->();
   select $oldFH;
   close $outputFH;
-  my $output_fixed = $output;  # do not delete!
-  utf8::decode($output_fixed); # this is a workarond for perl bug
-  return $output_fixed;        # otherwise UTF8 characters are SOMETIMES not decoded.
+  my $output_fixed = $output;        # Do not delete! This is a workarond for a perl bug.
+  return decode_utf8($output_fixed); # Otherwise UTF8 characters are SOMETIMES not decoded.
 }
 
 sub PageHtml {
@@ -1300,16 +1299,10 @@ sub Tss {
 sub GetId {
   my $id = UnquoteHtml(GetParam('id', GetParam('title', ''))); # id=x or title=x -> x
   if (not $id) {
-    my @keywords = $q->keywords;
-    foreach my $keyword (@keywords) {
-      utf8::decode($keyword);
-    }
-    $id ||= join('_', @keywords); # script?p+q -> p_q
+    $id ||= decode_utf8(join('_', $q->keywords)); # script?p+q -> p_q
   }
   if ($UsePathInfo) {
-    my $path = $q->path_info;
-    utf8::decode($path);
-    my @path = split(/\//, $path);
+    my @path = map { decode_utf8($_) } split(/\//, $q->path_info);
     $id ||= pop(@path); # script/p/q -> q
     foreach my $p (@path) {
       SetParam($p, 1);    # script/p/q -> p=1
@@ -1504,8 +1497,7 @@ sub GetRcLines { # starttime, hash of seen pages to use as a second return value
   my @result = ();
   my $ts;
   # check the first timestamp in the default file, maybe read old log file
-  utf8::encode($RcFile);
-  if (open(my $F, '<:encoding(UTF-8)', $RcFile)) {
+  if (open(my $F, '<:encoding(UTF-8)', encode_utf8($RcFile))) {
     my $line = <$F>;
     ($ts) = split(/$FS/, $line); # the first timestamp in the regular rc file
   }
@@ -1589,8 +1581,7 @@ sub GetRcLinesFor {
         rcclusteronly rcfilteronly match lang followup);
   # parsing and filtering
   my @result = ();
-  utf8::encode($file);
-  open(my $F, '<:encoding(UTF-8)', $file) or return ();
+  open(my $F, '<:encoding(UTF-8)', encode_utf8($file)) or return ();
   while (my $line = <$F>) {
     chomp($line);
     my ($ts, $id, $minor, $summary, $host, $username, $revision,
@@ -2606,10 +2597,9 @@ sub DoDiff {      # Actualy call the diff program
   RequestLockDir('diff') or return '';
   WriteStringToFile($oldName, $_[0]);
   WriteStringToFile($newName, $_[1]);
-  my $diff_out = `diff -- \Q$oldName\E \Q$newName\E`;
-  utf8::decode($diff_out); # needs decoding
-  $diff_out =~ s/\n\K\\ No newline.*\n//g; # Get rid of common complaint.
+  my $diff_out = decode_utf8(`diff -- \Q$oldName\E \Q$newName\E`);
   ReleaseLockDir('diff');
+  $diff_out =~ s/\n\K\\ No newline.*\n//g; # Get rid of common complaint.
   # No need to unlink temp files--next diff will just overwrite.
   return $diff_out;
 }
@@ -2850,9 +2840,7 @@ sub ExpireKeepFiles {   # call with opened page
 }
 
 sub ReadFile {
-  my $file = shift;
-  utf8::encode($file); # filenames are bytes!
-  if (open(my $IN, '<:encoding(UTF-8)', $file)) {
+  if (open(my $IN, '<:encoding(UTF-8)', encode_utf8(shift))) {
     local $/ = undef; # Read complete files
     my $data=<$IN>;
     close $IN;
@@ -2873,8 +2861,7 @@ sub ReadFileOrDie  {
 
 sub WriteStringToFile {
   my ($file, $string) = @_;
-  utf8::encode($file);
-  open(my $OUT, '>:encoding(UTF-8)', $file)
+  open(my $OUT, '>:encoding(UTF-8)', encode_utf8($file))
     or ReportError(Ts('Cannot write %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
   print $OUT  $string;
   close($OUT);
@@ -2882,69 +2869,29 @@ sub WriteStringToFile {
 
 sub AppendStringToFile {
   my ($file, $string) = @_;
-  utf8::encode($file);
-  open(my $OUT, '>>:encoding(UTF-8)', $file)
+  open(my $OUT, '>>:encoding(UTF-8)', encode_utf8($file))
     or ReportError(Ts('Cannot write %s', $file) . ": $!", '500 INTERNAL SERVER ERROR');
   print $OUT  $string;
   close($OUT);
 }
 
-sub IsFile {
-  my $file = shift;
-  utf8::encode($file);
-  return -f $file;
-}
-
-sub IsDir {
-  my $dir = shift;
-  utf8::encode($dir);
-  return -d $dir;
-}
-
-sub ZeroSize {
-  my $file = shift;
-  utf8::encode($file);
-  return -z $file;
-}
-
-sub Unlink {
-  my @files = @_; # copy
-  map { utf8::encode($_) } @files;
-  return unlink(@files); # lower case!
-}
-
-sub Modified {
-  my $file = shift;
-  utf8::encode($file);
-  return (stat($file))[9];
-}
+sub IsFile    { return -f encode_utf8(shift); }
+sub IsDir     { return -d encode_utf8(shift); }
+sub ZeroSize  { return -z encode_utf8(shift); }
+sub Unlink    { return unlink(map { encode_utf8($_) } @_); }
+sub Modified  { return (stat(encode_utf8(shift)))[9]; }
+sub Glob      { return map { decode_utf8($_) } bsd_glob(encode_utf8(shift)); }
+sub Rename    { return rename(encode_utf8($_[0]), encode_utf8($_[1])); }
+sub RemoveDir { return rmdir(encode_utf8(shift)); }
+sub ChangeDir { return chdir(encode_utf8(shift)); }
 
 sub CreateDir {
   my ($newdir) = @_;
-  utf8::encode($newdir);
-  return if -d $newdir;
-  mkdir($newdir, 0775)
+  return if IsDir($newdir);
+  mkdir(encode_utf8($newdir), 0775)
     or ReportError(Ts('Cannot create %s', $newdir) . ": $!", '500 INTERNAL SERVER ERROR');
 }
 
-sub RemoveDir {
-  my ($dir) = @_;
-  utf8::encode($dir);
-  rmdir($dir);
-}
-
-sub ChangeDir {
-  my ($dir) = @_;
-  utf8::encode($dir);
-  chdir($dir);
-}
-
-sub Glob {
-  my ($pattern) = @_;
-  utf8::encode($pattern);
-  return bsd_glob($pattern);
-}
-
 sub GetLockedPageFile {
   my $id = shift;
   return "$PageDir/$id.lck";
@@ -2956,9 +2903,10 @@ sub RequestLockDir {
   $wait ||= 2;
   CreateDir($TempDir);
   my $lock = $LockDir . $name;
-  utf8::encode($lock);
   my $n = 0;
-  while (mkdir($lock, 0555) == 0) {
+  # Cannot use CreateDir because we don't want to skip mkdir if the directory
+  # already exists.
+  while (mkdir(encode_utf8($lock), 0555) == 0) {
     if ($n++ >= $tries) {
       my $ts = Modified($lock);
       if ($Now - $ts > $LockExpiration and $LockExpires{$name} and not $retried) { # XXX should we remove this now?
@@ -3432,7 +3380,6 @@ sub RefreshIndex {
   foreach (Glob("$PageDir/*.pg"), Glob("$PageDir/.*.pg")) {
     next unless m|/.*/(.+)\.pg$|;
     my $id = $1;
-    utf8::decode($id);
     push(@IndexList, $id);
     $IndexHash{$id} = 1;
   }
@@ -3501,8 +3448,7 @@ sub PageIsUploadedFile {
   return if $OpenPageName eq $id;
   if ($IndexHash{$id}) {
     my $file = GetPageFile($id);
-    utf8::encode($file); # filenames are bytes!
-    open(my $FILE, '<:encoding(UTF-8)', $file)
+    open(my $FILE, '<:encoding(UTF-8)', encode_utf8($file))
       or ReportError(Ts('Cannot open %s', GetPageFile($id))
 		     . ": $!", '500 INTERNAL SERVER ERROR');
     while (defined($_ = <$FILE>) and $_ !~ /^text: /) {
@@ -3899,8 +3845,7 @@ sub MergeRevisions {   # merge change from file2 to file3 into file1
   WriteStringToFile($name2, $file2);
   WriteStringToFile($name3, $file3);
   my ($you, $ancestor, $other) = (T('you'), T('ancestor'), T('other'));
-  my $output = `diff3 -m -L \Q$you\E -L \Q$ancestor\E -L \Q$other\E -- \Q$name1\E \Q$name2\E \Q$name3\E`;
-  utf8::decode($output); # needs decoding
+  my $output = decode_utf8(`diff3 -m -L \Q$you\E -L \Q$ancestor\E -L \Q$other\E -- \Q$name1\E \Q$name2\E \Q$name3\E`);
   ReleaseLockDir('merge'); # don't unlink temp files--next merge will just overwrite.
   return $output;
 }