Merge pull request #2945 from splitbrain/phpcsCI

Check our code style for every PR via CI

9 files changed, 1333 insertions, 1222 deletions

diff --git a/.github/workflows/phpCS.yml b/.github/workflows/phpCS.yml
new file mode 100644
index 000000000..2699db6cd
--- /dev/null
+++ b/.github/workflows/phpCS.yml
@@ -0,0 +1,22 @@
+name: PHP Code Style
+
+on: [push]
+
+jobs:
+    phpcs:
+        name: PHP CodeSniffer
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v1
+
+            - name: Setup PHP
+              uses: shivammathur/setup-php@7b8bf7af6a42e028cbcccf623bb8499b4d6edf02
+              with:
+                  php-version: '7.4'
+                  coverage: none
+
+            - name: Download PHPCS
+              run: wget https://squizlabs.github.io/PHP_CodeSniffer/phpcs.phar
+
+            - name: run PHP codesniffer
+              run: php phpcs.phar -v --standard=_test/phpcs.xml
diff --git a/_test/phpcs.xml b/_test/phpcs.xml
index 4f38195a3..6e78784c8 100644
--- a/_test/phpcs.xml
+++ b/_test/phpcs.xml
@@ -35,12 +35,87 @@
     <exclude-pattern>*/inc/parser/*</exclude-pattern>
 
     <!-- rules on top of PSR-2 -->
-    <rule ref="PSR2">
+    <rule ref="PSR12">
         <!-- the following rule is not in PSR-2 and breaks the guardian pattern -->
         <exclude name="Generic.ControlStructures.InlineControlStructure.NotAllowed"/>
 
         <!-- we have lots of legacy classes without name spaces -->
         <exclude name="PSR1.Classes.ClassDeclaration.MissingNamespace"/>
+
+        <!-- Rules with automatic fixes that we want to adhere to, but currently don't -->
+        <exclude name="Generic.Formatting.DisallowMultipleStatements.SameLine"/>
+        <exclude name="Generic.Functions.FunctionCallArgumentSpacing.NoSpaceAfterComma"/>
+        <exclude name="Generic.Functions.FunctionCallArgumentSpacing.SpaceBeforeComma"/>
+        <exclude name="Generic.Functions.FunctionCallArgumentSpacing.TooMuchSpaceAfterComma"/>
+        <exclude name="Generic.PHP.LowerCaseKeyword.Found"/>
+        <exclude name="Generic.PHP.LowerCaseConstant.Found"/>
+        <exclude name="Generic.WhiteSpace.ScopeIndent.IncorrectExact"/>
+        <exclude name="Generic.WhiteSpace.ScopeIndent.Incorrect"/>
+        <exclude name="Generic.WhiteSpace.IncrementDecrementSpacing.SpaceAfterDecrement"/>
+        <exclude name="Generic.WhiteSpace.IncrementDecrementSpacing.SpaceAfterIncrement"/>
+
+        <exclude name="Squiz.ControlStructures.ControlSignature.SpaceAfterCloseParenthesis"/>
+        <exclude name="Squiz.ControlStructures.ControlSignature.SpaceAfterKeyword"/>
+        <exclude name="Squiz.ControlStructures.ControlSignature.SpaceAfterCloseBrace"/>
+        <exclude name="Squiz.ControlStructures.ControlSignature.NewlineAfterOpenBrace"/>
+        <exclude name="Squiz.ControlStructures.ForEachLoopDeclaration.NoSpaceBeforeArrow"/>
+        <exclude name="Squiz.ControlStructures.ForEachLoopDeclaration.NoSpaceAfterArrow"/>
+        <exclude name="Squiz.ControlStructures.ForEachLoopDeclaration.SpacingBeforeAs"/>
+        <exclude name="Squiz.ControlStructures.ForLoopDeclaration.SpacingAfterSecond"/>
+        <exclude name="Squiz.ControlStructures.ForLoopDeclaration.NoSpaceAfterSecond"/>
+        <exclude name="Squiz.ControlStructures.ForLoopDeclaration.NoSpaceAfterFirst"/>
+        <exclude name="Squiz.Functions.FunctionDeclarationArgumentSpacing.NoSpaceBeforeArg"/>
+        <exclude name="Squiz.Functions.FunctionDeclarationArgumentSpacing.SpaceAfterEquals"/>
+        <exclude name="Squiz.Functions.FunctionDeclarationArgumentSpacing.SpacingAfterReference"/>
+        <exclude name="Squiz.Functions.FunctionDeclarationArgumentSpacing.SpacingBeforeClose"/>
+        <exclude name="Squiz.Functions.FunctionDeclarationArgumentSpacing.SpaceBeforeEquals"/>
+        <exclude name="Squiz.Functions.MultiLineFunctionDeclaration.BraceOnSameLine"/>
+        <exclude name="Squiz.Functions.MultiLineFunctionDeclaration.NewlineBeforeOpenBrace"/>
+        <exclude name="Squiz.Functions.MultiLineFunctionDeclaration.CloseBracketLine"/>
+        <exclude name="Squiz.WhiteSpace.ControlStructureSpacing.SpacingAfterOpen"/>
+        <exclude name="Squiz.WhiteSpace.ControlStructureSpacing.SpacingBeforeClose"/>
+        <exclude name="Squiz.WhiteSpace.ScopeClosingBrace.ContentBefore"/>
+        <exclude name="Squiz.WhiteSpace.ScopeClosingBrace.Indent"/>
+        <exclude name="Squiz.WhiteSpace.SuperfluousWhitespace.EndLine"/>
+
+        <exclude name="PSR2.Classes.ClassDeclaration.CloseBraceAfterBody"/>
+        <exclude name="PSR2.Classes.ClassDeclaration.OpenBraceNewLine"/>
+        <exclude name="PSR2.Classes.PropertyDeclaration.StaticBeforeVisibility"/>
+        <exclude name="PSR2.ControlStructures.ControlStructureSpacing.SpacingAfterOpenBrace"/>
+        <exclude name="PSR2.ControlStructures.ControlStructureSpacing.SpaceBeforeCloseBrace"/>
+        <exclude name="PSR2.ControlStructures.ElseIfDeclaration.NotAllowed"/>
+        <exclude name="PSR2.ControlStructures.SwitchDeclaration.BreakIndent"/>
+        <exclude name="PSR2.ControlStructures.SwitchDeclaration.BreakNotNewLine"/>
+        <exclude name="PSR2.ControlStructures.SwitchDeclaration.BodyOnNextLineCASE"/>
+        <exclude name="PSR2.ControlStructures.SwitchDeclaration.SpaceBeforeColonDEFAULT"/>
+        <exclude name="PSR2.ControlStructures.SwitchDeclaration.SpaceBeforeColonCASE"/>
+        <exclude name="PSR2.Files.EndFileNewline.TooMany"/>
+        <exclude name="PSR2.Files.EndFileNewline.NoneFound"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.Indent"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.EmptyLine"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.ContentAfterOpenBracket"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.SpaceBeforeOpenBracket"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.CloseBracketLine"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.SpaceBeforeCloseBracket"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.SpaceAfterOpenBracket"/>
+        <exclude name="PSR2.Methods.FunctionCallSignature.MultipleArguments"/>
+        <exclude name="PSR2.Methods.FunctionClosingBrace.SpacingBeforeClose"/>
+        <exclude name="PSR2.Methods.MethodDeclaration.StaticBeforeVisibility"/>
+        <exclude name="PSR2.Namespaces.NamespaceDeclaration.BlankLineAfter"/>
+        <exclude name="PSR2.Namespaces.UseDeclaration.SpaceAfterLastUse"/>
+
+        <exclude name="PSR12.Classes.ClassInstantiation.MissingParentheses"/>
+        <exclude name="PSR12.ControlStructures.BooleanOperatorPlacement.FoundMixed"/>
+        <exclude name="PSR12.ControlStructures.ControlStructureSpacing.FirstExpressionLine"/>
+        <exclude name="PSR12.ControlStructures.ControlStructureSpacing.CloseParenthesisLine"/>
+        <exclude name="PSR12.ControlStructures.ControlStructureSpacing.LineIndent"/>
+        <exclude name="PSR12.ControlStructures.ControlStructureSpacing.SpacingAfterOpenBrace"/>
+        <exclude name="PSR12.ControlStructures.ControlStructureSpacing.SpaceBeforeCloseBrace"/>
+        <exclude name="PSR12.Files.FileHeader.SpacingAfterBlock"/>
+        <exclude name="PSR12.Operators.OperatorSpacing.NoSpaceBefore"/>
+        <exclude name="PSR12.Operators.OperatorSpacing.NoSpaceAfter"/>
+
+        <exclude name="PSR12.Properties.ConstantVisibility.NotFound"/>
     </rule>
 
     <!-- disable some rules for certain paths, for legacy support -->
@@ -84,7 +159,10 @@
         <exclude-pattern>*/inc/parserutils.php</exclude-pattern>
         <exclude-pattern>*/inc/mail.php</exclude-pattern>
         <exclude-pattern>*/inc/init.php</exclude-pattern>
+        <exclude-pattern>*/inc/farm.php</exclude-pattern>
         <exclude-pattern>*/inc/fulltext.php</exclude-pattern>
+        <exclude-pattern>*/inc/pluginutils.php</exclude-pattern>
+        <exclude-pattern>*/inc/indexer.php</exclude-pattern>
         <exclude-pattern>*/inc/Mailer.class.php</exclude-pattern>
         <exclude-pattern>*/doku.php</exclude-pattern>
         <exclude-pattern>*/install.php</exclude-pattern>
diff --git a/_test/tests/inc/indexer_rename.test.php b/_test/tests/inc/indexer_rename.test.php
index d8c456f8e..2dbe6dcad 100644
--- a/_test/tests/inc/indexer_rename.test.php
+++ b/_test/tests/inc/indexer_rename.test.php
@@ -3,7 +3,7 @@
  * Test cases for the Doku_Indexer::renamePage and Doku_Indexer::renameMetaValue methods
  */
 class indexer_rename_test extends DokuWikiTest {
-    /** @var Doku_Indexer $indexer */
+    /** @var \dokuwiki\Search\Indexer $indexer */
     private $indexer;
 
     private $old_id = 'old_testid';
diff --git a/inc/Cache/Cache.php b/inc/Cache/Cache.php
index 599dc5f59..ab64da5cb 100644
--- a/inc/Cache/Cache.php
+++ b/inc/Cache/Cache.php
@@ -2,7 +2,7 @@
 
 namespace dokuwiki\Cache;
 
-use \dokuwiki\Debug\PropertyDeprecationHelper;
+use dokuwiki\Debug\PropertyDeprecationHelper;
 use dokuwiki\Extension\Event;
 
 /**
diff --git a/inc/Extension/SyntaxPlugin.php b/inc/Extension/SyntaxPlugin.php
index e5dda9bdc..ea8f51b4d 100644
--- a/inc/Extension/SyntaxPlugin.php
+++ b/inc/Extension/SyntaxPlugin.php
@@ -2,8 +2,8 @@
 
 namespace dokuwiki\Extension;
 
-use \Doku_Handler;
-use \Doku_Renderer;
+use Doku_Handler;
+use Doku_Renderer;
 
 /**
  * Syntax Plugin Prototype
diff --git a/inc/Search/Indexer.php b/inc/Search/Indexer.php
new file mode 100644
index 000000000..8770ac794
--- /dev/null
+++ b/inc/Search/Indexer.php
@@ -0,0 +1,1214 @@
+<?php
+
+namespace dokuwiki\Search;
+
+use dokuwiki\Extension\Event;
+
+/**
+ * Class that encapsulates operations on the indexer database.
+ *
+ * @author Tom N Harris <tnharris@whoopdedo.org>
+ */
+class Indexer {
+    /**
+     * @var array $pidCache Cache for getPID()
+     */
+    protected $pidCache = array();
+
+    /**
+     * Adds the contents of a page to the fulltext index
+     *
+     * The added text replaces previous words for the same page.
+     * An empty value erases the page.
+     *
+     * @param string    $page   a page name
+     * @param string    $text   the body of the page
+     * @return string|boolean  the function completed successfully
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     * @author Andreas Gohr <andi@splitbrain.org>
+     */
+    public function addPageWords($page, $text) {
+        if (!$this->lock())
+            return "locked";
+
+        // load known documents
+        $pid = $this->getPIDNoLock($page);
+        if ($pid === false) {
+            $this->unlock();
+            return false;
+        }
+
+        $pagewords = array();
+        // get word usage in page
+        $words = $this->getPageWords($text);
+        if ($words === false) {
+            $this->unlock();
+            return false;
+        }
+
+        if (!empty($words)) {
+            foreach (array_keys($words) as $wlen) {
+                $index = $this->getIndex('i', $wlen);
+                foreach ($words[$wlen] as $wid => $freq) {
+                    $idx = ($wid<count($index)) ? $index[$wid] : '';
+                    $index[$wid] = $this->updateTuple($idx, $pid, $freq);
+                    $pagewords[] = "$wlen*$wid";
+                }
+                if (!$this->saveIndex('i', $wlen, $index)) {
+                    $this->unlock();
+                    return false;
+                }
+            }
+        }
+
+        // Remove obsolete index entries
+        $pageword_idx = $this->getIndexKey('pageword', '', $pid);
+        if ($pageword_idx !== '') {
+            $oldwords = explode(':',$pageword_idx);
+            $delwords = array_diff($oldwords, $pagewords);
+            $upwords = array();
+            foreach ($delwords as $word) {
+                if ($word != '') {
+                    list($wlen, $wid) = explode('*', $word);
+                    $wid = (int)$wid;
+                    $upwords[$wlen][] = $wid;
+                }
+            }
+            foreach ($upwords as $wlen => $widx) {
+                $index = $this->getIndex('i', $wlen);
+                foreach ($widx as $wid) {
+                    $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
+                }
+                $this->saveIndex('i', $wlen, $index);
+            }
+        }
+        // Save the reverse index
+        $pageword_idx = join(':', $pagewords);
+        if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) {
+            $this->unlock();
+            return false;
+        }
+
+        $this->unlock();
+        return true;
+    }
+
+    /**
+     * Split the words in a page and add them to the index.
+     *
+     * @param string    $text   content of the page
+     * @return array            list of word IDs and number of times used
+     *
+     * @author Andreas Gohr <andi@splitbrain.org>
+     * @author Christopher Smith <chris@jalakai.co.uk>
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function getPageWords($text) {
+
+        $tokens = $this->tokenizer($text);
+        $tokens = array_count_values($tokens);  // count the frequency of each token
+
+        $words = array();
+        foreach ($tokens as $w=>$c) {
+            $l = wordlen($w);
+            if (isset($words[$l])){
+                $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
+            }else{
+                $words[$l] = array($w => $c);
+            }
+        }
+
+        // arrive here with $words = array(wordlen => array(word => frequency))
+        $word_idx_modified = false;
+        $index = array();   //resulting index
+        foreach (array_keys($words) as $wlen) {
+            $word_idx = $this->getIndex('w', $wlen);
+            foreach ($words[$wlen] as $word => $freq) {
+                $word = (string)$word;
+                $wid = array_search($word, $word_idx, true);
+                if ($wid === false) {
+                    $wid = count($word_idx);
+                    $word_idx[] = $word;
+                    $word_idx_modified = true;
+                }
+                if (!isset($index[$wlen]))
+                    $index[$wlen] = array();
+                $index[$wlen][$wid] = $freq;
+            }
+            // save back the word index
+            if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx))
+                return false;
+        }
+
+        return $index;
+    }
+
+    /**
+     * Add/update keys to/of the metadata index.
+     *
+     * Adding new keys does not remove other keys for the page.
+     * An empty value will erase the key.
+     * The $key parameter can be an array to add multiple keys. $value will
+     * not be used if $key is an array.
+     *
+     * @param string    $page   a page name
+     * @param mixed     $key    a key string or array of key=>value pairs
+     * @param mixed     $value  the value or list of values
+     * @return boolean|string     the function completed successfully
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     * @author Michael Hamann <michael@content-space.de>
+     */
+    public function addMetaKeys($page, $key, $value=null) {
+        if (!is_array($key)) {
+            $key = array($key => $value);
+        } elseif (!is_null($value)) {
+            // $key is array, but $value is not null
+            trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING);
+        }
+
+        if (!$this->lock())
+            return "locked";
+
+        // load known documents
+        $pid = $this->getPIDNoLock($page);
+        if ($pid === false) {
+            $this->unlock();
+            return false;
+        }
+
+        // Special handling for titles so the index file is simpler
+        if (array_key_exists('title', $key)) {
+            $value = $key['title'];
+            if (is_array($value)) {
+                $value = $value[0];
+            }
+            $this->saveIndexKey('title', '', $pid, $value);
+            unset($key['title']);
+        }
+
+        foreach ($key as $name => $values) {
+            $metaname = idx_cleanName($name);
+            $this->addIndexKey('metadata', '', $metaname);
+            $metaidx = $this->getIndex($metaname.'_i', '');
+            $metawords = $this->getIndex($metaname.'_w', '');
+            $addwords = false;
+
+            if (!is_array($values)) $values = array($values);
+
+            $val_idx = $this->getIndexKey($metaname.'_p', '', $pid);
+            if ($val_idx !== '') {
+                $val_idx = explode(':', $val_idx);
+                // -1 means remove, 0 keep, 1 add
+                $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1));
+            } else {
+                $val_idx = array();
+            }
+
+            foreach ($values as $val) {
+                $val = (string)$val;
+                if ($val !== "") {
+                    $id = array_search($val, $metawords, true);
+                    if ($id === false) {
+                        // didn't find $val, so we'll add it to the end of metawords and create a placeholder in metaidx
+                        $id = count($metawords);
+                        $metawords[$id] = $val;
+                        $metaidx[$id] = '';
+                        $addwords = true;
+                    }
+                    // test if value is already in the index
+                    if (isset($val_idx[$id]) && $val_idx[$id] <= 0){
+                        $val_idx[$id] = 0;
+                    } else { // else add it
+                        $val_idx[$id] = 1;
+                    }
+                }
+            }
+
+            if ($addwords) {
+                $this->saveIndex($metaname.'_w', '', $metawords);
+            }
+            $vals_changed = false;
+            foreach ($val_idx as $id => $action) {
+                if ($action == -1) {
+                    $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0);
+                    $vals_changed = true;
+                    unset($val_idx[$id]);
+                } elseif ($action == 1) {
+                    $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1);
+                    $vals_changed = true;
+                }
+            }
+
+            if ($vals_changed) {
+                $this->saveIndex($metaname.'_i', '', $metaidx);
+                $val_idx = implode(':', array_keys($val_idx));
+                $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx);
+            }
+
+            unset($metaidx);
+            unset($metawords);
+        }
+
+        $this->unlock();
+        return true;
+    }
+
+    /**
+     * Rename a page in the search index without changing the indexed content. This function doesn't check if the
+     * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the
+     * indexer and it deletes all previously indexed content of the new page.
+     *
+     * @param string $oldpage The old page name
+     * @param string $newpage The new page name
+     * @return string|bool If the page was successfully renamed, can be a message in the case of an error
+     */
+    public function renamePage($oldpage, $newpage) {
+        if (!$this->lock()) return 'locked';
+
+        $pages = $this->getPages();
+
+        $id = array_search($oldpage, $pages, true);
+        if ($id === false) {
+            $this->unlock();
+            return 'page is not in index';
+        }
+
+        $new_id = array_search($newpage, $pages, true);
+        if ($new_id !== false) {
+            // make sure the page is not in the index anymore
+            if ($this->deletePageNoLock($newpage) !== true) {
+                return false;
+            }
+
+            $pages[$new_id] = 'deleted:'.time().rand(0, 9999);
+        }
+
+        $pages[$id] = $newpage;
+
+        // update index
+        if (!$this->saveIndex('page', '', $pages)) {
+            $this->unlock();
+            return false;
+        }
+
+        // reset the pid cache
+        $this->pidCache = array();
+
+        $this->unlock();
+        return true;
+    }
+
+    /**
+     * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages
+     * will be updated.
+     *
+     * @param string $key       The metadata key of which a value shall be changed
+     * @param string $oldvalue  The old value that shall be renamed
+     * @param string $newvalue  The new value to which the old value shall be renamed, if exists values will be merged
+     * @return bool|string      If renaming the value has been successful, false or error message on error.
+     */
+    public function renameMetaValue($key, $oldvalue, $newvalue) {
+        if (!$this->lock()) return 'locked';
+
+        // change the relation references index
+        $metavalues = $this->getIndex($key, '_w');
+        $oldid = array_search($oldvalue, $metavalues, true);
+        if ($oldid !== false) {
+            $newid = array_search($newvalue, $metavalues, true);
+            if ($newid !== false) {
+                // free memory
+                unset ($metavalues);
+
+                // okay, now we have two entries for the same value. we need to merge them.
+                $indexline = $this->getIndexKey($key.'_i', '', $oldid);
+                if ($indexline != '') {
+                    $newindexline = $this->getIndexKey($key.'_i', '', $newid);
+                    $pagekeys     = $this->getIndex($key.'_p', '');
+                    $parts = explode(':', $indexline);
+                    foreach ($parts as $part) {
+                        list($id, $count) = explode('*', $part);
+                        $newindexline =  $this->updateTuple($newindexline, $id, $count);
+
+                        $keyline = explode(':', $pagekeys[$id]);
+                        // remove old meta value
+                        $keyline = array_diff($keyline, array($oldid));
+                        // add new meta value when not already present
+                        if (!in_array($newid, $keyline)) {
+                            array_push($keyline, $newid);
+                        }
+                        $pagekeys[$id] = implode(':', $keyline);
+                    }
+                    $this->saveIndex($key.'_p', '', $pagekeys);
+                    unset($pagekeys);
+                    $this->saveIndexKey($key.'_i', '', $oldid, '');
+                    $this->saveIndexKey($key.'_i', '', $newid, $newindexline);
+                }
+            } else {
+                $metavalues[$oldid] = $newvalue;
+                if (!$this->saveIndex($key.'_w', '', $metavalues)) {
+                    $this->unlock();
+                    return false;
+                }
+            }
+        }
+
+        $this->unlock();
+        return true;
+    }
+
+    /**
+     * Remove a page from the index
+     *
+     * Erases entries in all known indexes.
+     *
+     * @param string    $page   a page name
+     * @return string|boolean  the function completed successfully
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    public function deletePage($page) {
+        if (!$this->lock())
+            return "locked";
+
+        $result = $this->deletePageNoLock($page);
+
+        $this->unlock();
+
+        return $result;
+    }
+
+    /**
+     * Remove a page from the index without locking the index, only use this function if the index is already locked
+     *
+     * Erases entries in all known indexes.
+     *
+     * @param string    $page   a page name
+     * @return boolean          the function completed successfully
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function deletePageNoLock($page) {
+        // load known documents
+        $pid = $this->getPIDNoLock($page);
+        if ($pid === false) {
+            return false;
+        }
+
+        // Remove obsolete index entries
+        $pageword_idx = $this->getIndexKey('pageword', '', $pid);
+        if ($pageword_idx !== '') {
+            $delwords = explode(':',$pageword_idx);
+            $upwords = array();
+            foreach ($delwords as $word) {
+                if ($word != '') {
+                    list($wlen,$wid) = explode('*', $word);
+                    $wid = (int)$wid;
+                    $upwords[$wlen][] = $wid;
+                }
+            }
+            foreach ($upwords as $wlen => $widx) {
+                $index = $this->getIndex('i', $wlen);
+                foreach ($widx as $wid) {
+                    $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
+                }
+                $this->saveIndex('i', $wlen, $index);
+            }
+        }
+        // Save the reverse index
+        if (!$this->saveIndexKey('pageword', '', $pid, "")) {
+            return false;
+        }
+
+        $this->saveIndexKey('title', '', $pid, "");
+        $keyidx = $this->getIndex('metadata', '');
+        foreach ($keyidx as $metaname) {
+            $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid));
+            $meta_idx = $this->getIndex($metaname.'_i', '');
+            foreach ($val_idx as $id) {
+                if ($id === '') continue;
+                $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0);
+            }
+            $this->saveIndex($metaname.'_i', '', $meta_idx);
+            $this->saveIndexKey($metaname.'_p', '', $pid, '');
+        }
+
+        return true;
+    }
+
+    /**
+     * Clear the whole index
+     *
+     * @return bool If the index has been cleared successfully
+     */
+    public function clear() {
+        global $conf;
+
+        if (!$this->lock()) return false;
+
+        @unlink($conf['indexdir'].'/page.idx');
+        @unlink($conf['indexdir'].'/title.idx');
+        @unlink($conf['indexdir'].'/pageword.idx');
+        @unlink($conf['indexdir'].'/metadata.idx');
+        $dir = @opendir($conf['indexdir']);
+        if($dir!==false){
+            while(($f = readdir($dir)) !== false){
+                if(substr($f,-4)=='.idx' &&
+                    (substr($f,0,1)=='i' || substr($f,0,1)=='w'
+                        || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx'))
+                    @unlink($conf['indexdir']."/$f");
+            }
+        }
+        @unlink($conf['indexdir'].'/lengths.idx');
+
+        // clear the pid cache
+        $this->pidCache = array();
+
+        $this->unlock();
+        return true;
+    }
+
+    /**
+     * Split the text into words for fulltext search
+     *
+     * TODO: does this also need &$stopwords ?
+     *
+     * @triggers INDEXER_TEXT_PREPARE
+     * This event allows plugins to modify the text before it gets tokenized.
+     * Plugins intercepting this event should also intercept INDEX_VERSION_GET
+     *
+     * @param string    $text   plain text
+     * @param boolean   $wc     are wildcards allowed?
+     * @return array            list of words in the text
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     * @author Andreas Gohr <andi@splitbrain.org>
+     */
+    public function tokenizer($text, $wc=false) {
+        $wc = ($wc) ? '' : '\*';
+        $stopwords =& idx_get_stopwords();
+
+        // prepare the text to be tokenized
+        $evt = new Event('INDEXER_TEXT_PREPARE', $text);
+        if ($evt->advise_before(true)) {
+            if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
+                $text = \dokuwiki\Utf8\Asian::separateAsianWords($text);
+            }
+        }
+        $evt->advise_after();
+        unset($evt);
+
+        $text = strtr($text,
+                      array(
+                          "\r" => ' ',
+                          "\n" => ' ',
+                          "\t" => ' ',
+                          "\xC2\xAD" => '', //soft-hyphen
+                      )
+        );
+        if (preg_match('/[^0-9A-Za-z ]/u', $text))
+            $text = \dokuwiki\Utf8\Clean::stripspecials($text, ' ', '\._\-:'.$wc);
+
+        $wordlist = explode(' ', $text);
+        foreach ($wordlist as $i => $word) {
+            $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ?
+                \dokuwiki\Utf8\PhpString::strtolower($word) : strtolower($word);
+        }
+
+        foreach ($wordlist as $i => $word) {
+            if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH)
+                || array_search($word, $stopwords, true) !== false)
+                unset($wordlist[$i]);
+        }
+        return array_values($wordlist);
+    }
+
+    /**
+     * Get the numeric PID of a page
+     *
+     * @param string $page The page to get the PID for
+     * @return bool|int The page id on success, false on error
+     */
+    public function getPID($page) {
+        // return PID without locking when it is in the cache
+        if (isset($this->pidCache[$page])) return $this->pidCache[$page];
+
+        if (!$this->lock())
+            return false;
+
+        // load known documents
+        $pid = $this->getPIDNoLock($page);
+        if ($pid === false) {
+            $this->unlock();
+            return false;
+        }
+
+        $this->unlock();
+        return $pid;
+    }
+
+    /**
+     * Get the numeric PID of a page without locking the index.
+     * Only use this function when the index is already locked.
+     *
+     * @param string $page The page to get the PID for
+     * @return bool|int The page id on success, false on error
+     */
+    protected function getPIDNoLock($page) {
+        // avoid expensive addIndexKey operation for the most recently requested pages by using a cache
+        if (isset($this->pidCache[$page])) return $this->pidCache[$page];
+        $pid = $this->addIndexKey('page', '', $page);
+        // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently
+        // added item will be requested again
+        if (count($this->pidCache) > 10) array_shift($this->pidCache);
+        $this->pidCache[$page] = $pid;
+        return $pid;
+    }
+
+    /**
+     * Get the page id of a numeric PID
+     *
+     * @param int $pid The PID to get the page id for
+     * @return string The page id
+     */
+    public function getPageFromPID($pid) {
+        return $this->getIndexKey('page', '', $pid);
+    }
+
+    /**
+     * Find pages in the fulltext index containing the words,
+     *
+     * The search words must be pre-tokenized, meaning only letters and
+     * numbers with an optional wildcard
+     *
+     * The returned array will have the original tokens as key. The values
+     * in the returned list is an array with the page names as keys and the
+     * number of times that token appears on the page as value.
+     *
+     * @param array  $tokens list of words to search for
+     * @return array         list of page names with usage counts
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     * @author Andreas Gohr <andi@splitbrain.org>
+     */
+    public function lookup(&$tokens) {
+        $result = array();
+        $wids = $this->getIndexWords($tokens, $result);
+        if (empty($wids)) return array();
+        // load known words and documents
+        $page_idx = $this->getIndex('page', '');
+        $docs = array();
+        foreach (array_keys($wids) as $wlen) {
+            $wids[$wlen] = array_unique($wids[$wlen]);
+            $index = $this->getIndex('i', $wlen);
+            foreach($wids[$wlen] as $ixid) {
+                if ($ixid < count($index))
+                    $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]);
+            }
+        }
+        // merge found pages into final result array
+        $final = array();
+        foreach ($result as $word => $res) {
+            $final[$word] = array();
+            foreach ($res as $wid) {
+                // handle the case when ($ixid < count($index)) has been false
+                // and thus $docs[$wid] hasn't been set.
+                if (!isset($docs[$wid])) continue;
+                $hits = &$docs[$wid];
+                foreach ($hits as $hitkey => $hitcnt) {
+                    // make sure the document still exists
+                    if (!page_exists($hitkey, '', false)) continue;
+                    if (!isset($final[$word][$hitkey]))
+                        $final[$word][$hitkey] = $hitcnt;
+                    else
+                        $final[$word][$hitkey] += $hitcnt;
+                }
+            }
+        }
+        return $final;
+    }
+
+    /**
+     * Find pages containing a metadata key.
+     *
+     * The metadata values are compared as case-sensitive strings. Pass a
+     * callback function that returns true or false to use a different
+     * comparison function. The function will be called with the $value being
+     * searched for as the first argument, and the word in the index as the
+     * second argument. The function preg_match can be used directly if the
+     * values are regexes.
+     *
+     * @param string    $key    name of the metadata key to look for
+     * @param string    $value  search term to look for, must be a string or array of strings
+     * @param callback  $func   comparison function
+     * @return array            lists with page names, keys are query values if $value is array
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     * @author Michael Hamann <michael@content-space.de>
+     */
+    public function lookupKey($key, &$value, $func=null) {
+        if (!is_array($value))
+            $value_array = array($value);
+        else
+            $value_array =& $value;
+
+        // the matching ids for the provided value(s)
+        $value_ids = array();
+
+        $metaname = idx_cleanName($key);
+
+        // get all words in order to search the matching ids
+        if ($key == 'title') {
+            $words = $this->getIndex('title', '');
+        } else {
+            $words = $this->getIndex($metaname.'_w', '');
+        }
+
+        if (!is_null($func)) {
+            foreach ($value_array as $val) {
+                foreach ($words as $i => $word) {
+                    if (call_user_func_array($func, array($val, $word)))
+                        $value_ids[$i][] = $val;
+                }
+            }
+        } else {
+            foreach ($value_array as $val) {
+                $xval = $val;
+                $caret = '^';
+                $dollar = '$';
+                // check for wildcards
+                if (substr($xval, 0, 1) == '*') {
+                    $xval = substr($xval, 1);
+                    $caret = '';
+                }
+                if (substr($xval, -1, 1) == '*') {
+                    $xval = substr($xval, 0, -1);
+                    $dollar = '';
+                }
+                if (!$caret || !$dollar) {
+                    $re = $caret.preg_quote($xval, '/').$dollar;
+                    foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i)
+                        $value_ids[$i][] = $val;
+                } else {
+                    if (($i = array_search($val, $words, true)) !== false)
+                        $value_ids[$i][] = $val;
+                }
+            }
+        }
+
+        unset($words); // free the used memory
+
+        // initialize the result so it won't be null
+        $result = array();
+        foreach ($value_array as $val) {
+            $result[$val] = array();
+        }
+
+        $page_idx = $this->getIndex('page', '');
+
+        // Special handling for titles
+        if ($key == 'title') {
+            foreach ($value_ids as $pid => $val_list) {
+                $page = $page_idx[$pid];
+                foreach ($val_list as $val) {
+                    $result[$val][] = $page;
+                }
+            }
+        } else {
+            // load all lines and pages so the used lines can be taken and matched with the pages
+            $lines = $this->getIndex($metaname.'_i', '');
+
+            foreach ($value_ids as $value_id => $val_list) {
+                // parse the tuples of the form page_id*1:page2_id*1 and so on, return value
+                // is an array with page_id => 1, page2_id => 1 etc. so take the keys only
+                $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id]));
+                foreach ($val_list as $val) {
+                    $result[$val] = array_merge($result[$val], $pages);
+                }
+            }
+        }
+        if (!is_array($value)) $result = $result[$value];
+        return $result;
+    }
+
+    /**
+     * Find the index ID of each search term.
+     *
+     * The query terms should only contain valid characters, with a '*' at
+     * either the beginning or end of the word (or both).
+     * The $result parameter can be used to merge the index locations with
+     * the appropriate query term.
+     *
+     * @param array  $words  The query terms.
+     * @param array  $result Set to word => array("length*id" ...)
+     * @return array         Set to length => array(id ...)
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function getIndexWords(&$words, &$result) {
+        $tokens = array();
+        $tokenlength = array();
+        $tokenwild = array();
+        foreach ($words as $word) {
+            $result[$word] = array();
+            $caret = '^';
+            $dollar = '$';
+            $xword = $word;
+            $wlen = wordlen($word);
+
+            // check for wildcards
+            if (substr($xword, 0, 1) == '*') {
+                $xword = substr($xword, 1);
+                $caret = '';
+                $wlen -= 1;
+            }
+            if (substr($xword, -1, 1) == '*') {
+                $xword = substr($xword, 0, -1);
+                $dollar = '';
+                $wlen -= 1;
+            }
+            if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword))
+                continue;
+            if (!isset($tokens[$xword]))
+                $tokenlength[$wlen][] = $xword;
+            if (!$caret || !$dollar) {
+                $re = $caret.preg_quote($xword, '/').$dollar;
+                $tokens[$xword][] = array($word, '/'.$re.'/');
+                if (!isset($tokenwild[$xword]))
+                    $tokenwild[$xword] = $wlen;
+            } else {
+                $tokens[$xword][] = array($word, null);
+            }
+        }
+        asort($tokenwild);
+        // $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
+        // $tokenlength = array( base word length => base word ... )
+        // $tokenwild = array( base word => base word length ... )
+        $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
+        $indexes_known = $this->indexLengths($length_filter);
+        if (!empty($tokenwild)) sort($indexes_known);
+        // get word IDs
+        $wids = array();
+        foreach ($indexes_known as $ixlen) {
+            $word_idx = $this->getIndex('w', $ixlen);
+            // handle exact search
+            if (isset($tokenlength[$ixlen])) {
+                foreach ($tokenlength[$ixlen] as $xword) {
+                    $wid = array_search($xword, $word_idx, true);
+                    if ($wid !== false) {
+                        $wids[$ixlen][] = $wid;
+                        foreach ($tokens[$xword] as $w)
+                            $result[$w[0]][] = "$ixlen*$wid";
+                    }
+                }
+            }
+            // handle wildcard search
+            foreach ($tokenwild as $xword => $wlen) {
+                if ($wlen >= $ixlen) break;
+                foreach ($tokens[$xword] as $w) {
+                    if (is_null($w[1])) continue;
+                    foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) {
+                        $wids[$ixlen][] = $wid;
+                        $result[$w[0]][] = "$ixlen*$wid";
+                    }
+                }
+            }
+        }
+        return $wids;
+    }
+
+    /**
+     * Return a list of all pages
+     * Warning: pages may not exist!
+     *
+     * @param string    $key    list only pages containing the metadata key (optional)
+     * @return array            list of page names
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    public function getPages($key=null) {
+        $page_idx = $this->getIndex('page', '');
+        if (is_null($key)) return $page_idx;
+
+        $metaname = idx_cleanName($key);
+
+        // Special handling for titles
+        if ($key == 'title') {
+            $title_idx = $this->getIndex('title', '');
+            array_splice($page_idx, count($title_idx));
+            foreach ($title_idx as $i => $title)
+                if ($title === "") unset($page_idx[$i]);
+            return array_values($page_idx);
+        }
+
+        $pages = array();
+        $lines = $this->getIndex($metaname.'_i', '');
+        foreach ($lines as $line) {
+            $pages = array_merge($pages, $this->parseTuples($page_idx, $line));
+        }
+        return array_keys($pages);
+    }
+
+    /**
+     * Return a list of words sorted by number of times used
+     *
+     * @param int       $min    bottom frequency threshold
+     * @param int       $max    upper frequency limit. No limit if $max<$min
+     * @param int       $minlen minimum length of words to count
+     * @param string    $key    metadata key to list. Uses the fulltext index if not given
+     * @return array            list of words as the keys and frequency as values
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    public function histogram($min=1, $max=0, $minlen=3, $key=null) {
+        if ($min < 1)
+            $min = 1;
+        if ($max < $min)
+            $max = 0;
+
+        $result = array();
+
+        if ($key == 'title') {
+            $index = $this->getIndex('title', '');
+            $index = array_count_values($index);
+            foreach ($index as $val => $cnt) {
+                if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen)
+                    $result[$val] = $cnt;
+            }
+        }
+        elseif (!is_null($key)) {
+            $metaname = idx_cleanName($key);
+            $index = $this->getIndex($metaname.'_i', '');
+            $val_idx = array();
+            foreach ($index as $wid => $line) {
+                $freq = $this->countTuples($line);
+                if ($freq >= $min && (!$max || $freq <= $max))
+                    $val_idx[$wid] = $freq;
+            }
+            if (!empty($val_idx)) {
+                $words = $this->getIndex($metaname.'_w', '');
+                foreach ($val_idx as $wid => $freq) {
+                    if (strlen($words[$wid]) >= $minlen)
+                        $result[$words[$wid]] = $freq;
+                }
+            }
+        }
+        else {
+            $lengths = idx_listIndexLengths();
+            foreach ($lengths as $length) {
+                if ($length < $minlen) continue;
+                $index = $this->getIndex('i', $length);
+                $words = null;
+                foreach ($index as $wid => $line) {
+                    $freq = $this->countTuples($line);
+                    if ($freq >= $min && (!$max || $freq <= $max)) {
+                        if ($words === null)
+                            $words = $this->getIndex('w', $length);
+                        $result[$words[$wid]] = $freq;
+                    }
+                }
+            }
+        }
+
+        arsort($result);
+        return $result;
+    }
+
+    /**
+     * Lock the indexer.
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     *
+     * @return bool|string
+     */
+    protected function lock() {
+        global $conf;
+        $status = true;
+        $run = 0;
+        $lock = $conf['lockdir'].'/_indexer.lock';
+        while (!@mkdir($lock, $conf['dmode'])) {
+            usleep(50);
+            if(is_dir($lock) && time()-@filemtime($lock) > 60*5){
+                // looks like a stale lock - remove it
+                if (!@rmdir($lock)) {
+                    $status = "removing the stale lock failed";
+                    return false;
+                } else {
+                    $status = "stale lock removed";
+                }
+            }elseif($run++ == 1000){
+                // we waited 5 seconds for that lock
+                return false;
+            }
+        }
+        if (!empty($conf['dperm'])) {
+            chmod($lock, $conf['dperm']);
+        }
+        return $status;
+    }
+
+    /**
+     * Release the indexer lock.
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     *
+     * @return bool
+     */
+    protected function unlock() {
+        global $conf;
+        @rmdir($conf['lockdir'].'/_indexer.lock');
+        return true;
+    }
+
+    /**
+     * Retrieve the entire index.
+     *
+     * The $suffix argument is for an index that is split into
+     * multiple parts. Different index files should use different
+     * base names.
+     *
+     * @param string    $idx    name of the index
+     * @param string    $suffix subpart identifier
+     * @return array            list of lines without CR or LF
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function getIndex($idx, $suffix) {
+        global $conf;
+        $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
+        if (!file_exists($fn)) return array();
+        return file($fn, FILE_IGNORE_NEW_LINES);
+    }
+
+    /**
+     * Replace the contents of the index with an array.
+     *
+     * @param string    $idx    name of the index
+     * @param string    $suffix subpart identifier
+     * @param array     $lines  list of lines without LF
+     * @return bool             If saving succeeded
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function saveIndex($idx, $suffix, &$lines) {
+        global $conf;
+        $fn = $conf['indexdir'].'/'.$idx.$suffix;
+        $fh = @fopen($fn.'.tmp', 'w');
+        if (!$fh) return false;
+        fwrite($fh, join("\n", $lines));
+        if (!empty($lines))
+            fwrite($fh, "\n");
+        fclose($fh);
+        if (isset($conf['fperm']))
+            chmod($fn.'.tmp', $conf['fperm']);
+        io_rename($fn.'.tmp', $fn.'.idx');
+        return true;
+    }
+
+    /**
+     * Retrieve a line from the index.
+     *
+     * @param string    $idx    name of the index
+     * @param string    $suffix subpart identifier
+     * @param int       $id     the line number
+     * @return string           a line with trailing whitespace removed
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function getIndexKey($idx, $suffix, $id) {
+        global $conf;
+        $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
+        if (!file_exists($fn)) return '';
+        $fh = @fopen($fn, 'r');
+        if (!$fh) return '';
+        $ln = -1;
+        while (($line = fgets($fh)) !== false) {
+            if (++$ln == $id) break;
+        }
+        fclose($fh);
+        return rtrim((string)$line);
+    }
+
+    /**
+     * Write a line into the index.
+     *
+     * @param string    $idx    name of the index
+     * @param string    $suffix subpart identifier
+     * @param int       $id     the line number
+     * @param string    $line   line to write
+     * @return bool             If saving succeeded
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function saveIndexKey($idx, $suffix, $id, $line) {
+        global $conf;
+        if (substr($line, -1) != "\n")
+            $line .= "\n";
+        $fn = $conf['indexdir'].'/'.$idx.$suffix;
+        $fh = @fopen($fn.'.tmp', 'w');
+        if (!$fh) return false;
+        $ih = @fopen($fn.'.idx', 'r');
+        if ($ih) {
+            $ln = -1;
+            while (($curline = fgets($ih)) !== false) {
+                fwrite($fh, (++$ln == $id) ? $line : $curline);
+            }
+            if ($id > $ln) {
+                while ($id > ++$ln)
+                    fwrite($fh, "\n");
+                fwrite($fh, $line);
+            }
+            fclose($ih);
+        } else {
+            $ln = -1;
+            while ($id > ++$ln)
+                fwrite($fh, "\n");
+            fwrite($fh, $line);
+        }
+        fclose($fh);
+        if (isset($conf['fperm']))
+            chmod($fn.'.tmp', $conf['fperm']);
+        io_rename($fn.'.tmp', $fn.'.idx');
+        return true;
+    }
+
+    /**
+     * Retrieve or insert a value in the index.
+     *
+     * @param string    $idx    name of the index
+     * @param string    $suffix subpart identifier
+     * @param string    $value  line to find in the index
+     * @return int|bool          line number of the value in the index or false if writing the index failed
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     */
+    protected function addIndexKey($idx, $suffix, $value) {
+        $index = $this->getIndex($idx, $suffix);
+        $id = array_search($value, $index, true);
+        if ($id === false) {
+            $id = count($index);
+            $index[$id] = $value;
+            if (!$this->saveIndex($idx, $suffix, $index)) {
+                trigger_error("Failed to write $idx index", E_USER_ERROR);
+                return false;
+            }
+        }
+        return $id;
+    }
+
+    /**
+     * Get the list of lengths indexed in the wiki.
+     *
+     * Read the index directory or a cache file and returns
+     * a sorted array of lengths of the words used in the wiki.
+     *
+     * @author YoBoY <yoboy.leguesh@gmail.com>
+     *
+     * @return array
+     */
+    protected function listIndexLengths() {
+        return idx_listIndexLengths();
+    }
+
+    /**
+     * Get the word lengths that have been indexed.
+     *
+     * Reads the index directory and returns an array of lengths
+     * that there are indices for.
+     *
+     * @author YoBoY <yoboy.leguesh@gmail.com>
+     *
+     * @param array|int $filter
+     * @return array
+     */
+    protected function indexLengths($filter) {
+        global $conf;
+        $idx = array();
+        if (is_array($filter)) {
+            // testing if index files exist only
+            $path = $conf['indexdir']."/i";
+            foreach ($filter as $key => $value) {
+                if (file_exists($path.$key.'.idx'))
+                    $idx[] = $key;
+            }
+        } else {
+            $lengths = idx_listIndexLengths();
+            foreach ($lengths as $key => $length) {
+                // keep all the values equal or superior
+                if ((int)$length >= (int)$filter)
+                    $idx[] = $length;
+            }
+        }
+        return $idx;
+    }
+
+    /**
+     * Insert or replace a tuple in a line.
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     *
+     * @param string $line
+     * @param string|int $id
+     * @param int    $count
+     * @return string
+     */
+    protected function updateTuple($line, $id, $count) {
+        if ($line != ''){
+            $line = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $line);
+        }
+        $line = trim($line, ':');
+        if ($count) {
+            if ($line) {
+                return "$id*$count:".$line;
+            } else {
+                return "$id*$count";
+            }
+        }
+        return $line;
+    }
+
+    /**
+     * Split a line into an array of tuples.
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     * @author Andreas Gohr <andi@splitbrain.org>
+     *
+     * @param array $keys
+     * @param string $line
+     * @return array
+     */
+    protected function parseTuples(&$keys, $line) {
+        $result = array();
+        if ($line == '') return $result;
+        $parts = explode(':', $line);
+        foreach ($parts as $tuple) {
+            if ($tuple === '') continue;
+            list($key, $cnt) = explode('*', $tuple);
+            if (!$cnt) continue;
+            $key = $keys[$key];
+            if ($key === false || is_null($key)) continue;
+            $result[$key] = $cnt;
+        }
+        return $result;
+    }
+
+    /**
+     * Sum the counts in a list of tuples.
+     *
+     * @author Tom N Harris <tnharris@whoopdedo.org>
+     *
+     * @param string $line
+     * @return int
+     */
+    protected function countTuples($line) {
+        $freq = 0;
+        $parts = explode(':', $line);
+        foreach ($parts as $tuple) {
+            if ($tuple === '') continue;
+            list(/* $pid */, $cnt) = explode('*', $tuple);
+            $freq += (int)$cnt;
+        }
+        return $freq;
+    }
+}
diff --git a/inc/deprecated.php b/inc/deprecated.php
index d3de76080..205037344 100644
--- a/inc/deprecated.php
+++ b/inc/deprecated.php
@@ -563,3 +563,8 @@ class Subscription {
         $manager->notifyAddresses($data);
     }
 }
+
+/**
+ * @deprecated 2019-12-29 use \dokuwiki\Search\Indexer
+ */
+class Doku_Indexer extends \dokuwiki\Search\Indexer {};
diff --git a/inc/fulltext.php b/inc/fulltext.php
index 48fe28da2..670f048d3 100644
--- a/inc/fulltext.php
+++ b/inc/fulltext.php
@@ -629,8 +629,8 @@ function ft_resultComplement($args) {
  * @author Andreas Gohr <andi@splitbrain.org>
  * @author Kazutaka Miyasaka <kazmiya@gmail.com>
  *
- * @param Doku_Indexer $Indexer
- * @param string $query search query
+ * @param dokuwiki\Search\Indexer $Indexer
+ * @param string                  $query search query
  * @return array of search formulas
  */
 function ft_queryParser($Indexer, $query){
@@ -864,10 +864,10 @@ function ft_queryParser($Indexer, $query){
  *
  * @author Kazutaka Miyasaka <kazmiya@gmail.com>
  *
- * @param Doku_Indexer $Indexer
- * @param string       $term
- * @param bool         $consider_asian
- * @param bool         $phrase_mode
+ * @param dokuwiki\Search\Indexer $Indexer
+ * @param string                  $term
+ * @param bool                    $consider_asian
+ * @param bool                    $phrase_mode
  * @return string
  */
 function ft_termParser($Indexer, $term, $consider_asian = true, $phrase_mode = false) {
diff --git a/inc/indexer.php b/inc/indexer.php
index f2c8a9aa8..ab02b8ea2 100644
--- a/inc/indexer.php
+++ b/inc/indexer.php
@@ -8,6 +8,7 @@
  */
 
 use dokuwiki\Extension\Event;
+use dokuwiki\Search\Indexer;
 
 // Version tag used to force rebuild on upgrade
 define('INDEXER_VERSION', 8);
@@ -67,1225 +68,16 @@ function wordlen($w){
 }
 
 /**
- * Class that encapsulates operations on the indexer database.
- *
- * @author Tom N Harris <tnharris@whoopdedo.org>
- */
-class Doku_Indexer {
-    /**
-     * @var array $pidCache Cache for getPID()
-     */
-    protected $pidCache = array();
-
-    /**
-     * Adds the contents of a page to the fulltext index
-     *
-     * The added text replaces previous words for the same page.
-     * An empty value erases the page.
-     *
-     * @param string    $page   a page name
-     * @param string    $text   the body of the page
-     * @return string|boolean  the function completed successfully
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     * @author Andreas Gohr <andi@splitbrain.org>
-     */
-    public function addPageWords($page, $text) {
-        if (!$this->lock())
-            return "locked";
-
-        // load known documents
-        $pid = $this->getPIDNoLock($page);
-        if ($pid === false) {
-            $this->unlock();
-            return false;
-        }
-
-        $pagewords = array();
-        // get word usage in page
-        $words = $this->getPageWords($text);
-        if ($words === false) {
-            $this->unlock();
-            return false;
-        }
-
-        if (!empty($words)) {
-            foreach (array_keys($words) as $wlen) {
-                $index = $this->getIndex('i', $wlen);
-                foreach ($words[$wlen] as $wid => $freq) {
-                    $idx = ($wid<count($index)) ? $index[$wid] : '';
-                    $index[$wid] = $this->updateTuple($idx, $pid, $freq);
-                    $pagewords[] = "$wlen*$wid";
-                }
-                if (!$this->saveIndex('i', $wlen, $index)) {
-                    $this->unlock();
-                    return false;
-                }
-            }
-        }
-
-        // Remove obsolete index entries
-        $pageword_idx = $this->getIndexKey('pageword', '', $pid);
-        if ($pageword_idx !== '') {
-            $oldwords = explode(':',$pageword_idx);
-            $delwords = array_diff($oldwords, $pagewords);
-            $upwords = array();
-            foreach ($delwords as $word) {
-                if ($word != '') {
-                    list($wlen,$wid) = explode('*', $word);
-                    $wid = (int)$wid;
-                    $upwords[$wlen][] = $wid;
-                }
-            }
-            foreach ($upwords as $wlen => $widx) {
-                $index = $this->getIndex('i', $wlen);
-                foreach ($widx as $wid) {
-                    $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
-                }
-                $this->saveIndex('i', $wlen, $index);
-            }
-        }
-        // Save the reverse index
-        $pageword_idx = join(':', $pagewords);
-        if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) {
-            $this->unlock();
-            return false;
-        }
-
-        $this->unlock();
-        return true;
-    }
-
-    /**
-     * Split the words in a page and add them to the index.
-     *
-     * @param string    $text   content of the page
-     * @return array            list of word IDs and number of times used
-     *
-     * @author Andreas Gohr <andi@splitbrain.org>
-     * @author Christopher Smith <chris@jalakai.co.uk>
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function getPageWords($text) {
-
-        $tokens = $this->tokenizer($text);
-        $tokens = array_count_values($tokens);  // count the frequency of each token
-
-        $words = array();
-        foreach ($tokens as $w=>$c) {
-            $l = wordlen($w);
-            if (isset($words[$l])){
-                $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
-            }else{
-                $words[$l] = array($w => $c);
-            }
-        }
-
-        // arrive here with $words = array(wordlen => array(word => frequency))
-        $word_idx_modified = false;
-        $index = array();   //resulting index
-        foreach (array_keys($words) as $wlen) {
-            $word_idx = $this->getIndex('w', $wlen);
-            foreach ($words[$wlen] as $word => $freq) {
-                $word = (string)$word;
-                $wid = array_search($word, $word_idx, true);
-                if ($wid === false) {
-                    $wid = count($word_idx);
-                    $word_idx[] = $word;
-                    $word_idx_modified = true;
-                }
-                if (!isset($index[$wlen]))
-                    $index[$wlen] = array();
-                $index[$wlen][$wid] = $freq;
-            }
-            // save back the word index
-            if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx))
-                return false;
-        }
-
-        return $index;
-    }
-
-    /**
-     * Add/update keys to/of the metadata index.
-     *
-     * Adding new keys does not remove other keys for the page.
-     * An empty value will erase the key.
-     * The $key parameter can be an array to add multiple keys. $value will
-     * not be used if $key is an array.
-     *
-     * @param string    $page   a page name
-     * @param mixed     $key    a key string or array of key=>value pairs
-     * @param mixed     $value  the value or list of values
-     * @return boolean|string     the function completed successfully
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     * @author Michael Hamann <michael@content-space.de>
-     */
-    public function addMetaKeys($page, $key, $value=null) {
-        if (!is_array($key)) {
-            $key = array($key => $value);
-        } elseif (!is_null($value)) {
-            // $key is array, but $value is not null
-            trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING);
-        }
-
-        if (!$this->lock())
-            return "locked";
-
-        // load known documents
-        $pid = $this->getPIDNoLock($page);
-        if ($pid === false) {
-            $this->unlock();
-            return false;
-        }
-
-        // Special handling for titles so the index file is simpler
-        if (array_key_exists('title', $key)) {
-            $value = $key['title'];
-            if (is_array($value)) {
-                $value = $value[0];
-            }
-            $this->saveIndexKey('title', '', $pid, $value);
-            unset($key['title']);
-        }
-
-        foreach ($key as $name => $values) {
-            $metaname = idx_cleanName($name);
-            $this->addIndexKey('metadata', '', $metaname);
-            $metaidx = $this->getIndex($metaname.'_i', '');
-            $metawords = $this->getIndex($metaname.'_w', '');
-            $addwords = false;
-
-            if (!is_array($values)) $values = array($values);
-
-            $val_idx = $this->getIndexKey($metaname.'_p', '', $pid);
-            if ($val_idx !== '') {
-                $val_idx = explode(':', $val_idx);
-                // -1 means remove, 0 keep, 1 add
-                $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1));
-            } else {
-                $val_idx = array();
-            }
-
-            foreach ($values as $val) {
-                $val = (string)$val;
-                if ($val !== "") {
-                    $id = array_search($val, $metawords, true);
-                    if ($id === false) {
-                        // didn't find $val, so we'll add it to the end of metawords and create a placeholder in metaidx
-                        $id = count($metawords);
-                        $metawords[$id] = $val;
-                        $metaidx[$id] = '';
-                        $addwords = true;
-                    }
-                    // test if value is already in the index
-                    if (isset($val_idx[$id]) && $val_idx[$id] <= 0){
-                        $val_idx[$id] = 0;
-                    } else { // else add it
-                        $val_idx[$id] = 1;
-                    }
-                }
-            }
-
-            if ($addwords) {
-                $this->saveIndex($metaname.'_w', '', $metawords);
-            }
-            $vals_changed = false;
-            foreach ($val_idx as $id => $action) {
-                if ($action == -1) {
-                    $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0);
-                    $vals_changed = true;
-                    unset($val_idx[$id]);
-                } elseif ($action == 1) {
-                    $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1);
-                    $vals_changed = true;
-                }
-            }
-
-            if ($vals_changed) {
-                $this->saveIndex($metaname.'_i', '', $metaidx);
-                $val_idx = implode(':', array_keys($val_idx));
-                $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx);
-            }
-
-            unset($metaidx);
-            unset($metawords);
-        }
-
-        $this->unlock();
-        return true;
-    }
-
-    /**
-     * Rename a page in the search index without changing the indexed content. This function doesn't check if the
-     * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the
-     * indexer and it deletes all previously indexed content of the new page.
-     *
-     * @param string $oldpage The old page name
-     * @param string $newpage The new page name
-     * @return string|bool If the page was successfully renamed, can be a message in the case of an error
-     */
-    public function renamePage($oldpage, $newpage) {
-        if (!$this->lock()) return 'locked';
-
-        $pages = $this->getPages();
-
-        $id = array_search($oldpage, $pages, true);
-        if ($id === false) {
-            $this->unlock();
-            return 'page is not in index';
-        }
-
-        $new_id = array_search($newpage, $pages, true);
-        if ($new_id !== false) {
-            // make sure the page is not in the index anymore
-            if ($this->deletePageNoLock($newpage) !== true) {
-                return false;
-            }
-
-            $pages[$new_id] = 'deleted:'.time().rand(0, 9999);
-        }
-
-        $pages[$id] = $newpage;
-
-        // update index
-        if (!$this->saveIndex('page', '', $pages)) {
-            $this->unlock();
-            return false;
-        }
-
-        // reset the pid cache
-        $this->pidCache = array();
-
-        $this->unlock();
-        return true;
-    }
-
-    /**
-     * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages
-     * will be updated.
-     *
-     * @param string $key       The metadata key of which a value shall be changed
-     * @param string $oldvalue  The old value that shall be renamed
-     * @param string $newvalue  The new value to which the old value shall be renamed, if exists values will be merged
-     * @return bool|string      If renaming the value has been successful, false or error message on error.
-     */
-    public function renameMetaValue($key, $oldvalue, $newvalue) {
-        if (!$this->lock()) return 'locked';
-
-        // change the relation references index
-        $metavalues = $this->getIndex($key, '_w');
-        $oldid = array_search($oldvalue, $metavalues, true);
-        if ($oldid !== false) {
-            $newid = array_search($newvalue, $metavalues, true);
-            if ($newid !== false) {
-                // free memory
-                unset ($metavalues);
-
-                // okay, now we have two entries for the same value. we need to merge them.
-                $indexline = $this->getIndexKey($key.'_i', '', $oldid);
-                if ($indexline != '') {
-                    $newindexline = $this->getIndexKey($key.'_i', '', $newid);
-                    $pagekeys     = $this->getIndex($key.'_p', '');
-                    $parts = explode(':', $indexline);
-                    foreach ($parts as $part) {
-                        list($id, $count) = explode('*', $part);
-                        $newindexline =  $this->updateTuple($newindexline, $id, $count);
-
-                        $keyline = explode(':', $pagekeys[$id]);
-                        // remove old meta value
-                        $keyline = array_diff($keyline, array($oldid));
-                        // add new meta value when not already present
-                        if (!in_array($newid, $keyline)) {
-                            array_push($keyline, $newid);
-                        }
-                        $pagekeys[$id] = implode(':', $keyline);
-                    }
-                    $this->saveIndex($key.'_p', '', $pagekeys);
-                    unset($pagekeys);
-                    $this->saveIndexKey($key.'_i', '', $oldid, '');
-                    $this->saveIndexKey($key.'_i', '', $newid, $newindexline);
-                }
-            } else {
-                $metavalues[$oldid] = $newvalue;
-                if (!$this->saveIndex($key.'_w', '', $metavalues)) {
-                    $this->unlock();
-                    return false;
-                }
-            }
-        }
-
-        $this->unlock();
-        return true;
-    }
-
-    /**
-     * Remove a page from the index
-     *
-     * Erases entries in all known indexes.
-     *
-     * @param string    $page   a page name
-     * @return string|boolean  the function completed successfully
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    public function deletePage($page) {
-        if (!$this->lock())
-            return "locked";
-
-        $result = $this->deletePageNoLock($page);
-
-        $this->unlock();
-
-        return $result;
-    }
-
-    /**
-     * Remove a page from the index without locking the index, only use this function if the index is already locked
-     *
-     * Erases entries in all known indexes.
-     *
-     * @param string    $page   a page name
-     * @return boolean          the function completed successfully
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function deletePageNoLock($page) {
-        // load known documents
-        $pid = $this->getPIDNoLock($page);
-        if ($pid === false) {
-            return false;
-        }
-
-        // Remove obsolete index entries
-        $pageword_idx = $this->getIndexKey('pageword', '', $pid);
-        if ($pageword_idx !== '') {
-            $delwords = explode(':',$pageword_idx);
-            $upwords = array();
-            foreach ($delwords as $word) {
-                if ($word != '') {
-                    list($wlen,$wid) = explode('*', $word);
-                    $wid = (int)$wid;
-                    $upwords[$wlen][] = $wid;
-                }
-            }
-            foreach ($upwords as $wlen => $widx) {
-                $index = $this->getIndex('i', $wlen);
-                foreach ($widx as $wid) {
-                    $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
-                }
-                $this->saveIndex('i', $wlen, $index);
-            }
-        }
-        // Save the reverse index
-        if (!$this->saveIndexKey('pageword', '', $pid, "")) {
-            return false;
-        }
-
-        $this->saveIndexKey('title', '', $pid, "");
-        $keyidx = $this->getIndex('metadata', '');
-        foreach ($keyidx as $metaname) {
-            $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid));
-            $meta_idx = $this->getIndex($metaname.'_i', '');
-            foreach ($val_idx as $id) {
-                if ($id === '') continue;
-                $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0);
-            }
-            $this->saveIndex($metaname.'_i', '', $meta_idx);
-            $this->saveIndexKey($metaname.'_p', '', $pid, '');
-        }
-
-        return true;
-    }
-
-    /**
-     * Clear the whole index
-     *
-     * @return bool If the index has been cleared successfully
-     */
-    public function clear() {
-        global $conf;
-
-        if (!$this->lock()) return false;
-
-        @unlink($conf['indexdir'].'/page.idx');
-        @unlink($conf['indexdir'].'/title.idx');
-        @unlink($conf['indexdir'].'/pageword.idx');
-        @unlink($conf['indexdir'].'/metadata.idx');
-        $dir = @opendir($conf['indexdir']);
-        if($dir!==false){
-            while(($f = readdir($dir)) !== false){
-                if(substr($f,-4)=='.idx' &&
-                    (substr($f,0,1)=='i' || substr($f,0,1)=='w'
-                        || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx'))
-                    @unlink($conf['indexdir']."/$f");
-            }
-        }
-        @unlink($conf['indexdir'].'/lengths.idx');
-
-        // clear the pid cache
-        $this->pidCache = array();
-
-        $this->unlock();
-        return true;
-    }
-
-    /**
-     * Split the text into words for fulltext search
-     *
-     * TODO: does this also need &$stopwords ?
-     *
-     * @triggers INDEXER_TEXT_PREPARE
-     * This event allows plugins to modify the text before it gets tokenized.
-     * Plugins intercepting this event should also intercept INDEX_VERSION_GET
-     *
-     * @param string    $text   plain text
-     * @param boolean   $wc     are wildcards allowed?
-     * @return array            list of words in the text
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     * @author Andreas Gohr <andi@splitbrain.org>
-     */
-    public function tokenizer($text, $wc=false) {
-        $wc = ($wc) ? '' : '\*';
-        $stopwords =& idx_get_stopwords();
-
-        // prepare the text to be tokenized
-        $evt = new Event('INDEXER_TEXT_PREPARE', $text);
-        if ($evt->advise_before(true)) {
-            if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
-                $text = \dokuwiki\Utf8\Asian::separateAsianWords($text);
-            }
-        }
-        $evt->advise_after();
-        unset($evt);
-
-        $text = strtr($text,
-                       array(
-                           "\r" => ' ',
-                           "\n" => ' ',
-                           "\t" => ' ',
-                           "\xC2\xAD" => '', //soft-hyphen
-                       )
-                     );
-        if (preg_match('/[^0-9A-Za-z ]/u', $text))
-            $text = \dokuwiki\Utf8\Clean::stripspecials($text, ' ', '\._\-:'.$wc);
-
-        $wordlist = explode(' ', $text);
-        foreach ($wordlist as $i => $word) {
-            $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ?
-                \dokuwiki\Utf8\PhpString::strtolower($word) : strtolower($word);
-        }
-
-        foreach ($wordlist as $i => $word) {
-            if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH)
-              || array_search($word, $stopwords, true) !== false)
-                unset($wordlist[$i]);
-        }
-        return array_values($wordlist);
-    }
-
-    /**
-     * Get the numeric PID of a page
-     *
-     * @param string $page The page to get the PID for
-     * @return bool|int The page id on success, false on error
-     */
-    public function getPID($page) {
-        // return PID without locking when it is in the cache
-        if (isset($this->pidCache[$page])) return $this->pidCache[$page];
-
-        if (!$this->lock())
-            return false;
-
-        // load known documents
-        $pid = $this->getPIDNoLock($page);
-        if ($pid === false) {
-            $this->unlock();
-            return false;
-        }
-
-        $this->unlock();
-        return $pid;
-    }
-
-    /**
-     * Get the numeric PID of a page without locking the index.
-     * Only use this function when the index is already locked.
-     *
-     * @param string $page The page to get the PID for
-     * @return bool|int The page id on success, false on error
-     */
-    protected function getPIDNoLock($page) {
-        // avoid expensive addIndexKey operation for the most recently requested pages by using a cache
-        if (isset($this->pidCache[$page])) return $this->pidCache[$page];
-        $pid = $this->addIndexKey('page', '', $page);
-        // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently
-        // added item will be requested again
-        if (count($this->pidCache) > 10) array_shift($this->pidCache);
-        $this->pidCache[$page] = $pid;
-        return $pid;
-    }
-
-    /**
-     * Get the page id of a numeric PID
-     *
-     * @param int $pid The PID to get the page id for
-     * @return string The page id
-     */
-    public function getPageFromPID($pid) {
-        return $this->getIndexKey('page', '', $pid);
-    }
-
-    /**
-     * Find pages in the fulltext index containing the words,
-     *
-     * The search words must be pre-tokenized, meaning only letters and
-     * numbers with an optional wildcard
-     *
-     * The returned array will have the original tokens as key. The values
-     * in the returned list is an array with the page names as keys and the
-     * number of times that token appears on the page as value.
-     *
-     * @param array  $tokens list of words to search for
-     * @return array         list of page names with usage counts
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     * @author Andreas Gohr <andi@splitbrain.org>
-     */
-    public function lookup(&$tokens) {
-        $result = array();
-        $wids = $this->getIndexWords($tokens, $result);
-        if (empty($wids)) return array();
-        // load known words and documents
-        $page_idx = $this->getIndex('page', '');
-        $docs = array();
-        foreach (array_keys($wids) as $wlen) {
-            $wids[$wlen] = array_unique($wids[$wlen]);
-            $index = $this->getIndex('i', $wlen);
-            foreach($wids[$wlen] as $ixid) {
-                if ($ixid < count($index))
-                    $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]);
-            }
-        }
-        // merge found pages into final result array
-        $final = array();
-        foreach ($result as $word => $res) {
-            $final[$word] = array();
-            foreach ($res as $wid) {
-                // handle the case when ($ixid < count($index)) has been false
-                // and thus $docs[$wid] hasn't been set.
-                if (!isset($docs[$wid])) continue;
-                $hits = &$docs[$wid];
-                foreach ($hits as $hitkey => $hitcnt) {
-                    // make sure the document still exists
-                    if (!page_exists($hitkey, '', false)) continue;
-                    if (!isset($final[$word][$hitkey]))
-                        $final[$word][$hitkey] = $hitcnt;
-                    else
-                        $final[$word][$hitkey] += $hitcnt;
-                }
-            }
-        }
-        return $final;
-    }
-
-    /**
-     * Find pages containing a metadata key.
-     *
-     * The metadata values are compared as case-sensitive strings. Pass a
-     * callback function that returns true or false to use a different
-     * comparison function. The function will be called with the $value being
-     * searched for as the first argument, and the word in the index as the
-     * second argument. The function preg_match can be used directly if the
-     * values are regexes.
-     *
-     * @param string    $key    name of the metadata key to look for
-     * @param string    $value  search term to look for, must be a string or array of strings
-     * @param callback  $func   comparison function
-     * @return array            lists with page names, keys are query values if $value is array
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     * @author Michael Hamann <michael@content-space.de>
-     */
-    public function lookupKey($key, &$value, $func=null) {
-        if (!is_array($value))
-            $value_array = array($value);
-        else
-            $value_array =& $value;
-
-        // the matching ids for the provided value(s)
-        $value_ids = array();
-
-        $metaname = idx_cleanName($key);
-
-        // get all words in order to search the matching ids
-        if ($key == 'title') {
-            $words = $this->getIndex('title', '');
-        } else {
-            $words = $this->getIndex($metaname.'_w', '');
-        }
-
-        if (!is_null($func)) {
-            foreach ($value_array as $val) {
-                foreach ($words as $i => $word) {
-                    if (call_user_func_array($func, array($val, $word)))
-                        $value_ids[$i][] = $val;
-                }
-            }
-        } else {
-            foreach ($value_array as $val) {
-                $xval = $val;
-                $caret = '^';
-                $dollar = '$';
-                // check for wildcards
-                if (substr($xval, 0, 1) == '*') {
-                    $xval = substr($xval, 1);
-                    $caret = '';
-                }
-                if (substr($xval, -1, 1) == '*') {
-                    $xval = substr($xval, 0, -1);
-                    $dollar = '';
-                }
-                if (!$caret || !$dollar) {
-                    $re = $caret.preg_quote($xval, '/').$dollar;
-                    foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i)
-                        $value_ids[$i][] = $val;
-                } else {
-                    if (($i = array_search($val, $words, true)) !== false)
-                        $value_ids[$i][] = $val;
-                }
-            }
-        }
-
-        unset($words); // free the used memory
-
-        // initialize the result so it won't be null
-        $result = array();
-        foreach ($value_array as $val) {
-            $result[$val] = array();
-        }
-
-        $page_idx = $this->getIndex('page', '');
-
-        // Special handling for titles
-        if ($key == 'title') {
-            foreach ($value_ids as $pid => $val_list) {
-                $page = $page_idx[$pid];
-                foreach ($val_list as $val) {
-                    $result[$val][] = $page;
-                }
-            }
-        } else {
-            // load all lines and pages so the used lines can be taken and matched with the pages
-            $lines = $this->getIndex($metaname.'_i', '');
-
-            foreach ($value_ids as $value_id => $val_list) {
-                // parse the tuples of the form page_id*1:page2_id*1 and so on, return value
-                // is an array with page_id => 1, page2_id => 1 etc. so take the keys only
-                $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id]));
-                foreach ($val_list as $val) {
-                    $result[$val] = array_merge($result[$val], $pages);
-                }
-            }
-        }
-        if (!is_array($value)) $result = $result[$value];
-        return $result;
-    }
-
-    /**
-     * Find the index ID of each search term.
-     *
-     * The query terms should only contain valid characters, with a '*' at
-     * either the beginning or end of the word (or both).
-     * The $result parameter can be used to merge the index locations with
-     * the appropriate query term.
-     *
-     * @param array  $words  The query terms.
-     * @param array  $result Set to word => array("length*id" ...)
-     * @return array         Set to length => array(id ...)
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function getIndexWords(&$words, &$result) {
-        $tokens = array();
-        $tokenlength = array();
-        $tokenwild = array();
-        foreach ($words as $word) {
-            $result[$word] = array();
-            $caret = '^';
-            $dollar = '$';
-            $xword = $word;
-            $wlen = wordlen($word);
-
-            // check for wildcards
-            if (substr($xword, 0, 1) == '*') {
-                $xword = substr($xword, 1);
-                $caret = '';
-                $wlen -= 1;
-            }
-            if (substr($xword, -1, 1) == '*') {
-                $xword = substr($xword, 0, -1);
-                $dollar = '';
-                $wlen -= 1;
-            }
-            if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword))
-                continue;
-            if (!isset($tokens[$xword]))
-                $tokenlength[$wlen][] = $xword;
-            if (!$caret || !$dollar) {
-                $re = $caret.preg_quote($xword, '/').$dollar;
-                $tokens[$xword][] = array($word, '/'.$re.'/');
-                if (!isset($tokenwild[$xword]))
-                    $tokenwild[$xword] = $wlen;
-            } else {
-                $tokens[$xword][] = array($word, null);
-            }
-        }
-        asort($tokenwild);
-        // $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
-        // $tokenlength = array( base word length => base word ... )
-        // $tokenwild = array( base word => base word length ... )
-        $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
-        $indexes_known = $this->indexLengths($length_filter);
-        if (!empty($tokenwild)) sort($indexes_known);
-        // get word IDs
-        $wids = array();
-        foreach ($indexes_known as $ixlen) {
-            $word_idx = $this->getIndex('w', $ixlen);
-            // handle exact search
-            if (isset($tokenlength[$ixlen])) {
-                foreach ($tokenlength[$ixlen] as $xword) {
-                    $wid = array_search($xword, $word_idx, true);
-                    if ($wid !== false) {
-                        $wids[$ixlen][] = $wid;
-                        foreach ($tokens[$xword] as $w)
-                            $result[$w[0]][] = "$ixlen*$wid";
-                    }
-                }
-            }
-            // handle wildcard search
-            foreach ($tokenwild as $xword => $wlen) {
-                if ($wlen >= $ixlen) break;
-                foreach ($tokens[$xword] as $w) {
-                    if (is_null($w[1])) continue;
-                    foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) {
-                        $wids[$ixlen][] = $wid;
-                        $result[$w[0]][] = "$ixlen*$wid";
-                    }
-                }
-            }
-        }
-        return $wids;
-    }
-
-    /**
-     * Return a list of all pages
-     * Warning: pages may not exist!
-     *
-     * @param string    $key    list only pages containing the metadata key (optional)
-     * @return array            list of page names
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    public function getPages($key=null) {
-        $page_idx = $this->getIndex('page', '');
-        if (is_null($key)) return $page_idx;
-
-        $metaname = idx_cleanName($key);
-
-        // Special handling for titles
-        if ($key == 'title') {
-            $title_idx = $this->getIndex('title', '');
-            array_splice($page_idx, count($title_idx));
-            foreach ($title_idx as $i => $title)
-                if ($title === "") unset($page_idx[$i]);
-            return array_values($page_idx);
-        }
-
-        $pages = array();
-        $lines = $this->getIndex($metaname.'_i', '');
-        foreach ($lines as $line) {
-            $pages = array_merge($pages, $this->parseTuples($page_idx, $line));
-        }
-        return array_keys($pages);
-    }
-
-    /**
-     * Return a list of words sorted by number of times used
-     *
-     * @param int       $min    bottom frequency threshold
-     * @param int       $max    upper frequency limit. No limit if $max<$min
-     * @param int       $minlen minimum length of words to count
-     * @param string    $key    metadata key to list. Uses the fulltext index if not given
-     * @return array            list of words as the keys and frequency as values
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    public function histogram($min=1, $max=0, $minlen=3, $key=null) {
-        if ($min < 1)
-            $min = 1;
-        if ($max < $min)
-            $max = 0;
-
-        $result = array();
-
-        if ($key == 'title') {
-            $index = $this->getIndex('title', '');
-            $index = array_count_values($index);
-            foreach ($index as $val => $cnt) {
-                if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen)
-                    $result[$val] = $cnt;
-            }
-        }
-        elseif (!is_null($key)) {
-            $metaname = idx_cleanName($key);
-            $index = $this->getIndex($metaname.'_i', '');
-            $val_idx = array();
-            foreach ($index as $wid => $line) {
-                $freq = $this->countTuples($line);
-                if ($freq >= $min && (!$max || $freq <= $max))
-                    $val_idx[$wid] = $freq;
-            }
-            if (!empty($val_idx)) {
-                $words = $this->getIndex($metaname.'_w', '');
-                foreach ($val_idx as $wid => $freq) {
-                    if (strlen($words[$wid]) >= $minlen)
-                        $result[$words[$wid]] = $freq;
-                }
-            }
-        }
-        else {
-            $lengths = idx_listIndexLengths();
-            foreach ($lengths as $length) {
-                if ($length < $minlen) continue;
-                $index = $this->getIndex('i', $length);
-                $words = null;
-                foreach ($index as $wid => $line) {
-                    $freq = $this->countTuples($line);
-                    if ($freq >= $min && (!$max || $freq <= $max)) {
-                        if ($words === null)
-                            $words = $this->getIndex('w', $length);
-                        $result[$words[$wid]] = $freq;
-                    }
-                }
-            }
-        }
-
-        arsort($result);
-        return $result;
-    }
-
-    /**
-     * Lock the indexer.
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     *
-     * @return bool|string
-     */
-    protected function lock() {
-        global $conf;
-        $status = true;
-        $run = 0;
-        $lock = $conf['lockdir'].'/_indexer.lock';
-        while (!@mkdir($lock, $conf['dmode'])) {
-            usleep(50);
-            if(is_dir($lock) && time()-@filemtime($lock) > 60*5){
-                // looks like a stale lock - remove it
-                if (!@rmdir($lock)) {
-                    $status = "removing the stale lock failed";
-                    return false;
-                } else {
-                    $status = "stale lock removed";
-                }
-            }elseif($run++ == 1000){
-                // we waited 5 seconds for that lock
-                return false;
-            }
-        }
-        if (!empty($conf['dperm'])) {
-            chmod($lock, $conf['dperm']);
-        }
-        return $status;
-    }
-
-    /**
-     * Release the indexer lock.
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     *
-     * @return bool
-     */
-    protected function unlock() {
-        global $conf;
-        @rmdir($conf['lockdir'].'/_indexer.lock');
-        return true;
-    }
-
-    /**
-     * Retrieve the entire index.
-     *
-     * The $suffix argument is for an index that is split into
-     * multiple parts. Different index files should use different
-     * base names.
-     *
-     * @param string    $idx    name of the index
-     * @param string    $suffix subpart identifier
-     * @return array            list of lines without CR or LF
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function getIndex($idx, $suffix) {
-        global $conf;
-        $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
-        if (!file_exists($fn)) return array();
-        return file($fn, FILE_IGNORE_NEW_LINES);
-    }
-
-    /**
-     * Replace the contents of the index with an array.
-     *
-     * @param string    $idx    name of the index
-     * @param string    $suffix subpart identifier
-     * @param array     $lines  list of lines without LF
-     * @return bool             If saving succeeded
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function saveIndex($idx, $suffix, &$lines) {
-        global $conf;
-        $fn = $conf['indexdir'].'/'.$idx.$suffix;
-        $fh = @fopen($fn.'.tmp', 'w');
-        if (!$fh) return false;
-        fwrite($fh, join("\n", $lines));
-        if (!empty($lines))
-            fwrite($fh, "\n");
-        fclose($fh);
-        if (isset($conf['fperm']))
-            chmod($fn.'.tmp', $conf['fperm']);
-        io_rename($fn.'.tmp', $fn.'.idx');
-        return true;
-    }
-
-    /**
-     * Retrieve a line from the index.
-     *
-     * @param string    $idx    name of the index
-     * @param string    $suffix subpart identifier
-     * @param int       $id     the line number
-     * @return string           a line with trailing whitespace removed
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function getIndexKey($idx, $suffix, $id) {
-        global $conf;
-        $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx';
-        if (!file_exists($fn)) return '';
-        $fh = @fopen($fn, 'r');
-        if (!$fh) return '';
-        $ln = -1;
-        while (($line = fgets($fh)) !== false) {
-            if (++$ln == $id) break;
-        }
-        fclose($fh);
-        return rtrim((string)$line);
-    }
-
-    /**
-     * Write a line into the index.
-     *
-     * @param string    $idx    name of the index
-     * @param string    $suffix subpart identifier
-     * @param int       $id     the line number
-     * @param string    $line   line to write
-     * @return bool             If saving succeeded
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function saveIndexKey($idx, $suffix, $id, $line) {
-        global $conf;
-        if (substr($line, -1) != "\n")
-            $line .= "\n";
-        $fn = $conf['indexdir'].'/'.$idx.$suffix;
-        $fh = @fopen($fn.'.tmp', 'w');
-        if (!$fh) return false;
-        $ih = @fopen($fn.'.idx', 'r');
-        if ($ih) {
-            $ln = -1;
-            while (($curline = fgets($ih)) !== false) {
-                fwrite($fh, (++$ln == $id) ? $line : $curline);
-            }
-            if ($id > $ln) {
-                while ($id > ++$ln)
-                    fwrite($fh, "\n");
-                fwrite($fh, $line);
-            }
-            fclose($ih);
-        } else {
-            $ln = -1;
-            while ($id > ++$ln)
-                fwrite($fh, "\n");
-            fwrite($fh, $line);
-        }
-        fclose($fh);
-        if (isset($conf['fperm']))
-            chmod($fn.'.tmp', $conf['fperm']);
-        io_rename($fn.'.tmp', $fn.'.idx');
-        return true;
-    }
-
-    /**
-     * Retrieve or insert a value in the index.
-     *
-     * @param string    $idx    name of the index
-     * @param string    $suffix subpart identifier
-     * @param string    $value  line to find in the index
-     * @return int|bool          line number of the value in the index or false if writing the index failed
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     */
-    protected function addIndexKey($idx, $suffix, $value) {
-        $index = $this->getIndex($idx, $suffix);
-        $id = array_search($value, $index, true);
-        if ($id === false) {
-            $id = count($index);
-            $index[$id] = $value;
-            if (!$this->saveIndex($idx, $suffix, $index)) {
-                trigger_error("Failed to write $idx index", E_USER_ERROR);
-                return false;
-            }
-        }
-        return $id;
-    }
-
-    /**
-     * Get the list of lengths indexed in the wiki.
-     *
-     * Read the index directory or a cache file and returns
-     * a sorted array of lengths of the words used in the wiki.
-     *
-     * @author YoBoY <yoboy.leguesh@gmail.com>
-     *
-     * @return array
-     */
-    protected function listIndexLengths() {
-        return idx_listIndexLengths();
-    }
-
-    /**
-     * Get the word lengths that have been indexed.
-     *
-     * Reads the index directory and returns an array of lengths
-     * that there are indices for.
-     *
-     * @author YoBoY <yoboy.leguesh@gmail.com>
-     *
-     * @param array|int $filter
-     * @return array
-     */
-    protected function indexLengths($filter) {
-        global $conf;
-        $idx = array();
-        if (is_array($filter)) {
-            // testing if index files exist only
-            $path = $conf['indexdir']."/i";
-            foreach ($filter as $key => $value) {
-                if (file_exists($path.$key.'.idx'))
-                    $idx[] = $key;
-            }
-        } else {
-            $lengths = idx_listIndexLengths();
-            foreach ($lengths as $key => $length) {
-                // keep all the values equal or superior
-                if ((int)$length >= (int)$filter)
-                    $idx[] = $length;
-            }
-        }
-        return $idx;
-    }
-
-    /**
-     * Insert or replace a tuple in a line.
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     *
-     * @param string $line
-     * @param string|int $id
-     * @param int    $count
-     * @return string
-     */
-    protected function updateTuple($line, $id, $count) {
-        if ($line != ''){
-            $line = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $line);
-        }
-        $line = trim($line, ':');
-        if ($count) {
-            if ($line) {
-                return "$id*$count:".$line;
-            } else {
-                return "$id*$count";
-            }
-        }
-        return $line;
-    }
-
-    /**
-     * Split a line into an array of tuples.
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     * @author Andreas Gohr <andi@splitbrain.org>
-     *
-     * @param array $keys
-     * @param string $line
-     * @return array
-     */
-    protected function parseTuples(&$keys, $line) {
-        $result = array();
-        if ($line == '') return $result;
-        $parts = explode(':', $line);
-        foreach ($parts as $tuple) {
-            if ($tuple === '') continue;
-            list($key, $cnt) = explode('*', $tuple);
-            if (!$cnt) continue;
-            $key = $keys[$key];
-            if ($key === false || is_null($key)) continue;
-            $result[$key] = $cnt;
-        }
-        return $result;
-    }
-
-    /**
-     * Sum the counts in a list of tuples.
-     *
-     * @author Tom N Harris <tnharris@whoopdedo.org>
-     *
-     * @param string $line
-     * @return int
-     */
-    protected function countTuples($line) {
-        $freq = 0;
-        $parts = explode(':', $line);
-        foreach ($parts as $tuple) {
-            if ($tuple === '') continue;
-            list(/* $pid */, $cnt) = explode('*', $tuple);
-            $freq += (int)$cnt;
-        }
-        return $freq;
-    }
-}
-
-/**
  * Create an instance of the indexer.
  *
- * @return Doku_Indexer    a Doku_Indexer
+ * @return Indexer    an Indexer
  *
  * @author Tom N Harris <tnharris@whoopdedo.org>
  */
 function idx_get_indexer() {
     static $Indexer;
     if (!isset($Indexer)) {
-        $Indexer = new Doku_Indexer();
+        $Indexer = new Indexer();
     }
     return $Indexer;
 }