From e3cafdc28a52ba17ed1f5ec78020911c95292ca0 Mon Sep 17 00:00:00 2001 From: Michael Große Date: Sat, 28 Dec 2019 13:49:11 +0100 Subject: Check our code style for every PR This excludes the currently failing checks with automatic fixes, as they are out of scope of this commit. The current goal is automatically reject pull requests that don't me the standards that the rest of our code base already follows. --- .github/workflows/phpCS.yml | 22 ++++++++++++++++ _test/phpcs.xml | 62 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/phpCS.yml diff --git a/.github/workflows/phpCS.yml b/.github/workflows/phpCS.yml new file mode 100644 index 000000000..2699db6cd --- /dev/null +++ b/.github/workflows/phpCS.yml @@ -0,0 +1,22 @@ +name: PHP Code Style + +on: [push] + +jobs: + phpcs: + name: PHP CodeSniffer + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + + - name: Setup PHP + uses: shivammathur/setup-php@7b8bf7af6a42e028cbcccf623bb8499b4d6edf02 + with: + php-version: '7.4' + coverage: none + + - name: Download PHPCS + run: wget https://squizlabs.github.io/PHP_CodeSniffer/phpcs.phar + + - name: run PHP codesniffer + run: php phpcs.phar -v --standard=_test/phpcs.xml diff --git a/_test/phpcs.xml b/_test/phpcs.xml index 4f38195a3..b24e9f5a0 100644 --- a/_test/phpcs.xml +++ b/_test/phpcs.xml @@ -41,6 +41,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- cgit v1.2.3 From cc8fc76154c110896331e7dee2af72b7adb89d96 Mon Sep 17 00:00:00 2001 From: Michael Große Date: Sun, 29 Dec 2019 00:37:01 +0100 Subject: Ignore some more definitions Some of the files are still having both logic and side-effects for legacy reasons. We are excluding them for now and might deal with them later. --- _test/phpcs.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/_test/phpcs.xml b/_test/phpcs.xml index b24e9f5a0..78bfad995 100644 --- a/_test/phpcs.xml +++ b/_test/phpcs.xml @@ -146,7 +146,9 @@ */inc/parserutils.php */inc/mail.php */inc/init.php + */inc/farm.php */inc/fulltext.php + */inc/pluginutils.php */inc/Mailer.class.php */doku.php */install.php -- cgit v1.2.3 From 6225b270e09ba8d1cb10e70c9cbdbd0fe690c253 Mon Sep 17 00:00:00 2001 From: Michael Große Date: Sun, 29 Dec 2019 00:52:19 +0100 Subject: Extract dokuwiki\Search\Indexer class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not sure why Doku_Indexer caused phpcs to complain about the class name not being in PascalCase, but Doku_Handler didn't. 🤷 The namespace and new class name was selected to be compatible with the upcoming changes in #2943. This should hpopefully reduce the overall hassle of touching the same code base. --- _test/phpcs.xml | 1 + _test/tests/inc/indexer_rename.test.php | 2 +- inc/Search/Indexer.php | 1214 +++++++++++++++++++++++++++++++ inc/deprecated.php | 5 + inc/fulltext.php | 12 +- inc/indexer.php | 1214 +------------------------------ 6 files changed, 1230 insertions(+), 1218 deletions(-) create mode 100644 inc/Search/Indexer.php diff --git a/_test/phpcs.xml b/_test/phpcs.xml index 78bfad995..fbc079018 100644 --- a/_test/phpcs.xml +++ b/_test/phpcs.xml @@ -149,6 +149,7 @@ */inc/farm.php */inc/fulltext.php */inc/pluginutils.php + */inc/indexer.php */inc/Mailer.class.php */doku.php */install.php diff --git a/_test/tests/inc/indexer_rename.test.php b/_test/tests/inc/indexer_rename.test.php index d8c456f8e..2dbe6dcad 100644 --- a/_test/tests/inc/indexer_rename.test.php +++ b/_test/tests/inc/indexer_rename.test.php @@ -3,7 +3,7 @@ * Test cases for the Doku_Indexer::renamePage and Doku_Indexer::renameMetaValue methods */ class indexer_rename_test extends DokuWikiTest { - /** @var Doku_Indexer $indexer */ + /** @var \dokuwiki\Search\Indexer $indexer */ private $indexer; private $old_id = 'old_testid'; diff --git a/inc/Search/Indexer.php b/inc/Search/Indexer.php new file mode 100644 index 000000000..8770ac794 --- /dev/null +++ b/inc/Search/Indexer.php @@ -0,0 +1,1214 @@ + + */ +class Indexer { + /** + * @var array $pidCache Cache for getPID() + */ + protected $pidCache = array(); + + /** + * Adds the contents of a page to the fulltext index + * + * The added text replaces previous words for the same page. + * An empty value erases the page. + * + * @param string $page a page name + * @param string $text the body of the page + * @return string|boolean the function completed successfully + * + * @author Tom N Harris + * @author Andreas Gohr + */ + public function addPageWords($page, $text) { + if (!$this->lock()) + return "locked"; + + // load known documents + $pid = $this->getPIDNoLock($page); + if ($pid === false) { + $this->unlock(); + return false; + } + + $pagewords = array(); + // get word usage in page + $words = $this->getPageWords($text); + if ($words === false) { + $this->unlock(); + return false; + } + + if (!empty($words)) { + foreach (array_keys($words) as $wlen) { + $index = $this->getIndex('i', $wlen); + foreach ($words[$wlen] as $wid => $freq) { + $idx = ($widupdateTuple($idx, $pid, $freq); + $pagewords[] = "$wlen*$wid"; + } + if (!$this->saveIndex('i', $wlen, $index)) { + $this->unlock(); + return false; + } + } + } + + // Remove obsolete index entries + $pageword_idx = $this->getIndexKey('pageword', '', $pid); + if ($pageword_idx !== '') { + $oldwords = explode(':',$pageword_idx); + $delwords = array_diff($oldwords, $pagewords); + $upwords = array(); + foreach ($delwords as $word) { + if ($word != '') { + list($wlen, $wid) = explode('*', $word); + $wid = (int)$wid; + $upwords[$wlen][] = $wid; + } + } + foreach ($upwords as $wlen => $widx) { + $index = $this->getIndex('i', $wlen); + foreach ($widx as $wid) { + $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); + } + $this->saveIndex('i', $wlen, $index); + } + } + // Save the reverse index + $pageword_idx = join(':', $pagewords); + if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) { + $this->unlock(); + return false; + } + + $this->unlock(); + return true; + } + + /** + * Split the words in a page and add them to the index. + * + * @param string $text content of the page + * @return array list of word IDs and number of times used + * + * @author Andreas Gohr + * @author Christopher Smith + * @author Tom N Harris + */ + protected function getPageWords($text) { + + $tokens = $this->tokenizer($text); + $tokens = array_count_values($tokens); // count the frequency of each token + + $words = array(); + foreach ($tokens as $w=>$c) { + $l = wordlen($w); + if (isset($words[$l])){ + $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0); + }else{ + $words[$l] = array($w => $c); + } + } + + // arrive here with $words = array(wordlen => array(word => frequency)) + $word_idx_modified = false; + $index = array(); //resulting index + foreach (array_keys($words) as $wlen) { + $word_idx = $this->getIndex('w', $wlen); + foreach ($words[$wlen] as $word => $freq) { + $word = (string)$word; + $wid = array_search($word, $word_idx, true); + if ($wid === false) { + $wid = count($word_idx); + $word_idx[] = $word; + $word_idx_modified = true; + } + if (!isset($index[$wlen])) + $index[$wlen] = array(); + $index[$wlen][$wid] = $freq; + } + // save back the word index + if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx)) + return false; + } + + return $index; + } + + /** + * Add/update keys to/of the metadata index. + * + * Adding new keys does not remove other keys for the page. + * An empty value will erase the key. + * The $key parameter can be an array to add multiple keys. $value will + * not be used if $key is an array. + * + * @param string $page a page name + * @param mixed $key a key string or array of key=>value pairs + * @param mixed $value the value or list of values + * @return boolean|string the function completed successfully + * + * @author Tom N Harris + * @author Michael Hamann + */ + public function addMetaKeys($page, $key, $value=null) { + if (!is_array($key)) { + $key = array($key => $value); + } elseif (!is_null($value)) { + // $key is array, but $value is not null + trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING); + } + + if (!$this->lock()) + return "locked"; + + // load known documents + $pid = $this->getPIDNoLock($page); + if ($pid === false) { + $this->unlock(); + return false; + } + + // Special handling for titles so the index file is simpler + if (array_key_exists('title', $key)) { + $value = $key['title']; + if (is_array($value)) { + $value = $value[0]; + } + $this->saveIndexKey('title', '', $pid, $value); + unset($key['title']); + } + + foreach ($key as $name => $values) { + $metaname = idx_cleanName($name); + $this->addIndexKey('metadata', '', $metaname); + $metaidx = $this->getIndex($metaname.'_i', ''); + $metawords = $this->getIndex($metaname.'_w', ''); + $addwords = false; + + if (!is_array($values)) $values = array($values); + + $val_idx = $this->getIndexKey($metaname.'_p', '', $pid); + if ($val_idx !== '') { + $val_idx = explode(':', $val_idx); + // -1 means remove, 0 keep, 1 add + $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1)); + } else { + $val_idx = array(); + } + + foreach ($values as $val) { + $val = (string)$val; + if ($val !== "") { + $id = array_search($val, $metawords, true); + if ($id === false) { + // didn't find $val, so we'll add it to the end of metawords and create a placeholder in metaidx + $id = count($metawords); + $metawords[$id] = $val; + $metaidx[$id] = ''; + $addwords = true; + } + // test if value is already in the index + if (isset($val_idx[$id]) && $val_idx[$id] <= 0){ + $val_idx[$id] = 0; + } else { // else add it + $val_idx[$id] = 1; + } + } + } + + if ($addwords) { + $this->saveIndex($metaname.'_w', '', $metawords); + } + $vals_changed = false; + foreach ($val_idx as $id => $action) { + if ($action == -1) { + $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0); + $vals_changed = true; + unset($val_idx[$id]); + } elseif ($action == 1) { + $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1); + $vals_changed = true; + } + } + + if ($vals_changed) { + $this->saveIndex($metaname.'_i', '', $metaidx); + $val_idx = implode(':', array_keys($val_idx)); + $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx); + } + + unset($metaidx); + unset($metawords); + } + + $this->unlock(); + return true; + } + + /** + * Rename a page in the search index without changing the indexed content. This function doesn't check if the + * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the + * indexer and it deletes all previously indexed content of the new page. + * + * @param string $oldpage The old page name + * @param string $newpage The new page name + * @return string|bool If the page was successfully renamed, can be a message in the case of an error + */ + public function renamePage($oldpage, $newpage) { + if (!$this->lock()) return 'locked'; + + $pages = $this->getPages(); + + $id = array_search($oldpage, $pages, true); + if ($id === false) { + $this->unlock(); + return 'page is not in index'; + } + + $new_id = array_search($newpage, $pages, true); + if ($new_id !== false) { + // make sure the page is not in the index anymore + if ($this->deletePageNoLock($newpage) !== true) { + return false; + } + + $pages[$new_id] = 'deleted:'.time().rand(0, 9999); + } + + $pages[$id] = $newpage; + + // update index + if (!$this->saveIndex('page', '', $pages)) { + $this->unlock(); + return false; + } + + // reset the pid cache + $this->pidCache = array(); + + $this->unlock(); + return true; + } + + /** + * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages + * will be updated. + * + * @param string $key The metadata key of which a value shall be changed + * @param string $oldvalue The old value that shall be renamed + * @param string $newvalue The new value to which the old value shall be renamed, if exists values will be merged + * @return bool|string If renaming the value has been successful, false or error message on error. + */ + public function renameMetaValue($key, $oldvalue, $newvalue) { + if (!$this->lock()) return 'locked'; + + // change the relation references index + $metavalues = $this->getIndex($key, '_w'); + $oldid = array_search($oldvalue, $metavalues, true); + if ($oldid !== false) { + $newid = array_search($newvalue, $metavalues, true); + if ($newid !== false) { + // free memory + unset ($metavalues); + + // okay, now we have two entries for the same value. we need to merge them. + $indexline = $this->getIndexKey($key.'_i', '', $oldid); + if ($indexline != '') { + $newindexline = $this->getIndexKey($key.'_i', '', $newid); + $pagekeys = $this->getIndex($key.'_p', ''); + $parts = explode(':', $indexline); + foreach ($parts as $part) { + list($id, $count) = explode('*', $part); + $newindexline = $this->updateTuple($newindexline, $id, $count); + + $keyline = explode(':', $pagekeys[$id]); + // remove old meta value + $keyline = array_diff($keyline, array($oldid)); + // add new meta value when not already present + if (!in_array($newid, $keyline)) { + array_push($keyline, $newid); + } + $pagekeys[$id] = implode(':', $keyline); + } + $this->saveIndex($key.'_p', '', $pagekeys); + unset($pagekeys); + $this->saveIndexKey($key.'_i', '', $oldid, ''); + $this->saveIndexKey($key.'_i', '', $newid, $newindexline); + } + } else { + $metavalues[$oldid] = $newvalue; + if (!$this->saveIndex($key.'_w', '', $metavalues)) { + $this->unlock(); + return false; + } + } + } + + $this->unlock(); + return true; + } + + /** + * Remove a page from the index + * + * Erases entries in all known indexes. + * + * @param string $page a page name + * @return string|boolean the function completed successfully + * + * @author Tom N Harris + */ + public function deletePage($page) { + if (!$this->lock()) + return "locked"; + + $result = $this->deletePageNoLock($page); + + $this->unlock(); + + return $result; + } + + /** + * Remove a page from the index without locking the index, only use this function if the index is already locked + * + * Erases entries in all known indexes. + * + * @param string $page a page name + * @return boolean the function completed successfully + * + * @author Tom N Harris + */ + protected function deletePageNoLock($page) { + // load known documents + $pid = $this->getPIDNoLock($page); + if ($pid === false) { + return false; + } + + // Remove obsolete index entries + $pageword_idx = $this->getIndexKey('pageword', '', $pid); + if ($pageword_idx !== '') { + $delwords = explode(':',$pageword_idx); + $upwords = array(); + foreach ($delwords as $word) { + if ($word != '') { + list($wlen,$wid) = explode('*', $word); + $wid = (int)$wid; + $upwords[$wlen][] = $wid; + } + } + foreach ($upwords as $wlen => $widx) { + $index = $this->getIndex('i', $wlen); + foreach ($widx as $wid) { + $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); + } + $this->saveIndex('i', $wlen, $index); + } + } + // Save the reverse index + if (!$this->saveIndexKey('pageword', '', $pid, "")) { + return false; + } + + $this->saveIndexKey('title', '', $pid, ""); + $keyidx = $this->getIndex('metadata', ''); + foreach ($keyidx as $metaname) { + $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid)); + $meta_idx = $this->getIndex($metaname.'_i', ''); + foreach ($val_idx as $id) { + if ($id === '') continue; + $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0); + } + $this->saveIndex($metaname.'_i', '', $meta_idx); + $this->saveIndexKey($metaname.'_p', '', $pid, ''); + } + + return true; + } + + /** + * Clear the whole index + * + * @return bool If the index has been cleared successfully + */ + public function clear() { + global $conf; + + if (!$this->lock()) return false; + + @unlink($conf['indexdir'].'/page.idx'); + @unlink($conf['indexdir'].'/title.idx'); + @unlink($conf['indexdir'].'/pageword.idx'); + @unlink($conf['indexdir'].'/metadata.idx'); + $dir = @opendir($conf['indexdir']); + if($dir!==false){ + while(($f = readdir($dir)) !== false){ + if(substr($f,-4)=='.idx' && + (substr($f,0,1)=='i' || substr($f,0,1)=='w' + || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx')) + @unlink($conf['indexdir']."/$f"); + } + } + @unlink($conf['indexdir'].'/lengths.idx'); + + // clear the pid cache + $this->pidCache = array(); + + $this->unlock(); + return true; + } + + /** + * Split the text into words for fulltext search + * + * TODO: does this also need &$stopwords ? + * + * @triggers INDEXER_TEXT_PREPARE + * This event allows plugins to modify the text before it gets tokenized. + * Plugins intercepting this event should also intercept INDEX_VERSION_GET + * + * @param string $text plain text + * @param boolean $wc are wildcards allowed? + * @return array list of words in the text + * + * @author Tom N Harris + * @author Andreas Gohr + */ + public function tokenizer($text, $wc=false) { + $wc = ($wc) ? '' : '\*'; + $stopwords =& idx_get_stopwords(); + + // prepare the text to be tokenized + $evt = new Event('INDEXER_TEXT_PREPARE', $text); + if ($evt->advise_before(true)) { + if (preg_match('/[^0-9A-Za-z ]/u', $text)) { + $text = \dokuwiki\Utf8\Asian::separateAsianWords($text); + } + } + $evt->advise_after(); + unset($evt); + + $text = strtr($text, + array( + "\r" => ' ', + "\n" => ' ', + "\t" => ' ', + "\xC2\xAD" => '', //soft-hyphen + ) + ); + if (preg_match('/[^0-9A-Za-z ]/u', $text)) + $text = \dokuwiki\Utf8\Clean::stripspecials($text, ' ', '\._\-:'.$wc); + + $wordlist = explode(' ', $text); + foreach ($wordlist as $i => $word) { + $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ? + \dokuwiki\Utf8\PhpString::strtolower($word) : strtolower($word); + } + + foreach ($wordlist as $i => $word) { + if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH) + || array_search($word, $stopwords, true) !== false) + unset($wordlist[$i]); + } + return array_values($wordlist); + } + + /** + * Get the numeric PID of a page + * + * @param string $page The page to get the PID for + * @return bool|int The page id on success, false on error + */ + public function getPID($page) { + // return PID without locking when it is in the cache + if (isset($this->pidCache[$page])) return $this->pidCache[$page]; + + if (!$this->lock()) + return false; + + // load known documents + $pid = $this->getPIDNoLock($page); + if ($pid === false) { + $this->unlock(); + return false; + } + + $this->unlock(); + return $pid; + } + + /** + * Get the numeric PID of a page without locking the index. + * Only use this function when the index is already locked. + * + * @param string $page The page to get the PID for + * @return bool|int The page id on success, false on error + */ + protected function getPIDNoLock($page) { + // avoid expensive addIndexKey operation for the most recently requested pages by using a cache + if (isset($this->pidCache[$page])) return $this->pidCache[$page]; + $pid = $this->addIndexKey('page', '', $page); + // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently + // added item will be requested again + if (count($this->pidCache) > 10) array_shift($this->pidCache); + $this->pidCache[$page] = $pid; + return $pid; + } + + /** + * Get the page id of a numeric PID + * + * @param int $pid The PID to get the page id for + * @return string The page id + */ + public function getPageFromPID($pid) { + return $this->getIndexKey('page', '', $pid); + } + + /** + * Find pages in the fulltext index containing the words, + * + * The search words must be pre-tokenized, meaning only letters and + * numbers with an optional wildcard + * + * The returned array will have the original tokens as key. The values + * in the returned list is an array with the page names as keys and the + * number of times that token appears on the page as value. + * + * @param array $tokens list of words to search for + * @return array list of page names with usage counts + * + * @author Tom N Harris + * @author Andreas Gohr + */ + public function lookup(&$tokens) { + $result = array(); + $wids = $this->getIndexWords($tokens, $result); + if (empty($wids)) return array(); + // load known words and documents + $page_idx = $this->getIndex('page', ''); + $docs = array(); + foreach (array_keys($wids) as $wlen) { + $wids[$wlen] = array_unique($wids[$wlen]); + $index = $this->getIndex('i', $wlen); + foreach($wids[$wlen] as $ixid) { + if ($ixid < count($index)) + $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]); + } + } + // merge found pages into final result array + $final = array(); + foreach ($result as $word => $res) { + $final[$word] = array(); + foreach ($res as $wid) { + // handle the case when ($ixid < count($index)) has been false + // and thus $docs[$wid] hasn't been set. + if (!isset($docs[$wid])) continue; + $hits = &$docs[$wid]; + foreach ($hits as $hitkey => $hitcnt) { + // make sure the document still exists + if (!page_exists($hitkey, '', false)) continue; + if (!isset($final[$word][$hitkey])) + $final[$word][$hitkey] = $hitcnt; + else + $final[$word][$hitkey] += $hitcnt; + } + } + } + return $final; + } + + /** + * Find pages containing a metadata key. + * + * The metadata values are compared as case-sensitive strings. Pass a + * callback function that returns true or false to use a different + * comparison function. The function will be called with the $value being + * searched for as the first argument, and the word in the index as the + * second argument. The function preg_match can be used directly if the + * values are regexes. + * + * @param string $key name of the metadata key to look for + * @param string $value search term to look for, must be a string or array of strings + * @param callback $func comparison function + * @return array lists with page names, keys are query values if $value is array + * + * @author Tom N Harris + * @author Michael Hamann + */ + public function lookupKey($key, &$value, $func=null) { + if (!is_array($value)) + $value_array = array($value); + else + $value_array =& $value; + + // the matching ids for the provided value(s) + $value_ids = array(); + + $metaname = idx_cleanName($key); + + // get all words in order to search the matching ids + if ($key == 'title') { + $words = $this->getIndex('title', ''); + } else { + $words = $this->getIndex($metaname.'_w', ''); + } + + if (!is_null($func)) { + foreach ($value_array as $val) { + foreach ($words as $i => $word) { + if (call_user_func_array($func, array($val, $word))) + $value_ids[$i][] = $val; + } + } + } else { + foreach ($value_array as $val) { + $xval = $val; + $caret = '^'; + $dollar = '$'; + // check for wildcards + if (substr($xval, 0, 1) == '*') { + $xval = substr($xval, 1); + $caret = ''; + } + if (substr($xval, -1, 1) == '*') { + $xval = substr($xval, 0, -1); + $dollar = ''; + } + if (!$caret || !$dollar) { + $re = $caret.preg_quote($xval, '/').$dollar; + foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i) + $value_ids[$i][] = $val; + } else { + if (($i = array_search($val, $words, true)) !== false) + $value_ids[$i][] = $val; + } + } + } + + unset($words); // free the used memory + + // initialize the result so it won't be null + $result = array(); + foreach ($value_array as $val) { + $result[$val] = array(); + } + + $page_idx = $this->getIndex('page', ''); + + // Special handling for titles + if ($key == 'title') { + foreach ($value_ids as $pid => $val_list) { + $page = $page_idx[$pid]; + foreach ($val_list as $val) { + $result[$val][] = $page; + } + } + } else { + // load all lines and pages so the used lines can be taken and matched with the pages + $lines = $this->getIndex($metaname.'_i', ''); + + foreach ($value_ids as $value_id => $val_list) { + // parse the tuples of the form page_id*1:page2_id*1 and so on, return value + // is an array with page_id => 1, page2_id => 1 etc. so take the keys only + $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id])); + foreach ($val_list as $val) { + $result[$val] = array_merge($result[$val], $pages); + } + } + } + if (!is_array($value)) $result = $result[$value]; + return $result; + } + + /** + * Find the index ID of each search term. + * + * The query terms should only contain valid characters, with a '*' at + * either the beginning or end of the word (or both). + * The $result parameter can be used to merge the index locations with + * the appropriate query term. + * + * @param array $words The query terms. + * @param array $result Set to word => array("length*id" ...) + * @return array Set to length => array(id ...) + * + * @author Tom N Harris + */ + protected function getIndexWords(&$words, &$result) { + $tokens = array(); + $tokenlength = array(); + $tokenwild = array(); + foreach ($words as $word) { + $result[$word] = array(); + $caret = '^'; + $dollar = '$'; + $xword = $word; + $wlen = wordlen($word); + + // check for wildcards + if (substr($xword, 0, 1) == '*') { + $xword = substr($xword, 1); + $caret = ''; + $wlen -= 1; + } + if (substr($xword, -1, 1) == '*') { + $xword = substr($xword, 0, -1); + $dollar = ''; + $wlen -= 1; + } + if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword)) + continue; + if (!isset($tokens[$xword])) + $tokenlength[$wlen][] = $xword; + if (!$caret || !$dollar) { + $re = $caret.preg_quote($xword, '/').$dollar; + $tokens[$xword][] = array($word, '/'.$re.'/'); + if (!isset($tokenwild[$xword])) + $tokenwild[$xword] = $wlen; + } else { + $tokens[$xword][] = array($word, null); + } + } + asort($tokenwild); + // $tokens = array( base word => array( [ query term , regexp ] ... ) ... ) + // $tokenlength = array( base word length => base word ... ) + // $tokenwild = array( base word => base word length ... ) + $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); + $indexes_known = $this->indexLengths($length_filter); + if (!empty($tokenwild)) sort($indexes_known); + // get word IDs + $wids = array(); + foreach ($indexes_known as $ixlen) { + $word_idx = $this->getIndex('w', $ixlen); + // handle exact search + if (isset($tokenlength[$ixlen])) { + foreach ($tokenlength[$ixlen] as $xword) { + $wid = array_search($xword, $word_idx, true); + if ($wid !== false) { + $wids[$ixlen][] = $wid; + foreach ($tokens[$xword] as $w) + $result[$w[0]][] = "$ixlen*$wid"; + } + } + } + // handle wildcard search + foreach ($tokenwild as $xword => $wlen) { + if ($wlen >= $ixlen) break; + foreach ($tokens[$xword] as $w) { + if (is_null($w[1])) continue; + foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) { + $wids[$ixlen][] = $wid; + $result[$w[0]][] = "$ixlen*$wid"; + } + } + } + } + return $wids; + } + + /** + * Return a list of all pages + * Warning: pages may not exist! + * + * @param string $key list only pages containing the metadata key (optional) + * @return array list of page names + * + * @author Tom N Harris + */ + public function getPages($key=null) { + $page_idx = $this->getIndex('page', ''); + if (is_null($key)) return $page_idx; + + $metaname = idx_cleanName($key); + + // Special handling for titles + if ($key == 'title') { + $title_idx = $this->getIndex('title', ''); + array_splice($page_idx, count($title_idx)); + foreach ($title_idx as $i => $title) + if ($title === "") unset($page_idx[$i]); + return array_values($page_idx); + } + + $pages = array(); + $lines = $this->getIndex($metaname.'_i', ''); + foreach ($lines as $line) { + $pages = array_merge($pages, $this->parseTuples($page_idx, $line)); + } + return array_keys($pages); + } + + /** + * Return a list of words sorted by number of times used + * + * @param int $min bottom frequency threshold + * @param int $max upper frequency limit. No limit if $max<$min + * @param int $minlen minimum length of words to count + * @param string $key metadata key to list. Uses the fulltext index if not given + * @return array list of words as the keys and frequency as values + * + * @author Tom N Harris + */ + public function histogram($min=1, $max=0, $minlen=3, $key=null) { + if ($min < 1) + $min = 1; + if ($max < $min) + $max = 0; + + $result = array(); + + if ($key == 'title') { + $index = $this->getIndex('title', ''); + $index = array_count_values($index); + foreach ($index as $val => $cnt) { + if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen) + $result[$val] = $cnt; + } + } + elseif (!is_null($key)) { + $metaname = idx_cleanName($key); + $index = $this->getIndex($metaname.'_i', ''); + $val_idx = array(); + foreach ($index as $wid => $line) { + $freq = $this->countTuples($line); + if ($freq >= $min && (!$max || $freq <= $max)) + $val_idx[$wid] = $freq; + } + if (!empty($val_idx)) { + $words = $this->getIndex($metaname.'_w', ''); + foreach ($val_idx as $wid => $freq) { + if (strlen($words[$wid]) >= $minlen) + $result[$words[$wid]] = $freq; + } + } + } + else { + $lengths = idx_listIndexLengths(); + foreach ($lengths as $length) { + if ($length < $minlen) continue; + $index = $this->getIndex('i', $length); + $words = null; + foreach ($index as $wid => $line) { + $freq = $this->countTuples($line); + if ($freq >= $min && (!$max || $freq <= $max)) { + if ($words === null) + $words = $this->getIndex('w', $length); + $result[$words[$wid]] = $freq; + } + } + } + } + + arsort($result); + return $result; + } + + /** + * Lock the indexer. + * + * @author Tom N Harris + * + * @return bool|string + */ + protected function lock() { + global $conf; + $status = true; + $run = 0; + $lock = $conf['lockdir'].'/_indexer.lock'; + while (!@mkdir($lock, $conf['dmode'])) { + usleep(50); + if(is_dir($lock) && time()-@filemtime($lock) > 60*5){ + // looks like a stale lock - remove it + if (!@rmdir($lock)) { + $status = "removing the stale lock failed"; + return false; + } else { + $status = "stale lock removed"; + } + }elseif($run++ == 1000){ + // we waited 5 seconds for that lock + return false; + } + } + if (!empty($conf['dperm'])) { + chmod($lock, $conf['dperm']); + } + return $status; + } + + /** + * Release the indexer lock. + * + * @author Tom N Harris + * + * @return bool + */ + protected function unlock() { + global $conf; + @rmdir($conf['lockdir'].'/_indexer.lock'); + return true; + } + + /** + * Retrieve the entire index. + * + * The $suffix argument is for an index that is split into + * multiple parts. Different index files should use different + * base names. + * + * @param string $idx name of the index + * @param string $suffix subpart identifier + * @return array list of lines without CR or LF + * + * @author Tom N Harris + */ + protected function getIndex($idx, $suffix) { + global $conf; + $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; + if (!file_exists($fn)) return array(); + return file($fn, FILE_IGNORE_NEW_LINES); + } + + /** + * Replace the contents of the index with an array. + * + * @param string $idx name of the index + * @param string $suffix subpart identifier + * @param array $lines list of lines without LF + * @return bool If saving succeeded + * + * @author Tom N Harris + */ + protected function saveIndex($idx, $suffix, &$lines) { + global $conf; + $fn = $conf['indexdir'].'/'.$idx.$suffix; + $fh = @fopen($fn.'.tmp', 'w'); + if (!$fh) return false; + fwrite($fh, join("\n", $lines)); + if (!empty($lines)) + fwrite($fh, "\n"); + fclose($fh); + if (isset($conf['fperm'])) + chmod($fn.'.tmp', $conf['fperm']); + io_rename($fn.'.tmp', $fn.'.idx'); + return true; + } + + /** + * Retrieve a line from the index. + * + * @param string $idx name of the index + * @param string $suffix subpart identifier + * @param int $id the line number + * @return string a line with trailing whitespace removed + * + * @author Tom N Harris + */ + protected function getIndexKey($idx, $suffix, $id) { + global $conf; + $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; + if (!file_exists($fn)) return ''; + $fh = @fopen($fn, 'r'); + if (!$fh) return ''; + $ln = -1; + while (($line = fgets($fh)) !== false) { + if (++$ln == $id) break; + } + fclose($fh); + return rtrim((string)$line); + } + + /** + * Write a line into the index. + * + * @param string $idx name of the index + * @param string $suffix subpart identifier + * @param int $id the line number + * @param string $line line to write + * @return bool If saving succeeded + * + * @author Tom N Harris + */ + protected function saveIndexKey($idx, $suffix, $id, $line) { + global $conf; + if (substr($line, -1) != "\n") + $line .= "\n"; + $fn = $conf['indexdir'].'/'.$idx.$suffix; + $fh = @fopen($fn.'.tmp', 'w'); + if (!$fh) return false; + $ih = @fopen($fn.'.idx', 'r'); + if ($ih) { + $ln = -1; + while (($curline = fgets($ih)) !== false) { + fwrite($fh, (++$ln == $id) ? $line : $curline); + } + if ($id > $ln) { + while ($id > ++$ln) + fwrite($fh, "\n"); + fwrite($fh, $line); + } + fclose($ih); + } else { + $ln = -1; + while ($id > ++$ln) + fwrite($fh, "\n"); + fwrite($fh, $line); + } + fclose($fh); + if (isset($conf['fperm'])) + chmod($fn.'.tmp', $conf['fperm']); + io_rename($fn.'.tmp', $fn.'.idx'); + return true; + } + + /** + * Retrieve or insert a value in the index. + * + * @param string $idx name of the index + * @param string $suffix subpart identifier + * @param string $value line to find in the index + * @return int|bool line number of the value in the index or false if writing the index failed + * + * @author Tom N Harris + */ + protected function addIndexKey($idx, $suffix, $value) { + $index = $this->getIndex($idx, $suffix); + $id = array_search($value, $index, true); + if ($id === false) { + $id = count($index); + $index[$id] = $value; + if (!$this->saveIndex($idx, $suffix, $index)) { + trigger_error("Failed to write $idx index", E_USER_ERROR); + return false; + } + } + return $id; + } + + /** + * Get the list of lengths indexed in the wiki. + * + * Read the index directory or a cache file and returns + * a sorted array of lengths of the words used in the wiki. + * + * @author YoBoY + * + * @return array + */ + protected function listIndexLengths() { + return idx_listIndexLengths(); + } + + /** + * Get the word lengths that have been indexed. + * + * Reads the index directory and returns an array of lengths + * that there are indices for. + * + * @author YoBoY + * + * @param array|int $filter + * @return array + */ + protected function indexLengths($filter) { + global $conf; + $idx = array(); + if (is_array($filter)) { + // testing if index files exist only + $path = $conf['indexdir']."/i"; + foreach ($filter as $key => $value) { + if (file_exists($path.$key.'.idx')) + $idx[] = $key; + } + } else { + $lengths = idx_listIndexLengths(); + foreach ($lengths as $key => $length) { + // keep all the values equal or superior + if ((int)$length >= (int)$filter) + $idx[] = $length; + } + } + return $idx; + } + + /** + * Insert or replace a tuple in a line. + * + * @author Tom N Harris + * + * @param string $line + * @param string|int $id + * @param int $count + * @return string + */ + protected function updateTuple($line, $id, $count) { + if ($line != ''){ + $line = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $line); + } + $line = trim($line, ':'); + if ($count) { + if ($line) { + return "$id*$count:".$line; + } else { + return "$id*$count"; + } + } + return $line; + } + + /** + * Split a line into an array of tuples. + * + * @author Tom N Harris + * @author Andreas Gohr + * + * @param array $keys + * @param string $line + * @return array + */ + protected function parseTuples(&$keys, $line) { + $result = array(); + if ($line == '') return $result; + $parts = explode(':', $line); + foreach ($parts as $tuple) { + if ($tuple === '') continue; + list($key, $cnt) = explode('*', $tuple); + if (!$cnt) continue; + $key = $keys[$key]; + if ($key === false || is_null($key)) continue; + $result[$key] = $cnt; + } + return $result; + } + + /** + * Sum the counts in a list of tuples. + * + * @author Tom N Harris + * + * @param string $line + * @return int + */ + protected function countTuples($line) { + $freq = 0; + $parts = explode(':', $line); + foreach ($parts as $tuple) { + if ($tuple === '') continue; + list(/* $pid */, $cnt) = explode('*', $tuple); + $freq += (int)$cnt; + } + return $freq; + } +} diff --git a/inc/deprecated.php b/inc/deprecated.php index d3de76080..205037344 100644 --- a/inc/deprecated.php +++ b/inc/deprecated.php @@ -563,3 +563,8 @@ class Subscription { $manager->notifyAddresses($data); } } + +/** + * @deprecated 2019-12-29 use \dokuwiki\Search\Indexer + */ +class Doku_Indexer extends \dokuwiki\Search\Indexer {}; diff --git a/inc/fulltext.php b/inc/fulltext.php index 48fe28da2..670f048d3 100644 --- a/inc/fulltext.php +++ b/inc/fulltext.php @@ -629,8 +629,8 @@ function ft_resultComplement($args) { * @author Andreas Gohr * @author Kazutaka Miyasaka * - * @param Doku_Indexer $Indexer - * @param string $query search query + * @param dokuwiki\Search\Indexer $Indexer + * @param string $query search query * @return array of search formulas */ function ft_queryParser($Indexer, $query){ @@ -864,10 +864,10 @@ function ft_queryParser($Indexer, $query){ * * @author Kazutaka Miyasaka * - * @param Doku_Indexer $Indexer - * @param string $term - * @param bool $consider_asian - * @param bool $phrase_mode + * @param dokuwiki\Search\Indexer $Indexer + * @param string $term + * @param bool $consider_asian + * @param bool $phrase_mode * @return string */ function ft_termParser($Indexer, $term, $consider_asian = true, $phrase_mode = false) { diff --git a/inc/indexer.php b/inc/indexer.php index f2c8a9aa8..ab02b8ea2 100644 --- a/inc/indexer.php +++ b/inc/indexer.php @@ -8,6 +8,7 @@ */ use dokuwiki\Extension\Event; +use dokuwiki\Search\Indexer; // Version tag used to force rebuild on upgrade define('INDEXER_VERSION', 8); @@ -66,1226 +67,17 @@ function wordlen($w){ return $l; } -/** - * Class that encapsulates operations on the indexer database. - * - * @author Tom N Harris - */ -class Doku_Indexer { - /** - * @var array $pidCache Cache for getPID() - */ - protected $pidCache = array(); - - /** - * Adds the contents of a page to the fulltext index - * - * The added text replaces previous words for the same page. - * An empty value erases the page. - * - * @param string $page a page name - * @param string $text the body of the page - * @return string|boolean the function completed successfully - * - * @author Tom N Harris - * @author Andreas Gohr - */ - public function addPageWords($page, $text) { - if (!$this->lock()) - return "locked"; - - // load known documents - $pid = $this->getPIDNoLock($page); - if ($pid === false) { - $this->unlock(); - return false; - } - - $pagewords = array(); - // get word usage in page - $words = $this->getPageWords($text); - if ($words === false) { - $this->unlock(); - return false; - } - - if (!empty($words)) { - foreach (array_keys($words) as $wlen) { - $index = $this->getIndex('i', $wlen); - foreach ($words[$wlen] as $wid => $freq) { - $idx = ($widupdateTuple($idx, $pid, $freq); - $pagewords[] = "$wlen*$wid"; - } - if (!$this->saveIndex('i', $wlen, $index)) { - $this->unlock(); - return false; - } - } - } - - // Remove obsolete index entries - $pageword_idx = $this->getIndexKey('pageword', '', $pid); - if ($pageword_idx !== '') { - $oldwords = explode(':',$pageword_idx); - $delwords = array_diff($oldwords, $pagewords); - $upwords = array(); - foreach ($delwords as $word) { - if ($word != '') { - list($wlen,$wid) = explode('*', $word); - $wid = (int)$wid; - $upwords[$wlen][] = $wid; - } - } - foreach ($upwords as $wlen => $widx) { - $index = $this->getIndex('i', $wlen); - foreach ($widx as $wid) { - $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); - } - $this->saveIndex('i', $wlen, $index); - } - } - // Save the reverse index - $pageword_idx = join(':', $pagewords); - if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) { - $this->unlock(); - return false; - } - - $this->unlock(); - return true; - } - - /** - * Split the words in a page and add them to the index. - * - * @param string $text content of the page - * @return array list of word IDs and number of times used - * - * @author Andreas Gohr - * @author Christopher Smith - * @author Tom N Harris - */ - protected function getPageWords($text) { - - $tokens = $this->tokenizer($text); - $tokens = array_count_values($tokens); // count the frequency of each token - - $words = array(); - foreach ($tokens as $w=>$c) { - $l = wordlen($w); - if (isset($words[$l])){ - $words[$l][$w] = $c + (isset($words[$l][$w]) ? $words[$l][$w] : 0); - }else{ - $words[$l] = array($w => $c); - } - } - - // arrive here with $words = array(wordlen => array(word => frequency)) - $word_idx_modified = false; - $index = array(); //resulting index - foreach (array_keys($words) as $wlen) { - $word_idx = $this->getIndex('w', $wlen); - foreach ($words[$wlen] as $word => $freq) { - $word = (string)$word; - $wid = array_search($word, $word_idx, true); - if ($wid === false) { - $wid = count($word_idx); - $word_idx[] = $word; - $word_idx_modified = true; - } - if (!isset($index[$wlen])) - $index[$wlen] = array(); - $index[$wlen][$wid] = $freq; - } - // save back the word index - if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx)) - return false; - } - - return $index; - } - - /** - * Add/update keys to/of the metadata index. - * - * Adding new keys does not remove other keys for the page. - * An empty value will erase the key. - * The $key parameter can be an array to add multiple keys. $value will - * not be used if $key is an array. - * - * @param string $page a page name - * @param mixed $key a key string or array of key=>value pairs - * @param mixed $value the value or list of values - * @return boolean|string the function completed successfully - * - * @author Tom N Harris - * @author Michael Hamann - */ - public function addMetaKeys($page, $key, $value=null) { - if (!is_array($key)) { - $key = array($key => $value); - } elseif (!is_null($value)) { - // $key is array, but $value is not null - trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING); - } - - if (!$this->lock()) - return "locked"; - - // load known documents - $pid = $this->getPIDNoLock($page); - if ($pid === false) { - $this->unlock(); - return false; - } - - // Special handling for titles so the index file is simpler - if (array_key_exists('title', $key)) { - $value = $key['title']; - if (is_array($value)) { - $value = $value[0]; - } - $this->saveIndexKey('title', '', $pid, $value); - unset($key['title']); - } - - foreach ($key as $name => $values) { - $metaname = idx_cleanName($name); - $this->addIndexKey('metadata', '', $metaname); - $metaidx = $this->getIndex($metaname.'_i', ''); - $metawords = $this->getIndex($metaname.'_w', ''); - $addwords = false; - - if (!is_array($values)) $values = array($values); - - $val_idx = $this->getIndexKey($metaname.'_p', '', $pid); - if ($val_idx !== '') { - $val_idx = explode(':', $val_idx); - // -1 means remove, 0 keep, 1 add - $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1)); - } else { - $val_idx = array(); - } - - foreach ($values as $val) { - $val = (string)$val; - if ($val !== "") { - $id = array_search($val, $metawords, true); - if ($id === false) { - // didn't find $val, so we'll add it to the end of metawords and create a placeholder in metaidx - $id = count($metawords); - $metawords[$id] = $val; - $metaidx[$id] = ''; - $addwords = true; - } - // test if value is already in the index - if (isset($val_idx[$id]) && $val_idx[$id] <= 0){ - $val_idx[$id] = 0; - } else { // else add it - $val_idx[$id] = 1; - } - } - } - - if ($addwords) { - $this->saveIndex($metaname.'_w', '', $metawords); - } - $vals_changed = false; - foreach ($val_idx as $id => $action) { - if ($action == -1) { - $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0); - $vals_changed = true; - unset($val_idx[$id]); - } elseif ($action == 1) { - $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1); - $vals_changed = true; - } - } - - if ($vals_changed) { - $this->saveIndex($metaname.'_i', '', $metaidx); - $val_idx = implode(':', array_keys($val_idx)); - $this->saveIndexKey($metaname.'_p', '', $pid, $val_idx); - } - - unset($metaidx); - unset($metawords); - } - - $this->unlock(); - return true; - } - - /** - * Rename a page in the search index without changing the indexed content. This function doesn't check if the - * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the - * indexer and it deletes all previously indexed content of the new page. - * - * @param string $oldpage The old page name - * @param string $newpage The new page name - * @return string|bool If the page was successfully renamed, can be a message in the case of an error - */ - public function renamePage($oldpage, $newpage) { - if (!$this->lock()) return 'locked'; - - $pages = $this->getPages(); - - $id = array_search($oldpage, $pages, true); - if ($id === false) { - $this->unlock(); - return 'page is not in index'; - } - - $new_id = array_search($newpage, $pages, true); - if ($new_id !== false) { - // make sure the page is not in the index anymore - if ($this->deletePageNoLock($newpage) !== true) { - return false; - } - - $pages[$new_id] = 'deleted:'.time().rand(0, 9999); - } - - $pages[$id] = $newpage; - - // update index - if (!$this->saveIndex('page', '', $pages)) { - $this->unlock(); - return false; - } - - // reset the pid cache - $this->pidCache = array(); - - $this->unlock(); - return true; - } - - /** - * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages - * will be updated. - * - * @param string $key The metadata key of which a value shall be changed - * @param string $oldvalue The old value that shall be renamed - * @param string $newvalue The new value to which the old value shall be renamed, if exists values will be merged - * @return bool|string If renaming the value has been successful, false or error message on error. - */ - public function renameMetaValue($key, $oldvalue, $newvalue) { - if (!$this->lock()) return 'locked'; - - // change the relation references index - $metavalues = $this->getIndex($key, '_w'); - $oldid = array_search($oldvalue, $metavalues, true); - if ($oldid !== false) { - $newid = array_search($newvalue, $metavalues, true); - if ($newid !== false) { - // free memory - unset ($metavalues); - - // okay, now we have two entries for the same value. we need to merge them. - $indexline = $this->getIndexKey($key.'_i', '', $oldid); - if ($indexline != '') { - $newindexline = $this->getIndexKey($key.'_i', '', $newid); - $pagekeys = $this->getIndex($key.'_p', ''); - $parts = explode(':', $indexline); - foreach ($parts as $part) { - list($id, $count) = explode('*', $part); - $newindexline = $this->updateTuple($newindexline, $id, $count); - - $keyline = explode(':', $pagekeys[$id]); - // remove old meta value - $keyline = array_diff($keyline, array($oldid)); - // add new meta value when not already present - if (!in_array($newid, $keyline)) { - array_push($keyline, $newid); - } - $pagekeys[$id] = implode(':', $keyline); - } - $this->saveIndex($key.'_p', '', $pagekeys); - unset($pagekeys); - $this->saveIndexKey($key.'_i', '', $oldid, ''); - $this->saveIndexKey($key.'_i', '', $newid, $newindexline); - } - } else { - $metavalues[$oldid] = $newvalue; - if (!$this->saveIndex($key.'_w', '', $metavalues)) { - $this->unlock(); - return false; - } - } - } - - $this->unlock(); - return true; - } - - /** - * Remove a page from the index - * - * Erases entries in all known indexes. - * - * @param string $page a page name - * @return string|boolean the function completed successfully - * - * @author Tom N Harris - */ - public function deletePage($page) { - if (!$this->lock()) - return "locked"; - - $result = $this->deletePageNoLock($page); - - $this->unlock(); - - return $result; - } - - /** - * Remove a page from the index without locking the index, only use this function if the index is already locked - * - * Erases entries in all known indexes. - * - * @param string $page a page name - * @return boolean the function completed successfully - * - * @author Tom N Harris - */ - protected function deletePageNoLock($page) { - // load known documents - $pid = $this->getPIDNoLock($page); - if ($pid === false) { - return false; - } - - // Remove obsolete index entries - $pageword_idx = $this->getIndexKey('pageword', '', $pid); - if ($pageword_idx !== '') { - $delwords = explode(':',$pageword_idx); - $upwords = array(); - foreach ($delwords as $word) { - if ($word != '') { - list($wlen,$wid) = explode('*', $word); - $wid = (int)$wid; - $upwords[$wlen][] = $wid; - } - } - foreach ($upwords as $wlen => $widx) { - $index = $this->getIndex('i', $wlen); - foreach ($widx as $wid) { - $index[$wid] = $this->updateTuple($index[$wid], $pid, 0); - } - $this->saveIndex('i', $wlen, $index); - } - } - // Save the reverse index - if (!$this->saveIndexKey('pageword', '', $pid, "")) { - return false; - } - - $this->saveIndexKey('title', '', $pid, ""); - $keyidx = $this->getIndex('metadata', ''); - foreach ($keyidx as $metaname) { - $val_idx = explode(':', $this->getIndexKey($metaname.'_p', '', $pid)); - $meta_idx = $this->getIndex($metaname.'_i', ''); - foreach ($val_idx as $id) { - if ($id === '') continue; - $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0); - } - $this->saveIndex($metaname.'_i', '', $meta_idx); - $this->saveIndexKey($metaname.'_p', '', $pid, ''); - } - - return true; - } - - /** - * Clear the whole index - * - * @return bool If the index has been cleared successfully - */ - public function clear() { - global $conf; - - if (!$this->lock()) return false; - - @unlink($conf['indexdir'].'/page.idx'); - @unlink($conf['indexdir'].'/title.idx'); - @unlink($conf['indexdir'].'/pageword.idx'); - @unlink($conf['indexdir'].'/metadata.idx'); - $dir = @opendir($conf['indexdir']); - if($dir!==false){ - while(($f = readdir($dir)) !== false){ - if(substr($f,-4)=='.idx' && - (substr($f,0,1)=='i' || substr($f,0,1)=='w' - || substr($f,-6)=='_w.idx' || substr($f,-6)=='_i.idx' || substr($f,-6)=='_p.idx')) - @unlink($conf['indexdir']."/$f"); - } - } - @unlink($conf['indexdir'].'/lengths.idx'); - - // clear the pid cache - $this->pidCache = array(); - - $this->unlock(); - return true; - } - - /** - * Split the text into words for fulltext search - * - * TODO: does this also need &$stopwords ? - * - * @triggers INDEXER_TEXT_PREPARE - * This event allows plugins to modify the text before it gets tokenized. - * Plugins intercepting this event should also intercept INDEX_VERSION_GET - * - * @param string $text plain text - * @param boolean $wc are wildcards allowed? - * @return array list of words in the text - * - * @author Tom N Harris - * @author Andreas Gohr - */ - public function tokenizer($text, $wc=false) { - $wc = ($wc) ? '' : '\*'; - $stopwords =& idx_get_stopwords(); - - // prepare the text to be tokenized - $evt = new Event('INDEXER_TEXT_PREPARE', $text); - if ($evt->advise_before(true)) { - if (preg_match('/[^0-9A-Za-z ]/u', $text)) { - $text = \dokuwiki\Utf8\Asian::separateAsianWords($text); - } - } - $evt->advise_after(); - unset($evt); - - $text = strtr($text, - array( - "\r" => ' ', - "\n" => ' ', - "\t" => ' ', - "\xC2\xAD" => '', //soft-hyphen - ) - ); - if (preg_match('/[^0-9A-Za-z ]/u', $text)) - $text = \dokuwiki\Utf8\Clean::stripspecials($text, ' ', '\._\-:'.$wc); - - $wordlist = explode(' ', $text); - foreach ($wordlist as $i => $word) { - $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ? - \dokuwiki\Utf8\PhpString::strtolower($word) : strtolower($word); - } - - foreach ($wordlist as $i => $word) { - if ((!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH) - || array_search($word, $stopwords, true) !== false) - unset($wordlist[$i]); - } - return array_values($wordlist); - } - - /** - * Get the numeric PID of a page - * - * @param string $page The page to get the PID for - * @return bool|int The page id on success, false on error - */ - public function getPID($page) { - // return PID without locking when it is in the cache - if (isset($this->pidCache[$page])) return $this->pidCache[$page]; - - if (!$this->lock()) - return false; - - // load known documents - $pid = $this->getPIDNoLock($page); - if ($pid === false) { - $this->unlock(); - return false; - } - - $this->unlock(); - return $pid; - } - - /** - * Get the numeric PID of a page without locking the index. - * Only use this function when the index is already locked. - * - * @param string $page The page to get the PID for - * @return bool|int The page id on success, false on error - */ - protected function getPIDNoLock($page) { - // avoid expensive addIndexKey operation for the most recently requested pages by using a cache - if (isset($this->pidCache[$page])) return $this->pidCache[$page]; - $pid = $this->addIndexKey('page', '', $page); - // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently - // added item will be requested again - if (count($this->pidCache) > 10) array_shift($this->pidCache); - $this->pidCache[$page] = $pid; - return $pid; - } - - /** - * Get the page id of a numeric PID - * - * @param int $pid The PID to get the page id for - * @return string The page id - */ - public function getPageFromPID($pid) { - return $this->getIndexKey('page', '', $pid); - } - - /** - * Find pages in the fulltext index containing the words, - * - * The search words must be pre-tokenized, meaning only letters and - * numbers with an optional wildcard - * - * The returned array will have the original tokens as key. The values - * in the returned list is an array with the page names as keys and the - * number of times that token appears on the page as value. - * - * @param array $tokens list of words to search for - * @return array list of page names with usage counts - * - * @author Tom N Harris - * @author Andreas Gohr - */ - public function lookup(&$tokens) { - $result = array(); - $wids = $this->getIndexWords($tokens, $result); - if (empty($wids)) return array(); - // load known words and documents - $page_idx = $this->getIndex('page', ''); - $docs = array(); - foreach (array_keys($wids) as $wlen) { - $wids[$wlen] = array_unique($wids[$wlen]); - $index = $this->getIndex('i', $wlen); - foreach($wids[$wlen] as $ixid) { - if ($ixid < count($index)) - $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]); - } - } - // merge found pages into final result array - $final = array(); - foreach ($result as $word => $res) { - $final[$word] = array(); - foreach ($res as $wid) { - // handle the case when ($ixid < count($index)) has been false - // and thus $docs[$wid] hasn't been set. - if (!isset($docs[$wid])) continue; - $hits = &$docs[$wid]; - foreach ($hits as $hitkey => $hitcnt) { - // make sure the document still exists - if (!page_exists($hitkey, '', false)) continue; - if (!isset($final[$word][$hitkey])) - $final[$word][$hitkey] = $hitcnt; - else - $final[$word][$hitkey] += $hitcnt; - } - } - } - return $final; - } - - /** - * Find pages containing a metadata key. - * - * The metadata values are compared as case-sensitive strings. Pass a - * callback function that returns true or false to use a different - * comparison function. The function will be called with the $value being - * searched for as the first argument, and the word in the index as the - * second argument. The function preg_match can be used directly if the - * values are regexes. - * - * @param string $key name of the metadata key to look for - * @param string $value search term to look for, must be a string or array of strings - * @param callback $func comparison function - * @return array lists with page names, keys are query values if $value is array - * - * @author Tom N Harris - * @author Michael Hamann - */ - public function lookupKey($key, &$value, $func=null) { - if (!is_array($value)) - $value_array = array($value); - else - $value_array =& $value; - - // the matching ids for the provided value(s) - $value_ids = array(); - - $metaname = idx_cleanName($key); - - // get all words in order to search the matching ids - if ($key == 'title') { - $words = $this->getIndex('title', ''); - } else { - $words = $this->getIndex($metaname.'_w', ''); - } - - if (!is_null($func)) { - foreach ($value_array as $val) { - foreach ($words as $i => $word) { - if (call_user_func_array($func, array($val, $word))) - $value_ids[$i][] = $val; - } - } - } else { - foreach ($value_array as $val) { - $xval = $val; - $caret = '^'; - $dollar = '$'; - // check for wildcards - if (substr($xval, 0, 1) == '*') { - $xval = substr($xval, 1); - $caret = ''; - } - if (substr($xval, -1, 1) == '*') { - $xval = substr($xval, 0, -1); - $dollar = ''; - } - if (!$caret || !$dollar) { - $re = $caret.preg_quote($xval, '/').$dollar; - foreach(array_keys(preg_grep('/'.$re.'/', $words)) as $i) - $value_ids[$i][] = $val; - } else { - if (($i = array_search($val, $words, true)) !== false) - $value_ids[$i][] = $val; - } - } - } - - unset($words); // free the used memory - - // initialize the result so it won't be null - $result = array(); - foreach ($value_array as $val) { - $result[$val] = array(); - } - - $page_idx = $this->getIndex('page', ''); - - // Special handling for titles - if ($key == 'title') { - foreach ($value_ids as $pid => $val_list) { - $page = $page_idx[$pid]; - foreach ($val_list as $val) { - $result[$val][] = $page; - } - } - } else { - // load all lines and pages so the used lines can be taken and matched with the pages - $lines = $this->getIndex($metaname.'_i', ''); - - foreach ($value_ids as $value_id => $val_list) { - // parse the tuples of the form page_id*1:page2_id*1 and so on, return value - // is an array with page_id => 1, page2_id => 1 etc. so take the keys only - $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id])); - foreach ($val_list as $val) { - $result[$val] = array_merge($result[$val], $pages); - } - } - } - if (!is_array($value)) $result = $result[$value]; - return $result; - } - - /** - * Find the index ID of each search term. - * - * The query terms should only contain valid characters, with a '*' at - * either the beginning or end of the word (or both). - * The $result parameter can be used to merge the index locations with - * the appropriate query term. - * - * @param array $words The query terms. - * @param array $result Set to word => array("length*id" ...) - * @return array Set to length => array(id ...) - * - * @author Tom N Harris - */ - protected function getIndexWords(&$words, &$result) { - $tokens = array(); - $tokenlength = array(); - $tokenwild = array(); - foreach ($words as $word) { - $result[$word] = array(); - $caret = '^'; - $dollar = '$'; - $xword = $word; - $wlen = wordlen($word); - - // check for wildcards - if (substr($xword, 0, 1) == '*') { - $xword = substr($xword, 1); - $caret = ''; - $wlen -= 1; - } - if (substr($xword, -1, 1) == '*') { - $xword = substr($xword, 0, -1); - $dollar = ''; - $wlen -= 1; - } - if ($wlen < IDX_MINWORDLENGTH && $caret && $dollar && !is_numeric($xword)) - continue; - if (!isset($tokens[$xword])) - $tokenlength[$wlen][] = $xword; - if (!$caret || !$dollar) { - $re = $caret.preg_quote($xword, '/').$dollar; - $tokens[$xword][] = array($word, '/'.$re.'/'); - if (!isset($tokenwild[$xword])) - $tokenwild[$xword] = $wlen; - } else { - $tokens[$xword][] = array($word, null); - } - } - asort($tokenwild); - // $tokens = array( base word => array( [ query term , regexp ] ... ) ... ) - // $tokenlength = array( base word length => base word ... ) - // $tokenwild = array( base word => base word length ... ) - $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength)); - $indexes_known = $this->indexLengths($length_filter); - if (!empty($tokenwild)) sort($indexes_known); - // get word IDs - $wids = array(); - foreach ($indexes_known as $ixlen) { - $word_idx = $this->getIndex('w', $ixlen); - // handle exact search - if (isset($tokenlength[$ixlen])) { - foreach ($tokenlength[$ixlen] as $xword) { - $wid = array_search($xword, $word_idx, true); - if ($wid !== false) { - $wids[$ixlen][] = $wid; - foreach ($tokens[$xword] as $w) - $result[$w[0]][] = "$ixlen*$wid"; - } - } - } - // handle wildcard search - foreach ($tokenwild as $xword => $wlen) { - if ($wlen >= $ixlen) break; - foreach ($tokens[$xword] as $w) { - if (is_null($w[1])) continue; - foreach(array_keys(preg_grep($w[1], $word_idx)) as $wid) { - $wids[$ixlen][] = $wid; - $result[$w[0]][] = "$ixlen*$wid"; - } - } - } - } - return $wids; - } - - /** - * Return a list of all pages - * Warning: pages may not exist! - * - * @param string $key list only pages containing the metadata key (optional) - * @return array list of page names - * - * @author Tom N Harris - */ - public function getPages($key=null) { - $page_idx = $this->getIndex('page', ''); - if (is_null($key)) return $page_idx; - - $metaname = idx_cleanName($key); - - // Special handling for titles - if ($key == 'title') { - $title_idx = $this->getIndex('title', ''); - array_splice($page_idx, count($title_idx)); - foreach ($title_idx as $i => $title) - if ($title === "") unset($page_idx[$i]); - return array_values($page_idx); - } - - $pages = array(); - $lines = $this->getIndex($metaname.'_i', ''); - foreach ($lines as $line) { - $pages = array_merge($pages, $this->parseTuples($page_idx, $line)); - } - return array_keys($pages); - } - - /** - * Return a list of words sorted by number of times used - * - * @param int $min bottom frequency threshold - * @param int $max upper frequency limit. No limit if $max<$min - * @param int $minlen minimum length of words to count - * @param string $key metadata key to list. Uses the fulltext index if not given - * @return array list of words as the keys and frequency as values - * - * @author Tom N Harris - */ - public function histogram($min=1, $max=0, $minlen=3, $key=null) { - if ($min < 1) - $min = 1; - if ($max < $min) - $max = 0; - - $result = array(); - - if ($key == 'title') { - $index = $this->getIndex('title', ''); - $index = array_count_values($index); - foreach ($index as $val => $cnt) { - if ($cnt >= $min && (!$max || $cnt <= $max) && strlen($val) >= $minlen) - $result[$val] = $cnt; - } - } - elseif (!is_null($key)) { - $metaname = idx_cleanName($key); - $index = $this->getIndex($metaname.'_i', ''); - $val_idx = array(); - foreach ($index as $wid => $line) { - $freq = $this->countTuples($line); - if ($freq >= $min && (!$max || $freq <= $max)) - $val_idx[$wid] = $freq; - } - if (!empty($val_idx)) { - $words = $this->getIndex($metaname.'_w', ''); - foreach ($val_idx as $wid => $freq) { - if (strlen($words[$wid]) >= $minlen) - $result[$words[$wid]] = $freq; - } - } - } - else { - $lengths = idx_listIndexLengths(); - foreach ($lengths as $length) { - if ($length < $minlen) continue; - $index = $this->getIndex('i', $length); - $words = null; - foreach ($index as $wid => $line) { - $freq = $this->countTuples($line); - if ($freq >= $min && (!$max || $freq <= $max)) { - if ($words === null) - $words = $this->getIndex('w', $length); - $result[$words[$wid]] = $freq; - } - } - } - } - - arsort($result); - return $result; - } - - /** - * Lock the indexer. - * - * @author Tom N Harris - * - * @return bool|string - */ - protected function lock() { - global $conf; - $status = true; - $run = 0; - $lock = $conf['lockdir'].'/_indexer.lock'; - while (!@mkdir($lock, $conf['dmode'])) { - usleep(50); - if(is_dir($lock) && time()-@filemtime($lock) > 60*5){ - // looks like a stale lock - remove it - if (!@rmdir($lock)) { - $status = "removing the stale lock failed"; - return false; - } else { - $status = "stale lock removed"; - } - }elseif($run++ == 1000){ - // we waited 5 seconds for that lock - return false; - } - } - if (!empty($conf['dperm'])) { - chmod($lock, $conf['dperm']); - } - return $status; - } - - /** - * Release the indexer lock. - * - * @author Tom N Harris - * - * @return bool - */ - protected function unlock() { - global $conf; - @rmdir($conf['lockdir'].'/_indexer.lock'); - return true; - } - - /** - * Retrieve the entire index. - * - * The $suffix argument is for an index that is split into - * multiple parts. Different index files should use different - * base names. - * - * @param string $idx name of the index - * @param string $suffix subpart identifier - * @return array list of lines without CR or LF - * - * @author Tom N Harris - */ - protected function getIndex($idx, $suffix) { - global $conf; - $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; - if (!file_exists($fn)) return array(); - return file($fn, FILE_IGNORE_NEW_LINES); - } - - /** - * Replace the contents of the index with an array. - * - * @param string $idx name of the index - * @param string $suffix subpart identifier - * @param array $lines list of lines without LF - * @return bool If saving succeeded - * - * @author Tom N Harris - */ - protected function saveIndex($idx, $suffix, &$lines) { - global $conf; - $fn = $conf['indexdir'].'/'.$idx.$suffix; - $fh = @fopen($fn.'.tmp', 'w'); - if (!$fh) return false; - fwrite($fh, join("\n", $lines)); - if (!empty($lines)) - fwrite($fh, "\n"); - fclose($fh); - if (isset($conf['fperm'])) - chmod($fn.'.tmp', $conf['fperm']); - io_rename($fn.'.tmp', $fn.'.idx'); - return true; - } - - /** - * Retrieve a line from the index. - * - * @param string $idx name of the index - * @param string $suffix subpart identifier - * @param int $id the line number - * @return string a line with trailing whitespace removed - * - * @author Tom N Harris - */ - protected function getIndexKey($idx, $suffix, $id) { - global $conf; - $fn = $conf['indexdir'].'/'.$idx.$suffix.'.idx'; - if (!file_exists($fn)) return ''; - $fh = @fopen($fn, 'r'); - if (!$fh) return ''; - $ln = -1; - while (($line = fgets($fh)) !== false) { - if (++$ln == $id) break; - } - fclose($fh); - return rtrim((string)$line); - } - - /** - * Write a line into the index. - * - * @param string $idx name of the index - * @param string $suffix subpart identifier - * @param int $id the line number - * @param string $line line to write - * @return bool If saving succeeded - * - * @author Tom N Harris - */ - protected function saveIndexKey($idx, $suffix, $id, $line) { - global $conf; - if (substr($line, -1) != "\n") - $line .= "\n"; - $fn = $conf['indexdir'].'/'.$idx.$suffix; - $fh = @fopen($fn.'.tmp', 'w'); - if (!$fh) return false; - $ih = @fopen($fn.'.idx', 'r'); - if ($ih) { - $ln = -1; - while (($curline = fgets($ih)) !== false) { - fwrite($fh, (++$ln == $id) ? $line : $curline); - } - if ($id > $ln) { - while ($id > ++$ln) - fwrite($fh, "\n"); - fwrite($fh, $line); - } - fclose($ih); - } else { - $ln = -1; - while ($id > ++$ln) - fwrite($fh, "\n"); - fwrite($fh, $line); - } - fclose($fh); - if (isset($conf['fperm'])) - chmod($fn.'.tmp', $conf['fperm']); - io_rename($fn.'.tmp', $fn.'.idx'); - return true; - } - - /** - * Retrieve or insert a value in the index. - * - * @param string $idx name of the index - * @param string $suffix subpart identifier - * @param string $value line to find in the index - * @return int|bool line number of the value in the index or false if writing the index failed - * - * @author Tom N Harris - */ - protected function addIndexKey($idx, $suffix, $value) { - $index = $this->getIndex($idx, $suffix); - $id = array_search($value, $index, true); - if ($id === false) { - $id = count($index); - $index[$id] = $value; - if (!$this->saveIndex($idx, $suffix, $index)) { - trigger_error("Failed to write $idx index", E_USER_ERROR); - return false; - } - } - return $id; - } - - /** - * Get the list of lengths indexed in the wiki. - * - * Read the index directory or a cache file and returns - * a sorted array of lengths of the words used in the wiki. - * - * @author YoBoY - * - * @return array - */ - protected function listIndexLengths() { - return idx_listIndexLengths(); - } - - /** - * Get the word lengths that have been indexed. - * - * Reads the index directory and returns an array of lengths - * that there are indices for. - * - * @author YoBoY - * - * @param array|int $filter - * @return array - */ - protected function indexLengths($filter) { - global $conf; - $idx = array(); - if (is_array($filter)) { - // testing if index files exist only - $path = $conf['indexdir']."/i"; - foreach ($filter as $key => $value) { - if (file_exists($path.$key.'.idx')) - $idx[] = $key; - } - } else { - $lengths = idx_listIndexLengths(); - foreach ($lengths as $key => $length) { - // keep all the values equal or superior - if ((int)$length >= (int)$filter) - $idx[] = $length; - } - } - return $idx; - } - - /** - * Insert or replace a tuple in a line. - * - * @author Tom N Harris - * - * @param string $line - * @param string|int $id - * @param int $count - * @return string - */ - protected function updateTuple($line, $id, $count) { - if ($line != ''){ - $line = preg_replace('/(^|:)'.preg_quote($id,'/').'\*\d*/', '', $line); - } - $line = trim($line, ':'); - if ($count) { - if ($line) { - return "$id*$count:".$line; - } else { - return "$id*$count"; - } - } - return $line; - } - - /** - * Split a line into an array of tuples. - * - * @author Tom N Harris - * @author Andreas Gohr - * - * @param array $keys - * @param string $line - * @return array - */ - protected function parseTuples(&$keys, $line) { - $result = array(); - if ($line == '') return $result; - $parts = explode(':', $line); - foreach ($parts as $tuple) { - if ($tuple === '') continue; - list($key, $cnt) = explode('*', $tuple); - if (!$cnt) continue; - $key = $keys[$key]; - if ($key === false || is_null($key)) continue; - $result[$key] = $cnt; - } - return $result; - } - - /** - * Sum the counts in a list of tuples. - * - * @author Tom N Harris - * - * @param string $line - * @return int - */ - protected function countTuples($line) { - $freq = 0; - $parts = explode(':', $line); - foreach ($parts as $tuple) { - if ($tuple === '') continue; - list(/* $pid */, $cnt) = explode('*', $tuple); - $freq += (int)$cnt; - } - return $freq; - } -} - /** * Create an instance of the indexer. * - * @return Doku_Indexer a Doku_Indexer + * @return Indexer an Indexer * * @author Tom N Harris */ function idx_get_indexer() { static $Indexer; if (!isset($Indexer)) { - $Indexer = new Doku_Indexer(); + $Indexer = new Indexer(); } return $Indexer; } -- cgit v1.2.3 From 903d43fe60f76e180b5d8ef35db7007bee2a8e5e Mon Sep 17 00:00:00 2001 From: Michael Große Date: Tue, 31 Dec 2019 12:15:25 +0100 Subject: Check code style against PSR12 In 2019 PSR-12 was officially released and PSR-2 deprecated. We should adapt to that and check our files against that new coding standard. --- _test/phpcs.xml | 15 ++++++++++++++- inc/Cache/Cache.php | 2 +- inc/Extension/SyntaxPlugin.php | 4 ++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/_test/phpcs.xml b/_test/phpcs.xml index fbc079018..6e78784c8 100644 --- a/_test/phpcs.xml +++ b/_test/phpcs.xml @@ -35,7 +35,7 @@ */inc/parser/* - + @@ -103,6 +103,19 @@ + + + + + + + + + + + + + diff --git a/inc/Cache/Cache.php b/inc/Cache/Cache.php index 599dc5f59..ab64da5cb 100644 --- a/inc/Cache/Cache.php +++ b/inc/Cache/Cache.php @@ -2,7 +2,7 @@ namespace dokuwiki\Cache; -use \dokuwiki\Debug\PropertyDeprecationHelper; +use dokuwiki\Debug\PropertyDeprecationHelper; use dokuwiki\Extension\Event; /** diff --git a/inc/Extension/SyntaxPlugin.php b/inc/Extension/SyntaxPlugin.php index e5dda9bdc..ea8f51b4d 100644 --- a/inc/Extension/SyntaxPlugin.php +++ b/inc/Extension/SyntaxPlugin.php @@ -2,8 +2,8 @@ namespace dokuwiki\Extension; -use \Doku_Handler; -use \Doku_Renderer; +use Doku_Handler; +use Doku_Renderer; /** * Syntax Plugin Prototype -- cgit v1.2.3