Merge branch 'mk-stricter-tokenizer' into 'master'

use stricter unicode-aware tokenizer separator

See merge request lcsb/sps/reploy!4

2 files changed, 4 insertions, 1 deletions

diff --git a/scripts/make-search-index.js b/scripts/make-search-index.js
index 1acc7cd..f923b54 100644
--- a/scripts/make-search-index.js
+++ b/scripts/make-search-index.js
@@ -26,7 +26,7 @@
  * yarnpkg add lunr
  *
  * Usage:
- * site ....some args.... --search-data-output tmp/search-raw.json
+ * reploy ....some args.... --search-data-output tmp/search-raw.json
  * node scripts/make-search-index.js tmp/search-raw.json _site/search-index.json _site/search-metadata.json
  */
 
@@ -40,6 +40,8 @@ if(process.argv.length !== 5) {
 
 documents = JSON.parse(fs.readFileSync(process.argv[2], {encoding: 'utf8'}))
 
+lunr.tokenizer.separator = /(\p{P}|\p{S}|\p{Z}|\p{C})+/u
+
 var idx = lunr(function () {
   this.ref('link')
   this.field('title', {boost: 9})
diff --git a/templates/searchresults.html b/templates/searchresults.html
index 5044d0b..5ced869 100644
--- a/templates/searchresults.html
+++ b/templates/searchresults.html
@@ -31,6 +31,7 @@
 <script src="{{root}}static/lunr.min.js"></script>
 
 <script>
+  lunr.QueryLexer.termSeparator = /(\p{P}|\p{S}|\p{Z}|\p{C})+/u;
   var el_query = document.getElementById('search_query');
   var el_placeholder = document.getElementById('search_placeholder');
   var el_noquery = document.getElementById('search_noquery');