From 97e0f73d88d0b3ba3e905ec354a8490cfc79873f Mon Sep 17 00:00:00 2001
From: alex <alex@alexloehr.net>
Date: Fri, 06 Jun 2025 14:26:14 +0000
Subject: [PATCH] adding search
---
vue/index.html | 2
app.js | 34 +++++++-
package-lock.json | 33 ++++++++
package.json | 1
lib/search.js | 158 +++++++++++++++++++++++++++++++++++++++
5 files changed, 221 insertions(+), 7 deletions(-)
diff --git a/app.js b/app.js
index a9633f7..7b99446 100644
--- a/app.js
+++ b/app.js
@@ -3,10 +3,11 @@
logger: true
})
const _ = require("lodash")
-const db = require("./lib/db")
-
-const settings = require("./settings")
const fs = require("node:fs")
+
+const db = require("./lib/db")
+const settings = require("./settings")
+const search = require("./lib/search.js")
/////////////////////////////////////////////////////////////////////////
@@ -27,7 +28,7 @@
return res.send({status: "error", error: "access denied"})
}
else {
- console.log("NO AUTH FOR ",req.url)
+ console.log("NO AUTH FOR ", req.url)
}
})
@@ -54,7 +55,7 @@
})
.get("/api/user/userid/:userid", async function (req, res) {
const {userid} = req.params
- if(!userid || isNaN(Number(userid))) {
+ if (!userid || isNaN(Number(userid))) {
return res.code(500).send({status: "error", msg: "userid error"})
}
const user = await db.getUserByUserId(userid)
@@ -68,7 +69,7 @@
.get("/api/user/teilnahmen/:userId", async function (req, res) {
let userId = req.params.userId
console.log(`--------${userId}-----------`, typeof userId)
- if(!userId || isNaN(Number(userId))) {
+ if (!userId || isNaN(Number(userId))) {
return res.code(500).send({status: "error", msg: "userId error"})
}
const tn = await db.getUserTeilnahmen(userId)
@@ -104,6 +105,7 @@
})
/////// Kurs ////////////////////////////////////////////////////////////////
+
.get("/api/kurs", async function (req, res) {
let data = await db.getKurse()
if (data) {
@@ -154,6 +156,26 @@
}
})
+/////// SEARCH ////////////////////////////////////////////////////////////////
+
+const searchLib = require("./lib/search")
+searchLib.doIndex().catch(console.error)
+fastify.get("/api/search/user", async function (req, res) {
+ console.log(req.query)
+ const search = req.query?.search
+ if (!search) {
+ return res.code(422).send({status: "error", msg: "no search"})
+ }
+ else {
+ console.log(search)
+ const data = await searchLib.search(search)
+ return res.send(data)
+ }
+})
+
+/////// STATIC ////////////////////////////////////////////////////////////////
+
+
fastify.register(require('@fastify/static'), {
root: path.join(__dirname, 'vue/dist'),
prefix: '/ui/', // optional: default '/'
diff --git a/lib/search.js b/lib/search.js
new file mode 100644
index 0000000..d054498
--- /dev/null
+++ b/lib/search.js
@@ -0,0 +1,158 @@
+const _ = require("lodash")
+const {Index, Document, Worker} = require("flexsearch")
+
+/////////////////////////////////////////////////////////////////////////
+
+// Message index
+const options = {
+ tokenize: "full",
+ split: true,
+}
+
+const idxUser = new Index(options)
+
+// Tag index
+const optionsTag = {
+ tokenize: "forward", // nur vorwärts indexieren bei den tags
+ split: true, // ein Tag ist immer nur ein Wort
+ // split: true, // doc
+ encode: function (it) {
+ // return it
+ return it.split(" ")
+ // return [it]
+ },
+ // encode: it => function (it) {
+ // return it.split(" ")
+ // },
+ // encode: "default",
+ stemmer: false,
+ matcher: false,
+ context: false,
+}
+const idxTags = new Index(optionsTag)
+// const idxTags = new Document({
+// document: {
+// id: "_id",
+// index: [
+// {
+// field: "tags",
+// tokenize: "forward",
+// // encode: it => it, // encode sorgt dafür, dass die Suche nach "+" funktioniert, aber auch dass komische Ergebnisse erscheinen
+// }
+// ],
+// },
+// })
+
+/////////////////////////////////////////////////////////////////////////
+
+module.exports = {
+ // idxMessage: idxUser,
+ // idxTags,
+
+ doIndex,
+ search,
+
+ // searchUsers,
+ // searchTags,
+
+ // addMessage: addUser,
+ // updateMessage: updateUser,
+ // deleteMessage: removeUser,
+
+ // addTags,
+ // removeTags,
+}
+
+// run()
+// .then(console.log)
+// .catch(console.error)
+
+async function run() {
+ await doIndex()
+ console.log(search("latu"))
+}
+
+/////////////////////////////////////////////////////////////////////////
+
+async function doIndex () {
+ const start = Date.now()
+ console.log("++ START indexing Users...")
+ let users = require("../users.json")
+ // users = users.slice(10)
+
+ for (const user of users) {
+ addUser(user)
+ // addTags(user)
+ }
+ console.log(`++ END indexing Users in ${Date.now() - start}ms`)
+}
+
+function search (query) {
+ return idxUser.search(query)
+}
+
+function searchUsers (query, user) {
+ // query = query.split(" ").join(" OR ") // ohne das "OR" scheint immer nur "AND" zu sein | die search option {bool:"or"} wird ignoriert
+ // console.log(`searching messages for "${query}"`)
+ return idxUser.search(`${user} ${query}`, {suggest: true})
+}
+
+function searchTags (query, user) {
+ const limit = 100000 // todo das mit dem Limit anders lösen // count? nein siehe https://github.com/nextapps-de/flexsearch?tab=readme-ov-file#limit--offset
+ const results = idxTags.search(`${user} ${query}`, limit)
+ return results
+ // format is now [{field,result:[_id]}] because using document index
+ // return results.length ? results[0].result : []
+}
+
+/////// idxMessage FNS ////////////////////////////////////////////////////////////////
+
+function getUserString (user) {
+ const {usr_id, firstname, lastname} = user
+ return `${usr_id} ${firstname} ${lastname}`.trim()
+}
+
+function addUser (user) {
+ add(idxUser, user.usr_id, getUserString(user))
+}
+
+function updateUser ({_id, title, tags, user}) {
+ update(idxUser, user.usr_id, getUserString(user))
+}
+
+function removeUser (usr_id) {
+ remove(idxUser, usr_id)
+}
+
+
+/////// idxTags FNS ////////////////////////////////////////////////////////////////
+
+/**
+ * add Tags from a message
+ * @param _id
+ * @param tags
+ */
+function addTags (msg) {
+ let {_id, tags, user} = msg
+ if (!tags) throw new Error("tags must be an array")
+ if (_.isString(tags)) tags = [tags]
+ add(idxTags, _id.toString(), `${user} ${tags.join(" ")}`) // muss erst gejoined werden, dann später in encode wird noch mal gesplittet - anders rum geht es nicht!
+}
+
+function removeTags (_id) {
+ remove(idxTags, _id.toString())
+}
+
+/////// FNS ////////////////////////////////////////////////////////////////
+
+function add (index, key, value) {
+ index.add(key, value)
+}
+
+function update (index, key, value) {
+ index.update(key, value)
+}
+
+function remove (index, key) {
+ index.remove(key)
+}
diff --git a/package-lock.json b/package-lock.json
index 7861474..a215029 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -15,6 +15,7 @@
"dayjs": "^1.11.13",
"dotenv": "^16.5.0",
"fastify": "^5.3.3",
+ "flexsearch": "^0.8.205",
"lodash": "^4.17.21",
"mysql2": "^3.14.1",
"nconf": "^0.13.0",
@@ -2869,6 +2870,38 @@
"node": ">=8"
}
},
+ "node_modules/flexsearch": {
+ "version": "0.8.205",
+ "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.205.tgz",
+ "integrity": "sha512-REFjMqy86DKkCTJ4gIE42c9MVm9t1vUWfEub/8taixYuhvyu4jd4XmFALk5VuKW4GH4VLav8A4BJboTsslHF1w==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/ts-thomas"
+ },
+ {
+ "type": "opencollective",
+ "url": "https://opencollective.com/flexsearch"
+ },
+ {
+ "type": "patreon",
+ "url": "https://patreon.com/user?u=96245532"
+ },
+ {
+ "type": "liberapay",
+ "url": "https://liberapay.com/ts-thomas"
+ },
+ {
+ "type": "paypal",
+ "url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW"
+ },
+ {
+ "type": "bountysource",
+ "url": "https://salt.bountysource.com/teams/ts-thomas"
+ }
+ ],
+ "license": "Apache-2.0"
+ },
"node_modules/foreground-child": {
"version": "3.3.1",
"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz",
diff --git a/package.json b/package.json
index 551c938..984f248 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
"dayjs": "^1.11.13",
"dotenv": "^16.5.0",
"fastify": "^5.3.3",
+ "flexsearch": "^0.8.205",
"lodash": "^4.17.21",
"mysql2": "^3.14.1",
"nconf": "^0.13.0",
diff --git a/vue/index.html b/vue/index.html
index b19040a..3de5e11 100644
--- a/vue/index.html
+++ b/vue/index.html
@@ -4,7 +4,7 @@
<meta charset="UTF-8">
<link rel="icon" href="/favicon.ico">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <title>Vite App</title>
+ <title>globus-ilias-rest UI</title>
</head>
<body>
<div id="app"></div>
--
Gitblit v1.8.0