globus/globus-ilias-rest.git

parent: c82fec24 | patch | commit | ignore whitespace

alex

2025-06-06 97e0f73d88d0b3ba3e905ec354a8490cfc79873f

adding search

1 files added

4 files modified

	app.js	34 ●●●●● patch \| view \| raw \| blame \| history
	lib/search.js	158 ●●●●● patch \| view \| raw \| blame \| history
	package-lock.json	33 ●●●●● patch \| view \| raw \| blame \| history
	package.json	1 ●●●●● patch \| view \| raw \| blame \| history
	vue/index.html	2 ●●●●● patch \| view \| raw \| blame \| history

 app.js

@@ -3,10 +3,11 @@
   logger: true
})
const _ = require("lodash")
const db = require("./lib/db")

const settings = require("./settings")
const fs = require("node:fs")

const db = require("./lib/db")
const settings = require("./settings")
const search = require("./lib/search.js")

/////////////////////////////////////////////////////////////////////////

@@ -27,7 +28,7 @@
      return res.send({status: "error", error: "access denied"})
   }
   else {
      console.log("NO AUTH FOR ",req.url)
      console.log("NO AUTH FOR ", req.url)
   }
})

@@ -54,7 +55,7 @@
   })
   .get("/api/user/userid/:userid", async function (req, res) {
      const {userid} = req.params
      if(!userid || isNaN(Number(userid))) {
      if (!userid || isNaN(Number(userid))) {
         return res.code(500).send({status: "error", msg: "userid error"})
      }
      const user = await db.getUserByUserId(userid)
@@ -68,7 +69,7 @@
   .get("/api/user/teilnahmen/:userId", async function (req, res) {
      let userId = req.params.userId
      console.log(`--------${userId}-----------`, typeof userId)
      if(!userId || isNaN(Number(userId))) {
      if (!userId || isNaN(Number(userId))) {
         return res.code(500).send({status: "error", msg: "userId error"})
      }
      const tn = await db.getUserTeilnahmen(userId)
@@ -104,6 +105,7 @@
   })

   /////// Kurs ////////////////////////////////////////////////////////////////

   .get("/api/kurs", async function (req, res) {
      let data = await db.getKurse()
      if (data) {
@@ -154,6 +156,26 @@
      }
   })

/////// SEARCH ////////////////////////////////////////////////////////////////

const searchLib = require("./lib/search")
searchLib.doIndex().catch(console.error)
fastify.get("/api/search/user", async function (req, res) {
   console.log(req.query)
   const search = req.query?.search
   if (!search) {
      return res.code(422).send({status: "error", msg: "no search"})
   }
   else {
      console.log(search)
      const data = await searchLib.search(search)
      return res.send(data)
   }
})

/////// STATIC ////////////////////////////////////////////////////////////////


fastify.register(require('@fastify/static'), {
   root: path.join(__dirname, 'vue/dist'),
   prefix: '/ui/', // optional: default '/'

 lib/search.js

New file
@@ -0,0 +1,158 @@
const _ = require("lodash")
const {Index, Document, Worker} = require("flexsearch")

/////////////////////////////////////////////////////////////////////////

// Message index
const options = {
   tokenize: "full",
   split: true,
}

const idxUser = new Index(options)

// Tag index
const optionsTag = {
   tokenize: "forward", // nur vorwärts indexieren bei den tags
   split: true, // ein Tag ist immer nur ein Wort
   // split: true, // doc
   encode: function (it) {
      // return it
      return it.split(" ")
      // return [it]
   },
   // encode: it => function (it) {
   //    return it.split(" ")
   // },
   // encode: "default",
   stemmer: false,
   matcher: false,
   context: false,
}
const idxTags = new Index(optionsTag)
// const idxTags = new Document({
//    document: {
//       id: "_id",
//       index: [
//          {
//             field: "tags",
//             tokenize: "forward",
//             // encode: it => it, // encode sorgt dafür, dass die Suche nach "+" funktioniert, aber auch dass komische Ergebnisse erscheinen
//          }
//       ],
//    },
// })

/////////////////////////////////////////////////////////////////////////

module.exports = {
   // idxMessage: idxUser,
   // idxTags,

   doIndex,
   search,

   // searchUsers,
   // searchTags,

   // addMessage: addUser,
   // updateMessage: updateUser,
   // deleteMessage: removeUser,

   // addTags,
   // removeTags,
}

// run()
// .then(console.log)
// .catch(console.error)

async function run() {
   await doIndex()
   console.log(search("latu"))
}

/////////////////////////////////////////////////////////////////////////

async function doIndex () {
   const start = Date.now()
   console.log("++ START indexing Users...")
   let users = require("../users.json")
   // users = users.slice(10)

   for (const user of users) {
      addUser(user)
      // addTags(user)
   }
   console.log(`++ END indexing Users in ${Date.now() - start}ms`)
}

function search (query) {
   return idxUser.search(query)
}

function searchUsers (query, user) {
   // query = query.split(" ").join(" OR ") // ohne das "OR" scheint immer nur "AND" zu sein | die search option {bool:"or"} wird ignoriert
   // console.log(`searching messages for "${query}"`)
   return idxUser.search(`${user} ${query}`, {suggest: true})
}

function searchTags (query, user) {
   const limit = 100000 // todo das mit dem Limit anders lösen // count? nein siehe https://github.com/nextapps-de/flexsearch?tab=readme-ov-file#limit--offset
   const results = idxTags.search(`${user} ${query}`, limit)
   return results
   // format is now [{field,result:[_id]}] because using document index
   // return results.length ? results[0].result : []
}

/////// idxMessage FNS ////////////////////////////////////////////////////////////////

function getUserString (user) {
   const {usr_id, firstname, lastname} = user
   return `${usr_id} ${firstname} ${lastname}`.trim()
}

function addUser (user) {
   add(idxUser, user.usr_id, getUserString(user))
}

function updateUser ({_id, title, tags, user}) {
   update(idxUser, user.usr_id, getUserString(user))
}

function removeUser (usr_id) {
   remove(idxUser, usr_id)
}


/////// idxTags FNS ////////////////////////////////////////////////////////////////

/**
 * add Tags from a message
 * @param _id
 * @param tags
 */
function addTags (msg) {
   let {_id, tags, user} = msg
   if (!tags) throw new Error("tags must be an array")
   if (_.isString(tags)) tags = [tags]
   add(idxTags, _id.toString(), `${user} ${tags.join(" ")}`) // muss erst gejoined werden, dann später in encode wird noch mal gesplittet - anders rum geht es nicht!
}

function removeTags (_id) {
   remove(idxTags, _id.toString())
}

/////// FNS ////////////////////////////////////////////////////////////////

function add (index, key, value) {
   index.add(key, value)
}

function update (index, key, value) {
   index.update(key, value)
}

function remove (index, key) {
   index.remove(key)
}

 package-lock.json

@@ -15,6 +15,7 @@
        "dayjs": "^1.11.13",
        "dotenv": "^16.5.0",
        "fastify": "^5.3.3",
        "flexsearch": "^0.8.205",
        "lodash": "^4.17.21",
        "mysql2": "^3.14.1",
        "nconf": "^0.13.0",
@@ -2869,6 +2870,38 @@
        "node": ">=8"
      }
    },
    "node_modules/flexsearch": {
      "version": "0.8.205",
      "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.205.tgz",
      "integrity": "sha512-REFjMqy86DKkCTJ4gIE42c9MVm9t1vUWfEub/8taixYuhvyu4jd4XmFALk5VuKW4GH4VLav8A4BJboTsslHF1w==",
      "funding": [
        {
          "type": "github",
          "url": "https://github.com/ts-thomas"
        },
        {
          "type": "opencollective",
          "url": "https://opencollective.com/flexsearch"
        },
        {
          "type": "patreon",
          "url": "https://patreon.com/user?u=96245532"
        },
        {
          "type": "liberapay",
          "url": "https://liberapay.com/ts-thomas"
        },
        {
          "type": "paypal",
          "url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW"
        },
        {
          "type": "bountysource",
          "url": "https://salt.bountysource.com/teams/ts-thomas"
        }
      ],
      "license": "Apache-2.0"
    },
    "node_modules/foreground-child": {
      "version": "3.3.1",
      "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz",

 package.json

@@ -19,6 +19,7 @@
    "dayjs": "^1.11.13",
    "dotenv": "^16.5.0",
    "fastify": "^5.3.3",
    "flexsearch": "^0.8.205",
    "lodash": "^4.17.21",
    "mysql2": "^3.14.1",
    "nconf": "^0.13.0",

 vue/index.html

@@ -4,7 +4,7 @@
    <meta charset="UTF-8">
    <link rel="icon" href="/favicon.ico">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Vite App</title>
    <title>globus-ilias-rest UI</title>
  </head>
  <body>
    <div id="app"></div>

			@@ -3,10 +3,11 @@
			logger: true
			})
			const _ = require("lodash")
			const db = require("./lib/db")

			const settings = require("./settings")
			const fs = require("node:fs")

			const db = require("./lib/db")
			const settings = require("./settings")
			const search = require("./lib/search.js")

			/////////////////////////////////////////////////////////////////////////

			@@ -27,7 +28,7 @@
			return res.send({status: "error", error: "access denied"})
			}
			else {
			console.log("NO AUTH FOR ",req.url)
			console.log("NO AUTH FOR ", req.url)
			}
			})

			@@ -54,7 +55,7 @@
			})
			.get("/api/user/userid/:userid", async function (req, res) {
			const {userid} = req.params
			if(!userid \|\| isNaN(Number(userid))) {
			if (!userid \|\| isNaN(Number(userid))) {
			return res.code(500).send({status: "error", msg: "userid error"})
			}
			const user = await db.getUserByUserId(userid)
			@@ -68,7 +69,7 @@
			.get("/api/user/teilnahmen/:userId", async function (req, res) {
			let userId = req.params.userId
			console.log(`--------${userId}-----------`, typeof userId)
			if(!userId \|\| isNaN(Number(userId))) {
			if (!userId \|\| isNaN(Number(userId))) {
			return res.code(500).send({status: "error", msg: "userId error"})
			}
			const tn = await db.getUserTeilnahmen(userId)
			@@ -104,6 +105,7 @@
			})

			/////// Kurs ////////////////////////////////////////////////////////////////

			.get("/api/kurs", async function (req, res) {
			let data = await db.getKurse()
			if (data) {
			@@ -154,6 +156,26 @@
			}
			})

			/////// SEARCH ////////////////////////////////////////////////////////////////

			const searchLib = require("./lib/search")
			searchLib.doIndex().catch(console.error)
			fastify.get("/api/search/user", async function (req, res) {
			console.log(req.query)
			const search = req.query?.search
			if (!search) {
			return res.code(422).send({status: "error", msg: "no search"})
			}
			else {
			console.log(search)
			const data = await searchLib.search(search)
			return res.send(data)
			}
			})

			/////// STATIC ////////////////////////////////////////////////////////////////


			fastify.register(require('@fastify/static'), {
			root: path.join(__dirname, 'vue/dist'),
			prefix: '/ui/', // optional: default '/'

New file
			@@ -0,0 +1,158 @@
			const _ = require("lodash")
			const {Index, Document, Worker} = require("flexsearch")

			/////////////////////////////////////////////////////////////////////////

			// Message index
			const options = {
			tokenize: "full",
			split: true,
			}

			const idxUser = new Index(options)

			// Tag index
			const optionsTag = {
			tokenize: "forward", // nur vorwärts indexieren bei den tags
			split: true, // ein Tag ist immer nur ein Wort
			// split: true, // doc
			encode: function (it) {
			// return it
			return it.split(" ")
			// return [it]
			},
			// encode: it => function (it) {
			// return it.split(" ")
			// },
			// encode: "default",
			stemmer: false,
			matcher: false,
			context: false,
			}
			const idxTags = new Index(optionsTag)
			// const idxTags = new Document({
			// document: {
			// id: "_id",
			// index: [
			// {
			// field: "tags",
			// tokenize: "forward",
			// // encode: it => it, // encode sorgt dafür, dass die Suche nach "+" funktioniert, aber auch dass komische Ergebnisse erscheinen
			// }
			// ],
			// },
			// })

			/////////////////////////////////////////////////////////////////////////

			module.exports = {
			// idxMessage: idxUser,
			// idxTags,

			doIndex,
			search,

			// searchUsers,
			// searchTags,

			// addMessage: addUser,
			// updateMessage: updateUser,
			// deleteMessage: removeUser,

			// addTags,
			// removeTags,
			}

			// run()
			// .then(console.log)
			// .catch(console.error)

			async function run() {
			await doIndex()
			console.log(search("latu"))
			}

			/////////////////////////////////////////////////////////////////////////

			async function doIndex () {
			const start = Date.now()
			console.log("++ START indexing Users...")
			let users = require("../users.json")
			// users = users.slice(10)

			for (const user of users) {
			addUser(user)
			// addTags(user)
			}
			console.log(`++ END indexing Users in ${Date.now() - start}ms`)
			}

			function search (query) {
			return idxUser.search(query)
			}

			function searchUsers (query, user) {
			// query = query.split(" ").join(" OR ") // ohne das "OR" scheint immer nur "AND" zu sein \| die search option {bool:"or"} wird ignoriert
			// console.log(`searching messages for "${query}"`)
			return idxUser.search(`${user} ${query}`, {suggest: true})
			}

			function searchTags (query, user) {
			const limit = 100000 // todo das mit dem Limit anders lösen // count? nein siehe https://github.com/nextapps-de/flexsearch?tab=readme-ov-file#limit--offset
			const results = idxTags.search(`${user} ${query}`, limit)
			return results
			// format is now [{field,result:[_id]}] because using document index
			// return results.length ? results[0].result : []
			}

			/////// idxMessage FNS ////////////////////////////////////////////////////////////////

			function getUserString (user) {
			const {usr_id, firstname, lastname} = user
			return `${usr_id} ${firstname} ${lastname}`.trim()
			}

			function addUser (user) {
			add(idxUser, user.usr_id, getUserString(user))
			}

			function updateUser ({_id, title, tags, user}) {
			update(idxUser, user.usr_id, getUserString(user))
			}

			function removeUser (usr_id) {
			remove(idxUser, usr_id)
			}


			/////// idxTags FNS ////////////////////////////////////////////////////////////////

			/**
			* add Tags from a message
			* @param _id
			* @param tags
			*/
			function addTags (msg) {
			let {_id, tags, user} = msg
			if (!tags) throw new Error("tags must be an array")
			if (_.isString(tags)) tags = [tags]
			add(idxTags, _id.toString(), `${user} ${tags.join(" ")}`) // muss erst gejoined werden, dann später in encode wird noch mal gesplittet - anders rum geht es nicht!
			}

			function removeTags (_id) {
			remove(idxTags, _id.toString())
			}

			/////// FNS ////////////////////////////////////////////////////////////////

			function add (index, key, value) {
			index.add(key, value)
			}

			function update (index, key, value) {
			index.update(key, value)
			}

			function remove (index, key) {
			index.remove(key)
			}

			@@ -15,6 +15,7 @@
			"dayjs": "^1.11.13",
			"dotenv": "^16.5.0",
			"fastify": "^5.3.3",
			"flexsearch": "^0.8.205",
			"lodash": "^4.17.21",
			"mysql2": "^3.14.1",
			"nconf": "^0.13.0",
			@@ -2869,6 +2870,38 @@
			"node": ">=8"
			}
			},
			"node_modules/flexsearch": {
			"version": "0.8.205",
			"resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.205.tgz",
			"integrity": "sha512-REFjMqy86DKkCTJ4gIE42c9MVm9t1vUWfEub/8taixYuhvyu4jd4XmFALk5VuKW4GH4VLav8A4BJboTsslHF1w==",
			"funding": [
			{
			"type": "github",
			"url": "https://github.com/ts-thomas"
			},
			{
			"type": "opencollective",
			"url": "https://opencollective.com/flexsearch"
			},
			{
			"type": "patreon",
			"url": "https://patreon.com/user?u=96245532"
			},
			{
			"type": "liberapay",
			"url": "https://liberapay.com/ts-thomas"
			},
			{
			"type": "paypal",
			"url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW"
			},
			{
			"type": "bountysource",
			"url": "https://salt.bountysource.com/teams/ts-thomas"
			}
			],
			"license": "Apache-2.0"
			},
			"node_modules/foreground-child": {
			"version": "3.3.1",
			"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz",

			@@ -19,6 +19,7 @@
			"dayjs": "^1.11.13",
			"dotenv": "^16.5.0",
			"fastify": "^5.3.3",
			"flexsearch": "^0.8.205",
			"lodash": "^4.17.21",
			"mysql2": "^3.14.1",
			"nconf": "^0.13.0",

			@@ -4,7 +4,7 @@
			<meta charset="UTF-8">
			<link rel="icon" href="/favicon.ico">
			<meta name="viewport" content="width=device-width, initial-scale=1.0">
			<title>Vite App</title>
			<title>globus-ilias-rest UI</title>
			</head>
			<body>
			<div id="app"></div>