Lukas/rkllm-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

remove ningle and create simple router

Browse source
This commit is contained in:
Lukas Nöllemeyer 2025-02-04 13:16:32 +01:00
parent 0aac3aae10
commit 63d079340b
3 changed files with 47 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
rkllm-server.asd
View file

@ -7,7 +7,7 @@
:license "MIT License"
;; Dependencies
:depends-on (:cl-autowrap :woo :clack :ningle :metabang-bind)
:depends-on (:cl-autowrap :woo :clack :metabang-bind :cl-json :str)
;; Components
:components ((:file "server")

46
server.lisp
View file

@ -111,21 +111,61 @@
(rkllm:input.prompt-input input) prompt))
(rkllm:run *model* input iparam nil)
(autowrap:free prompt)))
;(init (cffi:mem-aptr *model* :pointer) *model-param* (cffi:callback get-data-cb))
(defparameter *msg-start* "<|im_start|>")
(defparameter *msg-end* "<|im_end|>")
(defun message->prompt (role &optional message)
(let ((*print-case* :downcase)) (format nil "~a~a~%~a~a~%" *msg-start* role (or message "") (if message *msg-end* ""))))
(let ((*print-case* :downcase))
(format nil "~a~a~%~a~a~%" *msg-start* role (or message "") (if message *msg-end* ""))))
(defun messages->prompt (messages)
(apply #'concatenate 'string
(mapcar (lambda (msg)
(message->prompt (car msg) (cadr msg)))
(append messages '((:assistant))))))
(defvar *clack-server* (if (boundp '*clack-server*) *clack-server* nil))
(defun chat-handler (env)
(bind (((&key request-method &allow-other-keys) env))
(if (eq :post request-method)
(bind (((&key raw-body &allow-other-keys) env)
(post-data (yason:parse
(flexi-streams:make-flexi-stream raw-body :external-format :utf-8)))
(resp nil))
(format t "~&; INFO recieved ~a~%" (alexandria:hash-table-plist post-data))
(lambda (responder)
(let ((writer (funcall responder '(200 (:content-type "application/json"))))
(yason:*symbol-encoder* 'yason:encode-symbol-as-lowercase))
(loop :for chunk :across "Hello!"
:for i :upfrom 0
:do (funcall writer
(with-output-to-string (str)
(yason:encode
`(:id "bla" :object "blub" :choices ,(reverse (push `(:index ,i :data ,(format nil "~a" chunk)) resp)))
str)))
(sleep 0.5)
:while chunk
:finally (funcall writer nil :close t)))))
'(405 nil ("Only POST")))))
(defun server-handler (env)
(bind (((&key path-info remote-addr &allow-other-keys) env)
(path-sym (intern (if (string= path-info "") "/" (string-upcase path-info)) :keyword)))
(case path-sym
(:/rkllm-chat (chat-handler env))
(:/rkllm-abort `(200 nil (,(format nil "Welcome ~a. You should not have come here (~a) " remote-addr path-sym))))
(t `(200 nil ,(asdf:system-relative-pathname :rkllm-server (format nil "static~a" (if (eq path-sym :/) "/index.html" path-info))))))))
(defun start ()
(clack:clackup (lambda (env) (format nil "~a" env)) :server :woo :address "0.0.0.0"))
(when *clack-server* (clack:stop *clack-server*))
(setf *clack-server*
(clack:clackup
(labels ((caller (env)
(restart-case (funcall 'server-handler env)
(retry () (caller env))
(return-500 () '(500 nil nil)))))
(lambda (env)
(caller env)))
:server :woo :address "0.0.0.0")))
(export '(start))

6
static/v2.js
View file

@ -31,7 +31,7 @@ function messageListener(elem, idx, type) {
}
async function sendAbort() {
fetch('/rkllm_abort', {
fetch('/rkllm-abort', {
method: 'POST',
body: true
}).catch(error => alert(error))
@ -73,7 +73,7 @@ async function sendMessage() {
const withSysMsg = (sysText ? [{role: 'system', 'content': sysText}] : []).concat(messagesToSend);
const response = await fetch('/rkllm_chat', {
const response = await fetch('/rkllm-chat', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
@ -98,7 +98,7 @@ async function sendMessage() {
let chunks = '';
function double_try() {
function display(n) {
const ccs = chunks.split(/\n/).filter(e => e !== "");
const ccs = chunks.trim().split(/\n/).filter(e => e !== "");
const json = JSON.parse(ccs[ccs.length - n]);
const text = `${json.choices.map((c) => c.delta.content).join('')}`;
while (cur_idx >= answers.length) answers.push("");