Lukas/rkllm-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

remove ningle and create simple router

Browse source
This commit is contained in:
Lukas Nöllemeyer 2025-02-04 13:16:32 +01:00
parent 0aac3aae10
commit 63d079340b
3 changed files with 47 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

46
server.lisp
View file

@ -111,21 +111,61 @@
(rkllm:input.prompt-input input) prompt))
(rkllm:run *model* input iparam nil)
(autowrap:free prompt)))
;(init (cffi:mem-aptr *model* :pointer) *model-param* (cffi:callback get-data-cb))
(defparameter *msg-start* "<|im_start|>")
(defparameter *msg-end* "<|im_end|>")
(defun message->prompt (role &optional message)
(let ((*print-case* :downcase)) (format nil "~a~a~%~a~a~%" *msg-start* role (or message "") (if message *msg-end* ""))))
(let ((*print-case* :downcase))
(format nil "~a~a~%~a~a~%" *msg-start* role (or message "") (if message *msg-end* ""))))
(defun messages->prompt (messages)
(apply #'concatenate 'string
(mapcar (lambda (msg)
(message->prompt (car msg) (cadr msg)))
(append messages '((:assistant))))))
(defvar *clack-server* (if (boundp '*clack-server*) *clack-server* nil))
(defun chat-handler (env)
(bind (((&key request-method &allow-other-keys) env))
(if (eq :post request-method)
(bind (((&key raw-body &allow-other-keys) env)
(post-data (yason:parse
(flexi-streams:make-flexi-stream raw-body :external-format :utf-8)))
(resp nil))
(format t "~&; INFO recieved ~a~%" (alexandria:hash-table-plist post-data))
(lambda (responder)
(let ((writer (funcall responder '(200 (:content-type "application/json"))))
(yason:*symbol-encoder* 'yason:encode-symbol-as-lowercase))
(loop :for chunk :across "Hello!"
:for i :upfrom 0
:do (funcall writer
(with-output-to-string (str)
(yason:encode
`(:id "bla" :object "blub" :choices ,(reverse (push `(:index ,i :data ,(format nil "~a" chunk)) resp)))
str)))
(sleep 0.5)
:while chunk
:finally (funcall writer nil :close t)))))
'(405 nil ("Only POST")))))
(defun server-handler (env)
(bind (((&key path-info remote-addr &allow-other-keys) env)
(path-sym (intern (if (string= path-info "") "/" (string-upcase path-info)) :keyword)))
(case path-sym
(:/rkllm-chat (chat-handler env))
(:/rkllm-abort `(200 nil (,(format nil "Welcome ~a. You should not have come here (~a) " remote-addr path-sym))))
(t `(200 nil ,(asdf:system-relative-pathname :rkllm-server (format nil "static~a" (if (eq path-sym :/) "/index.html" path-info))))))))
(defun start ()
(clack:clackup (lambda (env) (format nil "~a" env)) :server :woo :address "0.0.0.0"))
(when *clack-server* (clack:stop *clack-server*))
(setf *clack-server*
(clack:clackup
(labels ((caller (env)
(restart-case (funcall 'server-handler env)
(retry () (caller env))
(return-500 () '(500 nil nil)))))
(lambda (env)
(caller env)))
:server :woo :address "0.0.0.0")))
(export '(start))