remove ningle and create simple router

rkllm-server.asd

@@ -7,7 +7,7 @@
   :license "MIT License"
 
   ;; Dependencies
-  :depends-on (:cl-autowrap :woo :clack :ningle :metabang-bind)
+  :depends-on (:cl-autowrap :woo :clack :metabang-bind :cl-json :str)
 
   ;; Components
   :components ((:file "server")

server.lisp

@@ -111,21 +111,61 @@
             (rkllm:input.prompt-input input) prompt))
     (rkllm:run *model* input iparam nil)
     (autowrap:free prompt)))
-;(init (cffi:mem-aptr *model* :pointer) *model-param* (cffi:callback get-data-cb))
 
 (defparameter *msg-start* "<|im_start|>")
 (defparameter *msg-end* "<|im_end|>")
 
 (defun message->prompt (role &optional message)
-  (let ((*print-case* :downcase)) (format nil "~a~a~%~a~a~%" *msg-start* role (or message "") (if message *msg-end* ""))))
+  (let ((*print-case* :downcase))
+    (format nil "~a~a~%~a~a~%" *msg-start* role (or message "") (if message *msg-end* ""))))
 (defun messages->prompt (messages)
   (apply #'concatenate 'string
          (mapcar (lambda (msg)
                    (message->prompt (car msg) (cadr msg)))
                  (append messages '((:assistant))))))
 
+(defvar *clack-server* (if (boundp '*clack-server*) *clack-server* nil))
 
+(defun chat-handler (env)
+  (bind (((&key request-method &allow-other-keys) env))
+    (if (eq :post request-method)
+        (bind (((&key raw-body &allow-other-keys) env)
+               (post-data (yason:parse
+                           (flexi-streams:make-flexi-stream raw-body :external-format :utf-8)))
+               (resp nil))
+          (format t "~&; INFO recieved ~a~%" (alexandria:hash-table-plist post-data))
+          (lambda (responder)
+            (let ((writer (funcall responder '(200 (:content-type "application/json"))))
+                  (yason:*symbol-encoder* 'yason:encode-symbol-as-lowercase))
+              (loop :for chunk :across "Hello!"
+                    :for i :upfrom 0
+                    :do (funcall writer
+                                 (with-output-to-string (str)
+                                   (yason:encode
+                                    `(:id "bla" :object "blub" :choices ,(reverse (push `(:index ,i :data ,(format nil "~a" chunk)) resp)))
+                                    str)))
+                        (sleep 0.5)
+                    :while chunk
+                    :finally (funcall writer nil :close t)))))
+      '(405 nil ("Only POST")))))
+
+(defun server-handler (env)
+  (bind (((&key path-info remote-addr &allow-other-keys) env)
+         (path-sym (intern (if (string= path-info "") "/" (string-upcase path-info)) :keyword)))
+    (case path-sym
+      (:/rkllm-chat (chat-handler env))
+      (:/rkllm-abort `(200 nil (,(format nil "Welcome ~a. You should not have come here (~a) " remote-addr path-sym))))
+      (t `(200 nil ,(asdf:system-relative-pathname :rkllm-server (format nil "static~a" (if (eq path-sym :/) "/index.html" path-info))))))))
 
 (defun start ()
-  (clack:clackup (lambda (env)  (format nil "~a" env)) :server :woo :address "0.0.0.0"))
+  (when *clack-server* (clack:stop *clack-server*))
+  (setf *clack-server* 
+        (clack:clackup
+         (labels ((caller (env)
+                    (restart-case (funcall 'server-handler env)
+                      (retry () (caller env))
+                      (return-500 () '(500 nil nil)))))
+           (lambda (env)
+             (caller env)))
+         :server :woo :address "0.0.0.0")))
 (export '(start))

static/v2.js

@@ -31,7 +31,7 @@ function messageListener(elem, idx, type) {
 }
 
 async function sendAbort() {
-    fetch('/rkllm_abort', {
+    fetch('/rkllm-abort', {
         method: 'POST',
         body: true
     }).catch(error => alert(error))
@@ -73,7 +73,7 @@ async function sendMessage() {
 
     const withSysMsg = (sysText ? [{role: 'system', 'content': sysText}] : []).concat(messagesToSend);
 
-    const response = await fetch('/rkllm_chat', {
+    const response = await fetch('/rkllm-chat', {
         method: 'POST',
         headers: {
             'Content-Type': 'application/json'
@@ -98,7 +98,7 @@ async function sendMessage() {
     let chunks = '';
     function double_try() {
         function display(n) {
-            const ccs = chunks.split(/\n/).filter(e => e !== "");
+            const ccs = chunks.trim().split(/\n/).filter(e => e !== "");
             const json = JSON.parse(ccs[ccs.length - n]);
             const text = `${json.choices.map((c) => c.delta.content).join('')}`;
             while (cur_idx >= answers.length) answers.push("");