From 7862118886db0a6faff92677d5d394658df8820e Mon Sep 17 00:00:00 2001 From: Jhen Date: Wed, 2 Aug 2023 15:33:47 +0800 Subject: [PATCH 01/31] server : add n_probs param in chat UI --- examples/server/public/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 1db69a6bab024..5432e23119ac2 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -166,6 +166,7 @@ mirostat: 0, // 0/1/2 mirostat_tau: 5, // target entropy mirostat_eta: 0.1, // learning rate + n_probs: 0, // no completion_probabilities }) const llamaStats = signal(null) @@ -382,6 +383,9 @@ ${FloatField({label: "Mirostat tau", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau})} ${FloatField({label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta})} +
+ ${IntField({label: "Show Probabilities", max: 100, min: 0, name: "n_probs", value: params.value.n_probs})} +
` From 6a8b9c27d48f6e07c9c84e83b70d6ba7311cfe8d Mon Sep 17 00:00:00 2001 From: Jhen Date: Wed, 2 Aug 2023 15:43:33 +0800 Subject: [PATCH 02/31] server : keep message data array & show in probabilites component --- examples/server/public/index.html | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 5432e23119ac2..fa5d3632a6ace 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -206,7 +206,7 @@ history: session.value.transcript.flatMap(([name, message]) => template(session.value.historyTemplate, {name, message})).join("\n"), }); - let currentMessage = ''; + const currentMessages = []; const history = session.value.transcript const llamaParams = { @@ -216,15 +216,12 @@ for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) { const data = chunk.data; - currentMessage += data.content; + currentMessages.push(data); - // remove leading whitespace - currentMessage = currentMessage.replace(/^\s+/, "") - - transcriptUpdate([...history, ["{{char}}", currentMessage]]) + transcriptUpdate([...history, ["{{char}}", currentMessages]]) if (data.stop) { - console.log("Completion finished: '", currentMessage, "', summary: ", data); + console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data); } if (data.timings) { @@ -288,8 +285,18 @@ } }, [messages]) - const chatLine = ([user, msg]) => { - return html`

${template(user)}: <${Markdownish} text=${template(msg)} />

` + const chatLine = ([user, data], index) => { + let message + const isArrayMessage = Array.isArray(data) + if (isArrayMessage && data.every(msg => msg.completion_probabilities.length > 0)) { + message = html`<${Probabilites} data=${data} />` + } else { + const text = isArrayMessage ? + data.map(msg => msg.content).join('').replace(/^\s+/, '') : + data; + message = html`<${Markdownish} text=${template(text)} />` + } + return html`

${template(user)}: ${message}

` }; return html` @@ -390,6 +397,9 @@ ` } + + const Probabilites = (params) => { /* Not implemented yet */ } + // poor mans markdown replacement const Markdownish = (params) => { const md = params.text From 2a5bab4c9f7c8beb2e86ba64988a2bb0cf596a0f Mon Sep 17 00:00:00 2001 From: Jhen Date: Wed, 2 Aug 2023 15:46:49 +0800 Subject: [PATCH 03/31] server : add simple popover component --- examples/server/public/index.html | 115 ++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index fa5d3632a6ace..c6d674e16e846 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -428,6 +428,121 @@ ` } + // simple popover impl + const Popover = (props) => { + const isOpen = useSignal(false); + const position = useSignal({ top: '0px', left: '0px' }); + const buttonRef = useRef(null); + const popoverRef = useRef(null); + + const togglePopover = () => { + if (buttonRef.current) { + const rect = buttonRef.current.getBoundingClientRect(); + position.value = { + top: `${rect.bottom + window.scrollY}px`, + left: `${rect.left + window.scrollX}px`, + }; + } + isOpen.value = !isOpen.value; + }; + + const handleClickOutside = (event) => { + if (popoverRef.current && !popoverRef.current.contains(event.target) && !buttonRef.current.contains(event.target)) { + isOpen.value = false; + } + }; + + useEffect(() => { + document.addEventListener('mousedown', handleClickOutside); + return () => { + document.removeEventListener('mousedown', handleClickOutside); + }; + }, []); + + return html` + ${props.children} + ${isOpen.value && html` + <${Portal} into="#portal"> +
+ ${props.popoverChildren} +
+ + `} + `; + }; + + // Source: preact-portal (https://github.com/developit/preact-portal/blob/master/src/preact-portal.js) + /** Redirect rendering of descendants into the given CSS selector */ + class Portal extends Component { + componentDidUpdate(props) { + for (let i in props) { + if (props[i] !== this.props[i]) { + return setTimeout(this.renderLayer); + } + } + } + + componentDidMount() { + this.isMounted = true; + this.renderLayer = this.renderLayer.bind(this); + this.renderLayer(); + } + + componentWillUnmount() { + this.renderLayer(false); + this.isMounted = false; + if (this.remote && this.remote.parentNode) this.remote.parentNode.removeChild(this.remote); + } + + findNode(node) { + return typeof node === 'string' ? document.querySelector(node) : node; + } + + renderLayer(show = true) { + if (!this.isMounted) return; + + // clean up old node if moving bases: + if (this.props.into !== this.intoPointer) { + this.intoPointer = this.props.into; + if (this.into && this.remote) { + this.remote = render(html`<${PortalProxy} />`, this.into, this.remote); + } + this.into = this.findNode(this.props.into); + } + + this.remote = render(html` + <${PortalProxy} context=${this.context}> + ${show && this.props.children || null} + + `, this.into, this.remote); + } + + render() { + return null; + } + } + // high-order component that renders its first child if it exists. + // used as a conditional rendering proxy. + class PortalProxy extends Component { + getChildContext() { + return this.props.context; + } + render({ children }) { + return children || null; + } + } + function App(props) { return html` From b9b6cd2f219fab663ae7d1b6a48eb64902c4fec1 Mon Sep 17 00:00:00 2001 From: Jhen Date: Wed, 2 Aug 2023 15:53:10 +0800 Subject: [PATCH 04/31] server : fix completion_probabilities undefined if not set n_probs --- examples/server/public/index.html | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index c6d674e16e846..ff17c8a184fc9 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -137,7 +137,7 @@ +
+
From 7f02fead8cff21aecba6bc4e5665b48e9438302e Mon Sep 17 00:00:00 2001 From: Jhen Date: Wed, 2 Aug 2023 16:14:10 +0800 Subject: [PATCH 06/31] server : handle bytes --- examples/server/public/index.html | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 74015426258e7..1621a07fd39dd 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -424,17 +424,21 @@ const popoverChildren = html`
${completion_probabilities.map((cp, i) => { - return cp.probs.map((p, j) => { - return html` -
- ${p.tok_str}: - ${p.prob} -
- ` - }) + return html` +
+ ${cp.probs.map((p, j) => { + return html` +
+ ${p.tok_str}: + ${p.prob} +
+ ` + })} +
+ ` })}
` From 368c41cb5b8c441e3574259fe1af542f92a05be1 Mon Sep 17 00:00:00 2001 From: Jhen Date: Wed, 2 Aug 2023 16:14:40 +0800 Subject: [PATCH 07/31] server : make n_probs max to 10 for easy scroll --- examples/server/public/index.html | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 1621a07fd39dd..290a2c42f1450 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -102,6 +102,18 @@ padding: 0.5em; } + .prob-set { + padding: 0.3em; + border-bottom: 1px solid #ccc; + } + + .popover-content { + position: absolute; + background-color: #777; + border: 1px solid #ccc; + padding: 0.2em; + box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); + } textarea { padding: 5px; @@ -397,7 +409,7 @@ ${FloatField({label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta})}
- ${IntField({label: "Show Probabilities", max: 100, min: 0, name: "n_probs", value: params.value.n_probs})} + ${IntField({label: "Show Probabilities", max: 10, min: 0, name: "n_probs", value: params.value.n_probs})}
@@ -425,7 +437,7 @@
${completion_probabilities.map((cp, i) => { return html` -
+
${cp.probs.map((p, j) => { return html`
${props.popoverChildren} From 750299726d0f316c47922bcd75a6ff56d73e7b76 Mon Sep 17 00:00:00 2001 From: Jhen Date: Wed, 2 Aug 2023 16:29:15 +0800 Subject: [PATCH 08/31] server : adjust for dark/light mode --- examples/server/public/index.html | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/server/public/index.html b/examples/server/public/index.html index 290a2c42f1450..09f79019f495d 100644 --- a/examples/server/public/index.html +++ b/examples/server/public/index.html @@ -109,8 +109,7 @@ .popover-content { position: absolute; - background-color: #777; - border: 1px solid #ccc; + background-color: white; padding: 0.2em; box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); } @@ -145,6 +144,12 @@ font-size: 80%; color: #888; } + + @media (prefers-color-scheme: dark) { + .popover-content { + background-color: #777; + } + }