diff --git a/src/lib/ChatRequestPetals.svelte b/src/lib/ChatRequestPetals.svelte
index 15f0278..eefd31d 100644
--- a/src/lib/ChatRequestPetals.svelte
+++ b/src/lib/ChatRequestPetals.svelte
@@ -53,6 +53,14 @@ export const runPetalsCompletionRequest = async (
throw err
}
const rMessages = request.messages || [] as Message[]
+ // make sure top_p and temperature are set the way we need
+ let temperature = request.temperature || 0
+ if (isNaN(temperature as any) || temperature === 1) temperature = 1
+ if (temperature === 0) temperature = 0.0001
+ let topP = request.top_p
+ if (isNaN(topP as any) || topP === 1) topP = 1
+ if (topP === 0) topP = 0.0001
+ // build the message array
const inputArray = (rMessages).reduce((a, m) => {
const c = getRoleTag(m.role, model, chatRequest.chat) + m.content
a.push(c)
@@ -65,11 +73,11 @@ export const runPetalsCompletionRequest = async (
const petalsRequest = {
type: 'generate',
inputs: inputArray.join(stopSequence),
- max_new_tokens: 3, // wait for up to 3 tokens before displaying
+ max_new_tokens: 1, // wait for up to 1 tokens before displaying
stop_sequence: stopSequence,
- doSample: 1,
- temperature: request.temperature || 0,
- top_p: request.top_p || 0,
+ do_sample: 1, // enable top p and the like
+ temperature,
+ top_p: topP,
extra_stop_sequences: stopSequencesC
}
ws.send(JSON.stringify(petalsRequest))
diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte
index 29f5be2..5f01837 100644
--- a/src/lib/Settings.svelte
+++ b/src/lib/Settings.svelte
@@ -55,6 +55,14 @@ export const getExcludeFromProfile = () => {
return excludeFromProfile
}
+const isNotOpenAI = (chatId) => {
+ return getModelDetail(getChatSettings(chatId).model).type !== 'OpenAIChat'
+}
+
+const isNotPetals = (chatId) => {
+ return getModelDetail(getChatSettings(chatId).model).type !== 'Petals'
+}
+
const gptDefaults = {
model: defaultModel,
messages: [],
@@ -406,7 +414,13 @@ const modelSetting: ChatSetting & SettingSelect = {
key: 'model',
name: 'Model',
title: 'The model to use - GPT-3.5 is cheaper, but GPT-4 is more powerful.',
- header: 'Below are the settings that OpenAI allows to be changed for the API calls. See the OpenAI API docs for more details.',
+ header: (chatId) => {
+ if (isNotOpenAI(chatId)) {
+ return 'Below are the settings that can be changed for the API calls. See this overview to start, though not all settings translate to Petals.'
+ } else {
+ return 'Below are the settings that OpenAI allows to be changed for the API calls. See the OpenAI API docs for more details.'
+ }
+ },
headerClass: 'is-warning',
options: [],
type: 'select',
@@ -414,14 +428,6 @@ const modelSetting: ChatSetting & SettingSelect = {
afterChange: (chatId, setting) => true // refresh settings
}
-const isNotOpenAI = (chatId) => {
- return getModelDetail(getChatSettings(chatId).model).type !== 'OpenAIChat'
-}
-
-const isNotPetals = (chatId) => {
- return getModelDetail(getChatSettings(chatId).model).type !== 'Petals'
-}
-
const chatSettingsList: ChatSetting[] = [
profileSetting,
...systemPromptSettings,
@@ -448,7 +454,7 @@ const chatSettingsList: ChatSetting[] = [
},
{
key: 'top_p',
- name: 'Nucleus Sampling',
+ name: 'Nucleus Sampling (Top-p)',
title: 'An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n' +
'\n' +
'We generally recommend altering this or temperature but not both',
diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte
index 30e0319..3757c9f 100644
--- a/src/lib/Types.svelte
+++ b/src/lib/Types.svelte
@@ -259,8 +259,8 @@ export type ChatSetting = {
title: string;
forceApi?: boolean; // force in api requests, even if set to default
hidden?: boolean; // Hide from setting menus
- header?: string;
- headerClass?: string;
+ header?: string | ValueFn;
+ headerClass?: string | ValueFn;
placeholder?: string | ValueFn;
hide?: (chatId:number) => boolean;
apiTransform?: (chatId:number, setting:ChatSetting, value:any) => any;