mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 14:21:06 -10:00
Merge f-022-gpuroles: Main / Helper / Off per graphics card in the app, TUI, launch, setup, server and Docker
This commit is contained in:
Niko1221 committed
commit
e28889d120
48 files changed
+2482
-203
No files matched your search
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* The host's copy of the roles rules it needs to WRITE a model's settings (src/lib/gpu-roles.ts has the screen's, strata_control/gpu_roles.py the
|
||||
* Terminal's and the server's; tests/data/gpu_roles_vectors.json is run against all three).
|
||||
* Roles are { [card number]: 'main' | 'helper' | 'off' }. The engine takes the cards in stage order: Helpers, then the Main card LAST.
|
||||
*/
|
||||
export const ROLES = ['main', 'helper', 'off'];
|
||||
const mem = c => Number(c.vram_gb) || 0;
|
||||
const speed = c => (typeof c.speed === 'number' && c.speed > 0 ? c.speed : 0);
|
||||
const nums = roles => Object.keys(roles).map(Number).sort((a, b) => a - b);
|
||||
const onList = roles => nums(roles).filter(i => roles[i] !== 'off');
|
||||
const shortName = c => { let n = String(c.name || `GPU ${c.index + 1}`); for (const j of ['NVIDIA GeForce ', 'NVIDIA ', 'GeForce ', 'AMD Radeon ', 'AMD ', 'Intel(R) ', 'Intel ']) if (n.startsWith(j)) { n = n.slice(j.length); break; } return n.trim(); };
|
||||
|
||||
export function parseSaved(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
||||
const out = {};
|
||||
for (const [k, v] of Object.entries(value)) {
|
||||
if (!/^\d+$/.test(k.trim()) || !ROLES.includes(String(v).toLowerCase())) return null;
|
||||
out[Number(k)] = String(v).toLowerCase();
|
||||
}
|
||||
if (!Object.keys(out).length || Object.values(out).filter(r => r === 'main').length > 1) return null;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function normalize(roles, cards) {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const out = {};
|
||||
for (const i of nums(roles)) if (cs.has(i)) out[i] = ROLES.includes(roles[i]) ? roles[i] : 'off';
|
||||
const list = onList(out), mains = list.filter(i => out[i] === 'main');
|
||||
if (list.length && !mains.length) {
|
||||
const best = [...list].sort((a, b) => speed(cs.get(b)) - speed(cs.get(a)) || Math.round(mem(cs.get(b))) - Math.round(mem(cs.get(a))) || a - b)[0];
|
||||
out[best] = 'main';
|
||||
}
|
||||
for (const extra of mains.slice(1)) out[extra] = 'helper';
|
||||
if (list.length === 1) out[list[0]] = 'main';
|
||||
return out;
|
||||
}
|
||||
|
||||
export function order(roles, cards) {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const r = normalize(roles, cards);
|
||||
const helpers = nums(r).filter(i => r[i] === 'helper').sort((a, b) => speed(cs.get(a)) - speed(cs.get(b)) || a - b);
|
||||
return [...helpers, ...nums(r).filter(i => r[i] === 'main')];
|
||||
}
|
||||
|
||||
/** "gpu_roles" plus the legacy mirror ("gpu" Main last, "gpu_order": "as_given"); nothing on: "gpu": [] and "gpu_none": true. */
|
||||
export function mirror(roles, cards) {
|
||||
const r = normalize(roles, cards), lst = order(r, cards);
|
||||
const out = { gpu_roles: Object.fromEntries(nums(r).map(i => [String(i), r[i]])), gpu: [], gpu_cards: {} };
|
||||
if (lst.length) { out.gpu = lst.length === 1 ? lst[0] : lst; out.gpu_order = 'as_given'; } else out.gpu_none = true;
|
||||
for (const c of cards) if (c.index in r) out.gpu_cards[String(c.index)] = [shortName(c), c.bus ?? ''].filter(Boolean).join('|');
|
||||
return out;
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import { checkRunConfig, describeIssues } from './run-config-check.mjs';
|
||||
import { openTextFile } from './open-text-file.mjs';
|
||||
import { markEngine, engineRefusal } from './engine-version.mjs';
|
||||
import { friendlyModelName } from './model-names.mjs';
|
||||
import { parseSaved as parseRoles, mirror as mirrorRoles } from './gpu-roles.mjs';
|
||||
|
||||
const ROOT=resolve(dirname(fileURLToPath(import.meta.url)),'..');
|
||||
const safeId=value=>typeof value==='string'&&/^[a-z0-9_-]{1,80}$/i.test(value);
|
||||
@@ -207,8 +208,10 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};}
|
||||
const resources=args.resourceProfile?resourceSettings({...base0,[USER_KEY]:mine,vision},snapshot.hardware,args.resourceProfile,modelArena(args.model),values0):null;
|
||||
const cfg={...config,...visionPreferences,...(resources?{...resources,ui_resource_profile:args.resourceProfile}:{}),host:'127.0.0.1',open_browser:false,...validateNativeSettings(args.values??{}),vision};
|
||||
/* The app's suggested cards only fill a gap: a model whose settings file already names its GPUs ("gpu": [1, 0], with "gpu_order": "as_given") keeps them (#3256). A card the person picked in the screen always wins. */
|
||||
const namedInFile=Array.isArray(config.gpu)?config.gpu.length>0:config.gpu!==undefined&&config.gpu!==null&&config.gpu!=='';
|
||||
if(args.gpus!==undefined&&!(args.gpusFromRecommendation&&namedInFile)){const detected=new Set((snapshot.hardware?.gpus??[]).map(g=>g.index));if(!Array.isArray(args.gpus)||!args.gpus.length||new Set(args.gpus).size!==args.gpus.length||args.gpus.some(g=>!Number.isInteger(g)||!detected.has(g)))throw new Error('Choose distinct detected GPUs. CPU-only text inference is not supported by this native adapter.');cfg.gpu=args.gpus.length===1?args.gpus[0]:args.gpus;delete cfg.hip_ordinal;cfg.layer_split=args.gpus.length>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==args.gpus.length-1)throw new Error('Choose one layer split point between each pair of GPUs.');}
|
||||
const namedInFile=(Array.isArray(config.gpu)?config.gpu.length>0:config.gpu!==undefined&&config.gpu!==null&&config.gpu!=='')||!!config.gpu_roles;
|
||||
/* Roles chosen in the screen (Main / Helper / Off per card, gpu-roles.mjs): the card list is the roles' mirror - Helpers first, the Main card LAST, Off cards left out - with "gpu_roles" beside it. Every card Off is allowed: the server says plainly what the engine can do. */
|
||||
if(args.gpuRoles!==undefined){const make=snapshot.hardware?.engine_vendor,cards=(snapshot.hardware?.gpus??[]).filter(g=>!make||!g.vendor||g.vendor===make),named=parseRoles(args.gpuRoles);if(!named)throw new Error('Choose Main, Helper or Off for each graphics card, with one Main.');const known=new Set(cards.map(g=>g.index));if(Object.keys(named).some(k=>!known.has(Number(k))))throw new Error('Choose detected graphics cards.');const roles=Object.fromEntries(cards.map(g=>[g.index,named[g.index]??'off'])),m=mirrorRoles(roles,cards);for(const k of ['gpu','gpu_order','gpu_none','gpu_roles','gpu_cards','hip_ordinal'])delete cfg[k];Object.assign(cfg,m);const count=Array.isArray(m.gpu)?m.gpu.length:m.gpu_none?0:1;cfg.layer_split=count>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==count-1)throw new Error('Choose one layer split point between each pair of GPUs.');}
|
||||
else if(args.gpus!==undefined&&!(args.gpusFromRecommendation&&namedInFile)){const detected=new Set((snapshot.hardware?.gpus??[]).map(g=>g.index));if(!Array.isArray(args.gpus)||!args.gpus.length||new Set(args.gpus).size!==args.gpus.length||args.gpus.some(g=>!Number.isInteger(g)||!detected.has(g)))throw new Error('Choose distinct detected GPUs. CPU-only text inference is not supported by this native adapter.');cfg.gpu=args.gpus.length===1?args.gpus[0]:args.gpus;for(const k of ['gpu_roles','gpu_cards','gpu_none'])delete cfg[k];delete cfg.hip_ordinal;cfg.layer_split=args.gpus.length>1?validateNativeSettings({layer_split:args.layerSplit??'auto'}).layer_split:'auto';if(cfg.layer_split!=='auto'&&cfg.layer_split.split(',').length!==args.gpus.length-1)throw new Error('Choose one layer split point between each pair of GPUs.');}
|
||||
if(cfg.vram_reserve_mib!==undefined&&cfg.vram_reserve_mib!==null){cfg.args=[...(cfg.args??[])];const at=cfg.args.indexOf('--vram-reserve-mib');if(at>=0)cfg.args.splice(at,2);cfg.args.push('--vram-reserve-mib',String(cfg.vram_reserve_mib));}
|
||||
if(mine.length){cfg.args=mergeUserArgs(cfg.args??[],mine);cfg[USER_KEY]=mine;}else delete cfg[USER_KEY];
|
||||
if(cfg.vision){if(cfg.vision_tokens!=null)cfg.vision={...cfg.vision,max_tokens:cfg.vision_tokens};if(cfg.vision_min_tokens!=null)cfg.vision={...cfg.vision,min_tokens:cfg.vision_min_tokens};}cfg.mcp={...(cfg.mcp??{}),...(cfg.mcp_timeout_s!=null?{timeout_s:cfg.mcp_timeout_s}:{}),...(cfg.mcp_max_rounds!=null?{max_rounds:cfg.mcp_max_rounds}:{}),...(cfg.mcp_max_result_chars!=null?{max_result_chars:cfg.mcp_max_result_chars}:{})};
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { tr } from '../lib/i18n';
|
||||
import { useId, useState } from 'react';
|
||||
import { Btn, Icon } from './ui';
|
||||
import {
|
||||
CPU_ONLY_LINE, NO_CARD_LINE, OFF_AT_OR_BELOW_GB, gpuLabel, noteText, order, recommend, setRole, shortName, vramOn, warnings,
|
||||
type Role, type RoleCard, type Roles, type RoleWarning,
|
||||
} from '../lib/gpu-roles';
|
||||
|
||||
const ROLE_WORD: Record<Role, string> = { main: 'Main', helper: 'Helper', off: 'Off' };
|
||||
const HINT: Record<Role, string> = { main: 'Does the main work', helper: 'Holds part of the model', off: 'Not used' };
|
||||
|
||||
/** The one plain line above the cards, built from translated pieces. */
|
||||
export function recommendationLine(cards: RoleCard[], vendor?: string | null): string {
|
||||
const rec = recommend(cards, vendor);
|
||||
if (rec.main === null) return tr('No graphics card that Strata can use was found.');
|
||||
const byi = new Map(cards.map(c => [c.index, c]));
|
||||
const m = byi.get(rec.main)!;
|
||||
if (cards.length === 1) return tr('Strata will use your {0}.', [shortName(m)]);
|
||||
const why = { same: 'the cards are alike', memory: 'it has more memory', faster: 'it is faster', only: 'it is the only card', none: '' }[rec.why];
|
||||
let s = tr('{0} is Main: {1}.', [shortName(m), tr(why)]);
|
||||
const helpers = Object.keys(rec.roles).map(Number).filter(i => rec.roles[i] === 'helper').map(i => shortName(byi.get(i)!));
|
||||
if (helpers.length) s += ' ' + tr(helpers.length > 1 ? '{0} help.' : '{0} helps.', [helpers.join(', ')]);
|
||||
for (const i of Object.keys(rec.roles).map(Number)) if (rec.roles[i] === 'off') {
|
||||
const code = rec.reasons[i], c = byi.get(i)!;
|
||||
s += ' ' + tr('{0} starts Off: {1}', [shortName(c), code === 'cannot_run' ? tr('Strata cannot run it.') : code === 'igpu' ? tr('built-in graphics.') : tr('{0} GB or less.', [OFF_AT_OR_BELOW_GB])]);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
function warningText(w: RoleWarning, cards: RoleCard[]): string {
|
||||
const c = w.card !== null ? cards.find(x => x.index === w.card) : undefined;
|
||||
if (w.code === 'cannot_run') return tr('{0} cannot run Strata.', [c ? shortName(c) : '']);
|
||||
if (w.code === 'igpu_main') return tr('{0} is built-in graphics; it is slow as Main.', [c ? shortName(c) : '']);
|
||||
if (w.code === 'cpu_only') return tr(w.text === CPU_ONLY_LINE ? CPU_ONLY_LINE : NO_CARD_LINE);
|
||||
return tr('Not enough memory for this setup: about {0} GB is read from the drive.', [w.text.match(/about (\d+) GB/)?.[1] ?? '']);
|
||||
}
|
||||
|
||||
export function noteLine(code: string | null, cards: RoleCard[]): string {
|
||||
if (!code) return '';
|
||||
const kind = code.split(':')[0], c = cards.find(x => x.index === Number(code.split(':')[1]));
|
||||
if (kind === 'old_main_helper') return tr('{0} is now a Helper.', [c ? shortName(c) : '']);
|
||||
if (kind === 'promoted') return tr('{0} is now Main.', [c ? shortName(c) : '']);
|
||||
if (kind === 'only_main') return tr('The only card that is on has to be Main.');
|
||||
return noteText(code, cards);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main / Helper / Off for every graphics card (the mockup GpuRoles-mockup.html). One click changes a role; "Back to recommended" restores the
|
||||
* recommendation. Warnings are short and never block Load. `roles` are the roles in force (the recommendation until the person changes one).
|
||||
*/
|
||||
export function GpuRoles({ cards, roles, custom, recommendedRoles, vendor, onChange, onBack, memoryGb, ramGb, spillGb = 0, engineRunsCpuOnly = false, disabled = false, usedBy }: {
|
||||
cards: RoleCard[];
|
||||
roles: Roles;
|
||||
/** The person changed a role: "Your choice", and "Back to recommended" is offered. */
|
||||
custom: boolean;
|
||||
/** The roles Strata recommends (the tags and "Back to recommended" use them); the pure recommendation when not given. */
|
||||
recommendedRoles?: Roles;
|
||||
vendor?: string | null;
|
||||
onChange: (next: Roles) => void;
|
||||
onBack: () => void;
|
||||
memoryGb?: number | null;
|
||||
ramGb?: number | null;
|
||||
/** About how many GB of the model are read from the drive with these cards (0 when everything fits or it is unknown). */
|
||||
spillGb?: number;
|
||||
engineRunsCpuOnly?: boolean;
|
||||
disabled?: boolean;
|
||||
/** Names of the loaded models that already use a card ("In use by ..."). */
|
||||
usedBy?: (card: number) => string[];
|
||||
}) {
|
||||
const titleId = useId();
|
||||
const [note, setNote] = useState('');
|
||||
const rec = { ...recommend(cards, vendor), ...(recommendedRoles ? { roles: recommendedRoles } : {}) };
|
||||
const single = cards.length === 1;
|
||||
const warns = warnings(roles, cards, { spillGb, engineRunsCpuOnly });
|
||||
const cardWarn = (i: number) => warns.filter(w => w.card === i);
|
||||
const globalWarn = warns.filter(w => w.card === null);
|
||||
const lst = order(roles, cards);
|
||||
const change = (idx: number, role: Role) => {
|
||||
const out = setRole(roles, cards, idx, role);
|
||||
setNote(noteLine(out.note, cards));
|
||||
onChange(out.roles);
|
||||
};
|
||||
const back = () => { setNote(''); onBack(); };
|
||||
const fix = (w: RoleWarning) => w.fix && change(w.fix.card, w.fix.role);
|
||||
const fixLabel = (w: RoleWarning) => {
|
||||
if (!w.fix) return '';
|
||||
const target = cards.find(c => c.index === w.fix!.card);
|
||||
return w.fix.role === 'off' ? tr('Turn it off') : tr('Make {0} Main', [target ? shortName(target) : '']);
|
||||
};
|
||||
const have = vramOn(roles, cards) + Math.max(0, (ramGb ?? 0) - 10);
|
||||
return <div className="gpuroles" aria-labelledby={titleId} data-custom={custom}>
|
||||
<div className="gpuroles__head">
|
||||
<span className="native-choice__label" id={titleId}>{tr('Graphics cards')}</span>
|
||||
{custom ? <button type="button" className="gpuroles__link" disabled={disabled} onClick={back}>{tr('Back to recommended')}</button> : null}
|
||||
</div>
|
||||
<div className="gpuroles__reco" data-custom={custom}>
|
||||
<Icon name="bolt" size={18}/>
|
||||
<p><strong>{tr(custom ? 'Your choice.' : 'Recommended.')}</strong> {custom ? tr('You changed the roles. Strata will use exactly these.') : recommendationLine(cards, vendor)}</p>
|
||||
</div>
|
||||
<div className="gpuroles__rows" role="list">
|
||||
{cards.map(c => {
|
||||
const role = roles[c.index] ?? 'off', isRec = rec.roles[c.index] === role, w = cardWarn(c.index), other = usedBy?.(c.index) ?? [];
|
||||
const shared = !!(c.integrated || c.uma || c.unified_gb);
|
||||
const options: Role[] = single ? ['main', 'off'] : ['main', 'helper', 'off'];
|
||||
return <div className="gpuroles__row" role="listitem" data-role={role} key={c.index}>
|
||||
<div className="gpuroles__top">
|
||||
<span className="gpuroles__ico"><Icon name="gpu" size={20}/></span>
|
||||
<span className="gpuroles__name">
|
||||
<strong>{gpuLabel(c)} · {c.name}{isRec ? <span className="gpuroles__tag">{tr('Recommended')}</span> : <span className="gpuroles__tag gpuroles__tag--neutral">{tr('Recommended: {0}', [tr(ROLE_WORD[rec.roles[c.index] ?? 'off'])])}</span>}</strong>
|
||||
<small>{shared ? tr("Uses your computer's memory (built-in graphics)") : tr('{0} GB memory', [Math.round(c.vram_gb)])} · {tr(HINT[role])}</small>
|
||||
</span>
|
||||
<span className="gpuroles__roles" role="radiogroup" aria-label={tr('Role of {0}', [gpuLabel(c)])}>
|
||||
{options.map(r => <button type="button" role="radio" key={r} data-r={r} aria-checked={role === r} disabled={disabled} onClick={() => change(c.index, r)}>{tr(ROLE_WORD[r])}</button>)}
|
||||
</span>
|
||||
</div>
|
||||
{other.length ? <div className="gpuroles__note gpuroles__note--info"><Icon name="info" size={16}/><p>{tr('In use by {0}', [other.join(', ')])}</p></div> : null}
|
||||
{w.map(x => <div className="gpuroles__note gpuroles__note--warn" key={x.code}><Icon name="warning" size={16}/><p>{warningText(x, cards)}</p>{x.fix ? <Btn size="sm" disabled={disabled} onClick={() => fix(x)}>{fixLabel(x)}</Btn> : null}</div>)}
|
||||
</div>;
|
||||
})}
|
||||
</div>
|
||||
<div className="gpuroles__toast" role="status" aria-live="polite">{note ? <div className="gpuroles__note gpuroles__note--info"><Icon name="info" size={16}/><p>{note}</p></div> : null}</div>
|
||||
{globalWarn.map(x => <div className="gpuroles__note gpuroles__note--warn" key={x.code}><Icon name="warning" size={16}/><p>{warningText(x, cards)}</p>{x.code === 'cpu_only' && custom ? <Btn size="sm" disabled={disabled} onClick={back}>{tr('Back to recommended')}</Btn> : null}</div>)}
|
||||
<details className="gpuroles__details">
|
||||
<summary>{tr('Details')}</summary>
|
||||
<div className="gpuroles__body">
|
||||
<dl>
|
||||
<dt>{tr('Main')}</dt><dd>{tr('The card that does the final steps of every answer: it writes the next words, runs the draft-ahead step and checks it, and keeps your conversation. Pick your fastest card with plenty of memory. There is only ever one.')}</dd>
|
||||
<dt>{tr('Helper')}</dt><dd>{tr('Holds a share of the model and its fast memory, so more of it fits on graphics cards instead of being read from your drive. Prompts are read about 20% faster.')}</dd>
|
||||
<dt>{tr('Off')}</dt><dd>{tr('Strata does not touch this card. Good for a card you use for something else, such as a game or your screen.')}</dd>
|
||||
</dl>
|
||||
{memoryGb ? <div className="gpuroles__fit" data-state={lst.length ? (spillGb >= 1 ? 'bad' : 'ok') : 'bad'}><span className="gpuroles__dot" aria-hidden="true"/><span>{lst.length ? (spillGb >= 1 ? tr('The model needs about {0} GB. These cards and your memory give about {1} GB, so about {2} GB would be read from your drive.', [Math.round(memoryGb), Math.round(have), Math.round(spillGb)]) : tr('The model needs about {0} GB. These cards and your memory give about {1} GB. It fits.', [Math.round(memoryGb), Math.round(have)])) : tr('Only your processor and memory are used.')}</span></div> : null}
|
||||
<div className="gpuroles__expert xs muted">{lst.length ? tr('For experts: the engine starts on cards {0} (Main last).', [lst.join(',')]) : null} {custom ? '' : tr('Not saved while you keep the recommendation: Strata picks again if you add or remove a card.')}</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>;
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import { Screen } from './layout';
|
||||
import { VersionPicker } from './VersionPicker';
|
||||
import { FamilySwitch } from './FamilySwitch';
|
||||
import { PcFitCard } from './PcFitCard';
|
||||
import { fitOn, runCards } from '../lib/native-fit';
|
||||
import { fitOn, memoryNeeded } from '../lib/native-fit';
|
||||
import { GpuRoles } from './GpuRoles';
|
||||
import { fromLegacy, normalize as normalizeRoles, order as roleOrder, recommend as recommendRoles, rows as roleRows, sameRoles, vramOn, type RoleCard, type Roles } from '../lib/gpu-roles';
|
||||
import { Btn, Dialog, ErrorNotice, Icon, Select } from './ui';
|
||||
import { installProgress, installStage, stageText, useDownloadEta } from '../lib/download-eta';
|
||||
import { useKept } from '../lib/kept-state';
|
||||
@@ -37,8 +39,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
// The choices are kept while the page is closed (Models -> Monitor -> Models used to start them over and a person lost track of what was downloading).
|
||||
const [family, setFamily] = useKept('models.family', ''), [size, setSize] = useKept('models.size', ''), [context, setContext] = useKept('models.context', 32768), [vision, setVision] = useKept('models.vision', 'yes');
|
||||
const manual = useStore(s => s.setup?.choices?.onboarding?.manual ?? false);
|
||||
const [expert, setExpert] = useKept('models.expert', manual), [multiple, setMultiple] = useKept('models.multiple', false), [gpus, setGpus] = useKept<number[]>('models.gpus', []), [kv, setKv] = useKept('models.kv', 'int8'), [lowRam, setLowRam] = useKept('models.lowRam', 'auto'), [split, setSplit] = useKept('models.split', 'auto'), [deviceChoice, setDeviceChoice] = useKept('models.device', 'auto');
|
||||
const [cardsKey, setCardsKey] = useKept<string | null>('models.cards', null);
|
||||
const [expert, setExpert] = useKept('models.expert', manual), [multiple, setMultiple] = useKept('models.multiple', false), [roles, setRoles] = useKept<Roles | null>('models.roles', null), [kv, setKv] = useKept('models.kv', 'int8'), [lowRam, setLowRam] = useKept('models.lowRam', 'auto'), [split, setSplit] = useKept('models.split', 'auto');
|
||||
const [resourceProfile, setResourceProfile] = useKept<'work' | 'performance'>('models.profile', 'work');
|
||||
const [pending, setPending] = useState(false), [error, setError] = useState(''), [plan, setPlan] = useState<Record<string, any> | null>(null), [loadedError, setLoadedError] = useState(''), [tuneOpen, setTuneOpen] = useState(false), [jobId, setJobId] = useKept('models.job', ''), [manage, setManage] = useState(false);
|
||||
const [toastedJob, setToastedJob] = useKept('models.toasted', ''), [appliedSig, setAppliedSig] = useKept('models.applied', '');
|
||||
@@ -53,7 +54,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
// The model that is running is shown as running, with Unload: not offered for loading again.
|
||||
const downloaded = native?.installed?.models ?? [], allSizes = families.flatMap(f => f.sizes);
|
||||
// This computer for the fit figures: the biggest usable graphics card, or the ones chosen under Expert settings.
|
||||
const runGpus = runCards(native?.hardware?.gpus, native?.hardware?.engine_vendor);
|
||||
const engineVendor = native?.hardware?.engine_vendor ?? null, roleCards: RoleCard[] = roleRows(native?.hardware?.gpus as RoleCard[] | undefined, engineVendor);
|
||||
const pcGpus = (native?.hardware?.gpus ?? []).filter(g => g.usable !== false), pcGpu = [...pcGpus].sort((x, y) => y.vram_gb - x.vram_gb)[0] ?? null;
|
||||
const pcRam = native?.hardware?.ram_gb || hardware?.memory.totalGb || null, pcVram = pcGpu?.vram_gb ?? 0;
|
||||
// What the models that stay loaded (Run multiple models) already take: the fit figures are for what is left.
|
||||
@@ -85,19 +86,20 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
// Graphics cards (strata_control/gpu_fit.py: the one rule of the app, the Terminal and setup): the recommended option is pre-selected, the person may pick another
|
||||
// A card a loaded model already uses (a second model on it usually does not fit): said on its row, and not the pre-selected one while another is free.
|
||||
const usedBy = (cards: number[]) => (native?.instances ?? []).filter(i => i.status !== 'error' && i.model !== selectedSize?.id && i.gpus.some(g => cards.includes(g))).map(i => i.label);
|
||||
const cardOptions = selectedSize?.gpu_options && selectedSize.gpu_options.options.length >= 2 ? selectedSize.gpu_options : null;
|
||||
const loadedCards = loadedSelected ? cardOptions?.options.find(o => o.cards.length === loadedSelected.gpus.length && o.cards.every(c => loadedSelected.gpus.includes(c))) : undefined;
|
||||
const activeCards = cardOptions ? (cardOptions.options.find(o => o.key === cardsKey) ?? loadedCards ?? (multiple ? cardOptions.options.find(o => o.recommended && !usedBy(o.cards).length) ?? cardOptions.options.find(o => !usedBy(o.cards).length) : undefined) ?? cardOptions.options.find(o => o.recommended) ?? cardOptions.options[0]) : null;
|
||||
const cardsGpus: number[] = activeCards?.cards ?? [];
|
||||
const chosenCards = expert && gpus.length ? gpus : cardsGpus;
|
||||
const fitHardware = { ram_gb: pcRam ? Math.max(0, pcRam - keptRam) : pcRam, vram_gb: Math.max(0, pcVram - keptVramOn(chosenCards)) };
|
||||
// "(card 1)" in a card name is the number the screens call GPU 2 (Monitor, the model picker, the loaded list): one numbering everywhere.
|
||||
const gpuNumbers = (text: string) => text.replace(/\(card (\d+)\)/g, (_m, n) => `(GPU ${Number(n) + 1})`);
|
||||
// The fit sentence for a card choice, less what the kept models take of RAM and of that card's memory.
|
||||
const lessKept = (words: string, cards: number[]) => { const gone = Math.round(keptRam + keptVramOn(cards)); return gone > 0 ? words.replace(/of the (\d+) GB available/, (_m, n) => `of the ${Math.max(0, Number(n) - gone)} GB available`) : words; };
|
||||
const shownSize = useRef(selectedSize?.id);
|
||||
useEffect(() => { if (shownSize.current !== selectedSize?.id) { shownSize.current = selectedSize?.id; setCardsKey(null); } }, [selectedSize?.id]);
|
||||
const values = useMemo(() => ({ family: selectedSize?.sourceFamily, model: selectedSize?.model, context, vision: selectedSize?.images ? vision : 'no', kv, low_ram: lowRam, resourceProfile, ...(expert && gpus.length === 1 ? { gpu: gpus[0] } : expert && gpus.length ? { gpus: gpus.join(',') } : cardsGpus.length > 1 ? { gpus: cardsGpus.join(',') } : cardsGpus.length === 1 ? { gpu: cardsGpus[0] } : {}), loadAfter: true, multiple: expert && multiple }), [selectedSize?.sourceFamily, selectedSize?.model, context, vision, kv, lowRam, resourceProfile, gpus.join(','), cardsGpus.join(','), expert, multiple]);
|
||||
// Main / Helper / Off per card (src/lib/gpu-roles.ts, strata_control/gpu_roles.py): the recommendation until the person changes a role; a loaded model shows what it was started with.
|
||||
// With Run multiple models, a card another loaded model already uses starts Off for this one (while a free card exists).
|
||||
const busy = (i: number) => expert && multiple && usedBy([i]).length > 0, freeCards = roleCards.filter(c => !busy(c.index));
|
||||
const recommended = freeCards.length && freeCards.length < roleCards.length ? { ...recommendRoles(freeCards, engineVendor), roles: { ...Object.fromEntries(roleCards.map(c => [c.index, 'off'])), ...recommendRoles(freeCards, engineVendor).roles } as Roles } : recommendRoles(roleCards, engineVendor);
|
||||
const loadedRoles = loadedSelected && roleCards.length > 1 ? fromLegacy(loadedSelected.gpus, roleCards, 'as_given') : null;
|
||||
const effectiveRoles: Roles = roleCards.length > 1 ? (roles ? normalizeRoles(roles, roleCards) : loadedRoles ?? recommended.roles) : recommended.roles;
|
||||
const customRoles = !sameRoles(effectiveRoles, recommended.roles);
|
||||
const changeRoles = (next: Roles) => setRoles(sameRoles(next, recommended.roles) ? null : next);
|
||||
const cardsGpus: number[] = roleCards.length > 1 ? roleOrder(effectiveRoles, roleCards) : [];
|
||||
const chosenCards = cardsGpus;
|
||||
const fitHardware = { ram_gb: pcRam ? Math.max(0, pcRam - keptRam) : pcRam, vram_gb: Math.max(0, (roleCards.length > 1 ? vramOn(effectiveRoles, roleCards) : pcVram) - keptVramOn(chosenCards)) };
|
||||
// A layer split written for another number of cards no longer fits: back to automatic.
|
||||
useEffect(() => { if (cardsGpus.length <= 1 && split !== 'auto') setSplit('auto'); }, [cardsGpus.length]);
|
||||
const values = useMemo(() => ({ family: selectedSize?.sourceFamily, model: selectedSize?.model, context, vision: selectedSize?.images ? vision : 'no', kv, low_ram: lowRam, resourceProfile, ...(cardsGpus.length > 1 ? { gpus: cardsGpus.join(',') } : cardsGpus.length === 1 ? { gpu: cardsGpus[0] } : {}), loadAfter: true, multiple: expert && multiple }), [selectedSize?.sourceFamily, selectedSize?.model, context, vision, kv, lowRam, resourceProfile, cardsGpus.join(','), expert, multiple]);
|
||||
const downloadValues = { ...values, vision: selectedSize?.images && vision !== 'no' ? 'yes' : 'no', loadVision: selectedSize?.images ? vision : 'no' };
|
||||
useEffect(() => { if (!native?.catalog || family)
|
||||
return; const current = native.instances.find(i => i.id === native.defaultId); const rec = native.catalog.recommendation ?? { family: null, model: null }; const loadedFamily = families.find(f => f.sizes.some(s => s.id === current?.model)); const have = new Set((native.installed?.models ?? []).filter(m => m.ready).map(m => m.model)); const installedFamily = families.find(f => f.sizes.some(s => have.has(s.id))); const f = loadedFamily ?? installedFamily ?? families.find(f => f.family === rec.family || f.sizes.some(s => s.sourceFamily === rec.family)) ?? families[0]; if (f) {
|
||||
@@ -149,7 +151,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
} }
|
||||
async function load() { if (!selectedSize)
|
||||
return; await perform(async () => { if (installed?.ready && context === installed.context && (!selectedSize?.images || vision === 'no' || installed.images && installed.images !== 'off') && (!installed.kv || kv === installed.kv || kv === tunedNow?.fields.kv) && (lowRam === (installed.lowRamMode ?? 'auto') || lowRam === tunedNow?.fields.lowRam)) {
|
||||
const result = await nativeAction<NativeJob>('operations', { kind: 'load', args: { model: selectedSize.id, resourceProfile, vision: selectedSize?.images ? vision : 'no', multiple: expert && multiple, ...(expert && gpus.length ? { gpus, layerSplit: split } : cardsGpus.length ? { gpus: cardsGpus, gpusFromRecommendation: !cardsKey } : {}), values: {} } });
|
||||
const result = await nativeAction<NativeJob>('operations', { kind: 'load', args: { model: selectedSize.id, resourceProfile, vision: selectedSize?.images ? vision : 'no', multiple: expert && multiple, ...(roleCards.length > 1 ? (roles && customRoles ? { gpuRoles: effectiveRoles, layerSplit: expert && cardsGpus.length > 1 ? split : 'auto' } : cardsGpus.length ? { gpus: cardsGpus, gpusFromRecommendation: true, layerSplit: expert && cardsGpus.length > 1 ? split : 'auto' } : {}) : {}), values: {} } });
|
||||
setJobId(result.id);
|
||||
}
|
||||
else
|
||||
@@ -165,10 +167,10 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
setPending(false);
|
||||
} }
|
||||
async function download() { await perform(async () => { const result = await nativeAction<NativeJob>('operations', { kind: 'install', args: downloadValues }); setJobId(result.id); setPlan(null); }); }
|
||||
return <Screen plainHeader title={<span className="sr-only">{tr("Models")}</span>} titleDetail={<FamilySwitch families={families} selected={selectedFamily?.family} onPick={f => { setFamily(f.family); setSize((f.sizes.find(s => native?.installed?.models.some(m => m.model === s.id && m.ready)) ?? f.sizes[0])?.id ?? ''); setVision('yes'); setError(''); }}/>} actions={<Btn icon="sliders" aria-pressed={expert} aria-expanded={expert} aria-controls="native-expert-settings" onClick={() => { setExpert(v => !v); setMultiple(false); setGpus([]); setDeviceChoice('auto'); }}>{tr(expert ? 'Close expert mode' : 'Expert mode')}</Btn>}>
|
||||
return <Screen plainHeader title={<span className="sr-only">{tr("Models")}</span>} titleDetail={<FamilySwitch families={families} selected={selectedFamily?.family} onPick={f => { setFamily(f.family); setSize((f.sizes.find(s => native?.installed?.models.some(m => m.model === s.id && m.ready)) ?? f.sizes[0])?.id ?? ''); setVision('yes'); setError(''); }}/>} actions={<Btn icon="sliders" aria-pressed={expert} aria-expanded={expert} aria-controls="native-expert-settings" onClick={() => { setExpert(v => !v); setMultiple(false); }}>{tr(expert ? 'Close expert mode' : 'Expert mode')}</Btn>}>
|
||||
<div className="native-model-layout native-model-layout--fit">
|
||||
<div className="stack">
|
||||
{tr(native?.instances.length && expert ? <section className="st-card native-running"><h2 className="h4">{tr(native.instances.length === 1 ? 'Loaded model' : 'Loaded models')}</h2>{tr(native.instances.map(i => <div key={i.id}><span><strong>{tr(i.label)}</strong><small>{tr(i.gpus.length ? i.gpus.map(g => `GPU ${g + 1}`).join(' + ') : 'Automatic GPU')} · {tr(i.status === 'ready' ? i.phase : i.status)}{tr(native.defaultId === i.id ? ' · Chat default' : '')}</small>{tr(i.error ? <small className="inline-error">{tr(plainFailureText(i.error).message)}</small> : null)}</span>{tr(expert && multiple && native.defaultId !== i.id ? <Btn size="sm" variant="ghost" onClick={() => void loadedAction(`instances/${i.id}/default`)}>{tr("Use in chat")}</Btn> : null)}<Btn size="sm" disabled={pending || !!active} onClick={() => void loadedAction(`instances/${i.id}/unload`)}>{tr("Unload")}</Btn></div>))}{tr(loadedError ? <ErrorNotice message={loadedError} next={tr("Stop or answer the chat that is still using this model, then try again.")}/> : null)}</section> : null)}
|
||||
{tr(native?.instances.length && expert ? <section className="st-card native-running"><h2 className="h4">{tr(native.instances.length === 1 ? 'Loaded model' : 'Loaded models')}</h2>{tr(native.instances.map(i => <div key={i.id}><span><strong>{tr(i.label)}</strong><small>{tr(i.gpus.length ? i.gpus.map((g, n) => `GPU ${g + 1}${i.gpus.length > 1 && n === i.gpus.length - 1 ? ' (Main)' : ''}`).join(' + ') : 'Automatic GPU')} · {tr(i.status === 'ready' ? i.phase : i.status)}{tr(native.defaultId === i.id ? ' · Chat default' : '')}</small>{tr(i.error ? <small className="inline-error">{tr(plainFailureText(i.error).message)}</small> : null)}</span>{tr(expert && multiple && native.defaultId !== i.id ? <Btn size="sm" variant="ghost" onClick={() => void loadedAction(`instances/${i.id}/default`)}>{tr("Use in chat")}</Btn> : null)}<Btn size="sm" disabled={pending || !!active} onClick={() => void loadedAction(`instances/${i.id}/unload`)}>{tr("Unload")}</Btn></div>))}{tr(loadedError ? <ErrorNotice message={loadedError} next={tr("Stop or answer the chat that is still using this model, then try again.")}/> : null)}</section> : null)}
|
||||
<section className="st-card native-choice">{tr(selectedFamily && (!selectedSize || selectedSize.needsNewerEngine) ? <><h2 className="h3">{tr("This model needs a newer engine.")}</h2><p className="small muted">{tr("Update your engine or import a local GGUF for this model.")}</p><Btn onClick={() => { setSettingsSection('Engine'); go('settings'); }}>{tr("Update engine")}</Btn><Btn variant="ghost" onClick={onImport ?? onOtherModels}>{tr("Use a local GGUF\u2026")}</Btn></> : null)}{tr(!selectedFamily ? <><h2 className="h3">{tr("Set up Strata")}</h2><p className="small muted">{tr("Connect your local installation to choose an optimized model.")}</p><Btn variant="primary" onClick={onConnectEngine ?? (() => { setSettingsSection('Engine'); go('settings'); })}>{tr("Connect engine")}</Btn>{tr(native?.error ? <p className="small inline-error">{tr(native.error)}</p> : null)}<Btn variant="ghost" onClick={onOtherModels}>{tr("Other local models")}</Btn></> : null)}
|
||||
{tr(selectedSize ? <>
|
||||
<div className="native-choice__head"><span className="native-choice__label">{tr("Version")}</span><span className="native-choice__tools"><NativePresets values={values} onApply={v => { const f = families.find(f => f.family === v.family || f.sizes.some(s => s.sourceFamily === v.family)); if (!f)
|
||||
@@ -179,7 +181,8 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
<div className="native-choice__col"><span className="native-choice__label">{tr("Conversation context")}</span><Select ariaLabel={tr("Conversation context")} value={String(context)} options={contextChoices(native?.catalog?.contexts ?? [8192, 32768, 65536, 131072, 262144], context).map(n => ({ value: String(n), label: n % 1024 === 0 ? `${n / 1024}K` : `${n.toLocaleString()} tokens` }))} onChange={n => setContext(Number(n))}/></div>
|
||||
<div className="native-choice__col"><span className="native-choice__label">{tr("Computer use")}</span><div className="native-resource-choice"><button aria-pressed={resourceProfile === 'work'} data-selected={resourceProfile === 'work'} onClick={() => setResourceProfile('work')}><Icon name="computer" size={20}/><span><strong>{tr("Leave 20% for my work")}</strong><small>{tr("Headroom for your other apps")}</small></span>{tr(resourceProfile === 'work' ? <Icon name="check" size={15}/> : null)}</button><button aria-pressed={resourceProfile === 'performance'} data-selected={resourceProfile === 'performance'} onClick={() => setResourceProfile('performance')}><Icon name="bolt" size={20}/><span><strong>{tr("Full performance")}</strong><small>{tr("Give Strata priority")}</small></span>{tr(resourceProfile === 'performance' ? <Icon name="check" size={15}/> : null)}</button></div></div>
|
||||
</div>
|
||||
{tr(cardOptions ? <div className="native-cards"><span className="native-choice__label">{tr("Graphics cards")}</span><div className="native-cards__list" role="radiogroup" aria-label={tr("Graphics cards")}>{cardOptions.options.map(o => { const on = activeCards?.key === o.key; return <button key={o.key} type="button" role="radio" aria-checked={on} data-selected={on} disabled={!!active} className="native-cards__opt" onClick={() => setCardsKey(o.key)}><span className="native-cards__dot" aria-hidden="true"/><span><strong>{gpuNumbers(o.label)}</strong><small data-warn={o.verdict === 'spills' || o.verdict === 'tight' || usedBy(o.cards).length > 0}>{[lessKept(o.words, o.cards), ...o.notes, ...(usedBy(o.cards).length ? [tr('Already used by {0}', [usedBy(o.cards).join(', ')])] : [])].join('. ')}.</small></span></button>; })}{cardOptions.ignored.length ? <p className="small muted native-cards__note">{`Not used: ${cardOptions.ignored.map(g => `${g.name ?? 'a card'} (${g.why_ignored})`).join('; ')}.`}</p> : null}</div></div> : null)}
|
||||
{roleCards.length > 1 ? <GpuRoles cards={roleCards} roles={effectiveRoles} custom={customRoles} recommendedRoles={recommended.roles} vendor={engineVendor} disabled={!!active} onChange={changeRoles} onBack={() => setRoles(null)} memoryGb={selectedSize ? memoryNeeded(selectedSize) || null : null} ramGb={pcRam} spillGb={selectedSize ? fitOn(selectedSize, fitHardware).spillGb : 0} usedBy={i => usedBy([i])}/> : null}
|
||||
{tr(roleCards.length > 1 && (native?.hardware?.gpus ?? []).length > roleCards.length ? <p className="small muted native-cards__note">{tr('Not used: {0} (another make of graphics card than this engine drives).', [(native?.hardware?.gpus ?? []).filter(g => !roleCards.some(r => r.index === g.index && r.vendor === g.vendor)).map(g => g.name).join(', ')])}</p> : null)}
|
||||
{tr(selectedSize?.imagesCapable && !selectedSize.images ? <p className="small muted native-spill-note">{tr("This model can read pictures, but the engine installed here does not include the picture reader yet, so Read pictures is off. Update the engine in Settings → About to turn it on.")}</p> : null)}
|
||||
{tr(selectedSize?.images ? <><div className="native-image-reading"><label><input type="checkbox" checked={vision !== 'no'} disabled={!selectedSize?.images || !!active} onChange={e => setVision(e.target.checked ? 'yes' : 'no')}/><span><strong>{tr("Read pictures ")}<small>{tr("On by default")}</small></strong><small>{tr("Understand photos, screenshots and charts. The picture files download automatically. Turn it off to save memory.")}</small></span></label>{tr(expert && vision !== 'no' ? <Select ariaLabel={tr("Image encoder device")} value={vision === 'yes' ? 'yes' : vision} options={[{ value: 'yes', label: 'Automatic' }, { value: 'gpu', label: 'GPU' }, { value: 'cpu', label: 'CPU' }]} onChange={setVision}/> : null)}</div>{tr(vision !== 'no' && installed?.ready && installed.visionPrepared === false ? <div className="native-vision-missing" role="status"><span>{tr("Picture reading needs a few more files that are not on this computer yet.")}</span><Btn size="sm" disabled={pending || !!active} onClick={() => void download()}>{tr("Download picture-reading files")}</Btn></div> : null)}</> : null)}
|
||||
</> : null)}
|
||||
@@ -188,10 +191,10 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
{tr(job?.state === 'error' ? <ErrorNotice message={job.error ?? job.summary} action={tr("Retry")} onRetry={() => void load()}/> : null)}
|
||||
{tr(error ? <ErrorNotice message={error}/> : null)}
|
||||
{selectedSize && installed?.ready ? <NativeTuneBadge model={selectedSize.id} resourceProfile={resourceProfile} vision={selectedSize.images ? vision : 'no'}/> : null}
|
||||
<div className="native-model-actions">{tr(loadedSelected ? <><Btn disabled={pending || !!active} onClick={() => void perform(() => nativeAction(`instances/${loadedSelected.id}/unload`))}>{tr("Unload")}</Btn>{tr(onReady ? <Btn variant="primary" onClick={onReady}>{tr("Start chatting")}</Btn> : <span className="native-ready">{tr("Ready")}</span>)}</> : onReady && native?.instances.some(i => i.status === 'ready') ? <Btn variant="primary" onClick={onReady}>{tr("Start chatting")}</Btn> : <><Btn icon="gauge" disabled={!installed?.ready || !native?.available || !!active || !selectedSize || selectedSize.needsNewerEngine} onClick={() => setTuneOpen(true)}>{tr("Tune model")}</Btn><Btn variant="primary" icon="models" disabled={pending || !native?.available || !!active || !selectedSize || !!selectedSize.needsNewerEngine || expert && deviceChoice === 'split' && gpus.length < 2} onClick={() => void load()}>{tr(pending ? 'Checking…' : installed?.ready ? 'Load model' : 'Download & load')}</Btn></>)}</div>
|
||||
<div className="native-model-actions">{tr(loadedSelected ? <><Btn disabled={pending || !!active} onClick={() => void perform(() => nativeAction(`instances/${loadedSelected.id}/unload`))}>{tr("Unload")}</Btn>{tr(onReady ? <Btn variant="primary" onClick={onReady}>{tr("Start chatting")}</Btn> : <span className="native-ready">{tr("Ready")}</span>)}</> : onReady && native?.instances.some(i => i.status === 'ready') ? <Btn variant="primary" onClick={onReady}>{tr("Start chatting")}</Btn> : <><Btn icon="gauge" disabled={!installed?.ready || !native?.available || !!active || !selectedSize || selectedSize.needsNewerEngine} onClick={() => setTuneOpen(true)}>{tr("Tune model")}</Btn><Btn variant="primary" icon="models" disabled={pending || !native?.available || !!active || !selectedSize || !!selectedSize.needsNewerEngine} onClick={() => void load()}>{tr(pending ? 'Checking…' : installed?.ready ? 'Load model' : 'Download & load')}</Btn></>)}</div>
|
||||
</section>
|
||||
{tr(expert && IMAGE_GENERATION_VISIBLE ? <><details className="native-generation"><summary>{tr("Image generation ")}<span className="muted">{tr("Optional")}</span></summary><RunImageModel library={imageLibrary} disabled={pending || !!active} onChange={() => { }}/></details></> : null)}
|
||||
{tr(expert ? <section id="native-expert-settings" ref={expertSection} className="st-card native-execution"><h2 className="h4">{tr("Expert settings")}</h2>{tr(tunedNow ? <p className="small muted native-tuned-note">{tr("Showing the values of your tuning '{0}'.", [tunedNow.name])}</p> : null)}<label className="native-toggle"><span><strong>{tr("Run multiple models")}</strong><small>{tr("Loading another model keeps the others loaded.")}</small></span><button className="st-toggle" role="switch" aria-label={tr("Run multiple models")} aria-checked={multiple} onClick={() => setMultiple(v => !v)}/></label><h3 className="small strong">{tr("Run on")}</h3><p className="small muted">{tr("Automatic uses the installation\u2019s GPU placement. Choose a GPU, or split this model across several GPUs.")}</p><Select ariaLabel={tr("Run on")} value={deviceChoice} options={[{ value: 'auto', label: 'Automatic' }, ...runGpus.map(g => ({ value: String(g.index), label: `GPU ${g.index + 1} · ${g.name}` })), ...(runGpus.length > 1 ? [{ value: 'split', label: 'Split across GPUs…' }] : [])]} onChange={value => { setDeviceChoice(value); setGpus(value === 'auto' ? [] : value === 'split' ? runGpus.map(g => g.index) : [Number(value)]); }}/>{tr(deviceChoice === 'split' ? <div className="native-gpu-options">{tr(runGpus.map(g => <label key={g.index}><input type="checkbox" checked={gpus.includes(g.index)} onChange={e => setGpus(old => e.target.checked ? [...old, g.index] : old.filter(i => i !== g.index))}/><span>{tr("GPU ")}{tr(g.index + 1)} · {tr(g.name)}<small>{tr(g.vram_gb)}{tr(" GB VRAM")}</small></span></label>))}</div> : null)}{tr(gpus.length > 1 ? <label className="input-field"><span>{tr("Layer split")}</span><input className="st-input" value={split} placeholder={tr("auto or 16,32")} onChange={e => setSplit(e.target.value)}/></label> : null)}<div className="native-field-grid"><label className="input-field"><span>{tr("Context memory")}</span><Select ariaLabel={tr("Native KV format")} value={kv} options={[{ value: 'int8', label: 'INT8' }, { value: 'q4_0', label: 'Q4' }, { value: 'k8v4', label: 'K8 / V4' }]} onChange={setKv}/></label><label className="input-field"><span>{tr("Low-RAM mode")}</span><Select ariaLabel={tr("Native low RAM mode")} value={lowRam} options={['auto', 'resident', 'mmap'].map(v => ({ value: v, label: v === 'auto' ? 'Automatic' : v === 'resident' ? 'Keep experts in RAM' : 'Read experts from disk' }))} onChange={setLowRam}/></label></div>{tr(installed ? <><NativeSettingsFile model={installed.model}/><NativeSettings model={installed.model}/></> : <p className="small muted">{tr("More engine parameters become available after preparing this version.")}</p>)}</section> : null)}
|
||||
{tr(expert ? <section id="native-expert-settings" ref={expertSection} className="st-card native-execution"><h2 className="h4">{tr("Expert settings")}</h2>{tr(tunedNow ? <p className="small muted native-tuned-note">{tr("Showing the values of your tuning '{0}'.", [tunedNow.name])}</p> : null)}<label className="native-toggle"><span><strong>{tr("Run multiple models")}</strong><small>{tr("Loading another model keeps the others loaded.")}</small></span><button className="st-toggle" role="switch" aria-label={tr("Run multiple models")} aria-checked={multiple} onClick={() => setMultiple(v => !v)}/></label>{tr(cardsGpus.length > 1 ? <label className="input-field"><span>{tr("Layer split")}</span><input className="st-input" value={split} placeholder={tr("auto or 16,32")} onChange={e => setSplit(e.target.value)}/></label> : null)}<div className="native-field-grid"><label className="input-field"><span>{tr("Context memory")}</span><Select ariaLabel={tr("Native KV format")} value={kv} options={[{ value: 'int8', label: 'INT8' }, { value: 'q4_0', label: 'Q4' }, { value: 'k8v4', label: 'K8 / V4' }]} onChange={setKv}/></label><label className="input-field"><span>{tr("Low-RAM mode")}</span><Select ariaLabel={tr("Native low RAM mode")} value={lowRam} options={['auto', 'resident', 'mmap'].map(v => ({ value: v, label: v === 'auto' ? 'Automatic' : v === 'resident' ? 'Keep experts in RAM' : 'Read experts from disk' }))} onChange={setLowRam}/></label></div>{tr(installed ? <><NativeSettingsFile model={installed.model}/><NativeSettings model={installed.model}/></> : <p className="small muted">{tr("More engine parameters become available after preparing this version.")}</p>)}</section> : null)}
|
||||
</div>
|
||||
<aside className="stack stack--lg"><PcFitCard gpuName={pcGpu?.name} ramGb={pcRam} vramGb={pcVram} inUse={keptModels.length ? { ramGb: Math.round(keptRam), vramGb: Math.round(keptVramOn(chosenCards)) } : null} fit={selectedSize ? fitOn(selectedSize, fitHardware) : null}/><section className="st-card"><h2 className="h4">{tr("Model files")}</h2><p className="small muted">{tr("Strata prepares all shards, the draft layer and shared files. Completed downloads are reused.")}</p><code className="native-path">{installed?.dataFolder ?? native?.downloadDirectory ?? native?.installed?.data_folder ?? 'Choose an installation'}</code><Btn size="sm" variant="ghost" onClick={() => { setSettingsSection('Storage'); go('settings'); }}>{tr("Storage settings")}</Btn></section><Btn size="sm" variant="ghost" onClick={onOtherModels}>{tr("Other local models\u2026")}</Btn></aside>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
/**
|
||||
* Main / Helper / Off for every graphics card. The same rules as strata_control/gpu_roles.py (the Terminal, setup and the server); both run
|
||||
* tests/data/gpu_roles_vectors.json.
|
||||
*
|
||||
* - Main: the card that runs the LAST stage of the layer split (the output head, the draft layer and the check of its guesses). Exactly one.
|
||||
* - Helper: holds a share of the model, so more of it fits on graphics cards instead of being read from the drive.
|
||||
* - Off: Strata does not touch it.
|
||||
*
|
||||
* The engine takes the cards in stage order (`gpu: [helper, ..., main]`, `gpu_order: "as_given"`), so Main is the LAST one.
|
||||
*/
|
||||
export type Role = 'main' | 'helper' | 'off';
|
||||
export const ROLES: Role[] = ['main', 'helper', 'off'];
|
||||
export type Roles = Record<number, Role>;
|
||||
|
||||
export interface RoleCard {
|
||||
index: number;
|
||||
name: string;
|
||||
vram_gb: number;
|
||||
/** SMs x clock (or the AMD equivalent): only the order between cards matters. */
|
||||
speed?: number;
|
||||
usable?: boolean;
|
||||
problem?: string | null;
|
||||
vendor?: string;
|
||||
uma?: boolean;
|
||||
integrated?: boolean;
|
||||
unified_gb?: number;
|
||||
/** PCI bus id when known: used to notice a renumbering. */
|
||||
bus?: string;
|
||||
}
|
||||
|
||||
export const OFF_AT_OR_BELOW_GB = 8;
|
||||
const SAME_SPEED = 1.05;
|
||||
const MEMORY_GUARD_SPEED = 0.5;
|
||||
const MIN_CARD_GB = 4;
|
||||
const SPLIT_CARD_OVERHEAD_GB = 2;
|
||||
export const CPU_ONLY_LINE = 'Performance is degraded on CPU only.';
|
||||
export const NO_CARD_LINE = 'This engine needs a graphics card to start. Turn one on.';
|
||||
|
||||
const mem = (c: RoleCard) => Number(c.vram_gb) || 0;
|
||||
const speed = (c: RoleCard) => (typeof c.speed === 'number' && c.speed > 0 ? c.speed : 0);
|
||||
const shares = (c: RoleCard) => !!(c.integrated || c.uma || c.unified_gb || c.vendor === 'apple');
|
||||
const canRun = (c: RoleCard) => c.usable !== false;
|
||||
const rnd = (n: number) => Math.round(n);
|
||||
|
||||
export function shortName(c: RoleCard): string {
|
||||
let n = String(c.name || `GPU ${c.index + 1}`);
|
||||
for (const junk of ['NVIDIA GeForce ', 'NVIDIA ', 'GeForce ', 'AMD Radeon ', 'AMD ', 'Intel(R) ', 'Intel ']) if (n.startsWith(junk)) { n = n.slice(junk.length); break; }
|
||||
return n.trim();
|
||||
}
|
||||
/** The number the screens show: GPU 1 is card 0. */
|
||||
export const gpuLabel = (c: RoleCard) => `GPU ${c.index + 1}`;
|
||||
|
||||
/** The cards shown as rows: every card the engine's make can name, in card order. */
|
||||
export function rows<T extends RoleCard>(cards: T[] | null | undefined, vendor?: string | null): T[] {
|
||||
const list = cards ?? [];
|
||||
const make = vendor || pickVendor(list);
|
||||
const mine = make ? list.filter(c => !c.vendor || c.vendor === make) : list;
|
||||
return [...mine].sort((a, b) => a.index - b.index);
|
||||
}
|
||||
function pickVendor(cards: RoleCard[]): string | null {
|
||||
const makes = new Set(cards.filter(c => canRun(c) && c.vendor).map(c => c.vendor as string));
|
||||
if (makes.size <= 1) return [...makes][0] ?? null;
|
||||
return makes.has('nvidia') ? 'nvidia' : cards[0]?.vendor ?? null;
|
||||
}
|
||||
|
||||
function eligible(cs: RoleCard[]): RoleCard[] {
|
||||
const real = cs.filter(c => canRun(c) && !shares(c) && mem(c) >= MIN_CARD_GB);
|
||||
return real.length ? real : cs.filter(canRun);
|
||||
}
|
||||
|
||||
export type MainWhy = 'only' | 'faster' | 'same' | 'memory' | 'none';
|
||||
|
||||
export function pickMain(cs: RoleCard[]): { main: RoleCard | null; why: MainWhy } {
|
||||
const pool = eligible(cs);
|
||||
if (!pool.length) return { main: null, why: 'none' };
|
||||
if (pool.length === 1) return { main: pool[0], why: 'only' };
|
||||
const top = Math.max(...pool.map(speed));
|
||||
const group = top > 0 ? pool.filter(c => speed(c) * SAME_SPEED >= top) : [...pool];
|
||||
const byMem = (a: RoleCard, b: RoleCard) => rnd(mem(b)) - rnd(mem(a)) || a.index - b.index;
|
||||
let main = [...group].sort(byMem)[0];
|
||||
let why: MainWhy = group.length === 1 ? 'faster' : 'same';
|
||||
if (group.length > 1 && new Set(group.map(c => `${c.name}|${rnd(mem(c))}`)).size === 1) main = [...group].sort((a, b) => b.index - a.index)[0]; // identical cards: the HIGHEST number is Main
|
||||
if (top <= 0 && new Set(pool.map(c => rnd(mem(c)))).size > 1) why = 'memory';
|
||||
const bigger = pool.filter(c => rnd(mem(c)) > rnd(mem(main)) && (top <= 0 || speed(c) >= MEMORY_GUARD_SPEED * top));
|
||||
if (bigger.length) { main = [...bigger].sort(byMem)[0]; why = 'memory'; }
|
||||
return { main, why };
|
||||
}
|
||||
|
||||
export type OffWhy = 'cannot_run' | 'igpu' | 'small' | 'helper';
|
||||
export interface Recommendation { roles: Roles; main: number | null; why: MainWhy; reasons: Record<number, MainWhy | OffWhy> }
|
||||
|
||||
/** The best card is Main; every other card starts as Helper, EXCEPT a card with 8 GB or less, built-in graphics beside a real card, and a card the engine cannot run: those start Off. */
|
||||
export function recommend(cards: RoleCard[] | null | undefined, vendor?: string | null): Recommendation {
|
||||
const cs = rows(cards, vendor);
|
||||
const { main, why } = pickMain(cs);
|
||||
const roles: Roles = {}, reasons: Recommendation['reasons'] = {};
|
||||
for (const c of cs) {
|
||||
const i = c.index;
|
||||
if (main && c === main) { roles[i] = 'main'; reasons[i] = why; }
|
||||
else if (!canRun(c)) { roles[i] = 'off'; reasons[i] = 'cannot_run'; }
|
||||
else if (shares(c) && main) { roles[i] = 'off'; reasons[i] = 'igpu'; }
|
||||
else if (rnd(mem(c)) <= OFF_AT_OR_BELOW_GB) { roles[i] = 'off'; reasons[i] = 'small'; }
|
||||
else { roles[i] = 'helper'; reasons[i] = 'helper'; }
|
||||
}
|
||||
return { roles, main: main ? main.index : null, why, reasons };
|
||||
}
|
||||
|
||||
const on = (roles: Roles) => Object.keys(roles).map(Number).filter(i => roles[i] !== 'off');
|
||||
const byNumber = (roles: Roles) => Object.keys(roles).map(Number).sort((a, b) => a - b);
|
||||
|
||||
/** Exactly one Main among the cards that are on; a lone card that is on is Main. */
|
||||
export function normalize(roles: Roles, cards: RoleCard[]): Roles {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const out: Roles = {};
|
||||
for (const i of byNumber(roles)) if (cs.has(i)) out[i] = ROLES.includes(roles[i]) ? roles[i] : 'off';
|
||||
const onList = on(out), mains = onList.filter(i => out[i] === 'main');
|
||||
if (onList.length && !mains.length) {
|
||||
const best = [...onList].sort((a, b) => speed(cs.get(b)!) - speed(cs.get(a)!) || rnd(mem(cs.get(b)!)) - rnd(mem(cs.get(a)!)) || a - b)[0];
|
||||
out[best] = 'main';
|
||||
}
|
||||
for (const extra of mains.slice(1)) out[extra] = 'helper';
|
||||
if (onList.length === 1) out[onList[0]] = 'main';
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface RoleChange { roles: Roles; note: string | null }
|
||||
|
||||
/** One click: the new roles and a code for the quiet line (old_main_helper:<i>, promoted:<i>, only_main). */
|
||||
export function setRole(roles: Roles, cards: RoleCard[], idx: number, role: Role): RoleChange {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const out: Roles = { ...roles };
|
||||
let note: string | null = null;
|
||||
if (!cs.has(idx) || !ROLES.includes(role)) return { roles: out, note };
|
||||
const ids = byNumber(out);
|
||||
const fastest = (list: number[]) => [...list].sort((a, b) => speed(cs.get(b)!) - speed(cs.get(a)!) || a - b);
|
||||
if (role === 'main') {
|
||||
const old = ids.find(i => i !== idx && out[i] === 'main');
|
||||
out[idx] = 'main';
|
||||
if (old !== undefined) { out[old] = 'helper'; note = `old_main_helper:${old}`; }
|
||||
} else if (role === 'helper') {
|
||||
if (out[idx] === 'main') {
|
||||
const others = fastest(ids.filter(i => i !== idx && out[i] === 'helper'));
|
||||
if (!others.length) return { roles: out, note: 'only_main' };
|
||||
out[idx] = 'helper'; out[others[0]] = 'main'; note = `promoted:${others[0]}`;
|
||||
} else out[idx] = 'helper';
|
||||
} else {
|
||||
const wasMain = out[idx] === 'main';
|
||||
out[idx] = 'off';
|
||||
if (wasMain) {
|
||||
const next = fastest(ids.filter(i => out[i] === 'helper'));
|
||||
if (next.length) { out[next[0]] = 'main'; note = `promoted:${next[0]}`; }
|
||||
}
|
||||
}
|
||||
const onList = on(out);
|
||||
if (onList.length === 1) out[onList[0]] = 'main';
|
||||
else if (onList.length && !onList.some(i => out[i] === 'main')) out[onList[0]] = 'main';
|
||||
return { roles: out, note };
|
||||
}
|
||||
|
||||
/** The engine's card list: Helpers (slowest first, ties by number), then the Main card LAST; Off cards left out. */
|
||||
export function order(roles: Roles, cards: RoleCard[]): number[] {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const r = normalize(roles, cards);
|
||||
const helpers = byNumber(r).filter(i => r[i] === 'helper').sort((a, b) => speed(cs.get(a)!) - speed(cs.get(b)!) || a - b);
|
||||
return [...helpers, ...byNumber(r).filter(i => r[i] === 'main')];
|
||||
}
|
||||
|
||||
export interface RoleMirror { gpu_roles: Record<string, Role>; gpu: number | number[]; gpu_order?: 'as_given'; gpu_none?: true; gpu_cards: Record<string, string> }
|
||||
/** What a writer stores: "gpu_roles" plus the legacy mirror older engines and the Monitor read. Nothing on: "gpu": [] and "gpu_none": true. */
|
||||
export function mirror(roles: Roles, cards: RoleCard[]): RoleMirror {
|
||||
const r = normalize(roles, cards), lst = order(r, cards);
|
||||
const out: RoleMirror = { gpu_roles: Object.fromEntries(byNumber(r).map(i => [String(i), r[i]])), gpu: [], gpu_cards: {} };
|
||||
if (lst.length) { out.gpu = lst.length === 1 ? lst[0] : lst; out.gpu_order = 'as_given'; } else out.gpu_none = true;
|
||||
for (const c of cards) if (c.index in r) out.gpu_cards[String(c.index)] = [shortName(c), c.bus ?? ''].filter(Boolean).join('|');
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Roles of an old "gpu" setting, for display: the LAST card of the engine's order is Main, earlier ones Helpers, unlisted cards Off. */
|
||||
export function fromLegacy(gpu: unknown, cards: RoleCard[], gpuOrder?: string | null): Roles {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
let items: number[] = Array.isArray(gpu) ? gpu.map(Number) : gpu === null || gpu === undefined || gpu === '' ? [] : String(gpu).split(',').filter(x => x.trim() !== '').map(Number);
|
||||
items = items.filter(i => cs.has(i));
|
||||
if (!items.length) return { ...recommend(cards).roles };
|
||||
if (items.length > 1 && gpuOrder !== 'as_given') {
|
||||
const sp = items.map(i => speed(cs.get(i)!));
|
||||
if (sp.every(x => x > 0) && Math.max(...sp) >= SAME_SPEED * Math.min(...sp)) items = [...items].sort((a, b) => speed(cs.get(a)!) - speed(cs.get(b)!));
|
||||
}
|
||||
const roles: Roles = {};
|
||||
for (const c of cards) roles[c.index] = 'off';
|
||||
for (const i of items.slice(0, -1)) roles[i] = 'helper';
|
||||
roles[items[items.length - 1]] = 'main';
|
||||
return roles;
|
||||
}
|
||||
|
||||
export const sameRoles = (a: Roles, b: Roles) => { const ka = byNumber(a), kb = byNumber(b); return ka.length === kb.length && ka.every((k, n) => k === kb[n] && a[k] === b[k]); };
|
||||
|
||||
export interface RoleWarning { code: 'igpu_main' | 'cannot_run' | 'low_memory' | 'cpu_only'; card: number | null; text: string; fix: { card: number; role: Role } | null }
|
||||
|
||||
/** One short line each, only what a person could not know. Never blocking. */
|
||||
export function warnings(roles: Roles, cards: RoleCard[], opts: { spillGb?: number; engineRunsCpuOnly?: boolean } = {}): RoleWarning[] {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const out: RoleWarning[] = [];
|
||||
for (const i of byNumber(roles)) {
|
||||
const c = cs.get(i), role = roles[i];
|
||||
if (!c || role === 'off') continue;
|
||||
if (!canRun(c)) out.push({ code: 'cannot_run', card: i, text: `${shortName(c)} cannot run Strata.`, fix: { card: i, role: 'off' } });
|
||||
else if (role === 'main' && shares(c) && cards.some(x => !shares(x) && canRun(x))) {
|
||||
const real = cards.filter(x => !shares(x) && canRun(x)).sort((a, b) => speed(b) - speed(a) || mem(b) - mem(a) || a.index - b.index)[0];
|
||||
out.push({ code: 'igpu_main', card: i, text: `${shortName(c)} is built-in graphics; it is slow as Main.`, fix: { card: real.index, role: 'main' } });
|
||||
}
|
||||
}
|
||||
if (!on(roles).length) out.push({ code: 'cpu_only', card: null, text: opts.engineRunsCpuOnly ? CPU_ONLY_LINE : NO_CARD_LINE, fix: null });
|
||||
else if ((opts.spillGb ?? 0) >= 1) out.push({ code: 'low_memory', card: null, text: `Not enough memory for this setup: about ${Math.round(opts.spillGb!)} GB is read from the drive.`, fix: null });
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The graphics memory the cards that are on give for the model (an extra card costs its own dense layers); 0 when all are off. */
|
||||
export function vramOn(roles: Roles, cards: RoleCard[]): number {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const onCards = on(roles).map(i => cs.get(i)).filter((c): c is RoleCard => !!c && !shares(c));
|
||||
return Math.max(0, onCards.reduce((n, c) => n + mem(c), 0) - SPLIT_CARD_OVERHEAD_GB * Math.max(0, onCards.length - 1));
|
||||
}
|
||||
|
||||
export function noteText(code: string | null, cards: RoleCard[]): string {
|
||||
if (!code) return '';
|
||||
const [kind, arg] = code.split(':');
|
||||
const c = arg !== undefined ? cards.find(x => x.index === Number(arg)) : undefined;
|
||||
const name = c ? shortName(c) : 'That card';
|
||||
if (kind === 'old_main_helper') return `${name} is now a Helper.`;
|
||||
if (kind === 'promoted') return `${name} is now Main.`;
|
||||
if (kind === 'only_main') return 'The only card that is on has to be Main.';
|
||||
return '';
|
||||
}
|
||||
|
||||
/** The one plain line above the cards. */
|
||||
export function sentence(cards: RoleCard[] | null | undefined, vendor?: string | null): string {
|
||||
const cs = rows(cards, vendor), rec = recommend(cards, vendor);
|
||||
if (rec.main === null) return 'No graphics card that Strata can use was found.';
|
||||
const byi = new Map(cs.map(c => [c.index, c]));
|
||||
const m = byi.get(rec.main)!;
|
||||
if (cs.length === 1) return `Strata will use your ${shortName(m)}.`;
|
||||
const why = { same: 'the cards are alike', memory: 'it has more memory', faster: 'it is faster', only: 'it is the only card', none: '' }[rec.why];
|
||||
let s = `${shortName(m)} is Main: ${why}.`;
|
||||
const helpers = byNumber(rec.roles).filter(i => rec.roles[i] === 'helper').map(i => shortName(byi.get(i)!));
|
||||
if (helpers.length) s += ` ${helpers.join(' and ')} ${helpers.length > 1 ? 'help' : 'helps'}.`;
|
||||
for (const i of byNumber(rec.roles)) if (rec.roles[i] === 'off') {
|
||||
const code = rec.reasons[i], c = byi.get(i)!;
|
||||
s += ` ${shortName(c)} starts Off: ${code === 'cannot_run' ? 'Strata cannot run it.' : code === 'igpu' ? 'built-in graphics.' : `${OFF_AT_OR_BELOW_GB} GB or less.`}`;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/** `Main: GPU 2 RTX 4070 Ti. Helper: GPU 1 RTX 3060.` */
|
||||
export function describe(roles: Roles, cards: RoleCard[]): string {
|
||||
const cs = new Map(cards.map(c => [c.index, c]));
|
||||
const parts: string[] = [];
|
||||
for (const [role, word] of [['main', 'Main'], ['helper', 'Helper'], ['off', 'Off']] as const) {
|
||||
const names = byNumber(roles).filter(i => roles[i] === role).map(i => (cs.has(i) ? `GPU ${i + 1} ${shortName(cs.get(i)!)}` : `GPU ${i + 1}`));
|
||||
if (names.length) parts.push(`${word}: ${names.join(', ')}.`);
|
||||
}
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
/** A saved "gpu_roles" object -> roles; null when it is not one. */
|
||||
export function parseSaved(value: unknown): Roles | null {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
||||
const out: Roles = {};
|
||||
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
||||
if (!/^\d+$/.test(k.trim()) || !ROLES.includes(String(v).toLowerCase() as Role)) return null;
|
||||
out[Number(k)] = String(v).toLowerCase() as Role;
|
||||
}
|
||||
if (!Object.keys(out).length || Object.values(out).filter(r => r === 'main').length > 1) return null;
|
||||
return out;
|
||||
}
|
||||
@@ -12,7 +12,7 @@ export interface NativeState {
|
||||
/** The picture reader (strata-vision) is installed with the engine, so models marked for pictures can read them here. */
|
||||
/** Strata Control's view of the engine for this computer (the app): usable false = none found or recognised; `notice` says why in plain words. */
|
||||
engine?:{usable:boolean;notice?:string|null;wrongMake?:boolean;version?:string|null};
|
||||
visionReader?: boolean;available:boolean;repo:string;downloadDirectory?:string;/** Free bytes on the drive that holds the models folder. */downloadFreeBytes?:number|null;downloadTotalBytes?:number|null;version?:string|null;summary:string;error:string|null;catalog:{families:NativeFamily[];contexts:number[];recommendation:{family:string|null;model:string|null;context?:number;why?:string};context_note:string}|null;installed:{models:{model:string;config:string;model_name:string;ready:boolean;/** The picture-reading files are on this computer (notes-split-agent8.md, item 8). */visionPrepared?:boolean;context?:number;kv?:string;images?:string;visionEnabled?:boolean;visionDevice?:string;lowRamMode?:string;resourceProfile?:'work'|'performance';dataFolder?:string;diskGb?:number;diskBytes?:number;/** Files this model shares with another (the picture reader, the draft layer): counted once, in `usage`. */sharedBytes?:number;/** What Delete frees: files in Strata's own folders that no other model uses. */removableGb?:number;missing_files?:string[]}[];/** What all models take on disk, from their real files, shared files counted once. */usage?:{total_bytes:number;total_gb:number;shared_bytes:number;shared_gb:number;models:Record<string,{bytes:number;gb:number}>}|null;data_folder:string;engine:{version:string|null}}|null;hardware:{ram_gb:number;engine_vendor?:string;gpus:{index:number;name:string;vram_gb:number;usable:boolean;vendor?:string;uma?:boolean;integrated?:boolean;unified_gb?:number;problem?:string|null}[]}|null;fields:NativeField[];instances:NativeInstance[];defaultId:string|null;jobs:NativeJob[];capabilities:{responses?:boolean;updates:boolean;rollback:boolean;calibration:boolean;componentUpdates?:boolean}}
|
||||
visionReader?: boolean;available:boolean;repo:string;downloadDirectory?:string;/** Free bytes on the drive that holds the models folder. */downloadFreeBytes?:number|null;downloadTotalBytes?:number|null;version?:string|null;summary:string;error:string|null;catalog:{families:NativeFamily[];contexts:number[];recommendation:{family:string|null;model:string|null;context?:number;why?:string};context_note:string}|null;installed:{models:{model:string;config:string;model_name:string;ready:boolean;/** The picture-reading files are on this computer (notes-split-agent8.md, item 8). */visionPrepared?:boolean;context?:number;kv?:string;images?:string;visionEnabled?:boolean;visionDevice?:string;lowRamMode?:string;resourceProfile?:'work'|'performance';dataFolder?:string;diskGb?:number;diskBytes?:number;/** Files this model shares with another (the picture reader, the draft layer): counted once, in `usage`. */sharedBytes?:number;/** What Delete frees: files in Strata's own folders that no other model uses. */removableGb?:number;missing_files?:string[]}[];/** What all models take on disk, from their real files, shared files counted once. */usage?:{total_bytes:number;total_gb:number;shared_bytes:number;shared_gb:number;models:Record<string,{bytes:number;gb:number}>}|null;data_folder:string;engine:{version:string|null}}|null;hardware:{ram_gb:number;engine_vendor?:string;gpus:{index:number;name:string;vram_gb:number;usable:boolean;/** SMs x clock: only the order between cards matters (strata_control/gpu_roles.py). */speed?:number;vendor?:string;uma?:boolean;integrated?:boolean;unified_gb?:number;problem?:string|null}[]}|null;fields:NativeField[];instances:NativeInstance[];defaultId:string|null;jobs:NativeJob[];capabilities:{responses?:boolean;updates:boolean;rollback:boolean;calibration:boolean;componentUpdates?:boolean}}
|
||||
let state:NativeState|null=null;
|
||||
const listeners=new Set<()=>void>();
|
||||
function put(value:NativeState){if(state&&JSON.stringify(state)===JSON.stringify(value))return;state=value;listeners.forEach(fn=>fn());}
|
||||
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "Verschieben beenden",
|
||||
"s8420": "Belassen",
|
||||
"s8421": "Beenden legt alle Ihre Modelle in {1} ab. Belassen lässt sie in {0} und entfernt die unvollständigen Kopien aus {1}.",
|
||||
"s8422": "Die unvollständigen Kopien wurden entfernt. Ihre Modelle liegen in {0}."
|
||||
"s8422": "Die unvollständigen Kopien wurden entfernt. Ihre Modelle liegen in {0}.",
|
||||
"s8601": "Grafikkarten",
|
||||
"s8602": "Haupt",
|
||||
"s8603": "Helfer",
|
||||
"s8604": "Ihre Wahl.",
|
||||
"s8605": "Empfohlen.",
|
||||
"s8606": "Sie haben die Rollen geändert. Strata verwendet genau diese.",
|
||||
"s8607": "Empfohlen: {0}",
|
||||
"s8608": "Rolle von {0}",
|
||||
"s8609": "{0} GB Speicher",
|
||||
"s8610": "Nutzt den Arbeitsspeicher Ihres Computers (integrierte Grafik)",
|
||||
"s8611": "Macht die Hauptarbeit",
|
||||
"s8612": "Hält einen Teil des Modells",
|
||||
"s8613": "Nicht genutzt",
|
||||
"s8614": "Wird von {0} verwendet",
|
||||
"s8615": "Ausschalten",
|
||||
"s8616": "{0} zur Hauptkarte machen",
|
||||
"s8617": "{0} kann Strata nicht ausführen.",
|
||||
"s8618": "{0} ist integrierte Grafik; als Hauptkarte ist sie langsam.",
|
||||
"s8619": "Nur mit der CPU ist die Leistung eingeschränkt.",
|
||||
"s8620": "Diese Engine braucht zum Start eine Grafikkarte. Schalten Sie eine ein.",
|
||||
"s8621": "Nicht genug Speicher für diese Einstellung: etwa {0} GB werden vom Laufwerk gelesen.",
|
||||
"s8622": "{0} ist jetzt ein Helfer.",
|
||||
"s8623": "{0} ist jetzt die Hauptkarte.",
|
||||
"s8624": "Die einzige eingeschaltete Karte muss die Hauptkarte sein.",
|
||||
"s8625": "Es wurde keine Grafikkarte gefunden, die Strata nutzen kann.",
|
||||
"s8626": "Strata verwendet Ihre {0}.",
|
||||
"s8627": "{0} ist die Hauptkarte: {1}.",
|
||||
"s8628": "die Karten sind gleich",
|
||||
"s8629": "sie hat mehr Speicher",
|
||||
"s8630": "sie ist schneller",
|
||||
"s8631": "sie ist die einzige Karte",
|
||||
"s8632": "{0} helfen.",
|
||||
"s8633": "{0} hilft.",
|
||||
"s8634": "{0} startet ausgeschaltet: {1}",
|
||||
"s8635": "Strata kann sie nicht ausführen.",
|
||||
"s8636": "integrierte Grafik.",
|
||||
"s8637": "{0} GB oder weniger.",
|
||||
"s8638": "Die Karte, die die letzten Schritte jeder Antwort übernimmt: Sie schreibt die nächsten Wörter, führt den Vorausschau-Schritt aus, prüft ihn und hält Ihr Gespräch. Wählen Sie Ihre schnellste Karte mit viel Speicher. Es gibt immer nur eine.",
|
||||
"s8639": "Hält einen Teil des Modells und seines schnellen Speichers, sodass mehr davon auf den Grafikkarten Platz findet statt vom Laufwerk gelesen zu werden. Prompts werden etwa 20 % schneller gelesen.",
|
||||
"s8640": "Strata rührt diese Karte nicht an. Gut für eine Karte, die Sie anderweitig nutzen, etwa für ein Spiel oder Ihren Bildschirm.",
|
||||
"s8641": "Das Modell braucht etwa {0} GB. Diese Karten und Ihr Arbeitsspeicher bieten etwa {1} GB. Es passt.",
|
||||
"s8642": "Das Modell braucht etwa {0} GB. Diese Karten und Ihr Arbeitsspeicher bieten etwa {1} GB, daher würden etwa {2} GB vom Laufwerk gelesen.",
|
||||
"s8643": "Nur Ihr Prozessor und Arbeitsspeicher werden verwendet.",
|
||||
"s8644": "Für Experten: Die Engine startet auf den Karten {0} (Hauptkarte zuletzt).",
|
||||
"s8645": "Wird nicht gespeichert, solange Sie die Empfehlung behalten: Strata wählt neu, wenn Sie eine Karte hinzufügen oder entfernen.",
|
||||
"s8646": "Nicht genutzt: {0} (anderer Grafikkarten-Hersteller als der, den diese Engine ansteuert)."
|
||||
}
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "Finish the move",
|
||||
"s8420": "Leave it",
|
||||
"s8421": "Finishing puts all your models in {1}. Leaving it keeps them in {0} and removes the partial copies from {1}.",
|
||||
"s8422": "The partial copies were removed. Your models are in {0}."
|
||||
"s8422": "The partial copies were removed. Your models are in {0}.",
|
||||
"s8601": "Graphics cards",
|
||||
"s8602": "Main",
|
||||
"s8603": "Helper",
|
||||
"s8604": "Your choice.",
|
||||
"s8605": "Recommended.",
|
||||
"s8606": "You changed the roles. Strata will use exactly these.",
|
||||
"s8607": "Recommended: {0}",
|
||||
"s8608": "Role of {0}",
|
||||
"s8609": "{0} GB memory",
|
||||
"s8610": "Uses your computer's memory (built-in graphics)",
|
||||
"s8611": "Does the main work",
|
||||
"s8612": "Holds part of the model",
|
||||
"s8613": "Not used",
|
||||
"s8614": "In use by {0}",
|
||||
"s8615": "Turn it off",
|
||||
"s8616": "Make {0} Main",
|
||||
"s8617": "{0} cannot run Strata.",
|
||||
"s8618": "{0} is built-in graphics; it is slow as Main.",
|
||||
"s8619": "Performance is degraded on CPU only.",
|
||||
"s8620": "This engine needs a graphics card to start. Turn one on.",
|
||||
"s8621": "Not enough memory for this setup: about {0} GB is read from the drive.",
|
||||
"s8622": "{0} is now a Helper.",
|
||||
"s8623": "{0} is now Main.",
|
||||
"s8624": "The only card that is on has to be Main.",
|
||||
"s8625": "No graphics card that Strata can use was found.",
|
||||
"s8626": "Strata will use your {0}.",
|
||||
"s8627": "{0} is Main: {1}.",
|
||||
"s8628": "the cards are alike",
|
||||
"s8629": "it has more memory",
|
||||
"s8630": "it is faster",
|
||||
"s8631": "it is the only card",
|
||||
"s8632": "{0} help.",
|
||||
"s8633": "{0} helps.",
|
||||
"s8634": "{0} starts Off: {1}",
|
||||
"s8635": "Strata cannot run it.",
|
||||
"s8636": "built-in graphics.",
|
||||
"s8637": "{0} GB or less.",
|
||||
"s8638": "The card that does the final steps of every answer: it writes the next words, runs the draft-ahead step and checks it, and keeps your conversation. Pick your fastest card with plenty of memory. There is only ever one.",
|
||||
"s8639": "Holds a share of the model and its fast memory, so more of it fits on graphics cards instead of being read from your drive. Prompts are read about 20% faster.",
|
||||
"s8640": "Strata does not touch this card. Good for a card you use for something else, such as a game or your screen.",
|
||||
"s8641": "The model needs about {0} GB. These cards and your memory give about {1} GB. It fits.",
|
||||
"s8642": "The model needs about {0} GB. These cards and your memory give about {1} GB, so about {2} GB would be read from your drive.",
|
||||
"s8643": "Only your processor and memory are used.",
|
||||
"s8644": "For experts: the engine starts on cards {0} (Main last).",
|
||||
"s8645": "Not saved while you keep the recommendation: Strata picks again if you add or remove a card.",
|
||||
"s8646": "Not used: {0} (another make of graphics card than this engine drives)."
|
||||
}
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "Terminar el traslado",
|
||||
"s8420": "Dejarlo",
|
||||
"s8421": "Terminar pone todos sus modelos en {1}. Dejarlo los mantiene en {0} y elimina las copias parciales de {1}.",
|
||||
"s8422": "Se eliminaron las copias parciales. Sus modelos están en {0}."
|
||||
"s8422": "Se eliminaron las copias parciales. Sus modelos están en {0}.",
|
||||
"s8601": "Tarjetas gráficas",
|
||||
"s8602": "Principal",
|
||||
"s8603": "Ayudante",
|
||||
"s8604": "Tu elección.",
|
||||
"s8605": "Recomendado.",
|
||||
"s8606": "Cambiaste los roles. Strata usará exactamente estos.",
|
||||
"s8607": "Recomendado: {0}",
|
||||
"s8608": "Rol de {0}",
|
||||
"s8609": "{0} GB de memoria",
|
||||
"s8610": "Usa la memoria de tu equipo (gráficos integrados)",
|
||||
"s8611": "Hace el trabajo principal",
|
||||
"s8612": "Guarda parte del modelo",
|
||||
"s8613": "No se usa",
|
||||
"s8614": "En uso por {0}",
|
||||
"s8615": "Apagarla",
|
||||
"s8616": "Hacer principal a {0}",
|
||||
"s8617": "{0} no puede ejecutar Strata.",
|
||||
"s8618": "{0} son gráficos integrados; como principal es lenta.",
|
||||
"s8619": "El rendimiento se reduce al usar solo la CPU.",
|
||||
"s8620": "Este motor necesita una tarjeta gráfica para arrancar. Activa una.",
|
||||
"s8621": "Memoria insuficiente para esta configuración: unos {0} GB se leen del disco.",
|
||||
"s8622": "{0} ahora es ayudante.",
|
||||
"s8623": "{0} ahora es la principal.",
|
||||
"s8624": "La única tarjeta activada tiene que ser la principal.",
|
||||
"s8625": "No se encontró ninguna tarjeta gráfica que Strata pueda usar.",
|
||||
"s8626": "Strata usará tu {0}.",
|
||||
"s8627": "{0} es la principal: {1}.",
|
||||
"s8628": "las tarjetas son iguales",
|
||||
"s8629": "tiene más memoria",
|
||||
"s8630": "es más rápida",
|
||||
"s8631": "es la única tarjeta",
|
||||
"s8632": "{0} ayudan.",
|
||||
"s8633": "{0} ayuda.",
|
||||
"s8634": "{0} empieza apagada: {1}",
|
||||
"s8635": "Strata no puede ejecutarla.",
|
||||
"s8636": "gráficos integrados.",
|
||||
"s8637": "{0} GB o menos.",
|
||||
"s8638": "La tarjeta que hace los pasos finales de cada respuesta: escribe las siguientes palabras, ejecuta el paso de adelanto y lo comprueba, y guarda tu conversación. Elige tu tarjeta más rápida con mucha memoria. Solo hay una.",
|
||||
"s8639": "Guarda una parte del modelo y de su memoria rápida, así cabe más en las tarjetas gráficas en lugar de leerse del disco. Los prompts se leen un 20 % más rápido.",
|
||||
"s8640": "Strata no toca esta tarjeta. Útil para una tarjeta que usas para otra cosa, como un juego o tu pantalla.",
|
||||
"s8641": "El modelo necesita unos {0} GB. Estas tarjetas y tu memoria suman unos {1} GB. Cabe.",
|
||||
"s8642": "El modelo necesita unos {0} GB. Estas tarjetas y tu memoria suman unos {1} GB, así que unos {2} GB se leerían del disco.",
|
||||
"s8643": "Solo se usan tu procesador y tu memoria.",
|
||||
"s8644": "Para expertos: el motor arranca en las tarjetas {0} (la principal al final).",
|
||||
"s8645": "No se guarda mientras mantengas la recomendación: Strata elige de nuevo si añades o quitas una tarjeta.",
|
||||
"s8646": "No se usa: {0} (otra marca de tarjeta gráfica distinta de la que maneja este motor)."
|
||||
}
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "移動を完了する",
|
||||
"s8420": "中止する",
|
||||
"s8421": "完了すると、すべてのモデルが {1} に入ります。中止すると、モデルは {0} に残り、{1} の不完全なコピーが削除されます。",
|
||||
"s8422": "不完全なコピーを削除しました。モデルは {0} にあります。"
|
||||
"s8422": "不完全なコピーを削除しました。モデルは {0} にあります。",
|
||||
"s8601": "グラフィックカード",
|
||||
"s8602": "メイン",
|
||||
"s8603": "ヘルパー",
|
||||
"s8604": "あなたの選択です。",
|
||||
"s8605": "おすすめです。",
|
||||
"s8606": "役割を変更しました。Strata はこのとおりに使います。",
|
||||
"s8607": "おすすめ：{0}",
|
||||
"s8608": "{0} の役割",
|
||||
"s8609": "メモリ {0} GB",
|
||||
"s8610": "コンピューター本体のメモリを使います（内蔵グラフィックス）",
|
||||
"s8611": "主な処理を担当",
|
||||
"s8612": "モデルの一部を保持",
|
||||
"s8613": "使用しません",
|
||||
"s8614": "{0} が使用中",
|
||||
"s8615": "オフにする",
|
||||
"s8616": "{0} をメインにする",
|
||||
"s8617": "{0} では Strata を実行できません。",
|
||||
"s8618": "{0} は内蔵グラフィックスで、メインにすると遅くなります。",
|
||||
"s8619": "CPU のみの場合、性能が低下します。",
|
||||
"s8620": "このエンジンの起動にはグラフィックカードが必要です。1 枚オンにしてください。",
|
||||
"s8621": "この設定ではメモリが足りません。約 {0} GB をドライブから読み込みます。",
|
||||
"s8622": "{0} はヘルパーになりました。",
|
||||
"s8623": "{0} がメインになりました。",
|
||||
"s8624": "オンになっている唯一のカードはメインでなければなりません。",
|
||||
"s8625": "Strata が使えるグラフィックカードが見つかりません。",
|
||||
"s8626": "Strata は {0} を使います。",
|
||||
"s8627": "{0} がメインです：{1}。",
|
||||
"s8628": "カードが同じため",
|
||||
"s8629": "メモリが多いため",
|
||||
"s8630": "より高速なため",
|
||||
"s8631": "唯一のカードのため",
|
||||
"s8632": "{0} がサポートします。",
|
||||
"s8633": "{0} がサポートします。",
|
||||
"s8634": "{0} は最初オフです：{1}",
|
||||
"s8635": "Strata では実行できません。",
|
||||
"s8636": "内蔵グラフィックスです。",
|
||||
"s8637": "{0} GB 以下です。",
|
||||
"s8638": "回答の最後の工程を担当するカードです。次の単語を書き、先読みの工程を実行して確認し、会話を保持します。メモリに余裕のある一番速いカードを選んでください。常に 1 枚だけです。",
|
||||
"s8639": "モデルの一部と高速メモリを保持し、ドライブから読み込む代わりにより多くをグラフィックカード上に置けます。プロンプトの読み込みが約 20% 速くなります。",
|
||||
"s8640": "Strata はこのカードに触れません。ゲームや画面表示など、別の用途に使うカードに向いています。",
|
||||
"s8641": "モデルには約 {0} GB が必要です。これらのカードとメモリで約 {1} GB あり、収まります。",
|
||||
"s8642": "モデルには約 {0} GB が必要です。これらのカードとメモリは約 {1} GB のため、約 {2} GB がドライブから読み込まれます。",
|
||||
"s8643": "プロセッサとメモリのみを使います。",
|
||||
"s8644": "上級者向け：エンジンはカード {0} で起動します（メインが最後）。",
|
||||
"s8645": "おすすめのままの間は保存されません。カードを追加・取り外すと Strata が選び直します。",
|
||||
"s8646": "未使用：{0}（このエンジンが対応するのとは別メーカーのカードです）。"
|
||||
}
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "이동 마무리",
|
||||
"s8420": "그대로 두기",
|
||||
"s8421": "마무리하면 모든 모델이 {1}에 들어갑니다. 그대로 두면 모델은 {0}에 남고 {1}의 불완전한 복사본이 삭제됩니다.",
|
||||
"s8422": "불완전한 복사본을 삭제했습니다. 모델은 {0}에 있습니다."
|
||||
"s8422": "불완전한 복사본을 삭제했습니다. 모델은 {0}에 있습니다.",
|
||||
"s8601": "그래픽 카드",
|
||||
"s8602": "메인",
|
||||
"s8603": "헬퍼",
|
||||
"s8604": "직접 고른 설정입니다.",
|
||||
"s8605": "추천 설정입니다.",
|
||||
"s8606": "역할을 변경했습니다. Strata는 이 설정을 그대로 사용합니다.",
|
||||
"s8607": "추천: {0}",
|
||||
"s8608": "{0}의 역할",
|
||||
"s8609": "메모리 {0} GB",
|
||||
"s8610": "컴퓨터 본체 메모리를 사용합니다 (내장 그래픽)",
|
||||
"s8611": "주요 작업을 처리합니다",
|
||||
"s8612": "모델의 일부를 담당합니다",
|
||||
"s8613": "사용 안 함",
|
||||
"s8614": "{0}에서 사용 중",
|
||||
"s8615": "끄기",
|
||||
"s8616": "{0}을(를) 메인으로 지정",
|
||||
"s8617": "{0}에서는 Strata를 실행할 수 없습니다.",
|
||||
"s8618": "{0}은(는) 내장 그래픽이라 메인으로 쓰면 느립니다.",
|
||||
"s8619": "CPU만 사용하면 성능이 떨어집니다.",
|
||||
"s8620": "이 엔진은 시작하려면 그래픽 카드가 필요합니다. 하나를 켜세요.",
|
||||
"s8621": "이 설정에는 메모리가 부족합니다. 약 {0} GB를 드라이브에서 읽습니다.",
|
||||
"s8622": "{0}이(가) 이제 헬퍼입니다.",
|
||||
"s8623": "{0}이(가) 이제 메인입니다.",
|
||||
"s8624": "켜져 있는 유일한 카드는 메인이어야 합니다.",
|
||||
"s8625": "Strata가 사용할 수 있는 그래픽 카드를 찾지 못했습니다.",
|
||||
"s8626": "Strata가 {0}을(를) 사용합니다.",
|
||||
"s8627": "{0}이(가) 메인입니다: {1}.",
|
||||
"s8628": "카드가 같습니다",
|
||||
"s8629": "메모리가 더 많습니다",
|
||||
"s8630": "더 빠릅니다",
|
||||
"s8631": "유일한 카드입니다",
|
||||
"s8632": "{0}이(가) 돕습니다.",
|
||||
"s8633": "{0}이(가) 돕습니다.",
|
||||
"s8634": "{0}은(는) 기본적으로 꺼집니다: {1}",
|
||||
"s8635": "Strata에서 실행할 수 없습니다.",
|
||||
"s8636": "내장 그래픽입니다.",
|
||||
"s8637": "{0} GB 이하입니다.",
|
||||
"s8638": "모든 답변의 마지막 단계를 맡는 카드입니다. 다음 단어를 쓰고, 미리 계산하는 단계를 실행해 확인하며, 대화를 보관합니다. 메모리가 넉넉한 가장 빠른 카드를 고르세요. 항상 하나뿐입니다.",
|
||||
"s8639": "모델의 일부와 빠른 메모리를 담당해, 더 많은 부분이 드라이브가 아니라 그래픽 카드에 올라갑니다. 프롬프트를 약 20% 더 빠르게 읽습니다.",
|
||||
"s8640": "Strata가 이 카드를 건드리지 않습니다. 게임이나 화면 출력 등 다른 용도로 쓰는 카드에 알맞습니다.",
|
||||
"s8641": "모델에는 약 {0} GB가 필요합니다. 이 카드들과 메모리는 약 {1} GB를 제공하므로 충분합니다.",
|
||||
"s8642": "모델에는 약 {0} GB가 필요합니다. 이 카드들과 메모리는 약 {1} GB를 제공하므로 약 {2} GB를 드라이브에서 읽게 됩니다.",
|
||||
"s8643": "프로세서와 메모리만 사용합니다.",
|
||||
"s8644": "전문가용: 엔진은 카드 {0}에서 시작합니다 (메인이 마지막).",
|
||||
"s8645": "추천 설정을 유지하는 동안은 저장되지 않습니다. 카드를 추가하거나 빼면 Strata가 다시 고릅니다.",
|
||||
"s8646": "사용 안 함: {0} (이 엔진이 다루는 것과 다른 제조사의 그래픽 카드입니다)."
|
||||
}
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "Terminar a mudança",
|
||||
"s8420": "Deixar como está",
|
||||
"s8421": "Terminar coloca todos os seus modelos em {1}. Deixar como está mantém os modelos em {0} e remove as cópias parciais de {1}.",
|
||||
"s8422": "As cópias parciais foram removidas. Seus modelos estão em {0}."
|
||||
"s8422": "As cópias parciais foram removidas. Seus modelos estão em {0}.",
|
||||
"s8601": "Placas de vídeo",
|
||||
"s8602": "Principal",
|
||||
"s8603": "Auxiliar",
|
||||
"s8604": "Sua escolha.",
|
||||
"s8605": "Recomendado.",
|
||||
"s8606": "Você mudou as funções. O Strata usará exatamente estas.",
|
||||
"s8607": "Recomendado: {0}",
|
||||
"s8608": "Função de {0}",
|
||||
"s8609": "{0} GB de memória",
|
||||
"s8610": "Usa a memória do seu computador (gráficos integrados)",
|
||||
"s8611": "Faz o trabalho principal",
|
||||
"s8612": "Guarda parte do modelo",
|
||||
"s8613": "Não usada",
|
||||
"s8614": "Em uso por {0}",
|
||||
"s8615": "Desligar",
|
||||
"s8616": "Tornar {0} a principal",
|
||||
"s8617": "{0} não consegue executar o Strata.",
|
||||
"s8618": "{0} é um gráfico integrado; como principal é lenta.",
|
||||
"s8619": "O desempenho fica reduzido usando apenas a CPU.",
|
||||
"s8620": "Este motor precisa de uma placa de vídeo para iniciar. Ative uma.",
|
||||
"s8621": "Memória insuficiente para esta configuração: cerca de {0} GB são lidos do disco.",
|
||||
"s8622": "{0} agora é auxiliar.",
|
||||
"s8623": "{0} agora é a principal.",
|
||||
"s8624": "A única placa ligada tem de ser a principal.",
|
||||
"s8625": "Nenhuma placa de vídeo que o Strata possa usar foi encontrada.",
|
||||
"s8626": "O Strata usará a sua {0}.",
|
||||
"s8627": "{0} é a principal: {1}.",
|
||||
"s8628": "as placas são iguais",
|
||||
"s8629": "tem mais memória",
|
||||
"s8630": "é mais rápida",
|
||||
"s8631": "é a única placa",
|
||||
"s8632": "{0} ajudam.",
|
||||
"s8633": "{0} ajuda.",
|
||||
"s8634": "{0} começa desligada: {1}",
|
||||
"s8635": "O Strata não consegue executá-la.",
|
||||
"s8636": "gráficos integrados.",
|
||||
"s8637": "{0} GB ou menos.",
|
||||
"s8638": "A placa que faz os passos finais de cada resposta: escreve as próximas palavras, executa o passo de antecipação e o verifica, e guarda a sua conversa. Escolha a placa mais rápida com bastante memória. Há sempre só uma.",
|
||||
"s8639": "Guarda uma parte do modelo e da sua memória rápida, para que mais caiba nas placas de vídeo em vez de ser lido do disco. Os prompts são lidos cerca de 20% mais rápido.",
|
||||
"s8640": "O Strata não mexe nesta placa. Bom para uma placa que você usa para outra coisa, como um jogo ou a tela.",
|
||||
"s8641": "O modelo precisa de cerca de {0} GB. Estas placas e a sua memória dão cerca de {1} GB. Cabe.",
|
||||
"s8642": "O modelo precisa de cerca de {0} GB. Estas placas e a sua memória dão cerca de {1} GB, por isso cerca de {2} GB seriam lidos do disco.",
|
||||
"s8643": "Apenas o processador e a memória são usados.",
|
||||
"s8644": "Para especialistas: o motor inicia nas placas {0} (a principal por último).",
|
||||
"s8645": "Não é guardado enquanto mantiver a recomendação: o Strata escolhe de novo se adicionar ou remover uma placa.",
|
||||
"s8646": "Não usada: {0} (outra marca de placa de vídeo diferente da que este motor controla)."
|
||||
}
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "Завершить перенос",
|
||||
"s8420": "Оставить как есть",
|
||||
"s8421": "Если завершить, все модели окажутся в {1}. Если оставить, модели останутся в {0}, а неполные копии в {1} будут удалены.",
|
||||
"s8422": "Неполные копии удалены. Ваши модели находятся в {0}."
|
||||
"s8422": "Неполные копии удалены. Ваши модели находятся в {0}.",
|
||||
"s8601": "Видеокарты",
|
||||
"s8602": "Основная",
|
||||
"s8603": "Помощник",
|
||||
"s8604": "Ваш выбор.",
|
||||
"s8605": "Рекомендуется.",
|
||||
"s8606": "Вы изменили роли. Strata будет использовать именно их.",
|
||||
"s8607": "Рекомендуется: {0}",
|
||||
"s8608": "Роль {0}",
|
||||
"s8609": "{0} ГБ памяти",
|
||||
"s8610": "Использует память компьютера (встроенная графика)",
|
||||
"s8611": "Выполняет основную работу",
|
||||
"s8612": "Хранит часть модели",
|
||||
"s8613": "Не используется",
|
||||
"s8614": "Используется: {0}",
|
||||
"s8615": "Выключить",
|
||||
"s8616": "Сделать {0} основной",
|
||||
"s8617": "{0} не может запускать Strata.",
|
||||
"s8618": "{0} — встроенная графика; в роли основной она медленная.",
|
||||
"s8619": "Только на процессоре производительность снижена.",
|
||||
"s8620": "Для запуска этому движку нужна видеокарта. Включите одну.",
|
||||
"s8621": "Для этой настройки не хватает памяти: около {0} ГБ читается с диска.",
|
||||
"s8622": "{0} теперь помощник.",
|
||||
"s8623": "{0} теперь основная.",
|
||||
"s8624": "Единственная включённая карта должна быть основной.",
|
||||
"s8625": "Не найдено видеокарт, которые может использовать Strata.",
|
||||
"s8626": "Strata будет использовать ваш {0}.",
|
||||
"s8627": "{0} — основная: {1}.",
|
||||
"s8628": "карты одинаковые",
|
||||
"s8629": "у неё больше памяти",
|
||||
"s8630": "она быстрее",
|
||||
"s8631": "это единственная карта",
|
||||
"s8632": "{0} помогают.",
|
||||
"s8633": "{0} помогает.",
|
||||
"s8634": "{0} по умолчанию выключена: {1}",
|
||||
"s8635": "Strata не может её запустить.",
|
||||
"s8636": "встроенная графика.",
|
||||
"s8637": "{0} ГБ или меньше.",
|
||||
"s8638": "Карта, выполняющая последние шаги каждого ответа: пишет следующие слова, запускает шаг опережающего черновика и проверяет его, а также хранит вашу беседу. Выберите самую быструю карту с большим объёмом памяти. Основная карта всегда одна.",
|
||||
"s8639": "Хранит часть модели и её быстрой памяти, поэтому больше помещается на видеокартах и меньше читается с диска. Запросы читаются примерно на 20 % быстрее.",
|
||||
"s8640": "Strata не трогает эту карту. Подходит для карты, занятой другим, например игрой или выводом на экран.",
|
||||
"s8641": "Модели нужно около {0} ГБ. Эти карты и ваша память дают около {1} ГБ. Помещается.",
|
||||
"s8642": "Модели нужно около {0} ГБ. Эти карты и ваша память дают около {1} ГБ, поэтому около {2} ГБ придётся читать с диска.",
|
||||
"s8643": "Используются только процессор и память.",
|
||||
"s8644": "Для опытных: движок запускается на картах {0} (основная — последняя).",
|
||||
"s8645": "Не сохраняется, пока вы оставляете рекомендацию: Strata выберет заново, если вы добавите или уберёте карту.",
|
||||
"s8646": "Не используется: {0} (другой производитель видеокарт, чем тот, с которым работает этот движок)."
|
||||
}
|
||||
@@ -3443,5 +3443,51 @@
|
||||
"s8419": "完成移动",
|
||||
"s8420": "保持原样",
|
||||
"s8421": "完成后，所有模型都会放到 {1}。保持原样则模型留在 {0}，并删除 {1} 中不完整的副本。",
|
||||
"s8422": "已删除不完整的副本。你的模型在 {0}。"
|
||||
"s8422": "已删除不完整的副本。你的模型在 {0}。",
|
||||
"s8601": "显卡",
|
||||
"s8602": "主卡",
|
||||
"s8603": "辅助",
|
||||
"s8604": "你的选择。",
|
||||
"s8605": "推荐。",
|
||||
"s8606": "你更改了分工。Strata 将完全按此使用。",
|
||||
"s8607": "推荐：{0}",
|
||||
"s8608": "{0} 的分工",
|
||||
"s8609": "{0} GB 显存",
|
||||
"s8610": "使用电脑自身的内存（集成显卡）",
|
||||
"s8611": "承担主要工作",
|
||||
"s8612": "存放模型的一部分",
|
||||
"s8613": "不使用",
|
||||
"s8614": "正被 {0} 使用",
|
||||
"s8615": "关闭它",
|
||||
"s8616": "将 {0} 设为主卡",
|
||||
"s8617": "{0} 无法运行 Strata。",
|
||||
"s8618": "{0} 是集成显卡，作主卡会很慢。",
|
||||
"s8619": "仅使用 CPU 时性能会下降。",
|
||||
"s8620": "此引擎需要一张显卡才能启动。请开启一张。",
|
||||
"s8621": "此配置的内存不足：约 {0} GB 将从硬盘读取。",
|
||||
"s8622": "{0} 现在是辅助卡。",
|
||||
"s8623": "{0} 现在是主卡。",
|
||||
"s8624": "唯一开启的显卡必须是主卡。",
|
||||
"s8625": "未找到 Strata 可用的显卡。",
|
||||
"s8626": "Strata 将使用你的 {0}。",
|
||||
"s8627": "{0} 是主卡：{1}。",
|
||||
"s8628": "两张卡相同",
|
||||
"s8629": "它的显存更大",
|
||||
"s8630": "它更快",
|
||||
"s8631": "它是唯一的显卡",
|
||||
"s8632": "{0} 提供辅助。",
|
||||
"s8633": "{0} 提供辅助。",
|
||||
"s8634": "{0} 默认关闭：{1}",
|
||||
"s8635": "Strata 无法运行它。",
|
||||
"s8636": "集成显卡。",
|
||||
"s8637": "{0} GB 或更少。",
|
||||
"s8638": "负责每次回答最后步骤的显卡：写出下一个词，运行预测步骤并校验，同时保存你的对话。请选最快且显存充足的显卡。始终只有一张。",
|
||||
"s8639": "存放模型的一部分及其高速缓存，让更多内容放在显卡上而不是从硬盘读取。读取提示词约快 20%。",
|
||||
"s8640": "Strata 不会使用这张显卡。适合用于其他用途的显卡，例如游戏或显示器。",
|
||||
"s8641": "模型约需 {0} GB。这些显卡加上内存共约 {1} GB，可以容纳。",
|
||||
"s8642": "模型约需 {0} GB。这些显卡加上内存共约 {1} GB，因此约 {2} GB 将从硬盘读取。",
|
||||
"s8643": "只使用处理器和内存。",
|
||||
"s8644": "专家信息：引擎在显卡 {0} 上启动（主卡在最后）。",
|
||||
"s8645": "沿用推荐设置时不会保存：增减显卡后 Strata 会重新选择。",
|
||||
"s8646": "未使用：{0}（与此引擎驱动的显卡品牌不同）。"
|
||||
}
|
||||
@@ -26,3 +26,5 @@ import './styles/model-run.css';
|
||||
import './styles/onboarding.css';
|
||||
|
||||
import './styles/board.css';
|
||||
|
||||
import './styles/gpu-roles.css';
|
||||
@@ -9,6 +9,8 @@ import { api } from '../lib/client';
|
||||
import { RunningModels } from '../components/RunningModels';
|
||||
import { LocalModelImport } from '../components/LocalModelImport';
|
||||
import { gpuId } from '../lib/monitor-hardware';
|
||||
import { GpuRoles } from '../components/GpuRoles';
|
||||
import { normalize as normalizeRoles, order as roleOrder, recommend as recommendRoles, sameRoles, type RoleCard, type Roles } from '../lib/gpu-roles';
|
||||
import { RunImageModel } from '../components/RunImageModel';
|
||||
import { IMAGE_GENERATION_VISIBLE } from '../lib/features';
|
||||
import { FloatingMenu } from '../components/FloatingMenu';
|
||||
@@ -49,6 +51,8 @@ function LegacyModelRunScreen({ importFirst = false, initialVariant }: {
|
||||
const runtime = useStore(s => s.runtime);
|
||||
const [deviceChoice, setDeviceChoice] = useState('auto');
|
||||
const [customGpus, setCustomGpus] = useState<string[]>([]);
|
||||
// Two or more cards (Expert settings): a role for each card instead of the "Run on" list. null = the recommended roles.
|
||||
const [runRoles, setRunRoles] = useState<Roles | null>(null);
|
||||
const placementInitialized = useRef('');
|
||||
const imageModels = useStore(s => s.imageModels), imagePending = useStore(s => !!s.busy['image:select']);
|
||||
const imageChoice = imageModels?.selected ?? 'off';
|
||||
@@ -123,8 +127,11 @@ function LegacyModelRunScreen({ importFirst = false, initialVariant }: {
|
||||
const profileChanged = !!selectedProfile && !!config && (configIdentity(config) !== configIdentity(selectedProfile.config) || imageChoice !== (selectedProfile.imageModel ?? 'off'));
|
||||
const use = config?.headroom === 'full' ? 'full' : 'shared';
|
||||
const ctx = contextTokens(config?.beginner.contextLength ?? 'recommended');
|
||||
const roleCards: RoleCard[] = (hardware?.gpus ?? []).map((g, i) => ({ index: i, name: g.name, vram_gb: g.vramGb ?? 0, integrated: !!g.unified, vendor: g.vendor }));
|
||||
const rolesMode = expert && roleCards.length > 1;
|
||||
const recommendedRoles = recommendRoles(roleCards).roles, effectiveRoles: Roles = runRoles ? normalizeRoles(runRoles, roleCards) : recommendedRoles;
|
||||
const effectiveDevice = expert ? deviceChoice : 'auto';
|
||||
const gpuIds = hardware ? (effectiveDevice === 'auto' ? hardware.gpus.map(gpuId) : effectiveDevice === 'cpu' ? [] : effectiveDevice === 'custom' ? customGpus : [effectiveDevice]) : [];
|
||||
const gpuIds = hardware ? (rolesMode ? roleOrder(effectiveRoles, roleCards).map(i => gpuId(hardware.gpus[i])) : effectiveDevice === 'auto' ? hardware.gpus.map(gpuId) : effectiveDevice === 'cpu' ? [] : effectiveDevice === 'custom' ? customGpus : [effectiveDevice]) : [];
|
||||
const executionHardware = hardware ? { ...hardware, gpus: hardware.gpus.filter(g => gpuIds.includes(gpuId(g))), accelerator: { ...hardware.accelerator, backend: gpuIds.length ? hardware.accelerator.backend : 'cpu', unified: gpuIds.length ? hardware.accelerator.unified : false, usableVramGb: hardware.gpus.filter(g => gpuIds.includes(gpuId(g))).reduce((n, g) => n + g.vramGb, 0) } } : null;
|
||||
const cfg = execution({ ...config?.expert, ...(!expert ? { 'gpu.layers': gpuIds.length ? 'Automatic' : 'CPU only' } : !gpuIds.length ? { 'gpu.layers': 'CPU only' } : {}) }, ctx);
|
||||
const baseExperience = estimate(model, executionHardware, cfg, 'document', library?.mode === 'simulated' && !model?.local);
|
||||
@@ -213,7 +220,7 @@ function LegacyModelRunScreen({ importFirst = false, initialVariant }: {
|
||||
});
|
||||
}, 180);
|
||||
return () => { active = false; clearTimeout(timer); };
|
||||
}, [text, model?.id, deviceChoice, customGpus.join(','), runtime?.revision, connected, expert, executionMode]);
|
||||
}, [text, model?.id, deviceChoice, customGpus.join(','), JSON.stringify(runRoles), runtime?.revision, connected, expert, executionMode]);
|
||||
function loadPatch(doc: ConfigDocument) {
|
||||
const patch = configPatch(doc);
|
||||
if (!expert || !gpuIds.length)
|
||||
@@ -378,7 +385,7 @@ function LegacyModelRunScreen({ importFirst = false, initialVariant }: {
|
||||
else {
|
||||
if (!lifecycle.allowed)
|
||||
throw new Error(lifecycle.reason);
|
||||
if (expert && deviceChoice !== 'auto')
|
||||
if (expert && (rolesMode ? !!runRoles : deviceChoice !== 'auto'))
|
||||
throw new Error('GPU assignment requires a native engine adapter. Use Automatic with this host.');
|
||||
await saveEngine(patch, true, true);
|
||||
const ok = getState().library?.loaded === model.id ? await restartEngine() : await loadVariant(model.id);
|
||||
@@ -443,9 +450,11 @@ function LegacyModelRunScreen({ importFirst = false, initialVariant }: {
|
||||
{tr(expert ? <section className="run-card run-placement">
|
||||
<div className="run-card__head"><span><Icon name="gpu" size={19}/><h2>{tr("Model execution")}</h2></span></div>
|
||||
<label className="run-multiple-choice"><span><strong>{tr("Run multiple models")}</strong><small>{tr("Keep models loaded when you add another.")}</small></span><input type="checkbox" role="switch" aria-label={tr("Run multiple models")} checked={allowMultiple} disabled={pending || runtime?.capabilities.multipleModels === false} onChange={e => { setAllowMultiple(e.target.checked); setIssues([]); setStage('idle'); }}/></label>
|
||||
{rolesMode ? <GpuRoles cards={roleCards} roles={effectiveRoles} custom={!!runRoles && !sameRoles(effectiveRoles, recommendedRoles)} disabled={pending || runtime?.capabilities.deviceAssignment === false} onChange={next => { setRunRoles(sameRoles(next, recommendedRoles) ? null : next); setStage('idle'); setIssues([]); }} onBack={() => { setRunRoles(null); setStage('idle'); setIssues([]); }}/> : <>
|
||||
<span className="run-device-label">{tr("Run on")}</span>
|
||||
<Select ariaLabel={tr("Execution device")} value={deviceChoice} disabled={pending || runtime?.capabilities.deviceAssignment === false} options={[{ value: 'auto', label: hardware?.gpus.length ? hardware.gpus.length > 1 ? 'Automatic · all GPUs' : 'Automatic · GPU 1' : 'CPU only' }, { value: 'cpu', label: 'CPU only' }, ...(hardware?.gpus ?? []).map((g, i) => ({ value: gpuId(g), label: `GPU ${i + 1} · ${g.name}` })), ...((hardware?.gpus.length ?? 0) > 1 ? [{ value: 'custom', label: 'Choose GPUs…' }] : [])]} onChange={id => { setDeviceChoice(id); setStage('idle'); setIssues([]); }}/>
|
||||
{tr(deviceChoice === 'custom' ? <div className="run-gpu-options">{tr(hardware?.gpus.map((g, i) => <label key={gpuId(g)}><input type="checkbox" checked={customGpus.includes(gpuId(g))} onChange={e => { setCustomGpus(ids => e.target.checked ? [...ids, gpuId(g)] : ids.filter(id => id !== gpuId(g))); setStage('idle'); }}/><span>{tr("GPU ")}{tr(i + 1)} · {tr(g.name)}<small>{tr(g.unified ? 'Shared system memory' : `${sizeGb(g.vramGb)} VRAM`)}</small></span></label>))}</div> : null)}
|
||||
</>}
|
||||
<p className="xs muted">{tr(runtime?.capabilities.multipleModels === false ? 'This native host currently supports one model with automatic GPU placement.' : !allowMultiple ? 'One model at a time. Loading switches to the selected model.' : placementPlan?.reloads ? 'Loading applies these settings to this instance. Other models stay loaded.' : 'Loading adds a model. Existing models keep their settings and GPU assignments.')}</p>
|
||||
{tr(runtime?.instances.length ? <div className="run-memory-reservations"><span>{tr(placementPlan && !placementPlan.alreadyLoaded ? 'Shared RAM after load' : 'Shared RAM reserved')} <strong>{tr((placementPlan?.budget.ramGb ?? runtime.memory.ramGb).toFixed(1))} / {tr((placementPlan?.budget.budgetGb ?? runtime.memory.budgetGb).toFixed(1))}{tr(" GB")}</strong></span><small>{tr(runtime.memory.reserveGb.toFixed(0))}{tr(" GB RAM headroom \u00B7 estimates include all loaded models")}</small>{tr((placementPlan?.budget.devices ?? runtime.memory.devices).map(d => <span key={d.id}>{tr(d.name)}<strong>{tr(d.usedGb.toFixed(1))} / {tr(d.budgetGb.toFixed(1))}{tr(" GB")}</strong></span>))}</div> : null)}
|
||||
</section> : null)}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Main / Helper / Off for every graphics card (GpuRoles.tsx). Tokens are the app's (styles/kit/tokens.css). */
|
||||
.gpuroles { display:flex; flex-direction:column; gap:10px; margin-top:14px; }
|
||||
.gpuroles__head { display:flex; align-items:baseline; justify-content:space-between; gap:12px; flex-wrap:wrap; }
|
||||
.gpuroles__link { background:none; border:0; padding:0; color:var(--st-accent-text); font:600 var(--st-fs-sm)/1.4 var(--st-font); cursor:pointer; text-decoration:underline; text-underline-offset:2px; }
|
||||
.gpuroles__link:disabled { opacity:.5; cursor:default; }
|
||||
.gpuroles__reco { display:flex; gap:10px; align-items:flex-start; padding:12px 14px; border-radius:var(--st-r-md); background:var(--st-accent-tint); color:var(--st-ink); }
|
||||
.gpuroles__reco svg { flex:none; margin-top:2px; color:var(--st-accent-text); }
|
||||
.gpuroles__reco[data-custom="true"] { background:var(--st-surface-2); }
|
||||
.gpuroles__reco[data-custom="true"] svg { color:var(--st-ink-muted); }
|
||||
.gpuroles__reco p { margin:0; }
|
||||
.gpuroles__rows { display:grid; gap:10px; }
|
||||
.gpuroles__row { border:1px solid var(--st-line); border-radius:var(--st-r-md); padding:14px 16px; background:var(--st-surface); }
|
||||
.gpuroles__row[data-role="main"] { border-color:var(--st-accent); background:var(--st-accent-tint); }
|
||||
.gpuroles__row[data-role="off"] { background:var(--st-surface-2); }
|
||||
.gpuroles__top { display:flex; gap:14px; align-items:center; flex-wrap:wrap; }
|
||||
.gpuroles__ico { flex:none; width:38px; height:38px; border-radius:10px; background:var(--st-surface-2); display:grid; place-items:center; color:var(--st-icon); }
|
||||
.gpuroles__row[data-role="main"] .gpuroles__ico { background:var(--st-surface); }
|
||||
.gpuroles__name { flex:1 1 200px; min-width:0; }
|
||||
.gpuroles__name strong { display:block; overflow-wrap:anywhere; }
|
||||
.gpuroles__name small { display:block; color:var(--st-ink-muted); font-size:var(--st-fs-sm); }
|
||||
.gpuroles__tag { display:inline-block; margin-left:6px; padding:1px 8px; border-radius:var(--st-r-pill); font-size:11px; font-weight:700; vertical-align:1px; background:var(--st-accent-tint); color:var(--st-accent-text); border:1px solid var(--st-accent); }
|
||||
.gpuroles__row[data-role="main"] .gpuroles__tag { background:var(--st-surface); }
|
||||
.gpuroles__tag--neutral { background:var(--st-surface-2) !important; border-color:var(--st-line); color:var(--st-ink-muted); }
|
||||
.gpuroles__roles { display:inline-flex; padding:3px; border-radius:12px; background:var(--st-surface-2); border:1px solid var(--st-line); gap:2px; flex:none; }
|
||||
.gpuroles__row[data-role="main"] .gpuroles__roles { background:var(--st-surface); }
|
||||
.gpuroles__roles button { height:34px; min-width:76px; padding:0 14px; border-radius:9px; border:0; background:transparent; cursor:pointer; font:500 var(--st-fs-md)/1 var(--st-font); color:var(--st-ink-soft); }
|
||||
.gpuroles__roles button:hover:not([aria-checked="true"]):not(:disabled) { background:var(--st-line-soft); }
|
||||
.gpuroles__roles button[aria-checked="true"] { background:var(--st-accent); color:var(--st-accent-ink); font-weight:700; }
|
||||
.gpuroles__roles button[aria-checked="true"][data-r="helper"] { background:var(--st-info); color:#fff; }
|
||||
.gpuroles__roles button[aria-checked="true"][data-r="off"] { background:var(--st-ink-muted); color:var(--st-surface); }
|
||||
.gpuroles__roles button:disabled { opacity:.4; cursor:not-allowed; }
|
||||
.gpuroles__roles button:focus-visible, .gpuroles__link:focus-visible, .gpuroles__details summary:focus-visible { outline:2px solid var(--st-focus); outline-offset:2px; }
|
||||
.gpuroles__note { margin-top:10px; display:flex; gap:10px; align-items:flex-start; padding:10px 12px; border-radius:10px; font-size:var(--st-fs-sm); line-height:1.5; }
|
||||
.gpuroles__note svg { flex:none; margin-top:2px; }
|
||||
.gpuroles__note p { margin:0; flex:1; }
|
||||
.gpuroles__note--warn { background:var(--st-warn-tint); color:var(--st-warn-text); }
|
||||
.gpuroles__note--info { background:var(--st-info-tint); color:var(--st-info-text); }
|
||||
.gpuroles__note .st-btn { flex:none; align-self:center; }
|
||||
.gpuroles__toast:empty { display:none; }
|
||||
.gpuroles__toast { margin-top:2px; }
|
||||
.gpuroles__details { margin-top:4px; border-top:1px solid var(--st-line-soft); padding-top:12px; }
|
||||
.gpuroles__details summary { cursor:pointer; color:var(--st-accent-text); font-weight:600; font-size:var(--st-fs-sm); }
|
||||
.gpuroles__body { margin-top:10px; font-size:var(--st-fs-sm); color:var(--st-ink-soft); display:grid; gap:8px; }
|
||||
.gpuroles__body dl { margin:0; display:grid; grid-template-columns:auto 1fr; gap:6px 14px; }
|
||||
.gpuroles__body dt { font-weight:700; }
|
||||
.gpuroles__body dd { margin:0; }
|
||||
.gpuroles__fit { display:flex; gap:8px; align-items:center; }
|
||||
.gpuroles__dot { width:9px; height:9px; border-radius:50%; background:var(--st-accent); flex:none; }
|
||||
.gpuroles__fit[data-state="bad"] .gpuroles__dot { background:var(--st-danger); }
|
||||
@media (max-width:560px) { .gpuroles__roles { width:100%; } .gpuroles__roles button { flex:1; min-width:0; } }
|
||||
@@ -55,7 +55,7 @@ test('with an older engine installed the app refuses to download or load Gyro, w
|
||||
test('the screens use the existing "needs a newer engine" notice for a size, and the buttons are held back', async () => {
|
||||
const models = await readFile(new URL('../src/components/NativeModels.tsx', import.meta.url), 'utf8');
|
||||
assert.match(models, /selectedFamily && \(!selectedSize \|\| selectedSize\.needsNewerEngine\) \? <><h2 className="h3">\{tr\("This model needs a newer engine\."\)\}/);
|
||||
assert.match(models, /!!selectedSize\.needsNewerEngine \|\| expert && deviceChoice === 'split'/);
|
||||
assert.match(models, /!!selectedSize\.needsNewerEngine\}/);
|
||||
const guided = await readFile(new URL('../src/components/GuidedSetup.tsx', import.meta.url), 'utf8');
|
||||
assert.equal((guided.match(/needsNewerEngine/g) ?? []).length, 3);
|
||||
});
|
||||
@@ -7,6 +7,7 @@ const root=resolve(import.meta.dirname,'../../dist'),upstream=process.env.STRATA
|
||||
const native=process.env.STRATA_UI_PREVIEW==='1' ? JSON.parse(await readFile(new URL('./ui-preview-native.json',import.meta.url),'utf8')) : await (await fetch(upstream+'/api/native')).json();
|
||||
native.hardware={cpu:{cores:16},ram_gb:128,gpus:[{index:0,name:'NVIDIA RTX 4090',vram_gb:24,usable:true}],disk_free_gb:250};
|
||||
if(process.env.STRATA_ONBOARDING_SCENARIO==='multi-gpu')native.hardware.gpus.push({index:1,name:'NVIDIA RTX 3090',vram_gb:24,usable:true});
|
||||
if(process.env.STRATA_ONBOARDING_GPUS)native.hardware.gpus=JSON.parse(process.env.STRATA_ONBOARDING_GPUS);
|
||||
native.catalog.this_pc=native.hardware;
|
||||
native.catalog.recommendation={family:'qwen',model:'Q2_0',context:32768};
|
||||
native.catalog.summary='Supported hardware fixture';
|
||||
@@ -60,6 +61,7 @@ if(['make-it-yours','ready'].includes(process.env.STRATA_ONBOARDING_START)){
|
||||
}
|
||||
}
|
||||
}
|
||||
if(process.env.STRATA_ONBOARDING_GPUS){native.instances=[];native.defaultId=null;}/* GPU roles previews: the model is prepared, not loaded, so the roles shown are the recommendation */
|
||||
if(process.env.STRATA_PREVIEW_STATE_FILE){const saved=JSON.parse(await readFile(process.env.STRATA_PREVIEW_STATE_FILE,'utf8'));Object.assign(native,saved.native);setup=saved.setup;const running=native.instances.find(i=>i.id===native.defaultId),entry=native.installed.models.find(m=>m.model===running?.model),family=native.catalog.families.find(f=>f.sizes.some(s=>s.id===entry?.model)),size=family?.sizes.find(s=>s.id===entry?.model);if(family&&size){selected={family:family.family,model:size.model,context:entry.context??32768,vision:entry.images==='off'?'no':'yes',kv:entry.kv??'int8',low_ram:entry.lowRamMode??'auto'};loadVision=entry.visionEnabled===false?'no':selected.vision;}}
|
||||
const server=http.createServer(async(req,res)=>{try{
|
||||
const path=new URL(req.url,'http://fixture').pathname;let payload='';if(req.method!=='GET')for await(const chunk of req)payload+=chunk;const body=payload?JSON.parse(payload):{};
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/** GPU roles (Main / Helper / Off): src/lib/gpu-roles.ts and server/gpu-roles.mjs run the vectors strata_control/gpu_roles.py runs (tests/data/gpu_roles_vectors.json). */
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { transform } from 'esbuild';
|
||||
|
||||
const { code } = await transform(await readFile(new URL('../src/lib/gpu-roles.ts', import.meta.url), 'utf8'), { loader: 'ts', format: 'esm' });
|
||||
const R = await import(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`);
|
||||
const S = await import('../server/gpu-roles.mjs');
|
||||
const V = JSON.parse(await readFile(new URL('../../../tests/data/gpu_roles_vectors.json', import.meta.url), 'utf8'));
|
||||
const roles = d => d === null || d === undefined ? d : Object.fromEntries(Object.entries(d).map(([k, v]) => [Number(k), v]));
|
||||
|
||||
test('recommend: the best card is Main, others Helper, 8 GB or less / built-in / unusable start Off', () => {
|
||||
for (const c of V.recommend) {
|
||||
const r = R.recommend(V.hw[c.hw]);
|
||||
assert.deepEqual(r.roles, roles(c.roles), c.hw);
|
||||
assert.equal(r.main, c.main, c.hw);
|
||||
assert.equal(r.why, c.why, c.hw);
|
||||
}
|
||||
});
|
||||
|
||||
test('setRole: one click, exactly one Main among the cards that are on', () => {
|
||||
for (const c of V.set_role) {
|
||||
const out = R.setRole(roles(c.roles), V.hw[c.hw], c.card, c.role);
|
||||
assert.deepEqual(out.roles, roles(c.after), JSON.stringify(c));
|
||||
assert.equal(out.note, c.note, JSON.stringify(c));
|
||||
}
|
||||
});
|
||||
|
||||
test('order: Helpers first, Main LAST, Off left out; one card is a number in the mirror', () => {
|
||||
for (const c of V.order) {
|
||||
assert.deepEqual(R.order(roles(c.roles), V.hw[c.hw]), c.gpu, JSON.stringify(c));
|
||||
assert.deepEqual(S.order(roles(c.roles), V.hw[c.hw]), c.gpu, 'server copy: ' + JSON.stringify(c));
|
||||
}
|
||||
const m = R.mirror({ 0: 'off', 1: 'main' }, V.hw.two);
|
||||
assert.equal(m.gpu, 1);
|
||||
assert.equal(m.gpu_order, 'as_given');
|
||||
assert.deepEqual(R.mirror({ 0: 'helper', 1: 'main' }, V.hw.two).gpu, [0, 1]);
|
||||
const none = R.mirror({ 0: 'off', 1: 'off' }, V.hw.two);
|
||||
assert.deepEqual([none.gpu, none.gpu_none], [[], true]);
|
||||
assert.deepEqual(S.mirror({ 0: 'helper', 1: 'main' }, V.hw.two), R.mirror({ 0: 'helper', 1: 'main' }, V.hw.two));
|
||||
assert.deepEqual(S.mirror({ 0: 'off', 1: 'off' }, V.hw.two), none);
|
||||
});
|
||||
|
||||
test('warnings: only built-in graphics as Main, a card the engine cannot run, too little memory, CPU only', () => {
|
||||
for (const c of V.warnings) {
|
||||
const w = R.warnings(roles(c.roles), V.hw[c.hw], { spillGb: c.spill_gb ?? 0 });
|
||||
assert.deepEqual(w.map(x => x.code), c.codes, JSON.stringify(c));
|
||||
if (c.fix) assert.deepEqual(w[0].fix, c.fix);
|
||||
for (const x of w) assert.ok(x.text.length < 90, x.text);
|
||||
}
|
||||
// a slower card as Main is understood by people: no warning
|
||||
assert.deepEqual(R.warnings({ 0: 'main', 1: 'helper' }, V.hw.two), []);
|
||||
assert.equal(R.warnings({ 0: 'off', 1: 'off' }, V.hw.two, { engineRunsCpuOnly: true })[0].text, 'Performance is degraded on CPU only.');
|
||||
assert.equal(R.warnings({ 0: 'off', 1: 'off' }, V.hw.two)[0].text, R.NO_CARD_LINE);
|
||||
});
|
||||
|
||||
test('legacy gpu settings show the roles the engine really uses', () => {
|
||||
for (const c of V.legacy) assert.deepEqual(R.fromLegacy(c.gpu, V.hw[c.hw], c.gpu_order), roles(c.roles), JSON.stringify(c));
|
||||
});
|
||||
|
||||
test('saved roles: parse; the server copy agrees on which are usable', () => {
|
||||
assert.deepEqual(R.parseSaved({ 0: 'helper', 1: 'main' }), { 0: 'helper', 1: 'main' });
|
||||
for (const bad of [{ 0: 'boss' }, { 0: 'main', 1: 'main' }, 'x', {}, null]) { assert.equal(R.parseSaved(bad), null); assert.equal(S.parseSaved(bad), null); }
|
||||
});
|
||||
|
||||
test('words: the sentence and the one-line summary', () => {
|
||||
assert.equal(R.sentence(V.hw.one), 'Strata will use your RTX 4070 Ti.');
|
||||
assert.match(R.sentence(V.hw.three), /RTX 4070 Ti is Main: it is faster\./);
|
||||
assert.match(R.sentence(V.hw.three), /RTX 2060 starts Off: 8 GB or less\./);
|
||||
assert.equal(R.describe({ 0: 'helper', 1: 'main' }, V.hw.two), 'Main: GPU 2 RTX 4070 Ti. Helper: GPU 1 RTX 3060.');
|
||||
assert.equal(R.noteText('old_main_helper:1', V.hw.two), 'RTX 4070 Ti is now a Helper.');
|
||||
assert.equal(R.noteText('only_main', V.hw.two), 'The only card that is on has to be Main.');
|
||||
});
|
||||
@@ -4,10 +4,9 @@ import { readFile } from 'node:fs/promises';
|
||||
|
||||
test('the card list preselects the card the loaded model runs on, and the sidebar card says the version and the card', async () => {
|
||||
const models = await readFile(new URL('../src/components/NativeModels.tsx', import.meta.url), 'utf8');
|
||||
assert.ok(models.indexOf('const loadedSelected = ') < models.indexOf('const loadedCards = '));
|
||||
assert.ok(models.indexOf('const loadedCards = ') < models.indexOf('const activeCards = '));
|
||||
assert.match(models, /cardOptions\.options\.find\(o => o\.key === cardsKey\) \?\? loadedCards \?\?/);
|
||||
assert.match(models, /o\.cards\.every\(c => loadedSelected\.gpus\.includes\(c\)\)/);
|
||||
assert.ok(models.indexOf('const loadedSelected = ') < models.indexOf('const loadedRoles = '));
|
||||
assert.match(models, /fromLegacy\(loadedSelected\.gpus, roleCards, 'as_given'\)/);
|
||||
assert.match(models, /roles \? normalizeRoles\(roles, roleCards\) : loadedRoles \?\? recommended\.roles/);
|
||||
const side = await readFile(new URL('../src/app/Sidebar.tsx', import.meta.url), 'utf8');
|
||||
assert.match(side, /nativeSize\?\.model \?\? nativeCurrent\.model\.toUpperCase\(\)/);
|
||||
assert.match(side, /`GPU \$\{g \+ 1\}`/);
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/** GPU roles in the host: a load writes the roles and the legacy mirror (Main last), a file that names roles keeps them, every card Off is allowed. */
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { nativeFixture, finishJob } from './helpers/native-fixture.mjs';
|
||||
|
||||
const load = async (f, args) => { const done = await finishJob(f.control, f.control.operation('load', { model: 'iq2_xs', ...args }).id); return done; };
|
||||
const written = (f, done) => JSON.parse(readFileSync(f.control.resolveInstance(done.result.instanceId).configPath, 'utf8'));
|
||||
|
||||
test('roles chosen in the screen: Helpers first, Main LAST, Off left out, and "gpu_roles" beside the mirror', async () => {
|
||||
const f = nativeFixture();
|
||||
try {
|
||||
await f.control.refresh(true);
|
||||
const done = await load(f, { gpuRoles: { 0: 'main', 1: 'helper' } });
|
||||
assert.equal(done.state, 'done', done.error);
|
||||
const cfg = written(f, done);
|
||||
assert.deepEqual(cfg.gpu, [1, 0], 'card 0 is Main, so it is the last stage');
|
||||
assert.equal(cfg.gpu_order, 'as_given');
|
||||
assert.deepEqual(cfg.gpu_roles, { 0: 'main', 1: 'helper' });
|
||||
assert.equal(cfg.layer_split, 'auto');
|
||||
assert.deepEqual(f.control.resolveInstance(done.result.instanceId).gpus, [1, 0]);
|
||||
await f.control.unload(done.result.instanceId);
|
||||
const one = await load(f, { gpuRoles: { 0: 'off', 1: 'main' } });
|
||||
assert.equal(one.state, 'done', one.error);
|
||||
const c1 = written(f, one);
|
||||
assert.equal(c1.gpu, 1, 'one card is a number, never a list');
|
||||
assert.deepEqual(c1.gpu_roles, { 0: 'off', 1: 'main' });
|
||||
} finally { await f.cleanup(); }
|
||||
});
|
||||
|
||||
test('every card Off is allowed: no card in the list, "gpu_none" says it was meant', async () => {
|
||||
const f = nativeFixture();
|
||||
try {
|
||||
await f.control.refresh(true);
|
||||
const done = await load(f, { gpuRoles: { 0: 'off', 1: 'off' } });
|
||||
assert.equal(done.state, 'done', done.error);
|
||||
const cfg = written(f, done);
|
||||
assert.deepEqual(cfg.gpu, []);
|
||||
assert.equal(cfg.gpu_none, true);
|
||||
} finally { await f.cleanup(); }
|
||||
});
|
||||
|
||||
test('a settings file that names roles keeps them when the screen only suggests cards; a plain card list replaces them', async () => {
|
||||
const f = nativeFixture();
|
||||
try {
|
||||
await f.control.refresh(true);
|
||||
const path = join(f.repo, 'strata-iq2_xs.json');
|
||||
const config = JSON.parse(readFileSync(path, 'utf8'));
|
||||
Object.assign(config, { gpu: [0, 1], gpu_order: 'as_given', gpu_roles: { 0: 'helper', 1: 'main' } });
|
||||
writeFileSync(path, JSON.stringify(config));
|
||||
await f.control.refresh(true);
|
||||
const kept = await load(f, { gpus: [1, 0], gpusFromRecommendation: true });
|
||||
assert.equal(kept.state, 'done', kept.error);
|
||||
assert.deepEqual(written(f, kept).gpu, [0, 1]);
|
||||
await f.control.unload(kept.result.instanceId);
|
||||
const picked = await load(f, { gpus: [1] });
|
||||
assert.equal(picked.state, 'done', picked.error);
|
||||
const cfg = written(f, picked);
|
||||
assert.equal(cfg.gpu, 1);
|
||||
assert.equal('gpu_roles' in cfg, false, 'a card list picked by hand replaces the roles');
|
||||
} finally { await f.cleanup(); }
|
||||
});
|
||||
|
||||
test('roles that make no sense are refused with a plain line', async () => {
|
||||
const f = nativeFixture();
|
||||
try {
|
||||
await f.control.refresh(true);
|
||||
const two = await load(f, { gpuRoles: { 0: 'main', 1: 'main' } });
|
||||
assert.equal(two.state, 'error');
|
||||
assert.match(two.error, /Main, Helper or Off/);
|
||||
const nope = await load(f, { gpuRoles: { 5: 'main' } });
|
||||
assert.equal(nope.state, 'error');
|
||||
assert.match(nope.error, /detected graphics cards/);
|
||||
} finally { await f.cleanup(); }
|
||||
});
|
||||
@@ -18,8 +18,10 @@ test('a second model on a card another model uses still loads (recommend, never
|
||||
|
||||
test('the screens say which card a loaded model uses, prefer a free card for a second model, and show an unload refusal beside its button', async () => {
|
||||
const models = await readFile(new URL('../src/components/NativeModels.tsx', import.meta.url), 'utf8');
|
||||
assert.match(models, /tr\('Already used by \{0\}', \[usedBy\(o\.cards\)\.join\(', '\)\]\)/);
|
||||
assert.match(models, /multiple \? cardOptions\.options\.find\(o => o\.recommended && !usedBy\(o\.cards\)\.length\)/);
|
||||
assert.match(models, /usedBy=\{i => usedBy\(\[i\]\)\}/);
|
||||
assert.match(models, /const busy = \(i: number\) => expert && multiple && usedBy\(\[i\]\)\.length > 0/); // a card another loaded model uses starts Off
|
||||
const roles = await readFile(new URL('../src/components/GpuRoles.tsx', import.meta.url), 'utf8');
|
||||
assert.match(roles, /tr\('In use by \{0\}', \[other\.join\(', '\)\]\)/);
|
||||
assert.match(models, /loadedAction\(`instances\/\$\{i\.id\}\/unload`\)/);
|
||||
assert.match(models, /loadedError \? <ErrorNotice message=\{loadedError\}/);
|
||||
const control = await readFile(new URL('../server/native-control.mjs', import.meta.url), 'utf8');
|
||||
@@ -28,7 +30,7 @@ test('the screens say which card a loaded model uses, prefer a free card for a s
|
||||
|
||||
test('usedBy is declared before the card choice that calls it (a const used earlier broke the Models page when Run multiple models was on)', async () => {
|
||||
const models = await readFile(new URL('../src/components/NativeModels.tsx', import.meta.url), 'utf8');
|
||||
assert.ok(models.indexOf('const usedBy = ') > 0 && models.indexOf('const usedBy = ') < models.indexOf('const activeCards = '));
|
||||
assert.ok(models.indexOf('const usedBy = ') > 0 && models.indexOf('const usedBy = ') < models.indexOf('const recommended = '));
|
||||
});
|
||||
|
||||
test('a model that failed to start is not offered in the chat model picker', async () => {
|
||||
@@ -54,8 +56,9 @@ test('a loaded model is named from the catalog (family and version), the file id
|
||||
|
||||
test('card names use the GPU numbering of the other screens, and the fit figures subtract the models that stay loaded', async () => {
|
||||
const models = await readFile(new URL('../src/components/NativeModels.tsx', import.meta.url), 'utf8');
|
||||
assert.ok(models.includes('(GPU ${Number(n) + 1})') && models.includes('text.replace(/\\(card (\\d+)\\)/g'));
|
||||
assert.match(models, /<strong>\{gpuNumbers\(o\.label\)\}<\/strong>/);
|
||||
const roles = await readFile(new URL('../src/lib/gpu-roles.ts', import.meta.url), 'utf8');
|
||||
assert.ok(roles.includes('`GPU ${c.index + 1}`')); // GPU 1 is card 0, as on every other screen
|
||||
assert.match(await readFile(new URL('../src/components/GpuRoles.tsx', import.meta.url), 'utf8'), /\{gpuLabel\(c\)\} · \{c\.name\}/);
|
||||
assert.match(models, /const keptModels = expert && multiple/);
|
||||
assert.match(models, /inUse=\{keptModels\.length \?/);
|
||||
const card = await readFile(new URL('../src/components/PcFitCard.tsx', import.meta.url), 'utf8');
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
# VISION off | on | gpu | cpu (pictures; the picture reader file is downloaded with the model)
|
||||
# KV int8 | q4_0 | k8v4 | fp16 (conversation memory format)
|
||||
# GPU / GPUS card numbers, e.g. 0 or 0,1 (empty or "all": the engine's choice)
|
||||
# GPU_ROLES a role for each card, e.g. 1=main,0=helper (main, helper or off; cards you do not name are off; not together with GPU / GPUS)
|
||||
# TUNE quick | thorough | skip (done once; RETUNE=1 tunes again)
|
||||
# PORT port inside the container (default 8080)
|
||||
# API_KEY / API_KEY_FILE the key clients must send. The server listens on every address inside the container, and Strata does
|
||||
@@ -201,6 +202,13 @@ case "$gpus" in ""|all|ALL) ;; *)
|
||||
fi
|
||||
done ;;
|
||||
esac
|
||||
# GPU_ROLES=1=main,0=helper: a role for each card (main, helper or off; cards you do not name are off). `launch --gpu-roles` writes them into the
|
||||
# run settings at every start, so nothing saved has to be dropped.
|
||||
roles=$(printf '%s' "${GPU_ROLES:-}" | tr -d ' ')
|
||||
if [ -n "$roles" ]; then
|
||||
case "$gpus" in ""|all|ALL) ;; *) die "Set GPU_ROLES or GPU / GPUS, not both." 2 ;; esac
|
||||
set -- "$@" --gpu-roles "$roles"; have=1
|
||||
fi
|
||||
[ -n "${STRATA_SET:-}" ] && { for kv in $STRATA_SET; do set -- "$@" --set "$kv"; done; have=1; }
|
||||
[ -n "$key" ] && { set -- "$@" --api-key-env STRATA_LAUNCH_KEY; }
|
||||
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ the kernel's amdgpu driver (no ROCm install needed):
|
||||
no hipBLASLt kernels) the same role falls to a rocBLAS table, `tools/hip/<arch>-rocblas-<version>.txt`, which the
|
||||
engine picks for its own rocBLAS build when setup points it at the directory ([RDNA2](#rdna2-gfx1030)).
|
||||
- **Several cards:** setup takes one card (the one with the most VRAM, or `--gpu N`) unless you name more:
|
||||
`./setup.sh --backend hip --gpus 1,0` splits the model's layers across them, the first one the main card (numbers
|
||||
`./setup.sh --backend hip --gpus 1,0` splits the model's layers across them, the faster card goes last and is the Main card (`--gpu-roles 1=main,0=helper` says it yourself; docs/MULTI_GPU.md) (numbers
|
||||
as setup lists them; `--gpus all` = every supported card, the most VRAM first). Every chosen card must be one of the
|
||||
architectures above; the engine is compiled for each of them (cards of two families, e.g. gfx1100 + gfx1201, need
|
||||
a system ROCm 7: AMD's wheels hold one family). A split pays only when no single card holds the model's experts
|
||||
|
||||
@@ -121,6 +121,7 @@ screens; leave out what you do not care about.
|
||||
| `VISION` | `off`, `on` (the model reads pictures), `gpu`, `cpu`; the picture-reader file (about 0.9 GB) is downloaded with the model | `off` |
|
||||
| `KV` | conversation memory format: `int8`, `q4_0` (half the memory), `k8v4`, `fp16` | `int8` |
|
||||
| `GPU` / `GPUS` | card numbers as the monitor lists them, `0` or `0,1`; empty or `all` = the engine's choice. Changing it later makes the model's run settings again (the files stay) | all that fit |
|
||||
| `GPU_ROLES` | a role for each card, `1=main,0=helper` (`main`, `helper`, `off`; cards you do not name are off; not together with `GPU` / `GPUS`). Passed to `launch --gpu-roles`; see [MULTI_GPU.md](MULTI_GPU.md) |
|
||||
| `TUNE` | `quick` (up to about 20 minutes, once), `thorough`, `skip`; done once and kept in the preset (`RETUNE=1` tunes again) | `skip` |
|
||||
| `API_KEY` / `API_KEY_FILE` | the key clients must send ([below](#the-api-key)); the file form is for Docker secrets | made for you |
|
||||
| `PORT` | the port inside the container | `8080` |
|
||||
|
||||
+1
-1
@@ -221,7 +221,7 @@ pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the
|
||||
Every card must be an RTX 20 series or newer with 8 GB or more. See [MULTI_GPU.md](MULTI_GPU.md).
|
||||
|
||||
**Several AMD cards:** setup takes one card (the one with the most VRAM, or `--gpu N`) unless you name more:
|
||||
`--gpus 1,0` splits the model's layers across them, the first one the main card (numbers as setup lists them). A
|
||||
`--gpus 1,0` splits the model's layers across them, the faster card goes last and is the Main card (`--gpu-roles 1=main,0=helper` says it yourself; docs/MULTI_GPU.md; numbers as setup lists them). A
|
||||
split pays only when no single card holds the model's experts ([AMD_HIP.md](AMD_HIP.md#rdna4-gfx1201)).
|
||||
|
||||
## Docker (Linux)
|
||||
|
||||
+53
-11
@@ -13,7 +13,7 @@ is every card's share and skips those probes; there is no per-card setting yet.
|
||||
|
||||
**Two models on two cards (the app).** A layer split puts one model on several cards. The other way to use two cards is two
|
||||
models, one per card, each answering on its own. In the app, turn on **Run multiple models** on the Models page: the card
|
||||
list then says which card a loaded model uses ("Already used by ..."), preselects a free card for the next model, and
|
||||
list then says which card a loaded model uses ("In use by ..."), starts that card as Off for the next model, and
|
||||
explains a failed start on a card that is taken. Measured on a PC with Radeon AI PRO R9700 cards (two of four, Linux): Q2_0 on
|
||||
one card and IQ3_XXS on another both loaded (33 s and 59 s with the files cached, 114 s the first time), both answered chats
|
||||
(about 60 tokens/s each), the Monitor listed both, and unloading one left the other answering. This was run with the AMD build only;
|
||||
@@ -41,19 +41,61 @@ Strata can use it:
|
||||
Which GPUs? [1]:
|
||||
```
|
||||
|
||||
**Which cards a model uses (one rule, in setup, `strata launch` and the app).** Strata first tries the model on your
|
||||
**largest** card plus the computer's memory, the normal hybrid mode, and recommends that when it fits (with some RAM to
|
||||
spare). Only when it does not fit one card, or would be too tight, the layer split over the two largest cards is
|
||||
recommended instead. Every option is listed with plain numbers (what the model needs and what the cards and RAM give;
|
||||
how much would be read from the SSD when it does not fit) and the choice is yours. The cards that count: not an
|
||||
integrated chip (it shares the system memory and adds nothing beside a real card), at least 4 GB of graphics memory, and
|
||||
8 GB or more to share a model. The arithmetic is the app's `fitOn` (strata_control/gpu_fit.py keeps its constants equal);
|
||||
each extra card of a split costs an estimated 2 GB of its own buffers.
|
||||
## Main, Helper, Off
|
||||
|
||||
**Choosing yourself** (at setup or at any start):
|
||||
Every card has one of three roles:
|
||||
|
||||
| Role | What it does |
|
||||
|---|---|
|
||||
| **Main** | Runs the **last** stage of the split: the output head, the draft layer and the check of its guesses. A prompt chunk waits on it. Exactly one card is Main. |
|
||||
| **Helper** | Holds a share of the layers and of the expert cache, so more of the model fits on graphics cards instead of being read from the drive. Prompts are read about 20% faster. |
|
||||
| **Off** | Strata does not touch the card (a game, the screen, a card that is too small). |
|
||||
|
||||
The Main card is the **last** card in the engine's order, not the first. The first card runs the first layers and, by default,
|
||||
the picture reader. (Older versions of this page said "the first is the main one"; that has not been true since 0.1.41.1.)
|
||||
|
||||
**Recommended roles** (the same rule in the app, `strata launch` and setup; `strata_control/gpu_roles.py`): the faster card
|
||||
is Main (speed is multiprocessors x max clock, the engine's own measure; cards within 5% are the same card, and the one with
|
||||
more memory wins, then the lower number; two identical cards: the highest number is Main, so the engine order stays 0, 1 as it always was; a card with clearly more memory beats a faster one that is at least half as fast).
|
||||
Every other card starts as Helper, except a card with **8 GB or less** (Off), built-in graphics beside a card with memory of
|
||||
its own (Off) and a card the engine cannot run (Off). Every role is one click (the app), one key (`strata launch`: M, H, O;
|
||||
R goes back to the recommendation) or one line (below).
|
||||
|
||||
Only a few things are warned about, in one short line each: built-in graphics as Main, a card the engine cannot run, too
|
||||
little memory for the chosen setup (part of the model would be read from the drive). **Every card Off** is allowed. The CUDA,
|
||||
HIP and SYCL engines cannot start without a card, so the server then says so in one line and stops; an engine that can run on
|
||||
the processor (the Mac engine) says "Performance is degraded on CPU only." and goes on.
|
||||
|
||||
**Setting roles**
|
||||
|
||||
```
|
||||
--gpus 0,2 these cards together, as nvidia-smi numbers them; the first is the main one. Remembered.
|
||||
strata launch --gpu-roles 1=main,0=helper cards you do not name are Off
|
||||
./setup.sh --gpu-roles 1=main,0=helper (START-HERE.bat --gpu-roles ... on Windows); saved
|
||||
strata launch the Cards step: M Main, H Helper, O Off, R recommended, D details
|
||||
```
|
||||
|
||||
or in the run settings (`strata-*.json`), which is what the app writes:
|
||||
|
||||
```json
|
||||
"gpu_roles": {"1": "main", "0": "helper", "2": "off"},
|
||||
"gpu": [0, 1],
|
||||
"gpu_order": "as_given"
|
||||
```
|
||||
|
||||
`gpu` is the same choice for older engines and the Monitor: the Helpers (slowest first), then the Main card, with the Off
|
||||
cards left out. The server reads `gpu_roles`; when `gpu` was edited by hand and no longer matches, `gpu` wins and the log says
|
||||
so (a settings file that names its cards keeps them). Nothing is rewritten on upgrade: a file with only `"gpu": [1, 0]` shows
|
||||
as Main = the last card, Helper = the first, the rest Off. The recommendation itself is not saved, so a card you add later is
|
||||
recommended again; a card you add after choosing roles starts Off. Docker: `GPU_ROLES=1=main,0=helper` is passed as `--gpu-roles`.
|
||||
|
||||
**One rule for which cards count** (`strata_control/gpu_fit.py` keeps the memory arithmetic): not an integrated chip beside
|
||||
a real card, at least 4 GB of graphics memory to be listed, 8 GB or less starts Off. The arithmetic is the app's `fitOn`;
|
||||
each extra card of a split costs an estimated 2 GB of its own buffers.
|
||||
|
||||
**Choosing the cards without roles** (at setup or at any start):
|
||||
|
||||
```
|
||||
--gpus 0,2 these cards together, as nvidia-smi numbers them; Strata puts the faster card last. Remembered.
|
||||
--gpus all every card that can share the model
|
||||
--gpu 0 one card (at a start: for that start only)
|
||||
--layer-split auto (default) or the first layer of each later card, e.g. 18 or 16,32
|
||||
|
||||
@@ -22,7 +22,7 @@ The release folder is the latest GitHub release of Niko1221/Strata (`https://git
|
||||
1. **Start**: saved presets (name, model, context, parallel requests, tuned or not, last used) or "Set up from the beginning" or "Use model files I already have".
|
||||
2. **Model**: the catalog of the app and `setup.py` (one source), with download size, what it needs (graphics memory + RAM) and "fits / tight / does not fit this computer".
|
||||
A warning, never a block. The n-gram table stays on your SSD and is not counted. `o` = your own files (a folder or a .gguf; checked with the fingerprint table).
|
||||
*(Cards: only with two or more usable graphics cards.)*
|
||||
*(Cards: only with two or more usable graphics cards. Each card gets a role: M Main, H Helper, O Off, R back to the recommendation, D details. `--gpu-roles 1=main,0=helper` does the same from the command line; see [MULTI_GPU.md](MULTI_GPU.md).)*
|
||||
3. **Folder**: where model files are stored, free space shown. For a model that is already installed (also one the desktop app installed) it starts where the model is, says
|
||||
"Already downloaded" and lists only what is really missing with its size (the draft layer, about 5.2 GB; the picture reader, 0.9 GB). It never says "download the model" when only
|
||||
a helper file is missing.
|
||||
|
||||
+47
-4
@@ -70,6 +70,7 @@ from strata_control import hipblaslt_tuning # noqa: E402
|
||||
from strata_control import vision_reader # noqa: E402
|
||||
from strata_control import draft_layer # noqa: E402
|
||||
from strata_control import user_args # noqa: E402
|
||||
from strata_control import gpu_roles # noqa: E402
|
||||
from serve import ready as ready_file # noqa: E402
|
||||
from serve import owner_guard # noqa: E402
|
||||
from serve import launch_identity # noqa: E402
|
||||
@@ -2812,13 +2813,50 @@ class Vision:
|
||||
def gpu_list(cfg: dict) -> list[int]:
|
||||
"""The config's "gpu": one card (2), or several for a layer split ([0, 2] or "0,2"), numbered as nvidia-smi
|
||||
numbers them; [] when it names none."""
|
||||
g = cfg.get("gpu")
|
||||
roles, _ = gpu_roles.saved_roles(cfg) # "gpu_roles" (Main / Helper / Off per card) wins unless "gpu" was edited by hand
|
||||
if roles is not None:
|
||||
legacy = _gpu_numbers(cfg.get("gpu"))
|
||||
return legacy if legacy else gpu_roles.engine_order(roles) # Main LAST; Off cards left out; [] when every card is Off
|
||||
return _gpu_numbers(cfg.get("gpu"))
|
||||
|
||||
|
||||
def _gpu_numbers(g) -> list[int]:
|
||||
if g is None or g == "":
|
||||
return []
|
||||
items = g if isinstance(g, (list, tuple)) else str(g).split(",")
|
||||
return [int(str(x).strip()) for x in items if str(x).strip() != ""]
|
||||
|
||||
|
||||
def gpu_roles_active(cfg: dict) -> bool:
|
||||
"""True when the run settings carry usable "gpu_roles": the card order is the person's (Main last), never re-sorted by speed."""
|
||||
return gpu_roles.saved_roles(cfg)[0] is not None
|
||||
|
||||
|
||||
def gpu_all_off(cfg: dict) -> bool:
|
||||
"""Every card is set to Off ("gpu_roles" with no card on, or "gpu_none": true)."""
|
||||
roles, _ = gpu_roles.saved_roles(cfg)
|
||||
return roles is not None and not any(r != "off" for r in roles.values())
|
||||
|
||||
|
||||
def card_roles_lines(cfg: dict) -> list[str]:
|
||||
"""The plain lines printed at start: the roles (cards numbered as nvidia-smi numbers them), a hand-edited "gpu" that beat them, and the
|
||||
all-Off case. SystemExit when every card is Off and the engine cannot start without one (the CUDA, HIP and SYCL engines)."""
|
||||
roles, problem = gpu_roles.saved_roles(cfg)
|
||||
out = [f"[strata] {problem}"] if problem else []
|
||||
if roles is None:
|
||||
return out
|
||||
if gpu_all_off(cfg):
|
||||
if cfg.get("backend") in ("cpu", "metal") or sys.platform == "darwin":
|
||||
out.append("[strata] every graphics card is Off: " + gpu_roles.CPU_ONLY_LINE)
|
||||
return out
|
||||
raise SystemExit("[strata] every graphics card is set to Off, and this engine cannot start without one. Turn a card on "
|
||||
"(Main or Helper) in the app or with strata launch --gpu-roles.")
|
||||
word = {"main": "Main", "helper": "Helper", "off": "Off"}
|
||||
out.append("[strata] card roles: " + ", ".join(f"{word[r]} GPU {i}" for r in ("main", "helper", "off")
|
||||
for i in sorted(roles) if roles[i] == r) + " (numbers as nvidia-smi gives them)")
|
||||
return out
|
||||
|
||||
|
||||
def engine_binary(exe: str, cfg: dict | None = None) -> str:
|
||||
"""The file that holds the engine's own option strings, for a feature check that reads it. The Intel launcher
|
||||
(sycl/serve/strata-sycl.sh) is a shell script that runs its repo's STRATA_SYCL_BIN (default build-sycl-aot/strata)
|
||||
@@ -3109,7 +3147,7 @@ def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:
|
||||
draft and verify have a floor the smaller card cannot afford. scores/vrams: {index: value} (tests); None asks
|
||||
the driver - injected scores without injected vrams means no VRAM check (a test seam is a test seam)."""
|
||||
gl = gpu_list(cfg)
|
||||
if len(gl) < 2 or cfg.get("gpu_order") == "as_given":
|
||||
if len(gl) < 2 or cfg.get("gpu_order") == "as_given" or gpu_roles_active(cfg):
|
||||
return gl
|
||||
args = cfg.get("args") or []
|
||||
if "--peer-device" in args or "--layer-split" in args or layer_split_value(cfg) != "auto":
|
||||
@@ -3189,7 +3227,7 @@ def hip_auto_devices(cfg: dict, env: dict) -> list[int] | None:
|
||||
"""HIP_VISIBLE_DEVICES for a start that names no card (see pick_hip_devices), asked of the engine package's own `strata-device`.
|
||||
None leaves everything as it was: a card the config or the environment names, no probe next to the engine, or a probe that fails.
|
||||
SystemExit with the plain line when no card of this computer can run."""
|
||||
if cfg.get("backend") != "hip" or gpu_list(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \
|
||||
if cfg.get("backend") != "hip" or gpu_list(cfg) or gpu_all_off(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \
|
||||
or env.get("HIP_VISIBLE_DEVICES") or env.get("ROCR_VISIBLE_DEVICES"):
|
||||
return None
|
||||
exe = cfg.get("exe")
|
||||
@@ -3220,7 +3258,10 @@ def child_env(cfg: dict, *, offline: bool = False) -> dict:
|
||||
"""The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
|
||||
compiled it) first on the library search path."""
|
||||
env = dict(os.environ)
|
||||
if hip_visible(cfg) and cfg.get("backend") == "hip": # AMD: numbered as HIP numbers them (hip_visible)
|
||||
if gpu_all_off(cfg): # every card Off: the engine sees none (a CPU engine runs; the others say so at start)
|
||||
env["CUDA_VISIBLE_DEVICES"] = ""
|
||||
env["HIP_VISIBLE_DEVICES"] = ""
|
||||
elif hip_visible(cfg) and cfg.get("backend") == "hip": # AMD: numbered as HIP numbers them (hip_visible)
|
||||
env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in hip_visible(cfg))
|
||||
elif gpu_list(cfg): # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
|
||||
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # own default order (fastest first) can number the cards otherwise
|
||||
@@ -7954,6 +7995,8 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
|
||||
pretty = ", ".join(f"{k}={v}" for k, v in sampling_defaults.items())
|
||||
print(f"[strata] sampling defaults from the config: {pretty}", flush=True)
|
||||
lazy = a.lazy or cfg.get("lazy_load") is True
|
||||
for line in card_roles_lines(cfg): # the card roles: said first; every card Off stops here, before anything touches a card
|
||||
print(line, flush=True)
|
||||
if cfg.get("vision"):
|
||||
elastic = cfg["vision"].get("elastic") is True and cfg.get("vram_elastic") is True
|
||||
print("the vision encoder starts with the first picture (elastic) ..." if elastic else
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""serve/server.py: the run settings' "gpu_roles" (Main / Helper / Off) become the engine's card list, Main LAST, Off left out."""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from serve.server import card_roles_lines, child_env, gpu_list, ordered_gpus # noqa: E402
|
||||
|
||||
|
||||
class Roles(unittest.TestCase):
|
||||
def test_roles_alone_give_the_order_main_last_and_off_left_out(self):
|
||||
cfg = {"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "args": []}
|
||||
self.assertEqual(gpu_list(cfg), [0, 1])
|
||||
self.assertEqual(gpu_list({"gpu_roles": {"0": "main", "1": "helper"}, "args": []}), [1, 0])
|
||||
env = child_env({"gpu_roles": {"0": "main", "1": "helper", "2": "off"}, "args": []})
|
||||
self.assertEqual(env["CUDA_VISIBLE_DEVICES"], "1,0")
|
||||
|
||||
def test_the_mirror_is_used_and_never_re_sorted_by_speed(self):
|
||||
cfg = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}
|
||||
self.assertEqual(ordered_gpus(cfg, {0: 1.0, 1: 9.0}), [1, 0]) # the faster card is Helper on purpose
|
||||
cfg2 = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "args": []} # even without gpu_order
|
||||
self.assertEqual(ordered_gpus(cfg2, {0: 1.0, 1: 9.0}), [1, 0])
|
||||
|
||||
def test_one_card_on(self):
|
||||
cfg = {"gpu_roles": {"0": "off", "1": "main"}, "gpu": 1, "gpu_order": "as_given", "args": []}
|
||||
self.assertEqual(gpu_list(cfg), [1])
|
||||
self.assertEqual(child_env(cfg)["CUDA_VISIBLE_DEVICES"], "1")
|
||||
|
||||
def test_a_hand_edited_gpu_wins(self):
|
||||
cfg = {"gpu_roles": {"0": "helper", "1": "main"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}
|
||||
self.assertEqual(gpu_list(cfg), [1, 0])
|
||||
self.assertTrue(any("edited by hand" in l for l in card_roles_lines(cfg)))
|
||||
self.assertEqual(ordered_gpus({**cfg, "gpu_order": None}, {0: 1.0, 1: 9.0}), [0, 1]) # the legacy rule again
|
||||
|
||||
def test_bad_roles_are_ignored(self):
|
||||
for bad in ({"0": "boss"}, {"0": "main", "1": "main"}, "x", {}):
|
||||
self.assertEqual(gpu_list({"gpu_roles": bad, "gpu": 1, "args": []}), [1])
|
||||
|
||||
def test_legacy_settings_are_untouched(self):
|
||||
self.assertEqual(gpu_list({"gpu": "0,2", "args": []}), [0, 2])
|
||||
self.assertEqual(gpu_list({"args": []}), [])
|
||||
self.assertEqual(card_roles_lines({"gpu": [1, 0], "args": []}), [])
|
||||
|
||||
def test_the_log_line_names_every_role(self):
|
||||
lines = card_roles_lines({"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "gpu": [0, 1], "args": []})
|
||||
self.assertEqual(lines, ["[strata] card roles: Main GPU 1, Helper GPU 0, Off GPU 2 (numbers as nvidia-smi gives them)"])
|
||||
|
||||
def test_every_card_off(self):
|
||||
cfg = {"gpu_roles": {"0": "off", "1": "off"}, "gpu": [], "gpu_none": True, "args": []}
|
||||
self.assertEqual(gpu_list(cfg), [])
|
||||
env = child_env(cfg)
|
||||
self.assertEqual((env["CUDA_VISIBLE_DEVICES"], env["HIP_VISIBLE_DEVICES"]), ("", ""))
|
||||
with self.assertRaises(SystemExit) as e:
|
||||
card_roles_lines(cfg) # a CUDA engine cannot start without a card: said plainly
|
||||
self.assertIn("Turn a card on", str(e.exception))
|
||||
cpu = card_roles_lines({**cfg, "backend": "cpu"})
|
||||
self.assertIn("Performance is degraded on CPU only.", cpu[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1616,7 +1616,7 @@ def start_gpus(text):
|
||||
|
||||
def choose_gpus(a, found) -> list:
|
||||
"""Which cards this install uses: --gpus / --gpu, or asked when two or more can share the model (the two best
|
||||
together recommended), else the supported card with the most VRAM. Returns their numbers, the main one first."""
|
||||
together recommended), else the supported card with the most VRAM. Returns their numbers, in the engine order, the Main card last when roles are chosen."""
|
||||
if a.gpus:
|
||||
sel = parse_gpus(a.gpus, found)
|
||||
check_gpus(sel, found, yes=a.yes, named=str(a.gpus).strip().lower() != "all")
|
||||
@@ -1631,34 +1631,66 @@ def choose_gpus(a, found) -> list:
|
||||
can = together_ok(found)
|
||||
if not can:
|
||||
return [single[0]["index"]]
|
||||
# the owner's rule (strata_control/gpu_fit.py, shared with the app and `strata launch`): the model goes on the LARGEST card + RAM first;
|
||||
# the second card is offered as a layer split and recommended only when the model does not fit one card (with RAM) or would be too tight
|
||||
from strata_control import gpu_fit
|
||||
mem = None
|
||||
d = MODELS.get(getattr(a, "model", None) or "")
|
||||
if d:
|
||||
mem = float(d["ram_gb"]) if d.get("mem_total") else float(d.get("arena_gb") or d["ram_gb"]) + 1.0
|
||||
cards = [{"index": g["index"], "name": g["name"], "vram_gb": g["vram_gb"], "usable": gpu_problem(g) is None} for g in found]
|
||||
res = gpu_fit.options(mem if mem is not None else 0.0, ram_gb(), cards)
|
||||
by_key = {o["key"]: o for o in res["options"]}
|
||||
return choose_roles(a, found)
|
||||
|
||||
|
||||
def fallback_card(a, gpu, chosen):
|
||||
"""A model that cannot share its memory across cards keeps ONE card: the one setup always chose (the most VRAM, then the lower number),
|
||||
not the Main card of the roles - unless the person named the roles (--gpu-roles), then their Main card."""
|
||||
if getattr(a, "_roles_explicit", False) or not getattr(a, "_roles", None):
|
||||
return gpu
|
||||
return min(chosen, key=lambda g: (-round(g["vram_gb"]), g["index"]))
|
||||
|
||||
|
||||
def card_speeds() -> dict:
|
||||
"""{card number: SMs x max clock} of the NVIDIA cards (strata_control/gpu_roles.py); {} when it cannot say."""
|
||||
from strata_control import gpu_roles
|
||||
return gpu_roles.nvidia_speeds()
|
||||
|
||||
|
||||
def roles_cards(found) -> list:
|
||||
sp = card_speeds()
|
||||
return [{"index": g["index"], "name": g["name"], "vram_gb": g["vram_gb"], "usable": gpu_problem(g) is None, "vendor": "nvidia",
|
||||
**({"speed": sp[g["index"]]} if g["index"] in sp else {})} for g in found]
|
||||
|
||||
|
||||
def choose_roles(a, found) -> list:
|
||||
"""Two or more cards can share the model: the recommended role of each (the best card Main, the others Helper, a card with 8 GB or less
|
||||
Off) and ONE question, "Use these roles?". n takes the person's own: 1=main,0=helper (cards left out are Off). Returns the card
|
||||
numbers in the engine's order (Main last) and keeps the roles in a._roles for the config (docs/MULTI_GPU.md)."""
|
||||
from strata_control import gpu_roles
|
||||
cards = roles_cards(found)
|
||||
roles = dict(gpu_roles.recommend(cards)["roles"])
|
||||
say()
|
||||
say(" Strata first tries the model on your largest GPU plus the PC's memory; a second GPU is offered as a layer split")
|
||||
say(" (each card holds the experts of its own layers: more room, prompts read about 20% faster; details: docs/MULTI_GPU.md).")
|
||||
if mem is None:
|
||||
say(" (the model is chosen below; the numbers here are for the recommended size when you name one: --model)")
|
||||
opts = [[g for g in found if g["index"] in o["cards"]] for o in res["options"]]
|
||||
rec = next((i for i, o in enumerate(res["options"], 1) if o["recommended"]), 1) if mem is not None else 1
|
||||
for i, o in enumerate(res["options"], 1):
|
||||
say(f" {i}) {o['summary'] if mem is not None else o['label']}" + (" (recommended)" if i == rec else ""))
|
||||
for n in o["notes"]:
|
||||
say(f" {n}")
|
||||
say(" Strata recommends a role for each graphics card. Main does the last steps of every answer; a Helper holds part of the model;")
|
||||
say(" Off leaves the card alone (a card with 8 GB or less starts Off). Details: docs/MULTI_GPU.md.")
|
||||
by = {c["index"]: c for c in cards}
|
||||
for i in sorted(roles):
|
||||
say(f" GPU {i + 1} {gpu_roles.short_name(by[i]):<22} {by[i]['vram_gb']:>4.0f} GB {roles[i].capitalize()}")
|
||||
for g in found:
|
||||
if gpu_problem(g, together=True) is not None:
|
||||
say(f" (GPU {g['index']}, {g['name']}: {gpu_problem(g, together=True)})")
|
||||
for g in split_short(can[:2]): # #448: the note for a second card that cannot lend a 4096-token chunk
|
||||
say(f" ({split_short_note(g)})")
|
||||
pick = opts[int(ask("Which GPUs?", [str(i) for i in range(1, len(opts) + 1)], str(rec), a.yes or a.check)) - 1]
|
||||
return [g["index"] for g in pick]
|
||||
if gpu_problem(g, together=True) is not None and roles.get(g["index"]) == "off":
|
||||
say(f" (GPU {g['index'] + 1}, {g['name']}: {gpu_problem(g, together=True)})")
|
||||
if ask("Use these roles?", ["y", "n"], "y", a.yes or a.check) == "n":
|
||||
while True:
|
||||
try:
|
||||
text = input("Roles, e.g. " + ",".join(f"{i}={r}" for i, r in sorted(roles.items())) + " (cards you leave out are Off): ").strip()
|
||||
except EOFError:
|
||||
fail("input ended before a setup answer was received", "run setup in a terminal, or pass --yes to accept the recommended roles")
|
||||
if not text:
|
||||
break
|
||||
try:
|
||||
roles = gpu_roles.roles_from_text(text, cards)
|
||||
break
|
||||
except ValueError as e:
|
||||
say(f" {e}")
|
||||
roles = gpu_roles.normalize(roles, cards)
|
||||
lst = gpu_roles.order(roles, cards)
|
||||
if not lst:
|
||||
fail("every graphics card is Off", "setup needs one card for Main (the app and `strata launch` can start with all cards Off)")
|
||||
a._roles = roles
|
||||
a._roles_cards = cards
|
||||
ok(gpu_roles.describe(roles, cards))
|
||||
return lst
|
||||
|
||||
|
||||
def split_mmap(cfg: dict) -> bool:
|
||||
@@ -7466,8 +7498,10 @@ def main() -> int:
|
||||
ap.add_argument("--gpu", help="one GPU, numbered as nvidia-smi numbers them (AMD: as setup lists them) (default: asked when several can be "
|
||||
"used; with --setup it is saved, when starting it is for that start only)")
|
||||
ap.add_argument("--gpus", help="several GPUs sharing one model, as nvidia-smi numbers them (AMD: as setup lists "
|
||||
"them): \"0,2\", or \"all\" (every card that can); the first is the main one. "
|
||||
"Saved, also when starting (see docs/MULTI_GPU.md)")
|
||||
"them): \"0,2\", or \"all\" (every card that can); Strata puts the faster card last, "
|
||||
"and the last card is the Main one. Saved, also when starting (see docs/MULTI_GPU.md)")
|
||||
ap.add_argument("--gpu-roles", metavar="LIST", help="a role for each GPU: main, helper or off, e.g. 1=main,0=helper (cards you do not "
|
||||
"name are off). Main does the last steps of every answer. Saved (docs/MULTI_GPU.md)")
|
||||
ap.add_argument("--layer-split", help="with --gpus: where each later GPU's layers start (\"18\", \"16,32\"), one "
|
||||
"rising number per GPU after the first - not layers per card; default "
|
||||
"auto, placed from each GPU's free VRAM")
|
||||
@@ -7576,6 +7610,27 @@ def main() -> int:
|
||||
a.gpu = int(a.gpu)
|
||||
else:
|
||||
ap.error(f"--gpu takes a GPU number as nvidia-smi numbers them, e.g. --gpu 1 (or --gpus 0,2), not {a.gpu!r}")
|
||||
a._roles = None
|
||||
if a.gpu_roles:
|
||||
if a.gpu is not None or a.gpus:
|
||||
ap.error("use --gpu-roles (a role for each card) or --gpu / --gpus (which cards), not both")
|
||||
from strata_control import gpu_roles as GR
|
||||
try:
|
||||
named = GR.parse_roles(a.gpu_roles)
|
||||
except ValueError as e:
|
||||
ap.error(f"--gpu-roles: {e}")
|
||||
on = [i for i, r in named.items() if r != "off"]
|
||||
if not on:
|
||||
ap.error("--gpu-roles: setup needs at least one card that is Main or Helper")
|
||||
mains = [i for i, r in named.items() if r == "main"] or [sorted(on)[-1]]
|
||||
helpers = sorted(i for i in on if i != mains[0])
|
||||
a._roles_explicit = True
|
||||
a._roles = {**{i: "off" for i in named}, **{i: "helper" for i in helpers}, mains[0]: "main"}
|
||||
order = helpers + [mains[0]]
|
||||
if len(order) > 1:
|
||||
a.gpus = ",".join(str(i) for i in order)
|
||||
else:
|
||||
a.gpu = order[0]
|
||||
say("Strata - Qwen3.8-Flash-Next on a normal PC (a GPU + system RAM + CPU)")
|
||||
have = installed_configs()
|
||||
# A plain start of an installed model finds everything where it already is: it creates, moves and copies nothing
|
||||
@@ -7769,7 +7824,7 @@ def main() -> int:
|
||||
gpu_table(found)
|
||||
sel = choose_gpus(a, found) # asked when two or more cards can share the model
|
||||
multi = sel if len(sel) > 1 else []
|
||||
a.gpu = sel[0] # the main GPU: the checks and the sizing below are its
|
||||
a.gpu = sel[-1] if getattr(a, "_roles", None) else sel[0] # the main GPU: the checks and the sizing below are its (roles: Main is last)
|
||||
GPU_PICK = a.gpu
|
||||
gpu = gpu_info(a.gpu)
|
||||
chosen = [gpu_info(i) for i in sel]
|
||||
@@ -7971,7 +8026,8 @@ def main() -> int:
|
||||
warn(f"installing {model} with {ram:.0f} GB of RAM, as you chose")
|
||||
if in_budget:
|
||||
budget = budget_choice(model, ram, a.resident_budget_gib)
|
||||
if multi and not unsloth_together(a, model, ram, gpu, chosen):
|
||||
if multi and not unsloth_together(a, model, ram, fallback_card(a, gpu, chosen), chosen):
|
||||
gpu = fallback_card(a, gpu, chosen)
|
||||
multi, sel, chosen = [], [gpu["index"]], [gpu]
|
||||
q4_split = bool(multi) and not resident_split() # #498: on several GPUs without the RAM budget (#642: kept)
|
||||
if not q4_split:
|
||||
@@ -7987,7 +8043,8 @@ def main() -> int:
|
||||
"the low-RAM mode")
|
||||
low_ram = low_ram_wanted(model, ram, a.low_ram)
|
||||
# #642: the engines from RESIDENT_SPLIT_ENGINE run the low-RAM mode on the chosen cards as on one (decided below)
|
||||
if low_ram and multi and not resident_split() and not low_ram_together(a, model, ram, gpu, chosen):
|
||||
if low_ram and multi and not resident_split() and not low_ram_together(a, model, ram, fallback_card(a, gpu, chosen), chosen):
|
||||
gpu = fallback_card(a, gpu, chosen)
|
||||
multi, sel, chosen = [], [gpu["index"]], [gpu]
|
||||
# (the low-RAM mode's variant is decided once the context is known, below; on several GPUs it is the mapped one
|
||||
# before RESIDENT_SPLIT_ENGINE, #642)
|
||||
@@ -8592,8 +8649,18 @@ def main() -> int:
|
||||
if multi: # a layer split across these cards (the server adds the flag)
|
||||
cfg["gpu"] = multi
|
||||
cfg["layer_split"] = a.layer_split or "auto"
|
||||
if getattr(a, "_roles", None): # roles chosen: the card list is Main last, in this order, and the server keeps it
|
||||
cfg["gpu_order"] = "as_given"
|
||||
ok(f"layer split across GPUs {multi} ({cfg['layer_split']})")
|
||||
recommend_remote_expert_opt(cfg, off=a.no_remote_expert_opt)
|
||||
if getattr(a, "_roles", None): # the roles themselves (strata_control/gpu_roles.py); "gpu" above is their mirror
|
||||
final = cfg["gpu"] if isinstance(cfg.get("gpu"), list) else [cfg["gpu"]] # what the checks above left (a model that cannot share keeps one card)
|
||||
main = final[-1]
|
||||
cfg["gpu_roles"] = {str(i): ("main" if i == main else "helper" if i in final else "off") for i in sorted(set(a._roles) | set(final))}
|
||||
cfg["gpus_asked"] = True
|
||||
cfg.pop("gpu_none", None)
|
||||
if isinstance(cfg.get("gpu"), list) and len(final) > 1:
|
||||
cfg["gpu_order"] = "as_given"
|
||||
if a.replicas is not None and a.replicas >= 2: # data-parallel replicas (serve/replicas.py), opt-in
|
||||
cards = multi if multi else []
|
||||
if not cards or len(cards) % a.replicas:
|
||||
|
||||
@@ -99,6 +99,8 @@ def add_parsers(sp: argparse._SubParsersAction) -> list[argparse.ArgumentParser]
|
||||
help="the draft layer that makes answers faster. off starts without it: about 1 GB less graphics memory and less RAM, "
|
||||
"slower answers; use it when the engine stops at start on a small card or a PC with little RAM. Default: on")
|
||||
g.add_argument("--gpu", metavar="LIST", help="graphics card numbers to use, e.g. 0 or 0,1 (default: the engine's choice)")
|
||||
g.add_argument("--gpu-roles", metavar="LIST", help="a role for each graphics card: main, helper or off, e.g. 1=main,0=helper (cards you do not name are off). "
|
||||
"Main does the last steps of every answer; a helper holds part of the model")
|
||||
g.add_argument("--tune", type=_choice("a tuning mode", S.TUNE_MODES), metavar="quick|thorough|skip", help="find the fastest settings first (default: skip here; the guided screens suggest quick the first time)")
|
||||
g.add_argument("--tune-allow-small", action="store_true", help="let the tuner also try settings that can change answers very slightly")
|
||||
g.add_argument("--set", action="append", default=[], metavar="KEY=VALUE",
|
||||
@@ -209,8 +211,18 @@ def settings_from_args(a: argparse.Namespace, catalog=None) -> tuple[S.LaunchSet
|
||||
s.parallel = a.parallel
|
||||
if a.mtp:
|
||||
s.mtp = a.mtp
|
||||
if a.gpu and getattr(a, "gpu_roles", None):
|
||||
raise ValueError("Use --gpu (which cards) or --gpu-roles (a role for each card), not both.")
|
||||
if a.gpu:
|
||||
s.gpus = _gpus(a.gpu)
|
||||
s.gpu_roles = None
|
||||
if getattr(a, "gpu_roles", None):
|
||||
from strata_control import gpu_roles as GR
|
||||
try:
|
||||
s.gpu_roles = {str(k): v for k, v in GR.parse_roles(a.gpu_roles).items()}
|
||||
except ValueError as e:
|
||||
raise ValueError(f"--gpu-roles: {e}") from None
|
||||
s.gpus = None
|
||||
if a.tune:
|
||||
s.tune = a.tune
|
||||
if a.tune_allow_small:
|
||||
@@ -284,6 +296,24 @@ def apply_defaults(s: S.LaunchSettings, catalog) -> tuple[S.LaunchSettings, list
|
||||
return s, notices
|
||||
|
||||
|
||||
def roles_line(s: S.LaunchSettings, catalog) -> str | None:
|
||||
"""`Main: GPU 2 RTX 4070 Ti. Helper: GPU 1 RTX 3060.` for the cards this launch names (--gpu-roles, or --gpu: the faster card goes last), else None."""
|
||||
if not (s.gpu_roles or s.gpus) or catalog is None:
|
||||
return None
|
||||
try:
|
||||
from strata_control import gpu_roles
|
||||
comp = catalog.computer()
|
||||
cards = gpu_roles.rows(comp.gpus, comp.engine_vendor)
|
||||
if s.gpu_roles:
|
||||
named = {int(k): v for k, v in s.gpu_roles.items()}
|
||||
roles = gpu_roles.normalize({int(c.get("index", 0)): named.get(int(c.get("index", 0)), "off") for c in cards}, cards)
|
||||
else:
|
||||
roles = gpu_roles.from_legacy(list(s.gpus), cards)
|
||||
return gpu_roles.describe(roles, cards)
|
||||
except Exception: # noqa: BLE001 - a summary line must never stop a launch
|
||||
return None
|
||||
|
||||
|
||||
def gpu_notices(s: S.LaunchSettings, comp) -> list[S.Notice]:
|
||||
"""Cards of another make than the engine drives: never offered, never mixed into a split. Said in plain words: a named card of the other
|
||||
make is refused with the reason, and a better card of the other make is mentioned once (recommend, never force)."""
|
||||
@@ -304,6 +334,15 @@ def gpu_notices(s: S.LaunchSettings, comp) -> list[S.Notice]:
|
||||
f"A model cannot be split across {make} and {omake} cards. {gpu_fit.SECOND_MODEL_SENTENCE}",
|
||||
f"Name a {make} card (" + ", ".join(str(g.get("index")) for g in comp.usable_gpus) + f"), or install the {omake} "
|
||||
"build of Strata to use that card."))
|
||||
if s.gpu_roles and vendor:
|
||||
from strata_control import gpu_roles
|
||||
have = {int(c.get("index", -1)): c for c in gpu_roles.rows(comp.gpus, vendor)}
|
||||
for k in s.gpu_roles:
|
||||
if int(k) not in have:
|
||||
out.append(S.Notice("error", f"There is no card {k} that this build of Strata can use.",
|
||||
"Cards: " + (", ".join(str(i) for i in sorted(have)) or "none") + "."))
|
||||
if not any(r != "off" for r in s.gpu_roles.values()) and not out:
|
||||
out.append(S.Notice("warn", gpu_roles.NO_CARD_LINE, ""))
|
||||
note = comp.other_vendor_note
|
||||
if note:
|
||||
out.append(S.Notice("warn", note, ""))
|
||||
@@ -616,12 +655,19 @@ def run_launch(a: argparse.Namespace) -> int:
|
||||
+ (" API key on" if s.api_key else ""))
|
||||
for n in notes:
|
||||
_say(f" [!] {n.line()}")
|
||||
line = roles_line(s, catalog)
|
||||
if line:
|
||||
_say(f" cards {line}")
|
||||
_say(f" same as {S.flags_text(s)}")
|
||||
_say("Nothing was changed.")
|
||||
return 0
|
||||
for n in notes:
|
||||
if not machine:
|
||||
_say(f"[!] {n.line()}")
|
||||
if not machine:
|
||||
line = roles_line(s, catalog)
|
||||
if line:
|
||||
_say(line)
|
||||
from strata_cli.launch import access as _access
|
||||
fw = _access.firewall_note(s.access)
|
||||
if fw and not machine:
|
||||
|
||||
@@ -40,6 +40,7 @@ PLAIN = {
|
||||
"--suffix-draft": "Guess the next words from text already in the conversation.",
|
||||
"--port": "The network port the server listens on.",
|
||||
"--gpu": "Which graphics card to use.",
|
||||
"--gpu-roles": "A role for each graphics card: main, helper or off.",
|
||||
"--gpus": "Which graphics cards to use together.",
|
||||
}
|
||||
_CODE = re.compile(r"\b[RTS]\d{1,2}\b[:,)]?\s*")
|
||||
|
||||
+38
-11
@@ -206,11 +206,26 @@ def apply_choices(cfg: dict, s: S.LaunchSettings) -> dict:
|
||||
return cfg
|
||||
|
||||
|
||||
def apply_cards(cfg: dict, gpus: list[int] | None) -> None:
|
||||
def apply_cards(cfg: dict, gpus: "list[int] | dict | None") -> None:
|
||||
"""The graphics cards of THIS launch go into its run config, whatever card the model was installed with (no reinstall): one card is a number,
|
||||
several are a list with the automatic layer split. None: the installed config's own choice stays."""
|
||||
several are a list with the automatic layer split. None: the installed config's own choice stays. A dict is the roles of every card
|
||||
(strata_control/gpu_roles.mirror: "gpu_roles" plus the legacy "gpu" with the Main card last)."""
|
||||
if isinstance(gpus, dict):
|
||||
for k in ("gpu", "gpu_order", "gpu_none", "gpu_roles", "gpu_cards"):
|
||||
cfg.pop(k, None)
|
||||
cfg.update(gpus)
|
||||
cfg["gpus_asked"] = True
|
||||
cfg.pop("hip_ordinal", None)
|
||||
on = gpus.get("gpu")
|
||||
if isinstance(on, list) and len(on) > 1:
|
||||
cfg["layer_split"] = cfg.get("layer_split") or "auto"
|
||||
else:
|
||||
cfg.pop("layer_split", None)
|
||||
return
|
||||
if not gpus:
|
||||
return
|
||||
for k in ("gpu_roles", "gpu_cards", "gpu_none"): # a plain card list replaces an earlier roles choice
|
||||
cfg.pop(k, None)
|
||||
gpus = [int(g) for g in gpus]
|
||||
cfg["gpu"] = gpus[0] if len(gpus) == 1 else gpus
|
||||
cfg["gpus_asked"] = True
|
||||
@@ -224,7 +239,7 @@ def apply_cards(cfg: dict, gpus: list[int] | None) -> None:
|
||||
def installed_cards(base: dict | None) -> list[int] | None:
|
||||
"""The cards the installed model's own config names after a choice (setup --gpu / --gpus, the app's card pick: "gpus_asked"), in its order;
|
||||
None when it names none or was never asked."""
|
||||
if not isinstance(base, dict) or not base.get("gpus_asked"):
|
||||
if not isinstance(base, dict) or not (base.get("gpus_asked") or base.get("gpu_roles")):
|
||||
return None
|
||||
g = base.get("gpu")
|
||||
if g is None or g == "":
|
||||
@@ -236,27 +251,37 @@ def installed_cards(base: dict | None) -> list[int] | None:
|
||||
return cards or None
|
||||
|
||||
|
||||
def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None, base: dict | None = None) -> list[int] | None:
|
||||
def cards_for_launch(s: S.LaunchSettings, catalog, ram_gb: float | None = None, base: dict | None = None) -> "list[int] | dict | None":
|
||||
"""The cards this launch uses: the ones named (--gpu / --gpus, the Cards step), else the cards the installed config was set to (a person's
|
||||
earlier choice, "gpu": 1 or [1, 0] - never replaced by the automatic pick, which would put the first of equal cards back), else the
|
||||
recommended ones by the shared fit rule (strata_control/gpu_fit.py) when the computer has more than one card that counts, else None."""
|
||||
if s.gpu_roles:
|
||||
try:
|
||||
from strata_control import gpu_roles
|
||||
comp = catalog.computer()
|
||||
cards = gpu_roles.rows(comp.gpus, comp.engine_vendor)
|
||||
named = {int(k): v for k, v in s.gpu_roles.items()}
|
||||
roles = gpu_roles.normalize({int(c.get("index", 0)): named.get(int(c.get("index", 0)), "off") for c in cards}, cards)
|
||||
return gpu_roles.mirror(roles, cards)
|
||||
except Exception: # noqa: BLE001 - never block a launch over the choice of cards
|
||||
return None
|
||||
if s.gpus:
|
||||
return [int(g) for g in s.gpus]
|
||||
if installed_cards(base):
|
||||
if installed_cards(base) or (isinstance(base, dict) and base.get("gpu_roles")):
|
||||
return None
|
||||
try:
|
||||
from strata_control import gpu_fit
|
||||
from strata_control import gpu_roles
|
||||
comp = catalog.computer()
|
||||
if len(comp.usable_gpus) < 2:
|
||||
return None
|
||||
row = next((r for r in catalog.rows() if (r.family, r.model) == (s.family, s.model)), None)
|
||||
mem = float(row.memory_gb) if row is not None and row.memory_gb else 44.0
|
||||
return gpu_fit.recommended_cards(mem, comp.ram_gb, comp.gpus, comp.engine_vendor) or None
|
||||
cards = gpu_roles.rows(comp.gpus, comp.engine_vendor)
|
||||
rec = gpu_roles.recommend(cards, comp.engine_vendor)["roles"]
|
||||
return gpu_roles.order(rec, cards) or None
|
||||
except Exception: # noqa: BLE001 - never block a launch over the choice of cards
|
||||
return None
|
||||
|
||||
|
||||
def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str], log: str | None = None, gpus: list[int] | None = None) -> dict:
|
||||
def derive_config(base: dict, s: S.LaunchSettings, *, bool_flags: frozenset[str], log: str | None = None, gpus: "list[int] | dict | None" = None) -> dict:
|
||||
"""The private run config of this launch: the installed one plus the choices that are not setup's (host, port, key, cards, advanced)."""
|
||||
from strata_control import tune_knobs
|
||||
cfg = json.loads(json.dumps(base))
|
||||
@@ -578,7 +603,9 @@ class Runner:
|
||||
if s.model_path:
|
||||
p = Path(s.model_path)
|
||||
args += ["--gguf-dir", str(p if p.is_dir() else p.parent)]
|
||||
if s.gpus:
|
||||
if s.gpu_roles:
|
||||
args += ["--gpu-roles", ",".join(f"{k}={v}" for k, v in sorted(s.gpu_roles.items(), key=lambda kv: int(kv[0])))]
|
||||
elif s.gpus:
|
||||
args += ["--gpus", ",".join(str(g) for g in s.gpus)] if len(s.gpus) > 1 else ["--gpu", str(s.gpus[0])]
|
||||
if sys.platform == "darwin":
|
||||
# a Mac's setup (Control mode, strata_control/mac_fit.py) takes fewer options (no --kv, --gpu, speed projection)
|
||||
|
||||
+130
-37
@@ -529,13 +529,21 @@ class ModelScreen(Base):
|
||||
# ------------------------------------------------------------------------------------------------ 2 cards (only with 2+ cards)
|
||||
|
||||
class CardsScreen(Base):
|
||||
"""Main / Helper / Off for every card (strata_control/gpu_roles.py). The recommendation is already selected; M H O set the role of the
|
||||
card under the cursor, R goes back to the recommendation, D shows what the roles mean."""
|
||||
|
||||
step_id = "cards"
|
||||
help_text = ("Strata tries the model on your largest graphics card plus the computer's memory first. A second card is offered as a layer "
|
||||
"split: it adds room and reads prompts faster. The recommended choice is already selected; pick another if you prefer.")
|
||||
hints = "↑↓ choose Enter next Esc back ? help"
|
||||
help_text = ("Main does the last steps of every answer; Helper holds part of the model; Off leaves the card alone. "
|
||||
"Keys: M Main, H Helper, O Off, R back to recommended, D details.")
|
||||
hints = "↑↓ choose card M Main H Helper O Off R recommended D details Enter next Esc back ? help"
|
||||
BINDINGS = Base.BINDINGS + [Binding("m", "role('main')", show=False), Binding("h", "role('helper')", show=False),
|
||||
Binding("o", "role('off')", show=False), Binding("r", "recommended", show=False),
|
||||
Binding("d", "details", show=False), Binding("right", "cycle(1)", show=False),
|
||||
Binding("left", "cycle(-1)", show=False)]
|
||||
|
||||
def body(self) -> ComposeResult:
|
||||
yield Static(Text("Which graphics cards should it use?", style=f"bold {ACCENT}"), markup=False)
|
||||
yield Static("", id="reco", markup=False)
|
||||
yield Menu([("", 0)], id="menu")
|
||||
yield Static("", id="cardnotes", markup=False)
|
||||
|
||||
@@ -552,53 +560,138 @@ class CardsScreen(Base):
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
def choices(self) -> dict:
|
||||
from strata_control import gpu_fit
|
||||
return gpu_fit.options(self.model_memory(), self.ctx.computer.ram_gb, self.ctx.computer.gpus, self.ctx.computer.engine_vendor)
|
||||
def cards(self) -> list[dict]:
|
||||
from strata_control import gpu_roles
|
||||
c = self.ctx.computer
|
||||
return gpu_roles.rows(c.gpus, c.engine_vendor)
|
||||
|
||||
def start_roles(self) -> dict[int, str]:
|
||||
"""The roles the screen opens with: this launch's own, else the installed model's own choice, else the recommendation."""
|
||||
from strata_control import gpu_roles
|
||||
s, cards = self.ctx.settings, self.cards()
|
||||
if s.gpu_roles:
|
||||
return gpu_roles.normalize({int(k): v for k, v in s.gpu_roles.items()}, cards)
|
||||
if s.gpus:
|
||||
return gpu_roles.from_legacy(list(s.gpus), cards, "as_given")
|
||||
return self.baseline()
|
||||
|
||||
def baseline(self) -> dict[int, str]:
|
||||
"""What "no choice made here" means: the installed model's own cards, else the recommendation."""
|
||||
from strata_control import gpu_roles
|
||||
inst = self.installed()
|
||||
cards = self.cards()
|
||||
if inst:
|
||||
return gpu_roles.from_legacy(inst, cards, "as_given")
|
||||
return dict(gpu_roles.recommend(cards, self.ctx.computer.engine_vendor)["roles"])
|
||||
|
||||
def on_mount(self) -> None:
|
||||
self.res = self.choices()
|
||||
items = []
|
||||
sel = 0
|
||||
cur = self.ctx.settings.gpus
|
||||
inst = self.installed()
|
||||
for i, o in enumerate(self.res["options"]):
|
||||
label = o["label"] + (" (recommended)" if o["recommended"] else "") + (" (as installed)" if inst and sorted(o["cards"]) == sorted(inst) else "")
|
||||
items.append(Item([label, Cell(o["words"], WARN if o["verdict"] in ("spills", "tight") else "")], key=o["cards"]))
|
||||
if cur is None and (inst is None and o["recommended"] or inst is not None and sorted(o["cards"]) == sorted(inst)) or (cur is not None and o["cards"] == list(cur)):
|
||||
sel = i
|
||||
wide = max((len(str(i.cells[0])) for i in items), default=40)
|
||||
self.query_one(Menu).columns = [("", max(40, min(wide + 3, 70))), ("", 0)]
|
||||
self.query_one(Menu).set_items(items, sel)
|
||||
from strata_control import gpu_roles
|
||||
self.detail = False
|
||||
self.note = ""
|
||||
self.cardlist = self.cards()
|
||||
self.roles = self.start_roles()
|
||||
self.rec = gpu_roles.recommend(self.cardlist, self.ctx.computer.engine_vendor)
|
||||
wide = max((len(gpu_roles.short_name(c)) for c in self.cardlist), default=20)
|
||||
self.query_one(Menu).columns = [("", 8), ("", min(wide + 2, 34)), ("", 10), ("", 32), ("", 0)]
|
||||
self.fill(0)
|
||||
self.paint()
|
||||
|
||||
def refresh_body(self) -> None:
|
||||
lines = Text()
|
||||
def role_cell(self, role: str) -> str:
|
||||
words = [("main", "Main"), ("helper", "Helper"), ("off", "Off")]
|
||||
return "".join(f"[ {w} ] " if r == role else f" {w} " for r, w in words)
|
||||
|
||||
def fill(self, index: int = 0) -> None:
|
||||
from strata_control import gpu_roles
|
||||
items = []
|
||||
for c in self.cardlist:
|
||||
i = int(c.get("index", 0))
|
||||
role = self.roles.get(i, "off")
|
||||
rec = self.rec["roles"].get(i)
|
||||
tag = "recommended" if rec == role else f"recommended: {rec}"
|
||||
mem = "built-in" if gpu_roles._shares(c) else f"{float(c.get('vram_gb') or 0):.0f} GB"
|
||||
items.append(Item([gpu_roles.gpu_label(c), gpu_roles.short_name(c), mem, self.role_cell(role), Cell(tag, "" if rec == role else MUTE)], key=i))
|
||||
self.query_one(Menu).set_items(items, index)
|
||||
|
||||
def current_card(self) -> int | None:
|
||||
cur = self.menu().current if self.menu() else None
|
||||
opt = next((o for o in self.res["options"] if cur is not None and o["cards"] == cur.key), None)
|
||||
for n in (opt or {}).get("notes", []):
|
||||
lines.append(f" {n}\n", style=MUTE)
|
||||
for g in self.res["ignored"]:
|
||||
lines.append(f" Not used: {g.get('name')} ({g['why_ignored']}).\n", style=MUTE)
|
||||
return None if cur is None else int(cur.key)
|
||||
|
||||
def action_role(self, role: str) -> None:
|
||||
from strata_control import gpu_roles
|
||||
i = self.current_card()
|
||||
if i is None:
|
||||
return
|
||||
self.roles, code = gpu_roles.set_role(self.roles, self.cardlist, i, role)
|
||||
self.note = gpu_roles.note_text(code, self.cardlist)
|
||||
self.fill(self.menu().index)
|
||||
self.paint()
|
||||
|
||||
def action_cycle(self, step: int) -> None:
|
||||
order = ["main", "helper", "off"]
|
||||
i = self.current_card()
|
||||
if i is None:
|
||||
return
|
||||
self.action_role(order[(order.index(self.roles.get(i, "off")) + step) % 3])
|
||||
|
||||
def action_recommended(self) -> None:
|
||||
self.roles = dict(self.rec["roles"])
|
||||
self.note = ""
|
||||
self.fill(self.menu().index)
|
||||
self.paint()
|
||||
|
||||
def action_details(self) -> None:
|
||||
self.detail = not self.detail
|
||||
self.paint()
|
||||
|
||||
def is_cpu_engine(self) -> bool:
|
||||
try:
|
||||
return (self.ctx.catalog.installed_backend() or "") in ("cpu", "metal")
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
def refresh_body(self) -> None:
|
||||
if not hasattr(self, "roles"):
|
||||
return
|
||||
from strata_control import gpu_fit, gpu_roles
|
||||
c = self.ctx.computer
|
||||
try:
|
||||
self.query_one("#reco", Static).update(Text(gpu_roles.sentence(self.cardlist, c.engine_vendor), style=MUTE))
|
||||
except Exception: # noqa: BLE001
|
||||
return
|
||||
lines = Text()
|
||||
if self.note:
|
||||
lines.append(f" {self.note}\n", style=MUTE)
|
||||
spill = gpu_roles.spill_for(self.roles, self.cardlist, self.model_memory(), c.ram_gb)
|
||||
for w in gpu_roles.warnings(self.roles, self.cardlist, spill_gb=spill, engine_runs_cpu_only=self.is_cpu_engine()):
|
||||
fix = ""
|
||||
if w["fix"]:
|
||||
n = gpu_roles.gpu_label(next(x for x in self.cardlist if int(x.get("index", 0)) == w["fix"]["card"]))
|
||||
fix = f" (select {n} and press {w['fix']['role'][0].upper()})"
|
||||
lines.append(f" ! {w['text']}{fix}\n", style=WARN)
|
||||
if self.detail:
|
||||
for word, text in (("Main", "does the last steps of every answer and keeps the conversation. Only one."),
|
||||
("Helper", "holds a share of the model, so more of it fits on graphics cards."),
|
||||
("Off", "Strata does not touch the card.")):
|
||||
lines.append(f" {word}: {text}\n", style=MUTE)
|
||||
for g in gpu_fit.for_vendor(c.gpus, c.engine_vendor)[1]:
|
||||
lines.append(f" Not used: {g.get('name')} (another make of card than this engine drives).\n", style=MUTE)
|
||||
try:
|
||||
self.query_one("#cardnotes", Static).update(lines)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
def on_cursor(self) -> None:
|
||||
self.refresh_body()
|
||||
|
||||
def explain(self) -> Text:
|
||||
return Text("One card plus your RAM is the normal way and keeps things simple. Two cards share the layers: more room for the model and "
|
||||
"prompts read about 20% faster, but a small second card can slow it down. Strata recommends one of them; the choice is yours.",
|
||||
style=MUTE)
|
||||
return Text("Strata recommends the roles shown. A Helper holds part of the model; Off leaves a card alone (for a game or the screen). "
|
||||
"Any card is one key away; the choice is yours.", style=MUTE)
|
||||
|
||||
def commit(self) -> None:
|
||||
cur = self.menu().current
|
||||
rec = next((o["cards"] for o in self.res["options"] if o["recommended"]), None)
|
||||
inst = self.installed()
|
||||
keep = rec if inst is None else inst # "let Strata choose" = the recommendation, or the cards the installed model was set to
|
||||
self.ctx.settings.gpus = None if keep is not None and sorted(cur.key) == sorted(keep) else list(cur.key) # the recommended choice stays "let Strata choose" (the same rule in setup)
|
||||
if not hasattr(self, "roles"):
|
||||
return
|
||||
s = self.ctx.settings
|
||||
if self.roles == self.baseline():
|
||||
s.gpu_roles, s.gpus = None, None # the recommendation (or the installed choice) stays "not set": a card added later is re-recommended
|
||||
else:
|
||||
s.gpu_roles, s.gpus = {str(i): r for i, r in sorted(self.roles.items())}, None
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ 3 folder
|
||||
|
||||
@@ -82,6 +82,7 @@ class LaunchSettings:
|
||||
model_path: str | None = None # "Use model files I already have": a .gguf file or a folder
|
||||
data_dir: str | None = None # where model files are stored (None: the Strata default)
|
||||
gpus: list[int] | None = None # None = the engine's choice (all usable cards)
|
||||
gpu_roles: dict[str, str] | None = None # {"1": "main", "0": "helper", "2": "off"}: a role for every card; None = the recommendation
|
||||
context: int | None = None
|
||||
kv: str | None = None # None = int8 above 8K, as setup
|
||||
vision: str | None = None # None = the default (on when the model can read pictures and the engine has the reader), off | on | gpu | cpu
|
||||
@@ -289,7 +290,9 @@ def flags_text(s: LaunchSettings, *, preset: str | None = None) -> str:
|
||||
parts.append("--tune-allow-small")
|
||||
parts.append(f"--access {s.access}")
|
||||
parts.append(f"--port {s.port}")
|
||||
if s.gpus:
|
||||
if s.gpu_roles:
|
||||
parts.append("--gpu-roles " + ",".join(f"{k}={v}" for k, v in sorted(s.gpu_roles.items(), key=lambda kv: int(kv[0]))))
|
||||
elif s.gpus:
|
||||
parts.append("--gpu " + ",".join(str(g) for g in s.gpus))
|
||||
for k, v in sorted(s.extra.items()):
|
||||
parts.append(f"--set {k[2:] if k.startswith('--') else k}={v}" if v else f"--set {k[2:] if k.startswith('--') else k}")
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
"""Main / Helper / Off for every graphics card: ONE place for the app, `strata launch`, setup and the server.
|
||||
|
||||
A card has one of three roles:
|
||||
* Main - the card that runs the LAST stage of the layer split: the output head, the draft layer and the check of its guesses. Exactly one.
|
||||
* Helper - holds a share of the model, so more of it fits on graphics cards instead of being read from the drive.
|
||||
* Off - Strata does not touch it.
|
||||
|
||||
The engine takes the cards in stage order (`"gpu": [helper, ..., main]` with `"gpu_order": "as_given"`), so the Main card is the LAST one. This
|
||||
module turns roles into that list (and back), recommends roles, and words the few warnings a person could not work out alone.
|
||||
The app has the same rules in apps/host/src/lib/gpu-roles.ts; both run tests/data/gpu_roles_vectors.json.
|
||||
|
||||
Pure functions, no I/O.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from strata_control import gpu_fit
|
||||
|
||||
ROLES = ("main", "helper", "off")
|
||||
OFF_AT_OR_BELOW_GB = 8.0 # owner rule: a card with 8 GB or less starts Off (it is not a helper unless the person turns it on)
|
||||
SAME_SPEED = 1.05 # serve/server.py ordered_gpus: cards within 5% of each other are the same card
|
||||
MEMORY_GUARD_SPEED = 0.5 # a card with more memory takes Main from a faster one only when it is at least half as fast (#1576)
|
||||
|
||||
CPU_ONLY_LINE = "Performance is degraded on CPU only."
|
||||
NO_CARD_LINE = "This engine needs a graphics card to start. Turn one on."
|
||||
|
||||
|
||||
def _idx(c: dict) -> int:
|
||||
return int(c.get("index", 0))
|
||||
|
||||
|
||||
def _mem(c: dict) -> float:
|
||||
return float(c.get("vram_gb") or 0)
|
||||
|
||||
|
||||
def _speed(c: dict) -> float:
|
||||
v = c.get("speed")
|
||||
return float(v) if isinstance(v, (int, float)) and not isinstance(v, bool) and v > 0 else 0.0
|
||||
|
||||
|
||||
def rnd_(x: float) -> int:
|
||||
return round(x)
|
||||
|
||||
|
||||
def _shares(c: dict) -> bool:
|
||||
return gpu_fit._shares_memory(c)
|
||||
|
||||
|
||||
def _can_run(c: dict) -> bool:
|
||||
return c.get("usable") is not False
|
||||
|
||||
|
||||
def short_name(c: dict) -> str:
|
||||
n = str(c.get("name") or f"GPU {_idx(c) + 1}")
|
||||
for junk in ("NVIDIA GeForce ", "NVIDIA ", "GeForce ", "AMD Radeon ", "AMD ", "Intel(R) ", "Intel "):
|
||||
if n.startswith(junk):
|
||||
n = n[len(junk):]
|
||||
break
|
||||
return n.strip()
|
||||
|
||||
|
||||
def gpu_label(c: dict) -> str:
|
||||
"""The number the screens show: GPU 1 is card 0 (Monitor, the model picker, the loaded list)."""
|
||||
return f"GPU {_idx(c) + 1}"
|
||||
|
||||
|
||||
def rows(cards: list[dict], vendor: str | None = None) -> list[dict]:
|
||||
"""The cards shown as rows: every card the engine's make can name, in card order. The other make is `gpu_fit.for_vendor(...)[1]` (a muted line)."""
|
||||
vendor = vendor or gpu_fit.pick_vendor(cards)
|
||||
mine = gpu_fit.for_vendor(cards, vendor)[0]
|
||||
return sorted(mine, key=_idx)
|
||||
|
||||
|
||||
def _eligible(cs: list[dict]) -> list[dict]:
|
||||
"""The cards that can be Main: run Strata, have memory of their own and at least MIN_CARD_GB of it; an integrated chip only when it is the
|
||||
only device (a Strix Halo PC, a mini PC)."""
|
||||
real = [c for c in cs if _can_run(c) and not _shares(c) and _mem(c) >= gpu_fit.MIN_CARD_GB]
|
||||
if real:
|
||||
return real
|
||||
return [c for c in cs if _can_run(c)]
|
||||
|
||||
|
||||
def pick_main(cs: list[dict]) -> tuple[dict | None, str]:
|
||||
"""(the Main card, why) why: only | faster | same | memory. The faster card, cards within 5% being the same card (the one with more memory,
|
||||
then, for identical cards, the HIGHEST number - so the engine order stays [0, 1] - else the lower number); a card with clearly more memory wins when it is at least half as fast (#1576)."""
|
||||
pool = _eligible(cs)
|
||||
if not pool:
|
||||
return None, "none"
|
||||
if len(pool) == 1:
|
||||
return pool[0], "only"
|
||||
top = max(_speed(c) for c in pool)
|
||||
group = [c for c in pool if _speed(c) * SAME_SPEED >= top] if top > 0 else list(pool)
|
||||
main = sorted(group, key=lambda c: (-round(_mem(c)), _idx(c)))[0]
|
||||
why = "faster" if len(group) == 1 else "same"
|
||||
if len(group) > 1 and len({(str(c.get("name")), rnd_(_mem(c))) for c in group}) == 1:
|
||||
main = max(group, key=_idx) # identical cards: the HIGHEST number is Main, so the engine order stays [0, 1] as it always was
|
||||
if top <= 0 and len({round(_mem(c)) for c in pool}) > 1:
|
||||
why = "memory" # speeds unknown: the card with the most memory
|
||||
bigger = [c for c in pool if round(_mem(c)) > round(_mem(main)) and (top <= 0 or _speed(c) >= MEMORY_GUARD_SPEED * top)]
|
||||
if bigger:
|
||||
main = sorted(bigger, key=lambda c: (-round(_mem(c)), _idx(c)))[0]
|
||||
why = "memory"
|
||||
return main, why
|
||||
|
||||
|
||||
def recommend(cards: list[dict], vendor: str | None = None) -> dict[str, Any]:
|
||||
"""The recommended roles: {"roles": {index: role}, "main": index|None, "why": code, "reasons": {index: code}}.
|
||||
The best card is Main, every other card starts as Helper, EXCEPT a card with 8 GB or less, an integrated chip beside a card with memory of
|
||||
its own, and a card the engine cannot run: those start Off."""
|
||||
cs = rows(cards, vendor)
|
||||
main, why = pick_main(cs)
|
||||
roles: dict[int, str] = {}
|
||||
reasons: dict[int, str] = {}
|
||||
for c in cs:
|
||||
i = _idx(c)
|
||||
if main is not None and c is main:
|
||||
roles[i], reasons[i] = "main", why
|
||||
elif not _can_run(c):
|
||||
roles[i], reasons[i] = "off", "cannot_run"
|
||||
elif _shares(c) and main is not None:
|
||||
roles[i], reasons[i] = "off", "igpu"
|
||||
elif round(_mem(c)) <= OFF_AT_OR_BELOW_GB:
|
||||
roles[i], reasons[i] = "off", "small"
|
||||
else:
|
||||
roles[i], reasons[i] = "helper", "helper"
|
||||
return {"roles": roles, "main": _idx(main) if main is not None else None, "why": why, "reasons": reasons}
|
||||
|
||||
|
||||
def _on(roles: dict[int, str]) -> list[int]:
|
||||
return [i for i, r in roles.items() if r != "off"]
|
||||
|
||||
|
||||
def normalize(roles: dict[int, str], cards: list[dict]) -> dict[int, str]:
|
||||
"""Exactly one Main among the cards that are on: none -> the fastest card that is on; more than one -> the first keeps it. A lone card that
|
||||
is on is Main."""
|
||||
cs = {_idx(c): c for c in cards}
|
||||
out = {i: (r if r in ROLES else "off") for i, r in roles.items() if i in cs}
|
||||
on = _on(out)
|
||||
mains = [i for i in on if out[i] == "main"]
|
||||
if on and not mains:
|
||||
best = sorted(on, key=lambda i: (-_speed(cs[i]), -round(_mem(cs[i])), i))[0]
|
||||
out[best] = "main"
|
||||
for extra in mains[1:]:
|
||||
out[extra] = "helper"
|
||||
if len(on) == 1:
|
||||
out[on[0]] = "main"
|
||||
return out
|
||||
|
||||
|
||||
def set_role(roles: dict[int, str], cards: list[dict], idx: int, role: str) -> tuple[dict[int, str], str | None]:
|
||||
"""One click: the new roles and a code for the quiet line the app/Terminal may show (None: nothing to say).
|
||||
Codes: old_main_helper:<i> (the old Main became a Helper), promoted:<i> (this card became Main because the Main left), only_main (refused)."""
|
||||
cs = {_idx(c): c for c in cards}
|
||||
out = dict(roles)
|
||||
note: str | None = None
|
||||
if idx not in cs or role not in ROLES:
|
||||
return out, None
|
||||
if role == "main":
|
||||
old = next((i for i in out if i != idx and out[i] == "main"), None)
|
||||
out[idx] = "main"
|
||||
if old is not None:
|
||||
out[old] = "helper"
|
||||
note = f"old_main_helper:{old}"
|
||||
elif role == "helper":
|
||||
if out.get(idx) == "main":
|
||||
others = sorted([i for i in out if i != idx and out[i] == "helper"], key=lambda i: (-_speed(cs[i]), i))
|
||||
if not others:
|
||||
return out, "only_main"
|
||||
out[idx] = "helper"
|
||||
out[others[0]] = "main"
|
||||
note = f"promoted:{others[0]}"
|
||||
else:
|
||||
out[idx] = "helper"
|
||||
else:
|
||||
was_main = out.get(idx) == "main"
|
||||
out[idx] = "off"
|
||||
if was_main:
|
||||
nxt = sorted([i for i in out if out[i] == "helper"], key=lambda i: (-_speed(cs[i]), i))
|
||||
if nxt:
|
||||
out[nxt[0]] = "main"
|
||||
note = f"promoted:{nxt[0]}"
|
||||
on = _on(out)
|
||||
if len(on) == 1:
|
||||
out[on[0]] = "main"
|
||||
elif on and not any(out[i] == "main" for i in on):
|
||||
out[on[0]] = "main"
|
||||
return out, note
|
||||
|
||||
|
||||
def order(roles: dict[int, str], cards: list[dict]) -> list[int]:
|
||||
"""The engine's card list: Helpers (slowest first, ties by number), then the Main card LAST; Off cards left out."""
|
||||
cs = {_idx(c): c for c in cards}
|
||||
roles = normalize(roles, cards)
|
||||
helpers = sorted([i for i, r in roles.items() if r == "helper"], key=lambda i: (_speed(cs[i]), i))
|
||||
return helpers + [i for i, r in roles.items() if r == "main"]
|
||||
|
||||
|
||||
def mirror(roles: dict[int, str], cards: list[dict]) -> dict[str, Any]:
|
||||
"""What a writer stores: "gpu_roles" plus the legacy mirror older engines and the Monitor read. Nothing on: "gpu": [] and "gpu_none": true."""
|
||||
roles = normalize(roles, cards)
|
||||
lst = order(roles, cards)
|
||||
out: dict[str, Any] = {"gpu_roles": {str(i): roles[i] for i in sorted(roles)}}
|
||||
if lst:
|
||||
out["gpu"] = lst[0] if len(lst) == 1 else lst
|
||||
out["gpu_order"] = "as_given"
|
||||
else:
|
||||
out["gpu"] = []
|
||||
out["gpu_none"] = True
|
||||
out["gpu_cards"] = {str(_idx(c)): "|".join(x for x in (short_name(c), str(c.get("bus") or "")) if x) for c in cards if _idx(c) in roles}
|
||||
return out
|
||||
|
||||
|
||||
def parse_roles(text: str, cards: list[dict] | None = None) -> dict[int, str]:
|
||||
"""`1=main,0=helper,2=off` (also `1:main`) -> {1: "main", 0: "helper", 2: "off"}. ValueError with a plain message otherwise."""
|
||||
out: dict[int, str] = {}
|
||||
for part in str(text or "").replace(";", ",").split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
sep = "=" if "=" in part else ":" if ":" in part else None
|
||||
if sep is None:
|
||||
raise ValueError(f"{part!r}: write each card as number=role, e.g. 1=main,0=helper,2=off")
|
||||
k, v = (x.strip().lower() for x in part.split(sep, 1))
|
||||
if not k.isdigit() or v not in ROLES:
|
||||
raise ValueError(f"{part!r}: a card number and one of main, helper, off, e.g. 1=main")
|
||||
if int(k) in out:
|
||||
raise ValueError(f"card {k} is named twice")
|
||||
out[int(k)] = v
|
||||
if not out:
|
||||
raise ValueError("no cards named; write e.g. 1=main,0=helper")
|
||||
mains = [i for i, r in out.items() if r == "main"]
|
||||
if len(mains) > 1:
|
||||
raise ValueError("only one card can be Main")
|
||||
if cards is not None:
|
||||
known = {_idx(c) for c in cards}
|
||||
missing = [i for i in out if i not in known]
|
||||
if missing:
|
||||
raise ValueError("no card " + ", ".join(str(i) for i in missing) + " on this computer (cards: " + ", ".join(str(i) for i in sorted(known)) + ")")
|
||||
return out
|
||||
|
||||
|
||||
def roles_from_text(text: str, cards: list[dict]) -> dict[int, str]:
|
||||
"""A command line's roles: the cards named get their role; every card NOT named is Off (an explicit command means only what was named);
|
||||
no Main named -> the fastest named card that is on becomes Main."""
|
||||
named = parse_roles(text, cards)
|
||||
full = {_idx(c): named.get(_idx(c), "off") for c in cards}
|
||||
return normalize(full, cards)
|
||||
|
||||
|
||||
def from_legacy(gpu: Any, cards: list[dict], gpu_order: str | None = None, speed_order: bool = True) -> dict[int, str]:
|
||||
"""Roles of an old "gpu" setting, for display: the LAST card of the engine's order is Main, earlier ones are Helpers, unlisted cards are Off.
|
||||
`gpu` None / "" / []: the recommendation (nothing was chosen)."""
|
||||
items: list[int] = []
|
||||
if isinstance(gpu, (list, tuple)):
|
||||
items = [int(x) for x in gpu]
|
||||
elif gpu is not None and str(gpu).strip() != "":
|
||||
items = [int(x) for x in str(gpu).split(",") if x.strip() != ""]
|
||||
cs = {_idx(c): c for c in cards}
|
||||
items = [i for i in items if i in cs]
|
||||
if not items:
|
||||
return dict(recommend(cards)["roles"])
|
||||
if len(items) > 1 and gpu_order != "as_given" and speed_order:
|
||||
if max(_speed(cs[i]) for i in items) >= SAME_SPEED * max(1e-9, min(_speed(cs[i]) for i in items)) and all(_speed(cs[i]) for i in items):
|
||||
items = sorted(items, key=lambda i: _speed(cs[i])) # the engine puts the faster card last (#1352)
|
||||
roles = {i: "off" for i in cs}
|
||||
for i in items[:-1]:
|
||||
roles[i] = "helper"
|
||||
roles[items[-1]] = "main"
|
||||
return roles
|
||||
|
||||
|
||||
def is_recommended(roles: dict[int, str], cards: list[dict], vendor: str | None = None) -> bool:
|
||||
rec = recommend(cards, vendor)["roles"]
|
||||
return {i: r for i, r in roles.items() if i in rec} == rec
|
||||
|
||||
|
||||
# ---- warnings: only what a person could not know -------------------------------------------------------------------------------------------
|
||||
|
||||
def warnings(roles: dict[int, str], cards: list[dict], *, spill_gb: float = 0.0, engine_runs_cpu_only: bool = False) -> list[dict]:
|
||||
"""[{"code", "card"|None, "text", "fix": {"card": i, "role": r}|None}] - one short line each, never blocking.
|
||||
* igpu_main built-in graphics as Main (fix: the real card becomes Main)
|
||||
* cannot_run a card the engine cannot run is on (fix: turn it off)
|
||||
* low_memory too little memory for the chosen setup (part of the model would be read from the drive)
|
||||
* cpu_only every card is Off"""
|
||||
cs = {_idx(c): c for c in cards}
|
||||
out: list[dict] = []
|
||||
for i in sorted(roles):
|
||||
c, role = cs.get(i), roles[i]
|
||||
if c is None or role == "off":
|
||||
continue
|
||||
if not _can_run(c):
|
||||
out.append({"code": "cannot_run", "card": i, "text": f"{short_name(c)} cannot run Strata.", "fix": {"card": i, "role": "off"}})
|
||||
elif role == "main" and _shares(c) and any(not _shares(x) and _can_run(x) for x in cs.values()):
|
||||
real = sorted([x for x in cs.values() if not _shares(x) and _can_run(x)], key=lambda x: (-_speed(x), -_mem(x), _idx(x)))[0]
|
||||
out.append({"code": "igpu_main", "card": i, "text": f"{short_name(c)} is built-in graphics; it is slow as Main.",
|
||||
"fix": {"card": _idx(real), "role": "main"}})
|
||||
if not _on(roles):
|
||||
out.append({"code": "cpu_only", "card": None, "text": CPU_ONLY_LINE if engine_runs_cpu_only else NO_CARD_LINE, "fix": None})
|
||||
elif spill_gb >= 1.0:
|
||||
out.append({"code": "low_memory", "card": None, "text": f"Not enough memory for this setup: about {spill_gb:.0f} GB is read from the drive.", "fix": None})
|
||||
return out
|
||||
|
||||
|
||||
def spill_for(roles: dict[int, str], cards: list[dict], memory_gb: float | None, ram_gb: float | None) -> float:
|
||||
"""About how many GB of the model would be read from the drive with the cards that are on (0 when unknown): gpu_fit's arithmetic, every
|
||||
extra card costing its own dense layers."""
|
||||
if not memory_gb or not ram_gb:
|
||||
return 0.0
|
||||
cs = {_idx(c): c for c in cards}
|
||||
on = [cs[i] for i in _on(roles) if i in cs and not _shares(cs[i])]
|
||||
vram = sum(_mem(c) for c in on) - gpu_fit.SPLIT_CARD_OVERHEAD_GB * max(0, len(on) - 1)
|
||||
return float(gpu_fit.fit(float(memory_gb), float(ram_gb), max(0.0, vram))["spill_gb"])
|
||||
|
||||
|
||||
def note_text(code: str | None, cards: list[dict]) -> str:
|
||||
"""The quiet line for a `set_role` code."""
|
||||
if not code:
|
||||
return ""
|
||||
cs = {_idx(c): c for c in cards}
|
||||
kind, _, arg = code.partition(":")
|
||||
c = cs.get(int(arg)) if arg.isdigit() else None
|
||||
name = short_name(c) if c else "That card"
|
||||
if kind == "old_main_helper":
|
||||
return f"{name} is now a Helper."
|
||||
if kind == "promoted":
|
||||
return f"{name} is now Main."
|
||||
if kind == "only_main":
|
||||
return "The only card that is on has to be Main."
|
||||
return ""
|
||||
|
||||
|
||||
# ---- the sentence above the cards ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
def sentence(cards: list[dict], vendor: str | None = None) -> str:
|
||||
"""One plain line for the recommendation: who is Main and why, who helps, who starts Off."""
|
||||
cs = rows(cards, vendor)
|
||||
rec = recommend(cards, vendor)
|
||||
byi = {_idx(c): c for c in cs}
|
||||
if rec["main"] is None:
|
||||
return "No graphics card that Strata can use was found."
|
||||
m = byi[rec["main"]]
|
||||
if len(cs) == 1:
|
||||
return f"Strata will use your {short_name(m)}."
|
||||
helpers = [byi[i] for i, r in rec["roles"].items() if r == "helper"]
|
||||
why = {"same": "the cards are alike", "memory": "it has more memory", "faster": "it is faster", "only": "it is the only card"}[rec["why"]]
|
||||
s = f"{short_name(m)} is Main: {why}."
|
||||
if helpers:
|
||||
s += " " + " and ".join(short_name(h) for h in helpers) + (" help." if len(helpers) > 1 else " helps.")
|
||||
for i, r in rec["roles"].items():
|
||||
if r == "off":
|
||||
code = rec["reasons"][i]
|
||||
c = byi[i]
|
||||
s += f" {short_name(c)} starts Off: " + ("Strata cannot run it." if code == "cannot_run" else "built-in graphics." if code == "igpu" else f"{OFF_AT_OR_BELOW_GB:.0f} GB or less.")
|
||||
return s
|
||||
|
||||
|
||||
# ---- reading saved settings (server, launch, app) ---------------------------------------------------------------------------------------
|
||||
|
||||
def _int_list(g: Any) -> list[int]:
|
||||
if g is None or g == "":
|
||||
return []
|
||||
items = g if isinstance(g, (list, tuple)) else str(g).split(",")
|
||||
return [int(str(x).strip()) for x in items if str(x).strip() != ""]
|
||||
|
||||
|
||||
def parse_saved(value: Any) -> dict[int, str] | None:
|
||||
"""A saved "gpu_roles" object -> {index: role}; None when it is not one (the setting is then ignored)."""
|
||||
if not isinstance(value, dict) or not value:
|
||||
return None
|
||||
out: dict[int, str] = {}
|
||||
for k, v in value.items():
|
||||
try:
|
||||
i = int(str(k).strip())
|
||||
except ValueError:
|
||||
return None
|
||||
if str(v).lower() not in ROLES:
|
||||
return None
|
||||
out[i] = str(v).lower()
|
||||
if sum(1 for r in out.values() if r == "main") > 1:
|
||||
return None
|
||||
return out
|
||||
|
||||
|
||||
def saved_roles(cfg: dict) -> tuple[dict[int, str] | None, str | None]:
|
||||
"""(the roles a run settings dict carries, a problem line or None). Roles win when the legacy "gpu" is the mirror of them, or when "gpu" is
|
||||
missing; a "gpu" that was edited by hand (it names other cards, or another Main) wins and the roles are ignored (#3256: a file that names
|
||||
its GPUs keeps them)."""
|
||||
roles = parse_saved(cfg.get("gpu_roles"))
|
||||
if roles is None:
|
||||
return None, ("run settings: gpu_roles is not {card: main|helper|off} with one main; ignored" if cfg.get("gpu_roles") is not None else None)
|
||||
try:
|
||||
gl = _int_list(cfg.get("gpu"))
|
||||
except ValueError:
|
||||
return None, "run settings: gpu is not a card number or a list; gpu_roles ignored"
|
||||
on = [i for i, r in roles.items() if r != "off"]
|
||||
main = next((i for i, r in roles.items() if r == "main"), None)
|
||||
if cfg.get("gpu_none") and not on:
|
||||
return roles, None
|
||||
if not gl:
|
||||
return (roles, None) if on else (None, None)
|
||||
if sorted(gl) != sorted(on) or (main is not None and gl[-1] != main):
|
||||
return None, "run settings: gpu was edited by hand, gpu_roles ignored"
|
||||
return roles, None
|
||||
|
||||
|
||||
def engine_order(roles: dict[int, str], speeds: dict[int, float] | None = None) -> list[int]:
|
||||
"""The card list of saved roles for the server: Helpers slowest first (ties by number), then Main. Used where only numbers and
|
||||
(maybe) measured speeds are known."""
|
||||
sp = speeds or {}
|
||||
helpers = sorted([i for i, r in roles.items() if r == "helper"], key=lambda i: (sp.get(i, 0.0), i))
|
||||
return helpers + sorted(i for i, r in roles.items() if r == "main")
|
||||
|
||||
|
||||
def describe(roles: dict[int, str], cards: list[dict]) -> str:
|
||||
"""`Main: GPU 2 RTX 4070 Ti. Helper: GPU 1 RTX 3060.` (the line launch and setup print)"""
|
||||
cs = {_idx(c): c for c in cards}
|
||||
parts = []
|
||||
for role, word in (("main", "Main"), ("helper", "Helper"), ("off", "Off")):
|
||||
names = [f"GPU {i + 1} {short_name(cs[i])}" if i in cs else f"GPU {i + 1}" for i in sorted(roles) if roles[i] == role]
|
||||
if names:
|
||||
parts.append(f"{word}: {', '.join(names)}.")
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
# ---- speed of each card (the same measure the engine orders a split by: multiprocessors x max clock) ------------------------------------
|
||||
|
||||
def nvidia_speeds(timeout: float = 20.0) -> dict[int, float]:
|
||||
"""{nvidia-smi number: SMs x max clock} from the CUDA driver API in a child process (no toolkit needed); {} when it cannot say."""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
code = ("import ctypes,sys,os\n"
|
||||
"os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID'\n"
|
||||
"lib=ctypes.CDLL('nvcuda.dll' if os.name=='nt' else 'libcuda.so.1')\n"
|
||||
"assert lib.cuInit(0)==0\n"
|
||||
"n=ctypes.c_int()\n"
|
||||
"assert lib.cuDeviceGetCount(ctypes.byref(n))==0\n"
|
||||
"for i in range(n.value):\n"
|
||||
" d=ctypes.c_int()\n"
|
||||
" assert lib.cuDeviceGet(ctypes.byref(d),i)==0\n"
|
||||
" sm=ctypes.c_int(); mhz=ctypes.c_int()\n"
|
||||
" lib.cuDeviceGetAttribute(ctypes.byref(sm),16,d)\n"
|
||||
" lib.cuDeviceGetAttribute(ctypes.byref(mhz),13,d)\n"
|
||||
" print(i,sm.value,mhz.value)\n")
|
||||
try:
|
||||
env = dict(os.environ)
|
||||
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
||||
env.pop("CUDA_VISIBLE_DEVICES", None)
|
||||
r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=timeout, env=env)
|
||||
out = {}
|
||||
for line in r.stdout.splitlines():
|
||||
i, sm, khz = (int(x) for x in line.split())
|
||||
if sm > 0 and khz > 0:
|
||||
out[i] = float(sm) * float(khz)
|
||||
return out
|
||||
except (OSError, ValueError, subprocess.SubprocessError):
|
||||
return {}
|
||||
|
||||
|
||||
def add_speeds(gpus: list[dict]) -> None:
|
||||
"""Puts "speed" on the NVIDIA cards of a hardware report (in place); a card it cannot measure keeps none."""
|
||||
nv = [g for g in gpus if str(g.get("vendor")) == "nvidia"]
|
||||
if not nv:
|
||||
return
|
||||
sp = nvidia_speeds()
|
||||
for g in nv:
|
||||
if g.get("index") in sp:
|
||||
g["speed"] = sp[g["index"]]
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"_about": "Shared by tests/terminal/test_gpu_roles.py (strata_control/gpu_roles.py) and apps/host/tests/gpu-roles.test.mjs (apps/host/src/lib/gpu-roles.ts). Roles are keyed by card number.",
|
||||
"hw": {
|
||||
"one": [{"index": 0, "name": "NVIDIA GeForce RTX 4070 Ti", "vram_gb": 12, "speed": 100}],
|
||||
"two": [{"index": 0, "name": "NVIDIA GeForce RTX 3060", "vram_gb": 12, "speed": 60}, {"index": 1, "name": "NVIDIA GeForce RTX 4070 Ti", "vram_gb": 12, "speed": 100}],
|
||||
"equal": [{"index": 0, "name": "NVIDIA RTX A4000", "vram_gb": 16, "speed": 50}, {"index": 1, "name": "NVIDIA RTX A4000", "vram_gb": 16, "speed": 50.5}],
|
||||
"three": [{"index": 0, "name": "NVIDIA GeForce RTX 3060", "vram_gb": 12, "speed": 60}, {"index": 1, "name": "NVIDIA GeForce RTX 4070 Ti", "vram_gb": 16, "speed": 100}, {"index": 2, "name": "NVIDIA GeForce RTX 2060", "vram_gb": 6, "speed": 45}],
|
||||
"igpu": [{"index": 0, "name": "Intel(R) UHD Graphics 770", "vram_gb": 0, "integrated": true, "speed": 12}, {"index": 1, "name": "NVIDIA GeForce RTX 4060", "vram_gb": 8, "speed": 70}],
|
||||
"igpu_only": [{"index": 0, "name": "AMD Radeon 780M", "vram_gb": 0, "integrated": true, "speed": 12}],
|
||||
"bigger_slower": [{"index": 0, "name": "NVIDIA GeForce RTX 3090", "vram_gb": 24, "speed": 90}, {"index": 1, "name": "NVIDIA GeForce RTX 4070", "vram_gb": 12, "speed": 100}],
|
||||
"small_second": [{"index": 0, "name": "NVIDIA GeForce RTX 4070 Ti", "vram_gb": 12, "speed": 100}, {"index": 1, "name": "NVIDIA GeForce RTX 3070", "vram_gb": 8, "speed": 70}],
|
||||
"cannot": [{"index": 0, "name": "NVIDIA GeForce GTX 1060", "vram_gb": 6, "usable": false, "problem": "too old"}, {"index": 1, "name": "NVIDIA GeForce RTX 4070 Ti", "vram_gb": 12, "speed": 100}],
|
||||
"nospeed": [{"index": 0, "name": "NVIDIA GeForce RTX 3060", "vram_gb": 12}, {"index": 1, "name": "NVIDIA GeForce RTX 3090", "vram_gb": 24}]
|
||||
},
|
||||
"recommend": [
|
||||
{"hw": "one", "roles": {"0": "main"}, "main": 0, "why": "only"},
|
||||
{"hw": "two", "roles": {"0": "helper", "1": "main"}, "main": 1, "why": "faster"},
|
||||
{"hw": "equal", "roles": {"0": "helper", "1": "main"}, "main": 1, "why": "same"},
|
||||
{"hw": "three", "roles": {"0": "helper", "1": "main", "2": "off"}, "main": 1, "why": "faster"},
|
||||
{"hw": "igpu", "roles": {"0": "off", "1": "main"}, "main": 1, "why": "only"},
|
||||
{"hw": "igpu_only", "roles": {"0": "main"}, "main": 0, "why": "only"},
|
||||
{"hw": "bigger_slower", "roles": {"0": "main", "1": "helper"}, "main": 0, "why": "memory"},
|
||||
{"hw": "small_second", "roles": {"0": "main", "1": "off"}, "main": 0, "why": "faster"},
|
||||
{"hw": "cannot", "roles": {"0": "off", "1": "main"}, "main": 1, "why": "only"},
|
||||
{"hw": "nospeed", "roles": {"0": "helper", "1": "main"}, "main": 1, "why": "memory"}
|
||||
],
|
||||
"set_role": [
|
||||
{"hw": "two", "roles": {"0": "helper", "1": "main"}, "card": 0, "role": "main", "after": {"0": "main", "1": "helper"}, "note": "old_main_helper:1"},
|
||||
{"hw": "two", "roles": {"0": "helper", "1": "main"}, "card": 1, "role": "off", "after": {"0": "main", "1": "off"}, "note": "promoted:0"},
|
||||
{"hw": "two", "roles": {"0": "helper", "1": "main"}, "card": 1, "role": "helper", "after": {"0": "main", "1": "helper"}, "note": "promoted:0"},
|
||||
{"hw": "two", "roles": {"0": "off", "1": "main"}, "card": 1, "role": "helper", "after": {"0": "off", "1": "main"}, "note": "only_main"},
|
||||
{"hw": "two", "roles": {"0": "helper", "1": "main"}, "card": 0, "role": "off", "after": {"0": "off", "1": "main"}, "note": null},
|
||||
{"hw": "two", "roles": {"0": "off", "1": "main"}, "card": 0, "role": "helper", "after": {"0": "helper", "1": "main"}, "note": null},
|
||||
{"hw": "two", "roles": {"0": "off", "1": "off"}, "card": 0, "role": "helper", "after": {"0": "main", "1": "off"}, "note": null},
|
||||
{"hw": "three", "roles": {"0": "helper", "1": "main", "2": "off"}, "card": 2, "role": "helper", "after": {"0": "helper", "1": "main", "2": "helper"}, "note": null},
|
||||
{"hw": "three", "roles": {"0": "helper", "1": "main", "2": "helper"}, "card": 1, "role": "off", "after": {"0": "main", "1": "off", "2": "helper"}, "note": "promoted:0"}
|
||||
],
|
||||
"order": [
|
||||
{"hw": "two", "roles": {"0": "helper", "1": "main"}, "gpu": [0, 1]},
|
||||
{"hw": "two", "roles": {"0": "main", "1": "helper"}, "gpu": [1, 0]},
|
||||
{"hw": "two", "roles": {"0": "off", "1": "main"}, "gpu": [1]},
|
||||
{"hw": "three", "roles": {"0": "helper", "1": "main", "2": "helper"}, "gpu": [2, 0, 1]},
|
||||
{"hw": "two", "roles": {"0": "off", "1": "off"}, "gpu": []}
|
||||
],
|
||||
"warnings": [
|
||||
{"hw": "igpu", "roles": {"0": "main", "1": "helper"}, "codes": ["igpu_main"], "fix": {"card": 1, "role": "main"}},
|
||||
{"hw": "igpu", "roles": {"0": "off", "1": "main"}, "codes": []},
|
||||
{"hw": "cannot", "roles": {"0": "helper", "1": "main"}, "codes": ["cannot_run"]},
|
||||
{"hw": "two", "roles": {"0": "off", "1": "off"}, "codes": ["cpu_only"]},
|
||||
{"hw": "two", "roles": {"0": "helper", "1": "main"}, "spill_gb": 7, "codes": ["low_memory"]},
|
||||
{"hw": "two", "roles": {"0": "main", "1": "helper"}, "codes": []}
|
||||
],
|
||||
"saved": [
|
||||
{"cfg": {"gpu_roles": {"0": "helper", "1": "main"}, "gpu": [0, 1]}, "roles": {"0": "helper", "1": "main"}},
|
||||
{"cfg": {"gpu_roles": {"0": "helper", "1": "main"}}, "roles": {"0": "helper", "1": "main"}},
|
||||
{"cfg": {"gpu_roles": {"0": "helper", "1": "main"}, "gpu": [1, 0]}, "roles": null, "problem": "edited by hand"},
|
||||
{"cfg": {"gpu_roles": {"0": "off", "1": "main"}, "gpu": 1}, "roles": {"0": "off", "1": "main"}},
|
||||
{"cfg": {"gpu_roles": {"0": "off", "1": "off"}, "gpu": [], "gpu_none": true}, "roles": {"0": "off", "1": "off"}},
|
||||
{"cfg": {"gpu_roles": {"0": "main", "1": "main"}}, "roles": null, "problem": "ignored"},
|
||||
{"cfg": {"gpu": [1, 0]}, "roles": null}
|
||||
],
|
||||
"legacy": [
|
||||
{"hw": "two", "gpu": null, "roles": {"0": "helper", "1": "main"}},
|
||||
{"hw": "two", "gpu": [1, 0], "gpu_order": "as_given", "roles": {"0": "main", "1": "helper"}},
|
||||
{"hw": "two", "gpu": 1, "roles": {"0": "off", "1": "main"}},
|
||||
{"hw": "two", "gpu": [1, 0], "roles": {"0": "helper", "1": "main"}},
|
||||
{"hw": "equal", "gpu": [1, 0], "roles": {"0": "main", "1": "helper"}}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
"""strata_control/gpu_roles.py: Main / Helper / Off for every graphics card (the rules the app, `strata launch`, setup and the server share)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from strata_control import gpu_roles as R
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
V = json.loads((ROOT / "tests" / "data" / "gpu_roles_vectors.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def roles(d):
|
||||
return None if d is None else {int(k): v for k, v in d.items()}
|
||||
|
||||
|
||||
class Vectors(unittest.TestCase):
|
||||
"""tests/data/gpu_roles_vectors.json is also run by apps/host/tests/gpu-roles.test.mjs against gpu-roles.ts."""
|
||||
|
||||
def test_recommend(self):
|
||||
for c in V["recommend"]:
|
||||
with self.subTest(c["hw"]):
|
||||
r = R.recommend(V["hw"][c["hw"]])
|
||||
self.assertEqual(r["roles"], roles(c["roles"]))
|
||||
self.assertEqual((r["main"], r["why"]), (c["main"], c["why"]))
|
||||
|
||||
def test_set_role(self):
|
||||
for c in V["set_role"]:
|
||||
with self.subTest(c):
|
||||
out, note = R.set_role(roles(c["roles"]), V["hw"][c["hw"]], c["card"], c["role"])
|
||||
self.assertEqual(out, roles(c["after"]))
|
||||
self.assertEqual(note, c["note"])
|
||||
|
||||
def test_order(self):
|
||||
for c in V["order"]:
|
||||
with self.subTest(c):
|
||||
self.assertEqual(R.order(roles(c["roles"]), V["hw"][c["hw"]]), c["gpu"])
|
||||
|
||||
def test_warnings(self):
|
||||
for c in V["warnings"]:
|
||||
with self.subTest(c):
|
||||
w = R.warnings(roles(c["roles"]), V["hw"][c["hw"]], spill_gb=c.get("spill_gb", 0.0))
|
||||
self.assertEqual([x["code"] for x in w], c["codes"])
|
||||
if c.get("fix"):
|
||||
self.assertEqual(w[0]["fix"], c["fix"])
|
||||
|
||||
def test_saved(self):
|
||||
for c in V["saved"]:
|
||||
with self.subTest(c):
|
||||
got, problem = R.saved_roles(c["cfg"])
|
||||
self.assertEqual(got, roles(c["roles"]))
|
||||
if c.get("problem"):
|
||||
self.assertIn(c["problem"], problem or "")
|
||||
|
||||
def test_legacy(self):
|
||||
for c in V["legacy"]:
|
||||
with self.subTest(c):
|
||||
self.assertEqual(R.from_legacy(c["gpu"], V["hw"][c["hw"]], c.get("gpu_order")), roles(c["roles"]))
|
||||
|
||||
|
||||
class Words(unittest.TestCase):
|
||||
def test_warnings_are_one_short_line_and_say_only_what_a_person_cannot_know(self):
|
||||
two = V["hw"]["two"]
|
||||
# a slower card as Main is NOT warned about: people understand it (owner rule)
|
||||
self.assertEqual(R.warnings({0: "main", 1: "helper"}, two), [])
|
||||
for case in V["warnings"]:
|
||||
for w in R.warnings(roles(case["roles"]), V["hw"][case["hw"]], spill_gb=case.get("spill_gb", 0.0)):
|
||||
self.assertLess(len(w["text"]), 90, w["text"])
|
||||
self.assertNotIn("slower", w["text"].replace("is slow as Main", ""))
|
||||
|
||||
def test_cpu_only_line(self):
|
||||
w = R.warnings({0: "off", 1: "off"}, V["hw"]["two"], engine_runs_cpu_only=True)
|
||||
self.assertEqual(w[0]["text"], "Performance is degraded on CPU only.")
|
||||
w = R.warnings({0: "off", 1: "off"}, V["hw"]["two"])
|
||||
self.assertEqual(w[0]["text"], R.NO_CARD_LINE)
|
||||
|
||||
def test_sentence_and_describe(self):
|
||||
self.assertEqual(R.sentence(V["hw"]["one"]), "Strata will use your RTX 4070 Ti.")
|
||||
s = R.sentence(V["hw"]["three"])
|
||||
self.assertIn("RTX 4070 Ti is Main: it is faster.", s)
|
||||
self.assertIn("RTX 2060 starts Off: 8 GB or less.", s)
|
||||
self.assertEqual(R.describe({0: "helper", 1: "main"}, V["hw"]["two"]), "Main: GPU 2 RTX 4070 Ti. Helper: GPU 1 RTX 3060.")
|
||||
|
||||
def test_cards_with_8_gb_or_less_start_off_but_a_lone_8_gb_card_is_main(self):
|
||||
self.assertEqual(R.recommend([{"index": 0, "name": "RTX 4060", "vram_gb": 8}])["roles"], {0: "main"})
|
||||
r = R.recommend([{"index": 0, "name": "A", "vram_gb": 12, "speed": 9}, {"index": 1, "name": "B", "vram_gb": 8, "speed": 5}, {"index": 2, "name": "C", "vram_gb": 10, "speed": 5}])
|
||||
self.assertEqual(r["roles"], {0: "main", 1: "off", 2: "helper"})
|
||||
|
||||
|
||||
class MirrorAndParse(unittest.TestCase):
|
||||
def test_mirror_writes_roles_and_the_legacy_fields_with_main_last(self):
|
||||
m = R.mirror({0: "helper", 1: "main"}, V["hw"]["two"])
|
||||
self.assertEqual(m["gpu_roles"], {"0": "helper", "1": "main"})
|
||||
self.assertEqual((m["gpu"], m["gpu_order"]), ([0, 1], "as_given"))
|
||||
self.assertNotIn("gpu_none", m)
|
||||
one = R.mirror({0: "off", 1: "main"}, V["hw"]["two"])
|
||||
self.assertEqual(one["gpu"], 1) # one card is a number, never a list (#3203)
|
||||
none = R.mirror({0: "off", 1: "off"}, V["hw"]["two"])
|
||||
self.assertEqual((none["gpu"], none["gpu_none"]), ([], True))
|
||||
|
||||
def test_parse_roles(self):
|
||||
cards = V["hw"]["three"]
|
||||
self.assertEqual(R.parse_roles("1=main, 0=helper,2=off", cards), {1: "main", 0: "helper", 2: "off"})
|
||||
self.assertEqual(R.roles_from_text("1=main,0=helper", cards), {0: "helper", 1: "main", 2: "off"}) # unnamed cards are Off
|
||||
self.assertEqual(R.roles_from_text("0=helper", cards), {0: "main", 1: "off", 2: "off"}) # a lone card on is Main
|
||||
for bad in ("", "1", "1=boss", "x=main", "0=main,1=main", "7=main"):
|
||||
with self.subTest(bad), self.assertRaises(ValueError):
|
||||
R.parse_roles(bad, cards)
|
||||
|
||||
def test_normalize(self):
|
||||
two = V["hw"]["two"]
|
||||
self.assertEqual(R.normalize({0: "helper", 1: "helper"}, two), {0: "helper", 1: "main"})
|
||||
self.assertEqual(R.normalize({0: "main", 1: "main"}, two), {0: "main", 1: "helper"})
|
||||
self.assertEqual(R.normalize({0: "helper", 1: "off"}, two), {0: "main", 1: "off"})
|
||||
|
||||
def test_spill_uses_the_cards_that_are_on(self):
|
||||
two = [{"index": 0, "name": "A", "vram_gb": 12}, {"index": 1, "name": "B", "vram_gb": 12}]
|
||||
a = R.spill_for({0: "main", 1: "helper"}, two, 44.0, 32.0)
|
||||
b = R.spill_for({0: "main", 1: "off"}, two, 44.0, 32.0)
|
||||
self.assertLess(a, b)
|
||||
self.assertEqual(R.spill_for({0: "main", 1: "off"}, two, None, 32.0), 0.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -585,7 +585,7 @@ class AdoptRules(unittest.TestCase):
|
||||
# which cards: named ones win (one card too: --gpu 1 on a box that has them), else the fit rule on a box with several cards
|
||||
two = catalog(HW_2X16)
|
||||
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1, 0]), two), [1, 0])
|
||||
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), two), [0]) # fits one card + RAM
|
||||
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), two), [0, 1]) # roles: card 0 Main (last), card 1 Helper
|
||||
one = catalog(HW_12)
|
||||
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[1]), one), [1])
|
||||
self.assertIsNone(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), one))
|
||||
@@ -614,7 +614,7 @@ class AdoptRules(unittest.TestCase):
|
||||
self.assertIsNone(installed_cards({**chosen, "gpus_asked": False})) # never asked: setup's default, not a choice
|
||||
self.assertIsNone(installed_cards({"gpu": 1}))
|
||||
self.assertIsNone(cards_for_launch(plain, two, base=chosen)) # the installed choice stays
|
||||
self.assertEqual(cards_for_launch(plain, two, base={**chosen, "gpus_asked": False}), [0]) # no choice recorded: the fit rule
|
||||
self.assertEqual(cards_for_launch(plain, two, base={**chosen, "gpus_asked": False}), [0, 1]) # no choice recorded: the recommended roles
|
||||
self.assertEqual(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpus=[0]), two, base=chosen), [0]) # --gpu 0 wins
|
||||
if True:
|
||||
installed = h.base / "strata-iq3_xxs.json"
|
||||
@@ -1190,5 +1190,66 @@ class ConnectCommand(unittest.TestCase):
|
||||
self.assertIn('"reserveTokens": 50536', out)
|
||||
|
||||
|
||||
class GpuRolesFlag(unittest.TestCase):
|
||||
"""--gpu-roles: a role for each card (strata_control/gpu_roles.py); unnamed cards are Off."""
|
||||
|
||||
def setUp(self):
|
||||
self.home = Home().__enter__()
|
||||
self.addCleanup(self.home.__exit__, None, None, None)
|
||||
p = mock.patch.object(Catalog, "hardware", lambda self, fresh=False: dict(HW_2X16, disk_free_gb=500.0))
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def test_roles_reach_the_settings_and_the_plan_says_who_does_what(self):
|
||||
code, out, err = run_cli(["launch", "--plan", "--output", "json", "--model", "iq3_xxs", "--gpu-roles", "1=main,0=helper"])
|
||||
self.assertEqual(code, 0, err or out)
|
||||
s = json.loads(out)["plan"]["settings"]
|
||||
self.assertEqual(s["gpu_roles"], {"1": "main", "0": "helper"})
|
||||
self.assertIsNone(s["gpus"])
|
||||
code, out, err = run_cli(["launch", "--plan", "--model", "iq3_xxs", "--gpu-roles", "1=main,0=helper"])
|
||||
self.assertEqual(code, 0, err or out)
|
||||
self.assertIn("cards Main: GPU 2 RTX A4000. Helper: GPU 1 RTX A4000.", out)
|
||||
self.assertIn("--gpu-roles 0=helper,1=main", out) # the copyable line says the same
|
||||
|
||||
def test_one_card_named_leaves_the_other_off(self):
|
||||
code, out, err = run_cli(["launch", "--plan", "--model", "iq3_xxs", "--gpu-roles", "1=main"])
|
||||
self.assertEqual(code, 0, err or out)
|
||||
self.assertIn("Main: GPU 2 RTX A4000. Off: GPU 1 RTX A4000.", out)
|
||||
|
||||
def test_bad_roles_are_plain_sentences(self):
|
||||
for argv, word in ((["--gpu-roles", "1=boss"], "main, helper, off"), (["--gpu-roles", "0=main,1=main"], "only one card can be Main"),
|
||||
(["--gpu-roles", "1"], "number=role"), (["--gpu", "0", "--gpu-roles", "1=main"], "not both")):
|
||||
code, out, err = run_cli(["launch", "--plan", *argv])
|
||||
self.assertEqual(code, 2, argv)
|
||||
self.assertIn(word, err, argv)
|
||||
code, out, err = run_cli(["launch", "--plan", "--model", "iq3_xxs", "--gpu-roles", "7=main"])
|
||||
self.assertIn("There is no card 7", out + err)
|
||||
|
||||
def test_the_launch_config_gets_the_roles_and_the_mirror_with_main_last(self):
|
||||
from strata_cli.launch.runner import cards_for_launch
|
||||
two = catalog(HW_2X16)
|
||||
mirror = cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpu_roles={"1": "main", "0": "helper"}), two)
|
||||
self.assertEqual((mirror["gpu"], mirror["gpu_order"]), ([0, 1], "as_given"))
|
||||
self.assertEqual(mirror["gpu_roles"], {"0": "helper", "1": "main"})
|
||||
base = {"exe": "x", "args": [], "gpu": 1, "gpus_asked": True, "hip_ordinal": 0, "gpu_roles": {"0": "off", "1": "main"}}
|
||||
cfg = derive_config(base, S.LaunchSettings(family="qwen", model="IQ3_XXS"), bool_flags=frozenset(), gpus=mirror)
|
||||
self.assertEqual((cfg["gpu"], cfg["layer_split"], cfg["gpu_order"]), ([0, 1], "auto", "as_given"))
|
||||
self.assertNotIn("hip_ordinal", cfg)
|
||||
one = cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpu_roles={"0": "main"}), two)
|
||||
cfg = derive_config(base, S.LaunchSettings(family="qwen", model="IQ3_XXS"), bool_flags=frozenset(), gpus=one)
|
||||
self.assertEqual((cfg["gpu"], cfg["gpu_roles"], "layer_split" in cfg), (0, {"0": "main", "1": "off"}, False))
|
||||
none = cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS", gpu_roles={"0": "off", "1": "off"}), two)
|
||||
self.assertEqual((none["gpu"], none["gpu_none"]), ([], True))
|
||||
|
||||
def test_a_plain_card_list_replaces_an_earlier_roles_choice_and_the_installed_roles_are_kept(self):
|
||||
from strata_cli.launch.runner import apply_cards, cards_for_launch, installed_cards
|
||||
cfg = {"gpu": 1, "gpu_roles": {"0": "off", "1": "main"}, "gpu_cards": {}, "gpus_asked": True}
|
||||
apply_cards(cfg, [0, 1])
|
||||
self.assertEqual((cfg["gpu"], "gpu_roles" in cfg), ([0, 1], False))
|
||||
saved = {"gpu": [0, 1], "gpu_order": "as_given", "gpu_roles": {"0": "helper", "1": "main"}}
|
||||
self.assertEqual(installed_cards(saved), [0, 1])
|
||||
self.assertIsNone(cards_for_launch(S.LaunchSettings(family="qwen", model="IQ3_XXS"), catalog(HW_2X16), base=saved))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -859,38 +859,58 @@ class SaveAndRun(Base):
|
||||
|
||||
|
||||
class CardsStep(Base):
|
||||
def test_cards_step_defaults_to_the_largest_card_plus_ram_and_offers_the_split(self):
|
||||
def test_cards_step_shows_a_role_for_every_card_and_the_recommendation_is_selected(self):
|
||||
async def case(app, pilot, ctx):
|
||||
await press(pilot, "enter", "enter")
|
||||
t = text_of(app)
|
||||
self.assertIn("Which graphics cards should it use?", t)
|
||||
self.assertIn("Step 2 of 10", t)
|
||||
flat = " ".join(t.split())
|
||||
self.assertIn("+ RAM", flat) # every choice says what it uses
|
||||
self.assertIn("(layer split)", flat) # the second card is offered as a split
|
||||
self.assertIn("[ Main ]", flat) # equal cards: the highest number is Main (the engine order stays [0, 1])
|
||||
self.assertIn("[ Helper ]", flat)
|
||||
self.assertIn("recommended", flat)
|
||||
self.assertIn("fits: needs", flat) # plain numbers
|
||||
cur = app.screen.menu().current
|
||||
self.assertEqual(len(cur.key), 1) # the recommended choice is the single largest card
|
||||
await press(pilot, "enter") # keeping it stores "let Strata choose": the same rule runs in setup
|
||||
self.assertIn("M Main", flat)
|
||||
self.assertNotIn("slower", flat) # no explaining warning for a recommendation
|
||||
await press(pilot, "enter") # keeping the recommendation stores "not set": a card added later is re-recommended
|
||||
self.assertIsNone(ctx.settings.gpu_roles)
|
||||
self.assertIsNone(ctx.settings.gpus)
|
||||
self.assertIn("Where should the model files be stored?", text_of(app))
|
||||
self.run_app(case, ctx=make_ctx(HW_2X16))
|
||||
|
||||
def test_choosing_the_split_stores_both_cards(self):
|
||||
async def case(app, pilot, ctx):
|
||||
await press(pilot, "enter", "enter", "down", "enter")
|
||||
self.assertEqual(ctx.settings.gpus, [0, 1])
|
||||
self.run_app(case, ctx=make_ctx(HW_2X16))
|
||||
|
||||
def test_a_model_too_big_for_one_card_and_ram_recommends_the_split(self):
|
||||
def test_m_h_o_set_the_role_of_the_card_under_the_cursor_and_r_goes_back(self):
|
||||
async def case(app, pilot, ctx):
|
||||
await press(pilot, "enter", "enter")
|
||||
cur = app.screen.menu().current
|
||||
self.assertEqual(sorted(cur.key), [0, 1])
|
||||
hw = dict(HW_2X16)
|
||||
hw["ram_gb"] = 24.0
|
||||
self.run_app(case, ctx=make_ctx(hw))
|
||||
await press(pilot, "o") # card 1 (a Helper) Off
|
||||
flat = " ".join(text_of(app).split())
|
||||
self.assertIn("recommended: helper", flat) # the muted tag says what was recommended
|
||||
await press(pilot, "m") # Main on card 1: card 2 becomes a Helper, one quiet line says so
|
||||
flat = " ".join(text_of(app).split())
|
||||
self.assertIn("is now a Helper.", flat)
|
||||
await press(pilot, "enter")
|
||||
self.assertEqual(ctx.settings.gpu_roles, {"0": "main", "1": "helper"})
|
||||
self.assertIsNone(ctx.settings.gpus)
|
||||
self.run_app(case, ctx=make_ctx(HW_2X16))
|
||||
|
||||
def test_r_restores_the_recommendation(self):
|
||||
async def case(app, pilot, ctx):
|
||||
await press(pilot, "enter", "enter", "down", "o", "r", "enter")
|
||||
self.assertIsNone(ctx.settings.gpu_roles)
|
||||
self.run_app(case, ctx=make_ctx(HW_2X16))
|
||||
|
||||
def test_every_card_off_is_allowed_with_one_short_line(self):
|
||||
async def case(app, pilot, ctx):
|
||||
await press(pilot, "enter", "enter", "down", "o", "up", "o")
|
||||
flat = " ".join(text_of(app).split())
|
||||
self.assertIn("! This engine needs a graphics card to start. Turn one on.", flat)
|
||||
await press(pilot, "enter")
|
||||
self.assertEqual(ctx.settings.gpu_roles, {"0": "off", "1": "off"}) # allowed; the server says plainly what the engine can do
|
||||
self.run_app(case, ctx=make_ctx(HW_2X16))
|
||||
|
||||
def test_d_shows_what_the_roles_mean(self):
|
||||
async def case(app, pilot, ctx):
|
||||
await press(pilot, "enter", "enter", "d")
|
||||
self.assertIn("Helper: holds a share of the model", " ".join(text_of(app).split()))
|
||||
self.run_app(case, ctx=make_ctx(HW_2X16))
|
||||
|
||||
def test_an_integrated_chip_beside_one_card_gives_no_cards_step(self):
|
||||
async def case(app, pilot, ctx):
|
||||
|
||||
+6
-1
@@ -647,7 +647,7 @@ class Strata:
|
||||
"context": int(val("--max-context")) if str(val("--max-context") or "").isdigit() else None,
|
||||
"kv": val("--kv"), "images": ("gpu" if vis.get("gpu") else "cpu") if isinstance(vis, dict) else "off",
|
||||
"port": cfg.get("port", DEFAULT_PORT), "host": cfg.get("host", "127.0.0.1"),
|
||||
"gpu": cfg.get("gpu"), "api_key_set": bool(cfg.get("api_key")),
|
||||
"gpu": cfg.get("gpu"), "gpu_roles": cfg.get("gpu_roles"), "api_key_set": bool(cfg.get("api_key")),
|
||||
"low_ram": "--resident-experts" in a or "--mmap-experts" in a,
|
||||
"start_script": script.name if script.exists() else None,
|
||||
"ready": not missing and bool(cfg), "missing_files": missing[:3], "parts_missing": len(gaps),
|
||||
@@ -929,6 +929,11 @@ class Strata:
|
||||
except Exception: # noqa: BLE001 - unknown stays unknown: the app then keeps its safety default
|
||||
pass
|
||||
hw["gpus"] = nv + amd
|
||||
try: # how fast each card is (SMs x clock): the app and the Terminal pick the Main card by it
|
||||
from strata_control import gpu_roles
|
||||
gpu_roles.add_speeds(hw["gpus"])
|
||||
except Exception: # noqa: BLE001 - unknown speed: the card with the most memory is Main
|
||||
pass
|
||||
self._engine_code(hw)
|
||||
self._engine_vendor(hw)
|
||||
if WIN:
|
||||
|
||||
@@ -298,6 +298,11 @@
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
@@ -353,6 +358,11 @@
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
@@ -404,8 +414,17 @@
|
||||
],
|
||||
"cwd": "<T>",
|
||||
"exe": "<T>/engine/<EXE>",
|
||||
"gpu": 0,
|
||||
"gpu": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
"<cuda>"
|
||||
],
|
||||
@@ -448,15 +467,24 @@
|
||||
"131072",
|
||||
"--kv",
|
||||
"int8",
|
||||
"--mmap-experts",
|
||||
"--resident-experts",
|
||||
"--vision",
|
||||
"--vram-reserve-mib",
|
||||
"700"
|
||||
],
|
||||
"cwd": "<T>",
|
||||
"exe": "<T>/engine/<EXE>",
|
||||
"gpu": 0,
|
||||
"gpu": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
"<cuda>"
|
||||
],
|
||||
@@ -509,6 +537,10 @@
|
||||
},
|
||||
"exe": "<T>/engine/<EXE>",
|
||||
"gpu": 0,
|
||||
"gpu_roles": {
|
||||
"0": "main",
|
||||
"1": "off"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"lib_dirs": [
|
||||
"<cuda>"
|
||||
@@ -561,6 +593,11 @@
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
@@ -611,8 +648,17 @@
|
||||
],
|
||||
"cwd": "<T>",
|
||||
"exe": "<T>/engine/<EXE>",
|
||||
"gpu": 0,
|
||||
"gpu": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
"<cuda>"
|
||||
],
|
||||
@@ -660,8 +706,17 @@
|
||||
],
|
||||
"cwd": "<T>",
|
||||
"exe": "<T>/engine/<EXE>",
|
||||
"gpu": 0,
|
||||
"gpu": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
"<cuda>"
|
||||
],
|
||||
@@ -705,13 +760,22 @@
|
||||
"65536",
|
||||
"--kv",
|
||||
"int8",
|
||||
"--mmap-experts",
|
||||
"--resident-experts",
|
||||
"--vision"
|
||||
],
|
||||
"cwd": "<T>",
|
||||
"exe": "<T>/engine/<EXE>",
|
||||
"gpu": 0,
|
||||
"gpu": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"gpu_order": "as_given",
|
||||
"gpu_roles": {
|
||||
"0": "helper",
|
||||
"1": "main"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"layer_split": "auto",
|
||||
"lib_dirs": [
|
||||
"<cuda>"
|
||||
],
|
||||
@@ -763,6 +827,10 @@
|
||||
},
|
||||
"exe": "<T>/engine/<EXE>",
|
||||
"gpu": 0,
|
||||
"gpu_roles": {
|
||||
"0": "main",
|
||||
"1": "off"
|
||||
},
|
||||
"gpus_asked": true,
|
||||
"lib_dirs": [
|
||||
"<cuda>"
|
||||
|
||||
@@ -98,6 +98,7 @@ def install(ram, found, argv, answers=None, extra=(), avx512=False, configs=()):
|
||||
mock.patch.object(setup, "load_settings", lambda: {}),
|
||||
mock.patch.object(setup, "save_settings", lambda s: None),
|
||||
mock.patch.object(setup, "gpus", lambda: found),
|
||||
mock.patch.object(setup, "card_speeds", lambda: {}),
|
||||
mock.patch.object(setup, "amd_gpus", lambda: []),
|
||||
mock.patch.object(setup, "ram_gb", lambda: ram),
|
||||
mock.patch.object(setup, "cpu_info", lambda: ("Test CPU", True, avx512)),
|
||||
@@ -179,7 +180,7 @@ class Golden(unittest.TestCase):
|
||||
lowram_multi = self.golden["32GB-2x24GB qwen IQ3_XXS"]["config"] # #364 -> #642: low-RAM, two cards -> both
|
||||
self.assertIn("--resident-experts" if "--resident-experts" in lowram_multi["args"] else "--mmap-experts",
|
||||
lowram_multi["args"])
|
||||
self.assertEqual(lowram_multi.get("gpu"), [0, 1]) # 0.1.40 (#642): resident on both cards
|
||||
self.assertEqual(lowram_multi.get("gpu"), [0, 1]) # 0.1.40 (#642): resident on both cards; roles: Main 1 goes last, as before
|
||||
|
||||
def test_yes(self):
|
||||
for key, ram, found, family, model in self.cases():
|
||||
|
||||
+26
-12
@@ -1,5 +1,5 @@
|
||||
"""The owner's multi-GPU rule in setup (strata_control/gpu_fit.py): the model goes on the LARGEST card + RAM first; the second card is
|
||||
offered as a layer split and is the default only when the model does not fit one card (with RAM) or would be too tight.
|
||||
"""Graphics card roles in setup (strata_control/gpu_roles.py): the best card is Main, the others start as Helper, a card with 8 GB or less
|
||||
starts Off; one question, "Use these roles?"; --gpu-roles names them.
|
||||
|
||||
python tools/test_setup_gpu_fit.py
|
||||
"""
|
||||
@@ -28,28 +28,42 @@ class SetupFitRule(unittest.TestCase):
|
||||
self.assertEqual(cfg["gpu"], 0) if "gpu" in cfg else None
|
||||
self.assertFalse(any("Which GPUs" in q for q in asked), asked)
|
||||
|
||||
def test_two_unequal_cards_and_the_model_fits_one_defaults_to_the_largest_card_and_offers_the_split(self):
|
||||
def test_two_unequal_cards_the_big_one_is_main_and_a_card_with_8_gb_or_less_starts_off(self):
|
||||
found = [card(0, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86"), card(1, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86")]
|
||||
code, out, cfg, asked = run(62.0, found, "IQ3_XXS")
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(cfg["gpu"], 1) # the 16 GB card, alone
|
||||
self.assertEqual(cfg["gpu_roles"], {"0": "off", "1": "main"})
|
||||
self.assertNotIn("layer_split", cfg)
|
||||
self.assertIn("layer split", out) # the split was offered
|
||||
self.assertIn("1) RTX 3070 Ti + RAM", out.replace("NVIDIA GeForce ", ""))
|
||||
self.assertTrue(any("Which GPUs" in q and "[1]" in q for q in asked), asked)
|
||||
# choosing the split is one answer away
|
||||
code, out, cfg, _ = run(62.0, found, "IQ3_XXS", answers={"Which GPUs?": "2"})
|
||||
self.assertTrue(any("Use these roles?" in q and "[y]" in q for q in asked), asked)
|
||||
# one answer away: the 8 GB card as a Helper
|
||||
code, out, cfg, _ = run(62.0, found, "IQ3_XXS", answers={"Use these roles?": "n", "Roles, e.g.": "1=main,0=helper"})
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(sorted(cfg["gpu"]), [0, 1])
|
||||
self.assertEqual(cfg["gpu"], [0, 1]) # Main LAST: the engine's stage order
|
||||
self.assertEqual(cfg["gpu_order"], "as_given")
|
||||
self.assertEqual(cfg["gpu_roles"], {"0": "helper", "1": "main"})
|
||||
self.assertEqual(cfg["layer_split"], "auto")
|
||||
|
||||
def test_two_small_cards_and_the_model_does_not_fit_one_defaults_to_the_split(self):
|
||||
def test_two_equal_cards_start_with_the_highest_number_as_main(self):
|
||||
found = [card(0, "NVIDIA GeForce RTX 3060", 12.0, "86"), card(1, "NVIDIA GeForce RTX 3060", 12.0, "86")]
|
||||
code, out, cfg, asked = run(26.0, found, "IQ3_XXS")
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(sorted(cfg["gpu"]), [0, 1])
|
||||
self.assertEqual(cfg["gpu"], [0, 1])
|
||||
self.assertEqual(cfg["gpu_roles"], {"0": "helper", "1": "main"})
|
||||
self.assertEqual(cfg["gpu_order"], "as_given")
|
||||
self.assertEqual(cfg["layer_split"], "auto")
|
||||
self.assertTrue(any("Which GPUs" in q and "[2]" in q for q in asked), asked)
|
||||
|
||||
def test_gpu_roles_flag(self):
|
||||
found = [card(0, "NVIDIA GeForce RTX 3060", 12.0, "86"), card(1, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86")]
|
||||
code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu-roles", "0=main,1=helper"))
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(cfg["gpu"], [1, 0]) # the slower card is Main on purpose: it goes last
|
||||
self.assertEqual(cfg["gpu_roles"], {"0": "main", "1": "helper"})
|
||||
code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu-roles", "1=main,0=off"))
|
||||
self.assertEqual((cfg["gpu"], cfg["gpu_roles"]), (1, {"0": "off", "1": "main"}))
|
||||
self.assertNotIn("layer_split", cfg)
|
||||
code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu-roles", "0=off,1=off"))
|
||||
self.assertNotEqual(code, 0)
|
||||
|
||||
def test_naming_the_cards_still_wins(self):
|
||||
found = [card(0, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86"), card(1, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86")]
|
||||
|
||||
@@ -238,7 +238,7 @@ class LowRamGpus(unittest.TestCase):
|
||||
def test_asked_and_answered_two(self):
|
||||
ram, found = PROFILES["32GB-2x24GB"]
|
||||
code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
|
||||
answers={"Which GPUs?": "2", "Low-RAM mode: which GPUs?": "2"}) # the split asked for (the default is the largest card + RAM)
|
||||
answers={"Low-RAM mode: which GPUs?": "2"}) # the roles are the default (Main last); the low-RAM question still defaults to one card
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertTrue(any("Low-RAM mode: which GPUs? [1]" in q for q in asked), asked)
|
||||
self.assertEqual(cfg["gpu"], [0, 1])
|
||||
@@ -276,7 +276,7 @@ class LowRamGpus(unittest.TestCase):
|
||||
ram, found = PROFILES["32GB-2x24GB"]
|
||||
with mock.patch.object(setup, "MIN_ENGINE", setup.RESIDENT_SPLIT_ENGINE):
|
||||
code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
|
||||
answers={"Which GPUs?": "2"}) # the split chosen (the default is the largest card + RAM)
|
||||
answers={"Use these roles?": "y"}) # the recommended roles: Main 0 (last), Helper 1
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(cfg["gpu"], [0, 1])
|
||||
self.assertIn("--resident-experts", cfg["args"])
|
||||
@@ -401,11 +401,12 @@ class SplitShortCard(unittest.TestCase):
|
||||
eights = [card(i, "NVIDIA GeForce RTX 3070", 8.0, "86") for i in range(2)]
|
||||
self.assertEqual(setup.split_short(eights), []) # neither card alone reads more
|
||||
|
||||
def test_yes_takes_the_first_card(self):
|
||||
def test_yes_takes_the_recommended_roles(self):
|
||||
"""The 10 GB card is above the 8 GB line, so it starts as a Helper (the owner's rule); the first card is Main and goes last."""
|
||||
code, out, cfg, _ = install(127.8, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start"])
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(cfg["gpu"], 0)
|
||||
self.assertIn("too small to lend a split its prompt buffers", out)
|
||||
self.assertEqual(cfg["gpu"], [1, 0])
|
||||
self.assertEqual(cfg["gpu_roles"], {"0": "main", "1": "helper"})
|
||||
|
||||
def test_named_pair_is_kept(self):
|
||||
code, out, cfg, _ = install(127.8, self.FOUND, ["--family", "qwen", "--model", "Q2_0", "--no-start",
|
||||
|
||||
@@ -352,7 +352,7 @@ class Main(Base):
|
||||
"""#967: images are allowed with UD-Q4_K_XL (tested end to end since MODEL-UPDATE)."""
|
||||
code, out, cfg = self.main(["--context", "8192", "--vision", "yes", "--low-ram", "on"], n_gpus=2)
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("(NVIDIA GeForce RTX 5070, 12 GB) only (--gpus 0,1 shares it across", out)
|
||||
self.assertIn("(NVIDIA GeForce RTX 5070, 12 GB) only (--gpus 1,0 shares it across", out)
|
||||
self.assertNotIn("images are not available", out)
|
||||
self.assertIn("images: on", out)
|
||||
self.assertNotIn("are untested", out) # MODEL-UPDATE: run end to end (CUDA, HIP)
|
||||
@@ -385,8 +385,7 @@ class Main(Base):
|
||||
self.assertAlmostEqual(need, setup.MODELS[M]["download_gb"] + setup.UNSLOTH_RAM_LEFT_GB)
|
||||
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8) # no --gpus: stays on one GPU
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertNotIn("uses them together anyway", out) # one card + RAM is the default now: nothing to warn about
|
||||
self.assertNotIn("layer_split", cfg)
|
||||
self.assertNotIn("layer_split", cfg) # the model cannot share its RAM budget: one card (the Main one)
|
||||
self.assertEqual(cfg["args"][cfg["args"].index("--resident-budget-gib") + 1], "71") # all of them
|
||||
code, out, cfg = self.main(["--context", "8192", "--gpus", "0,1"], n_gpus=2, ram=127.8) # asked for by name
|
||||
self.assertEqual(code, 0, out)
|
||||
@@ -434,15 +433,14 @@ class Main(Base):
|
||||
self.assertIn("--resident-budget-gib", cfg["args"])
|
||||
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, answers={"which GPUs?": "2"})
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(cfg["gpu"], [0, 1])
|
||||
self.assertEqual(cfg["gpu"], [1, 0])
|
||||
self.assertIn("--resident-budget-gib", cfg["args"])
|
||||
|
||||
@budget_mode()
|
||||
def test_yes_alone_keeps_one_gpu(self):
|
||||
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=165.0)
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertIn("layer split", out) # the second card is offered as a split
|
||||
self.assertIn("(recommended)", out)
|
||||
self.assertIn("Main: GPU 1", out) # the roles are shown; the model keeps to one card
|
||||
self.assertNotIn("layer_split", cfg)
|
||||
self.assertIn("--resident-budget-gib", cfg["args"])
|
||||
|
||||
@@ -644,10 +642,10 @@ class LayerSplit(unittest.TestCase):
|
||||
argv = ["--family", "unsloth", "--model", M, "--context", "32768", "--no-start"]
|
||||
code, out, cfg, asked = install(165.0, self.found, argv, answers={"which GPUs?": "1", "Which GPUs?": "1"})
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertTrue(any("Which GPUs? [1]" in q for q in asked), asked) # the fit rule's question: the largest card + RAM is the default
|
||||
self.assertTrue(any("Use these roles? [y]" in q for q in asked), asked) # the roles question; the model's own question follows
|
||||
self.assertEqual(cfg["gpu"], 0)
|
||||
self.assertIn("--resident-budget-gib", cfg["args"])
|
||||
code, out, cfg, asked = install(165.0, self.found, argv, answers={"Which GPUs?": "2", f"{M}: which GPUs?": "2"})
|
||||
code, out, cfg, asked = install(165.0, self.found, argv, answers={f"{M}: which GPUs?": "2"})
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(cfg["gpu"], [0, 1])
|
||||
self.assertEqual(cfg["layer_split"], "auto")
|
||||
@@ -806,9 +804,9 @@ class Resident(Base):
|
||||
|
||||
def test_two_gpus_split_with_every_expert_in_ram(self):
|
||||
"""Where the experts fit, two cards are the GGUF models' layer split (no budget question, no #498 note)."""
|
||||
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8, answers={"Which GPUs?": "2"}) # the split chosen
|
||||
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8, answers={"Use these roles?": "y"}) # the recommended roles
|
||||
self.assertEqual(code, 0, out)
|
||||
self.assertEqual(cfg["gpu"], [0, 1])
|
||||
self.assertEqual(cfg["gpu"], [1, 0])
|
||||
self.assertEqual(cfg["layer_split"], "auto")
|
||||
self.assertNotIn("--resident-budget-gib", cfg["args"])
|
||||
self.assertNotIn("which GPUs?", out)
|
||||
|
||||
Reference in new issue
Block a user