mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 14:21:06 -10:00
DFlash2 graphics memory follows the pack in use: 0.72 GB (715 MiB) with the Q4_0 selector, 0.89 GB (890 MiB) otherwise; a SYCL build says it cannot read the Q4_0 selector
drafting.DRAFTER_MIB gains dflash2_selq4 (715); drafting.vram_gb(data_dir, engine) gives the true figure (an engine program that carries 'selector codebooks: Q4_0 rows' and a Q4_0 pack that setup / Load converts), strata drafting plan reports vram_gb and selq4 and its text says the same number, Control passes capabilities.dflash2Selq4 to the page and the fit (apps/host/src/lib/native-fit.ts DFLASH2_SELQ4_VRAM_GB, apps/host/server/native-parallel.mjs) uses it; apps/host/tests/drafting.test.mjs reads DRAFTER_MIB from drafting.py and fails when a copy differs. Marker per backend: CUDA and HIP builds carry the text (src/core/dflash2.cpp); the SYCL drafter (sycl/src/core/dflash2.cpp, bf16 select kernel) does not read Q4_0 rows, its built binary (P1006 rc1sycl) has no such text so it stays on the bf16 pack, and its loader now refuses a Q4_0 codebook with a plain message instead of tiling it as a matrix. Metal: DF2-METAL's f-0291-metal-selq4. The idea of the Q4_0 selector is nazeshinjite's (nazeshinjite/Qwen3.8-Flash-Next-DFlash2-GGUF).
This commit is contained in:
1 parent
ebac5686f9
commit
d4c8ca1f65
11 files changed
+320
-201
No files matched your search
@@ -69,9 +69,9 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
|
||||
const append=(instanceId,line)=>{logs.push({at:new Date().toISOString(),instanceId,line:String(line).slice(0,2000)});if(logs.length>400)logs.shift();};
|
||||
function diskSpace(path){try{const s=statfsSync(path);return {free:Number(s.bavail)*Number(s.bsize),total:Number(s.blocks)*Number(s.bsize)};}catch{return null;}}
|
||||
/* Does the engine in this release serve DFlash2? Control knows (strata_control/drafting.py); asked once, false until it answers. The Models screen offers the choice only when true. */
|
||||
let dflash2Served=null,dflash2Support=null;
|
||||
function probeDrafting(){if(!packaged||dflash2Served!==null||typeof control.draftingPlan!=='function')return;dflash2Served=false;control.draftingPlan({mode:'auto'}).then(p=>{dflash2Served=p.serves===true;dflash2Support=p.serves===true&&p.support&&p.support_line?{level:p.support,line:p.support_line}:null;emit();}).catch(()=>{});}
|
||||
function state(){const folder=preferences.downloadDirectory??(modelsDirectory?modelsDirectory():snapshot.installed?.data_folder),space=folder?diskSpace(folder):null;return {...snapshot,downloadDirectory:folder,downloadFreeBytes:space?.free??null,downloadTotalBytes:space?.total??null,fields:nativeFields(),instances:[...instances.values()].map(({key,configPath,fingerprint,mcpServers,...i})=>structuredClone(i)),defaultId,stoppedForUpdate:stoppedForUpdate?{updated:stoppedForUpdate.updated,at:stoppedForUpdate.at,models:stoppedForUpdate.instances.map(i=>({id:i.id,label:i.label}))}:null,jobs:[...jobs.values()].slice(-12).map(({process,abort,engineAbort,rawResult,...j})=>j),capabilities:{responses:sourceText().includes('/v1/responses'),updates:!!snapshot.available,rollback:previousEngine(),componentUpdates:packaged&&typeof control.update==='function',calibration:packaged?false:snapshot.hardware?.gpus?.some(g=>/nvidia/i.test(g.name??''))??false,dflash2:dflash2Served===true,dflash2Support}};}
|
||||
let dflash2Served=null,dflash2Support=null,dflash2Selq4=false;
|
||||
function probeDrafting(){if(!packaged||dflash2Served!==null||typeof control.draftingPlan!=='function')return;dflash2Served=false;control.draftingPlan({mode:'auto'}).then(p=>{dflash2Served=p.serves===true;dflash2Support=p.serves===true&&p.support&&p.support_line?{level:p.support,line:p.support_line}:null;dflash2Selq4=p.serves===true&&p.selq4===true;emit();}).catch(()=>{});}
|
||||
function state(){const folder=preferences.downloadDirectory??(modelsDirectory?modelsDirectory():snapshot.installed?.data_folder),space=folder?diskSpace(folder):null;return {...snapshot,downloadDirectory:folder,downloadFreeBytes:space?.free??null,downloadTotalBytes:space?.total??null,fields:nativeFields(),instances:[...instances.values()].map(({key,configPath,fingerprint,mcpServers,...i})=>structuredClone(i)),defaultId,stoppedForUpdate:stoppedForUpdate?{updated:stoppedForUpdate.updated,at:stoppedForUpdate.at,models:stoppedForUpdate.instances.map(i=>({id:i.id,label:i.label}))}:null,jobs:[...jobs.values()].slice(-12).map(({process,abort,engineAbort,rawResult,...j})=>j),capabilities:{responses:sourceText().includes('/v1/responses'),updates:!!snapshot.available,rollback:previousEngine(),componentUpdates:packaged&&typeof control.update==='function',calibration:packaged?false:snapshot.hardware?.gpus?.some(g=>/nvidia/i.test(g.name??''))??false,dflash2:dflash2Served===true,dflash2Support,dflash2Selq4}};}
|
||||
let sourceCache={path:'',time:0,text:''};
|
||||
function sourceText(){try{const path=join(packaged?control.control.dir:preferences.repo,'serve/server.py'),time=statSync(path).mtimeMs;if(sourceCache.path===path&&sourceCache.time===time)return sourceCache.text;sourceCache={path,time,text:readFileSync(path,'utf8')};return sourceCache.text;}catch{return '';}}
|
||||
function nativeFields(){const source=sourceText();return NATIVE_FIELDS.map(field=>({...field,supported:APP_OWNED_FIELDS.includes(field.id)||field.id.startsWith('vision_')||field.id.startsWith('mcp_')||field.id==='vram_reserve_mib'||source.includes(field.id)}));}
|
||||
|
||||
@@ -51,6 +51,8 @@ export const FIRST_CARD_OTHER_GB = 6.9;
|
||||
export const MTP_VRAM_GB = 1.08;
|
||||
/** Graphics memory of DFlash2 alone, with the MTP layer not loaded beside it (RESULTS-DF2MEM.md: 890 MiB; the MTP layer 1082 MiB). Fit, plan and slot estimates use the drafter that is chosen. */
|
||||
export const DFLASH2_VRAM_GB = 0.89;
|
||||
/** The same with the converted pack (the selector's codebooks as Q4_0 rows, 715 MiB; strata_control/drafting.py DRAFTER_MIB "dflash2_selq4"): `capabilities.dflash2Selq4` says it is the one in use. */
|
||||
export const DFLASH2_SELQ4_VRAM_GB = 0.72;
|
||||
export const NEXT_CARD_OTHER_GB = 2.0;
|
||||
/**
|
||||
* `--batch-mtp` lets each slot verify draft tokens. Where it pays and where it does not (all measured):
|
||||
|
||||
@@ -137,6 +137,8 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
const draftingChoices = selectedSize ? draftingOptions(selectedSize.sourceFamily ?? selectedFamily?.family, native?.capabilities?.dflash2) : null;
|
||||
// The drafter this run loads (one per run): the fit and the plan use its graphics memory (DFlash2 0.89 GB, MTP 1.08 GB).
|
||||
const nowDrafter = draftingChoices ? resolveDrafter(drafting, selectedSize?.sourceFamily ?? selectedFamily?.family, native?.capabilities?.dflash2) : 'mtp';
|
||||
// DFlash2 with the converted pack (the selector as Q4_0 rows) takes 175 MiB less graphics memory: the fit says the true number (Control's `selq4`, strata_control/drafting.py vram_gb).
|
||||
const fitDrafter = nowDrafter === 'dflash2' && native?.capabilities?.dflash2Selq4 === true ? 'dflash2_selq4' : nowDrafter;
|
||||
const draftingWords: Record<string, string> = { auto: 'Auto (recommended)', mtp: 'MTP', dflash2: 'DFlash2', off: 'Off' };
|
||||
const values = useMemo(() => ({ family: selectedSize?.sourceFamily, model: selectedSize?.model, context, vision: selectedSize?.images ? vision : 'no', kv, low_ram: lowRam, resourceProfile, ...(cardsGpus.length > 1 ? { gpus: cardsGpus.join(',') } : cardsGpus.length === 1 ? { gpu: cardsGpus[0] } : {}), loadAfter: true, multiple: expert && multiple }), [selectedSize?.sourceFamily, selectedSize?.model, context, vision, kv, lowRam, resourceProfile, cardsGpus.join(','), expert, multiple]);
|
||||
const downloadValues = { ...values, ...(draftingChoices ? { drafting } : {}), vision: selectedSize?.images && vision !== 'no' ? 'yes' : 'no', loadVision: selectedSize?.images ? vision : 'no', ...(noReader ? { visionUnavailable: true } : {}), ...(contextPicked ? { contextChosen: true } : {}) };
|
||||
@@ -245,7 +247,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
|
||||
{tr(selectedSize ? <>
|
||||
<div className="native-choice__head"><span className="native-choice__label">{tr("Version")}</span><span className="native-choice__tools"><NativePresets values={values} onApply={v => { const f = families.find(f => f.family === v.family || f.sizes.some(s => s.sourceFamily === v.family)); if (!f)
|
||||
return; presetPending.current = true; setFamily(f.family); setSize(f.sizes.find(s => s.model === v.model && (s.sourceFamily === v.family || f.family === v.family))?.id ?? ''); setContext(v.context ?? 32768); setVision(v.vision ?? 'no'); setKv(v.kv ?? 'int8'); setLowRam(v.low_ram ?? 'auto'); }}/><Btn icon="models" size="sm" variant="ghost" onClick={() => setManage(true)}>{tr("Manage models")}</Btn></span></div>
|
||||
<div className="native-version"><VersionPicker pick={native?.catalog?.recommendation} versions={selectedFamily?.sizes ?? []} details={expert} selectedId={selectedSize.id} hardware={fitHardware} cache={{ context, kv, drafter: nowDrafter }} downloaded={readyIds} onSelect={id => { const f = families.find(f => f.sizes.some(s => s.id === id)); if (f) { setFamily(f.family); setSize(id); setError(''); } }} onOwnGguf={() => (onImport ?? onOtherModels)()}/></div>
|
||||
<div className="native-version"><VersionPicker pick={native?.catalog?.recommendation} versions={selectedFamily?.sizes ?? []} details={expert} selectedId={selectedSize.id} hardware={fitHardware} cache={{ context, kv, drafter: fitDrafter }} downloaded={readyIds} onSelect={id => { const f = families.find(f => f.sizes.some(s => s.id === id)); if (f) { setFamily(f.family); setSize(id); setError(''); } }} onOwnGguf={() => (onImport ?? onOtherModels)()}/></div>
|
||||
{tr(fitState === 'ssd' ? <p className="small muted native-spill-note"><Icon name="disk" size={16}/>{tr("SSD streaming · will run slower")}</p> : fitState === 'no' ? <p className="small muted native-spill-note"><Icon name="warning" size={16}/>{tr("Doesn't fit")}</p> : null)}
|
||||
{tr(selectedSize?.multi_file && deepseekContextNote(context) ? <p className="small muted native-spill-note"><Icon name="info" size={16}/>{tr(deepseekContextNote(context))}</p> : null)}
|
||||
<div className="native-choice__row">
|
||||
|
||||
@@ -25,7 +25,9 @@ export { DENSE_GB };
|
||||
* (phaseA-tests/blackwell/RESULTS-DF2MEM.md section 1; strata_control/drafting.py DRAFTER_GB). */
|
||||
export const MTP_VRAM_GB = 1.08;
|
||||
export const DFLASH2_VRAM_GB = 0.89;
|
||||
export const DRAFTER_VRAM_GB: Record<string, number> = { mtp: MTP_VRAM_GB, dflash2: DFLASH2_VRAM_GB, off: 0 };
|
||||
/** DFlash2 alone with the converted pack (the selector's codebooks as Q4_0 rows: 715 MiB; strata_control/drafting.py DRAFTER_MIB "dflash2_selq4"). The page passes `dflash2_selq4` when `capabilities.dflash2Selq4` is true. */
|
||||
export const DFLASH2_SELQ4_VRAM_GB = 0.72;
|
||||
export const DRAFTER_VRAM_GB: Record<string, number> = { mtp: MTP_VRAM_GB, dflash2: DFLASH2_VRAM_GB, dflash2_selq4: DFLASH2_SELQ4_VRAM_GB, off: 0 };
|
||||
const round1 = (n: number) => Math.round(n * 10) / 10;
|
||||
|
||||
export function memoryNeeded(v: FitInput): number {
|
||||
|
||||
@@ -12,7 +12,7 @@ export interface NativeState {
|
||||
/** The picture reader (strata-vision) is installed with the engine, so models marked for pictures can read them here. */
|
||||
/** Strata Control's view of the engine for this computer (the app): usable false = none found or recognised; `notice` says why in plain words. */
|
||||
engine?:{usable:boolean;notice?:string|null;wrongMake?:boolean;version?:string|null;/** What the last update check found newer (kept on this PC, read without the network): the Qwen engine, the DeepSeek engine, the app. */updateAvailable?:{engine?:string|null;dsv41?:string|null;app?:string|null}|null};
|
||||
visionReader?: boolean;/** The installed app (Strata Control is bundled): picture files, engines and updates go through it. */packaged?: boolean;available:boolean;repo:string;downloadDirectory?:string;/** Free bytes on the drive that holds the models folder. */downloadFreeBytes?:number|null;downloadTotalBytes?:number|null;version?:string|null;summary:string;error:string|null;catalog:{families:NativeFamily[];contexts:number[];recommendation:{family:string|null;model:string|null;context?:number;why?:string};context_note:string}|null;installed:{models:{model:string;config:string;model_name:string;ready:boolean;/** The picture-reading files are on this computer (notes-split-agent8.md, item 8). */visionPrepared?:boolean;/** The data folder the model's own run config keeps its pack and draft layer in (not the app's models folder). */runDataFolder?:string;context?:number;kv?:string;images?:string;visionEnabled?:boolean;visionDevice?:string;lowRamMode?:string;resourceProfile?:'work'|'performance';/** The Drafting choice saved with the model: auto | mtp | dflash2 | off. */drafting?:string;/** DeepSeek: the most context its pack reads (max_position_embeddings in the pack's ds4.txt); absent when the pack does not say. */contextMax?:number;/** The context was picked by a person (a marker saved with the model); a saved 32K without it is the old default. */contextChosen?:boolean;dataFolder?:string;diskGb?:number;diskBytes?:number;/** Files this model shares with another (the picture reader, the draft layer): counted once, in `usage`. */sharedBytes?:number;/** What Delete frees: files in Strata's own folders that no other model uses. */removableGb?:number;missing_files?:string[]}[];/** What all models take on disk, from their real files, shared files counted once. */usage?:{total_bytes:number;total_gb:number;shared_bytes:number;shared_gb:number;models:Record<string,{bytes:number;gb:number}>}|null;data_folder:string;engine:{version:string|null}}|null;hardware:{ram_gb:number;engine_vendor?:string;gpus:{index:number;name:string;vram_gb:number;usable:boolean;/** SMs x clock: only the order between cards matters (strata_control/gpu_roles.py). */speed?:number;vendor?:string;uma?:boolean;integrated?:boolean;unified_gb?:number;problem?:string|null}[]}|null;fields:NativeField[];instances:NativeInstance[];defaultId:string|null;/** Models an engine update stopped, to offer starting them again with the same settings (updated: false = the update did not finish, the old engine is unchanged). */stoppedForUpdate?:{updated:boolean;at:string;models:{id:string;label:string}[]}|null;jobs:NativeJob[];capabilities:{/** The engine serves DFlash2 (Control says): the Drafting choice then offers it. */dflash2?:boolean;/** What was measured on this kind of card, in Control's words (null until Control answers). */dflash2Support?:{level:'tested'|'slower'|'untested';line:string}|null;responses?:boolean;updates:boolean;rollback:boolean;calibration:boolean;componentUpdates?:boolean}}
|
||||
visionReader?: boolean;/** The installed app (Strata Control is bundled): picture files, engines and updates go through it. */packaged?: boolean;available:boolean;repo:string;downloadDirectory?:string;/** Free bytes on the drive that holds the models folder. */downloadFreeBytes?:number|null;downloadTotalBytes?:number|null;version?:string|null;summary:string;error:string|null;catalog:{families:NativeFamily[];contexts:number[];recommendation:{family:string|null;model:string|null;context?:number;why?:string};context_note:string}|null;installed:{models:{model:string;config:string;model_name:string;ready:boolean;/** The picture-reading files are on this computer (notes-split-agent8.md, item 8). */visionPrepared?:boolean;/** The data folder the model's own run config keeps its pack and draft layer in (not the app's models folder). */runDataFolder?:string;context?:number;kv?:string;images?:string;visionEnabled?:boolean;visionDevice?:string;lowRamMode?:string;resourceProfile?:'work'|'performance';/** The Drafting choice saved with the model: auto | mtp | dflash2 | off. */drafting?:string;/** DeepSeek: the most context its pack reads (max_position_embeddings in the pack's ds4.txt); absent when the pack does not say. */contextMax?:number;/** The context was picked by a person (a marker saved with the model); a saved 32K without it is the old default. */contextChosen?:boolean;dataFolder?:string;diskGb?:number;diskBytes?:number;/** Files this model shares with another (the picture reader, the draft layer): counted once, in `usage`. */sharedBytes?:number;/** What Delete frees: files in Strata's own folders that no other model uses. */removableGb?:number;missing_files?:string[]}[];/** What all models take on disk, from their real files, shared files counted once. */usage?:{total_bytes:number;total_gb:number;shared_bytes:number;shared_gb:number;models:Record<string,{bytes:number;gb:number}>}|null;data_folder:string;engine:{version:string|null}}|null;hardware:{ram_gb:number;engine_vendor?:string;gpus:{index:number;name:string;vram_gb:number;usable:boolean;/** SMs x clock: only the order between cards matters (strata_control/gpu_roles.py). */speed?:number;vendor?:string;uma?:boolean;integrated?:boolean;unified_gb?:number;problem?:string|null}[]}|null;fields:NativeField[];instances:NativeInstance[];defaultId:string|null;/** Models an engine update stopped, to offer starting them again with the same settings (updated: false = the update did not finish, the old engine is unchanged). */stoppedForUpdate?:{updated:boolean;at:string;models:{id:string;label:string}[]}|null;jobs:NativeJob[];capabilities:{/** The engine serves DFlash2 (Control says): the Drafting choice then offers it. */dflash2?:boolean;/** Control's word that DFlash2 uses the converted pack (the selector as Q4_0 rows, 175 MiB less graphics memory): the fit says 0.72 GB instead of 0.89 */dflash2Selq4?:boolean;/** What was measured on this kind of card, in Control's words (null until Control answers). */dflash2Support?:{level:'tested'|'slower'|'untested';line:string}|null;responses?:boolean;updates:boolean;rollback:boolean;calibration:boolean;componentUpdates?:boolean}}
|
||||
let state:NativeState|null=null;
|
||||
const listeners=new Set<()=>void>();
|
||||
/** True once the first read of the native state has ended, whatever its answer: until then the Models page must not pick a model of its own (#3740). */
|
||||
|
||||
@@ -186,7 +186,7 @@ test('a new install fetches only the drafter it will use; the fit and the slot e
|
||||
assert.equal(chooseParallel({ config, hardware, drafter: 'dflash2', values: { parallel: 4 } }).slots, 4, 'a number the person chose stays (the start then loads the MTP and says so)');
|
||||
assert.notEqual(chooseParallel({ config: { ...config, gpu: [0, 1] }, hardware: { gpus: [{ index: 0, vram_gb: 24 }, { index: 1, vram_gb: 24 }] }, drafter: 'dflash2' }).slots, 1, 'a layer split cannot run DFlash2: the slots stay with the MTP');
|
||||
const fit = read('../src/lib/native-fit.ts');
|
||||
assert.match(fit, /DRAFTER_VRAM_GB: Record<string, number> = \{ mtp: MTP_VRAM_GB, dflash2: DFLASH2_VRAM_GB, off: 0 \}/);
|
||||
assert.match(fit, /DRAFTER_VRAM_GB: Record<string, number> = \{ mtp: MTP_VRAM_GB, dflash2: DFLASH2_VRAM_GB, dflash2_selq4: DFLASH2_SELQ4_VRAM_GB, off: 0 \}/);
|
||||
});
|
||||
|
||||
test('a failed MTP download says one short line, not the installer text, and the Load goes on', () => {
|
||||
@@ -195,3 +195,30 @@ test('a failed MTP download says one short line, not the installer text, and the
|
||||
assert.match(fn, /job\.drafterNote='The MTP drafter could not be downloaded\.'/);
|
||||
assert.doesNotMatch(fn, /error\.message/);
|
||||
});
|
||||
|
||||
test('the drafter memory figures in the app are the ones strata_control/drafting.py holds (DRAFTER_MIB), and the page uses the true one', async () => {
|
||||
const py = read('../../../strata_control/drafting.py');
|
||||
const table = JSON.parse(py.match(/^DRAFTER_MIB = (\{[^}]*\})/m)[1]);
|
||||
const gb = mib => Math.floor(mib / 10 + 0.5) / 100; // the same rounding as DRAFTER_GB (half up to 10 MiB)
|
||||
assert.deepEqual(Object.keys(table).sort(), ['dflash2', 'dflash2_selq4', 'mtp', 'off']);
|
||||
const parallel = await import('../server/native-parallel.mjs');
|
||||
assert.equal(parallel.MTP_VRAM_GB, gb(table.mtp));
|
||||
assert.equal(parallel.DFLASH2_VRAM_GB, gb(table.dflash2));
|
||||
assert.equal(parallel.DFLASH2_SELQ4_VRAM_GB, gb(table.dflash2_selq4));
|
||||
assert.deepEqual([table.dflash2, table.dflash2_selq4, table.mtp], [890, 715, 1082]);
|
||||
const fit = read('../src/lib/native-fit.ts');
|
||||
const num = name => Number(fit.match(new RegExp(`export const ${name} = ([0-9.]+);`))[1]);
|
||||
assert.equal(num('MTP_VRAM_GB'), gb(table.mtp));
|
||||
assert.equal(num('DFLASH2_VRAM_GB'), gb(table.dflash2));
|
||||
assert.equal(num('DFLASH2_SELQ4_VRAM_GB'), gb(table.dflash2_selq4));
|
||||
assert.match(fit, /DRAFTER_VRAM_GB: Record<string, number> = \{ mtp: MTP_VRAM_GB, dflash2: DFLASH2_VRAM_GB, dflash2_selq4: DFLASH2_SELQ4_VRAM_GB, off: 0 \}/);
|
||||
// Control's word reaches the page, and the page hands the fit the drafter it really loads
|
||||
const control = read('../server/native-control.mjs');
|
||||
assert.match(control, /dflash2Selq4=p\.serves===true&&p\.selq4===true/);
|
||||
assert.match(control, /dflash2Support,dflash2Selq4\}/);
|
||||
assert.match(read('../src/lib/native.ts'), /dflash2Selq4\?:boolean/);
|
||||
const page = read('../src/components/NativeModels.tsx');
|
||||
assert.match(page, /const fitDrafter = nowDrafter === 'dflash2' && native\?\.capabilities\?\.dflash2Selq4 === true \? 'dflash2_selq4' : nowDrafter;/);
|
||||
assert.match(page, /drafter: fitDrafter/);
|
||||
assert.doesNotMatch(page, /drafter: nowDrafter/);
|
||||
});
|
||||
@@ -69,7 +69,7 @@ def run(a) -> int:
|
||||
decision = drafting.decide(**want)
|
||||
support = drafting.hardware_support(want["backend"], want["archs"])
|
||||
# serves: this computer's engine can run DFlash2 (the build has the code and the card is one it is made for); the app offers the choice only then
|
||||
result = {**drafting_fetch.plan(folder), "needed": needed, "mtp_here": want["mtp_here"], "mtp_needed": mtp_needed, "mtp_gb": MTP_FETCH_GB if mtp_needed else 0.0,
|
||||
result = {**drafting_fetch.plan(folder, drafting.installed_engine_exe()), "needed": needed, "mtp_here": want["mtp_here"], "mtp_needed": mtp_needed, "mtp_gb": MTP_FETCH_GB if mtp_needed else 0.0,
|
||||
"serves": drafting.ENGINE_SERVES_DFLASH2 and support.can_run, "drafter": decision.drafter,
|
||||
"choice": drafting.choice_of(want["mode"], model_name=want["model_name"], family=want["family"], backend=want["backend"], archs=want["archs"]),
|
||||
"note": decision.note, "line": drafting.summary_line(decision), "support": support.level, "support_line": support.line}
|
||||
|
||||
@@ -42,8 +42,11 @@ DFLASH2_BACKENDS = ("cuda", "hip", "metal", "sycl")
|
||||
|
||||
#: Graphics memory each drafter takes on Qwen3.8-Flash-Next IQ3_S (GB; phaseA-tests/blackwell/RESULTS-DF2MEM.md section 1, l620, engine free memory at
|
||||
#: the first graph capture): DFlash2 alone 890 MiB, the MTP layer 1082 MiB. The fit, the plan and the slot estimates use the one that is chosen.
|
||||
DRAFTER_MIB = {"dflash2": 890, "mtp": 1082, "off": 0}
|
||||
DRAFTER_GB = {name: round(mib / 1000, 2) for name, mib in DRAFTER_MIB.items()} # 0.89 and 1.08, the figures the apps show
|
||||
#: `dflash2_selq4` is DFlash2 alone with the converted pack (the selector's codebooks as Q4_0 rows, 175 MiB less: the engine's drafter line 800 -> 625 MiB, section 16):
|
||||
#: 715 MiB. vram_gb() says which of the two is the true figure on this computer. The apps keep copies of these numbers (apps/host/server/native-parallel.mjs and
|
||||
#: apps/host/src/lib/native-fit.ts); apps/host/tests/drafting.test.mjs reads this table and fails when they differ.
|
||||
DRAFTER_MIB = {"dflash2": 890, "dflash2_selq4": 715, "mtp": 1082, "off": 0}
|
||||
DRAFTER_GB = {name: int(mib / 10 + 0.5) / 100 for name, mib in DRAFTER_MIB.items()} # 0.89, 0.72, 1.08, the figures the apps show (mib rounded half up to 10 MiB)
|
||||
|
||||
#: GPU architecture (what nvidia-smi's compute capability and the kernel driver report) -> card class
|
||||
ARCH_CLASS = (
|
||||
@@ -209,7 +212,7 @@ CONVERTED_BYTES = 339694592
|
||||
CONVERTED_SHA256 = "4a42b98220ea7880664002c8edda1563aecbae754b762a2e2f31b2b91ee93ecd"
|
||||
_MARK_SELQ4 = b"selector codebooks: Q4_0 rows" # text of an engine that reads such a pack (src/core/dflash2.cpp)
|
||||
LEGACY_PACK_NAME = "qwen3.8-flash-next-dflash2.q8.pack" # what Strata 0.2.9 test builds packed: still used when it is the one on disk
|
||||
VRAM_GB = DRAFTER_GB["dflash2"] # graphics memory with the MTP layer not loaded: 890 MiB (RESULTS-DF2MEM.md); the engine reports "drafter 799 MiB" for the pack and buffers alone
|
||||
VRAM_GB = DRAFTER_GB["dflash2"] # graphics memory with the MTP layer not loaded and the bf16-selector pack: 890 MiB (RESULTS-DF2MEM.md); vram_gb() has the true figure for this computer
|
||||
FOLDER = "dflash2" # under the model data folder, next to ``mtp``
|
||||
SPEC_WINDOW = 6 # --spec with DFlash2: the widest window of drafted words per round (measured best on Blackwell; the engine caps it per card)
|
||||
MTP_WINDOW = 4 # the MTP's usual --spec: the window it runs when DFlash2 falls back to it, unless the config names another
|
||||
@@ -329,6 +332,35 @@ def engine_reads_selector_q4(exe: str | os.PathLike[str] | None) -> bool:
|
||||
return _selq4[key]
|
||||
|
||||
|
||||
def uses_selector_q4(data_dir: str | Path | None, engine: str | os.PathLike[str] | None = None) -> bool:
|
||||
"""Is this computer's DFlash2 start the one with the Q4_0 selector (the converted pack, 175 MiB less)? True when the installed engine reads such a pack and the drafter is
|
||||
(or will be, at setup / Load) the Q4_0 pack it is made from; a Q8 pack an earlier Strata made is never converted. An engine that is not known: False (the larger figure)."""
|
||||
if not engine or not engine_reads_selector_q4(engine):
|
||||
return False
|
||||
if data_dir and not _here(Path(data_dir) / FOLDER / PACK_NAME) and _here(Path(data_dir) / FOLDER / LEGACY_PACK_NAME):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def vram_gb(data_dir: str | Path | None = None, engine: str | os.PathLike[str] | None = None) -> float:
|
||||
"""Graphics memory of DFlash2 alone on this computer, in GB: DRAFTER_GB["dflash2_selq4"] (0.72) with the converted pack, DRAFTER_GB["dflash2"] (0.89) otherwise. The fit,
|
||||
the plan text and the slot estimates say this number."""
|
||||
return DRAFTER_GB["dflash2_selq4" if uses_selector_q4(data_dir, engine) else "dflash2"]
|
||||
|
||||
|
||||
def installed_engine_exe() -> Path | None:
|
||||
"""The engine program of the newest installed engine component for this computer's backend (None: not found). A quick folder scan, nothing is hashed or started."""
|
||||
try:
|
||||
from strata_control import engine_component as ec
|
||||
found, _ = ec.scan(None)
|
||||
backend = (detect_hardware()[0] or "").lower()
|
||||
best = [c for c in found if not backend or str(c.backend).split(":")[0].lower() == backend] or list(found)
|
||||
best.sort(key=lambda c: ec._version_key(c.version), reverse=True)
|
||||
return best[0].exe if best and best[0].exe.is_file() else None
|
||||
except Exception: # noqa: BLE001 - informational: an unknown engine means the larger figure
|
||||
return None
|
||||
|
||||
|
||||
def pack_path(data_dir: str | Path, engine: str | os.PathLike[str] | None = None) -> Path:
|
||||
"""The pack the engine is started with: the converted pack (Q4_0 selector) when it is here, verified, and `engine` is a program that reads it; else the Q4_0 pack, else
|
||||
a Q8 pack an earlier Strata made (no download for it), else where the Q4_0 pack will be."""
|
||||
|
||||
@@ -29,17 +29,18 @@ class DraftingError(RuntimeError):
|
||||
"""The drafter could not be fetched or packed. The message is plain words and says what to do."""
|
||||
|
||||
|
||||
def plan(data_dir: str | Path) -> dict:
|
||||
"""What fetching would do, in numbers and words. ``gb`` is 0 when the drafter is already here."""
|
||||
def plan(data_dir: str | Path, engine: str | os.PathLike[str] | None = None) -> dict:
|
||||
"""What fetching would do, in numbers and words. ``gb`` is 0 when the drafter is already here. ``engine``: the installed engine program (None: not known); ``vram_gb`` is
|
||||
the graphics memory DFlash2 alone takes on this computer (0.72 with the Q4_0 selector, which the engine reads and setup makes, else 0.89) and ``selq4`` says which."""
|
||||
ready = drafting.pack_ready(data_dir)
|
||||
gb = 0.0 if ready else drafting.DOWNLOAD_GB
|
||||
if ready:
|
||||
text = "The DFlash2 drafter is already on this computer."
|
||||
else:
|
||||
text = (f"The DFlash2 drafter is {drafting.DOWNLOAD_GB:.1f} GB to download (it is packed to {drafting.PACK_GB:.2f} GB for the engine and the "
|
||||
f"download is removed), from {drafting.REPO}, {drafting.LICENSE}. It needs about {drafting.VRAM_GB:.1f} GB of graphics memory, "
|
||||
f"download is removed), from {drafting.REPO}, {drafting.LICENSE}. It needs about {drafting.vram_gb(data_dir, engine):.1f} GB of graphics memory, "
|
||||
"less than the MTP's layer, which is not loaded beside it.")
|
||||
return {"ready": ready, "gb": gb, "pack": str(drafting.pack_path(data_dir)), "repo": drafting.REPO, "revision": drafting.REVISION, "text": text}
|
||||
return {"ready": ready, "gb": gb, "vram_gb": drafting.vram_gb(data_dir, engine), "selq4": drafting.uses_selector_q4(data_dir, engine), "pack": str(drafting.pack_path(data_dir)), "repo": drafting.REPO, "revision": drafting.REVISION, "text": text}
|
||||
|
||||
|
||||
def _packer():
|
||||
|
||||
@@ -139,6 +139,13 @@ bool Dflash2Drafter::load(const std::string& path, int device, std::string& err)
|
||||
fseeko(f, data_off + (off_t) t.off, SEEK_SET);
|
||||
#endif
|
||||
if (std::fread(stage.data(), 1, t.bytes, f) != t.bytes) { err = "dflash2: the pack is shorter than its index"; return false; }
|
||||
if (t.dtype == "q4_0" && t.name.find("_codebook") != std::string::npos) {
|
||||
// The pack with the selector as Q4_0 rows (tools/dflash2_pack.py --q4_0-selector; strata_control/drafting.py CONVERTED_NAME) is for engines that print
|
||||
// "selector codebooks: Q4_0 rows" (CUDA, HIP, Metal). This build's select kernel reads bf16 rows only, so it says so instead of reading garbage; the app
|
||||
// never hands it that pack (it looks for that text in the engine program and this file does not have it).
|
||||
err = "dflash2: this build reads bf16 selector codebooks only; " + path + " has them as Q4_0 (use the pack without --q4_0-selector)";
|
||||
return false;
|
||||
}
|
||||
if (t.dtype == "q4_0") { // GGML Q4_0 blocks -> the tiled nibble plane at the tensor's offset, then the d plane (same total bytes)
|
||||
cs_->memcpy(tmp, stage.data(), t.bytes).wait();
|
||||
kernels::df2_q4_tile(tmp, blob_ + t.off, blob_ + t.off + (uint64_t) t.rows * (uint64_t) t.cols / 2, (int) t.cols, t.rows * t.cols / 32, cs_);
|
||||
|
||||
@@ -1,186 +1,232 @@
|
||||
"""The DFlash2 pack with the Q4_0 selector (174 MiB less graphics memory): made on this computer from the checked Q4_0 pack, used only when it verifies and the engine reads it.
|
||||
|
||||
The idea and the bytes are nazeshinjite's (https://huggingface.co/nazeshinjite/Qwen3.8-Flash-Next-DFlash2-GGUF, Apache-2.0): the community GGUF stores the selector's two
|
||||
codebooks as Q4_0, and tools/dflash2_pack.py --q4_0-selector writes exactly those bytes (checked against the GGUF with the real checkpoint: RESULTS-DF2MEM.md section 15).
|
||||
These tests use the small synthetic checkpoint of tools/test_dflash2_pack.py: the real pins cannot be exercised without the 1.2 GB download, so the pinned numbers are
|
||||
checked for consistency and the mechanism with pins of the synthetic pack.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from strata_control import drafting, drafting_fetch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VOCAB_REAL, RANK_REAL = 248320, 256
|
||||
|
||||
|
||||
def _load(name, rel):
|
||||
spec = importlib.util.spec_from_file_location(name, ROOT / rel)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules.setdefault(name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class PinnedNumbers(unittest.TestCase):
|
||||
def test_the_converted_pack_is_the_q4_pack_less_the_codebooks_difference(self):
|
||||
"""CONVERTED_BYTES follows from PACK_BYTES by arithmetic alone: two codebooks of 248,320 x 256 go from bf16 (2 bytes a value) to Q4_0 (18 bytes per 32 values)."""
|
||||
bf16 = 2 * VOCAB_REAL * RANK_REAL * 2
|
||||
q4 = 2 * VOCAB_REAL * (RANK_REAL // 32) * 18
|
||||
self.assertEqual(drafting.PACK_BYTES - drafting.CONVERTED_BYTES, bf16 - q4)
|
||||
self.assertEqual(bf16 - q4, 182763520)
|
||||
self.assertEqual(len(drafting.CONVERTED_SHA256), 64)
|
||||
self.assertNotEqual(drafting.CONVERTED_SHA256, drafting.PACK_SHA256)
|
||||
|
||||
|
||||
class SmallPacks(unittest.TestCase):
|
||||
"""A synthetic checkpoint packed as the Q4_0 pack, then converted."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.synth = _load("t_pack_sel", "tools/test_dflash2_pack.py")
|
||||
cls.packer = _load("strata_dflash2_pack_sel", "tools/dflash2_pack.py")
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.dir = Path(self.tmp.name)
|
||||
self.synth.write_safetensors(self.dir / "m.safetensors", self.synth.tensors())
|
||||
(self.dir / "c.json").write_text(json.dumps(self.synth.CFG))
|
||||
self.data = self.dir / "data"
|
||||
self.q4 = drafting.target_path(self.data)
|
||||
self.q4.parent.mkdir(parents=True)
|
||||
self.packer.pack(str(self.dir / "m.safetensors"), str(self.dir / "c.json"), str(self.q4), say=lambda *_: None, q4=True)
|
||||
self.src_pins = {"source_bytes": self.q4.stat().st_size, "source_sha256": hashlib.sha256(self.q4.read_bytes()).hexdigest()}
|
||||
direct = self.dir / "direct.pack"
|
||||
self.packer.pack(str(self.dir / "m.safetensors"), str(self.dir / "c.json"), str(direct), say=lambda *_: None, q4=True, q4_selector=True)
|
||||
self.pins = {"size": direct.stat().st_size, "sha256": hashlib.sha256(direct.read_bytes()).hexdigest(), **self.src_pins}
|
||||
self.direct = direct
|
||||
|
||||
def convert(self, **extra):
|
||||
return drafting_fetch.convert_selector(self.data, **{**self.pins, **extra})
|
||||
|
||||
def test_the_conversion_reproduces_the_pinned_bytes(self):
|
||||
"""The pack converted from the finished Q4_0 pack is byte-identical to the one packed straight from the checkpoint with --q4_0-selector (the pinned hash)."""
|
||||
out = self.convert()
|
||||
self.assertEqual(out, drafting.converted_path(self.data))
|
||||
self.assertEqual(out.read_bytes(), self.direct.read_bytes())
|
||||
self.assertLess(out.stat().st_size, self.q4.stat().st_size)
|
||||
self.assertTrue(drafting.converted_ok(self.data, size=self.pins["size"], sha256=self.pins["sha256"]))
|
||||
self.assertTrue(self.q4.is_file(), "the Q4_0 pack stays: it is the fallback")
|
||||
|
||||
def test_every_tensor_but_the_two_codebooks_is_carried_over_unchanged(self):
|
||||
out = self.convert()
|
||||
a, b = self.q4.read_bytes(), out.read_bytes()
|
||||
|
||||
def table(blob):
|
||||
n = int.from_bytes(blob[8:12], "little")
|
||||
rows = {}
|
||||
for line in blob[12:12 + n].decode().split("\n"):
|
||||
w = line.split()
|
||||
if w and w[0] == "t":
|
||||
rows[w[1]] = (w[2], int(w[5]), int(w[6]))
|
||||
return rows, 12 + n
|
||||
ta, base_a = table(a)
|
||||
tb, base_b = table(b)
|
||||
self.assertEqual(set(ta), set(tb))
|
||||
books = {"candidate_selector.predecessor_codebook", "candidate_selector.successor_codebook"}
|
||||
for name in ta:
|
||||
ba = a[base_a + ta[name][1]: base_a + ta[name][1] + ta[name][2]]
|
||||
bb = b[base_b + tb[name][1]: base_b + tb[name][1] + tb[name][2]]
|
||||
if name in books:
|
||||
self.assertEqual((ta[name][0], tb[name][0]), ("bf16", "q4_0"))
|
||||
self.assertEqual(len(bb) * 32, len(ba) * 9, name) # 18 bytes per 32 values against 2 bytes a value
|
||||
else:
|
||||
self.assertEqual(ba, bb, name)
|
||||
|
||||
def test_a_missing_corrupt_or_short_converted_pack_falls_back_to_the_q4_pack(self):
|
||||
eng = self.dir / "engine"
|
||||
eng.write_bytes(b"\0" * 64 + drafting._MARK_SELQ4 + b"\0" * 64)
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "not made yet")
|
||||
out = self.convert()
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]):
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), out)
|
||||
blob = bytearray(out.read_bytes())
|
||||
blob[len(blob) // 2] ^= 0xFF # one flipped byte, the same size, and the marker still describes the old file
|
||||
out.write_bytes(bytes(blob))
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "a corrupt file (the marker no longer matches it) is not used")
|
||||
out.write_bytes(bytes(blob)[:-1000])
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "a short file is not used")
|
||||
out.unlink()
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "a missing file is not used")
|
||||
|
||||
def test_a_q4_pack_that_is_not_the_pinned_one_is_left_alone(self):
|
||||
self.assertIsNone(self.convert(source_sha256="0" * 64))
|
||||
self.assertFalse(drafting.converted_path(self.data).exists())
|
||||
self.assertTrue(self.q4.is_file())
|
||||
|
||||
def test_a_result_that_does_not_match_the_pin_is_deleted(self):
|
||||
self.assertIsNone(self.convert(sha256="f" * 64))
|
||||
self.assertFalse(drafting.converted_path(self.data).exists())
|
||||
self.assertFalse(drafting.converted_path(self.data).with_suffix(".tmp").exists())
|
||||
|
||||
def test_a_failing_conversion_is_never_an_error_for_the_caller(self):
|
||||
with mock.patch.object(drafting_fetch, "_packer", side_effect=RuntimeError("no numpy")):
|
||||
self.assertIsNone(self.convert())
|
||||
self.assertFalse(drafting.converted_path(self.data).exists())
|
||||
|
||||
def test_the_engine_must_read_the_pack_before_it_is_used(self):
|
||||
out = self.convert()
|
||||
old = self.dir / "old-engine"
|
||||
old.write_bytes(b"\0" * 256)
|
||||
new = self.dir / "new-engine"
|
||||
new.write_bytes(b"\0" * 64 + drafting._MARK_SELQ4 + b"\0" * 64)
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]):
|
||||
self.assertEqual(drafting.pack_path(self.data), self.q4, "no engine named: the pack every engine reads")
|
||||
self.assertEqual(drafting.pack_path(self.data, old), self.q4, "an engine without the text would read the rows as garbage")
|
||||
self.assertEqual(drafting.pack_path(self.data, self.dir / "none"), self.q4)
|
||||
self.assertEqual(drafting.pack_path(self.data, new), out)
|
||||
|
||||
def test_auto_arguments_point_at_the_converted_pack(self):
|
||||
self.convert()
|
||||
new = self.dir / "new-engine"
|
||||
new.write_bytes(b"\0" * 64 + drafting._MARK_SELQ4 + b"\0" * 64 + b"--draft-fallback" + b"--mtp-draft-vocab FILE (or --mtp DIR")
|
||||
vocab = self.data / drafting.FOLDER / drafting.VOCAB_NAME
|
||||
vocab.write_bytes(b"\1\0\0\0" * 8)
|
||||
cfg = {"model_name": "qwen3.8-flash-next-iq3_s", "drafting": "dflash2", "args": ["--spec", "4", "--expert-cache", "1000"]}
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]), \
|
||||
mock.patch.object(drafting, "ENGINE_SERVES_DFLASH2", True), mock.patch.object(drafting, "pack_ready", return_value=True):
|
||||
drafting.apply(cfg, klass="blackwell", data_dir=self.data, backend="cuda", archs=["sm_120"], engine=new)
|
||||
args = cfg["args"]
|
||||
self.assertEqual(args[args.index("--draft-model") + 1], str(drafting.converted_path(self.data)))
|
||||
cfg2 = {"model_name": "qwen3.8-flash-next-iq3_s", "drafting": "dflash2", "args": ["--spec", "4", "--expert-cache", "1000"]}
|
||||
drafting.apply(cfg2, klass="blackwell", data_dir=self.data, backend="cuda", archs=["sm_120"], engine=None)
|
||||
self.assertEqual(cfg2["args"][cfg2["args"].index("--draft-model") + 1], str(self.q4), "an engine that is not known gets the pack every engine reads")
|
||||
|
||||
def test_needs_conversion_only_when_dflash2_is_chosen_and_the_q4_pack_is_here(self):
|
||||
want = dict(mode="dflash2", model_name="qwen3.8-flash-next-iq3_s", data_dir=self.data, backend="cuda", archs=["sm_120"])
|
||||
with mock.patch.object(drafting, "ENGINE_SERVES_DFLASH2", True), mock.patch.object(drafting, "_here", lambda p: Path(p).is_file()):
|
||||
self.assertTrue(drafting.needs_conversion(**want))
|
||||
self.assertFalse(drafting.needs_conversion(**{**want, "mode": "mtp"}))
|
||||
self.assertFalse(drafting.needs_conversion(**{**want, "data_dir": self.dir / "empty"}))
|
||||
self.convert()
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]):
|
||||
self.assertFalse(drafting.needs_conversion(**want), "already made")
|
||||
|
||||
def test_setup_and_load_convert_a_start_does_not(self):
|
||||
"""ensure() (strata drafting fetch, setup) converts a pack that is already here, with no network; the serving side only reads."""
|
||||
with mock.patch.object(drafting_fetch, "convert_selector") as convert, mock.patch.object(drafting, "pack_ready", return_value=True):
|
||||
drafting_fetch.ensure(self.data)
|
||||
convert.assert_called_once()
|
||||
source = (ROOT / "serve" / "server.py").read_text(encoding="utf-8")
|
||||
self.assertNotIn("convert_selector", source)
|
||||
self.assertNotIn("convert_selector", (ROOT / "strata_cli" / "launch" / "plan.py").read_text(encoding="utf-8") if (ROOT / "strata_cli" / "launch" / "plan.py").exists() else "")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
"""The DFlash2 pack with the Q4_0 selector (174 MiB less graphics memory): made on this computer from the checked Q4_0 pack, used only when it verifies and the engine reads it.
|
||||
|
||||
The idea and the bytes are nazeshinjite's (https://huggingface.co/nazeshinjite/Qwen3.8-Flash-Next-DFlash2-GGUF, Apache-2.0): the community GGUF stores the selector's two
|
||||
codebooks as Q4_0, and tools/dflash2_pack.py --q4_0-selector writes exactly those bytes (checked against the GGUF with the real checkpoint: RESULTS-DF2MEM.md section 15).
|
||||
These tests use the small synthetic checkpoint of tools/test_dflash2_pack.py: the real pins cannot be exercised without the 1.2 GB download, so the pinned numbers are
|
||||
checked for consistency and the mechanism with pins of the synthetic pack.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from strata_control import drafting, drafting_fetch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
VOCAB_REAL, RANK_REAL = 248320, 256
|
||||
|
||||
|
||||
def _load(name, rel):
|
||||
spec = importlib.util.spec_from_file_location(name, ROOT / rel)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules.setdefault(name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class PinnedNumbers(unittest.TestCase):
|
||||
def test_the_converted_pack_is_the_q4_pack_less_the_codebooks_difference(self):
|
||||
"""CONVERTED_BYTES follows from PACK_BYTES by arithmetic alone: two codebooks of 248,320 x 256 go from bf16 (2 bytes a value) to Q4_0 (18 bytes per 32 values)."""
|
||||
bf16 = 2 * VOCAB_REAL * RANK_REAL * 2
|
||||
q4 = 2 * VOCAB_REAL * (RANK_REAL // 32) * 18
|
||||
self.assertEqual(drafting.PACK_BYTES - drafting.CONVERTED_BYTES, bf16 - q4)
|
||||
self.assertEqual(bf16 - q4, 182763520)
|
||||
self.assertEqual(len(drafting.CONVERTED_SHA256), 64)
|
||||
self.assertNotEqual(drafting.CONVERTED_SHA256, drafting.PACK_SHA256)
|
||||
|
||||
|
||||
class SmallPacks(unittest.TestCase):
|
||||
"""A synthetic checkpoint packed as the Q4_0 pack, then converted."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.synth = _load("t_pack_sel", "tools/test_dflash2_pack.py")
|
||||
cls.packer = _load("strata_dflash2_pack_sel", "tools/dflash2_pack.py")
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.dir = Path(self.tmp.name)
|
||||
self.synth.write_safetensors(self.dir / "m.safetensors", self.synth.tensors())
|
||||
(self.dir / "c.json").write_text(json.dumps(self.synth.CFG))
|
||||
self.data = self.dir / "data"
|
||||
self.q4 = drafting.target_path(self.data)
|
||||
self.q4.parent.mkdir(parents=True)
|
||||
self.packer.pack(str(self.dir / "m.safetensors"), str(self.dir / "c.json"), str(self.q4), say=lambda *_: None, q4=True)
|
||||
self.src_pins = {"source_bytes": self.q4.stat().st_size, "source_sha256": hashlib.sha256(self.q4.read_bytes()).hexdigest()}
|
||||
direct = self.dir / "direct.pack"
|
||||
self.packer.pack(str(self.dir / "m.safetensors"), str(self.dir / "c.json"), str(direct), say=lambda *_: None, q4=True, q4_selector=True)
|
||||
self.pins = {"size": direct.stat().st_size, "sha256": hashlib.sha256(direct.read_bytes()).hexdigest(), **self.src_pins}
|
||||
self.direct = direct
|
||||
|
||||
def convert(self, **extra):
|
||||
return drafting_fetch.convert_selector(self.data, **{**self.pins, **extra})
|
||||
|
||||
def test_the_conversion_reproduces_the_pinned_bytes(self):
|
||||
"""The pack converted from the finished Q4_0 pack is byte-identical to the one packed straight from the checkpoint with --q4_0-selector (the pinned hash)."""
|
||||
out = self.convert()
|
||||
self.assertEqual(out, drafting.converted_path(self.data))
|
||||
self.assertEqual(out.read_bytes(), self.direct.read_bytes())
|
||||
self.assertLess(out.stat().st_size, self.q4.stat().st_size)
|
||||
self.assertTrue(drafting.converted_ok(self.data, size=self.pins["size"], sha256=self.pins["sha256"]))
|
||||
self.assertTrue(self.q4.is_file(), "the Q4_0 pack stays: it is the fallback")
|
||||
|
||||
def test_every_tensor_but_the_two_codebooks_is_carried_over_unchanged(self):
|
||||
out = self.convert()
|
||||
a, b = self.q4.read_bytes(), out.read_bytes()
|
||||
|
||||
def table(blob):
|
||||
n = int.from_bytes(blob[8:12], "little")
|
||||
rows = {}
|
||||
for line in blob[12:12 + n].decode().split("\n"):
|
||||
w = line.split()
|
||||
if w and w[0] == "t":
|
||||
rows[w[1]] = (w[2], int(w[5]), int(w[6]))
|
||||
return rows, 12 + n
|
||||
ta, base_a = table(a)
|
||||
tb, base_b = table(b)
|
||||
self.assertEqual(set(ta), set(tb))
|
||||
books = {"candidate_selector.predecessor_codebook", "candidate_selector.successor_codebook"}
|
||||
for name in ta:
|
||||
ba = a[base_a + ta[name][1]: base_a + ta[name][1] + ta[name][2]]
|
||||
bb = b[base_b + tb[name][1]: base_b + tb[name][1] + tb[name][2]]
|
||||
if name in books:
|
||||
self.assertEqual((ta[name][0], tb[name][0]), ("bf16", "q4_0"))
|
||||
self.assertEqual(len(bb) * 32, len(ba) * 9, name) # 18 bytes per 32 values against 2 bytes a value
|
||||
else:
|
||||
self.assertEqual(ba, bb, name)
|
||||
|
||||
def test_a_missing_corrupt_or_short_converted_pack_falls_back_to_the_q4_pack(self):
|
||||
eng = self.dir / "engine"
|
||||
eng.write_bytes(b"\0" * 64 + drafting._MARK_SELQ4 + b"\0" * 64)
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "not made yet")
|
||||
out = self.convert()
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]):
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), out)
|
||||
blob = bytearray(out.read_bytes())
|
||||
blob[len(blob) // 2] ^= 0xFF # one flipped byte, the same size, and the marker still describes the old file
|
||||
out.write_bytes(bytes(blob))
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "a corrupt file (the marker no longer matches it) is not used")
|
||||
out.write_bytes(bytes(blob)[:-1000])
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "a short file is not used")
|
||||
out.unlink()
|
||||
self.assertEqual(drafting.pack_path(self.data, eng), self.q4, "a missing file is not used")
|
||||
|
||||
def test_a_q4_pack_that_is_not_the_pinned_one_is_left_alone(self):
|
||||
self.assertIsNone(self.convert(source_sha256="0" * 64))
|
||||
self.assertFalse(drafting.converted_path(self.data).exists())
|
||||
self.assertTrue(self.q4.is_file())
|
||||
|
||||
def test_a_result_that_does_not_match_the_pin_is_deleted(self):
|
||||
self.assertIsNone(self.convert(sha256="f" * 64))
|
||||
self.assertFalse(drafting.converted_path(self.data).exists())
|
||||
self.assertFalse(drafting.converted_path(self.data).with_suffix(".tmp").exists())
|
||||
|
||||
def test_a_failing_conversion_is_never_an_error_for_the_caller(self):
|
||||
with mock.patch.object(drafting_fetch, "_packer", side_effect=RuntimeError("no numpy")):
|
||||
self.assertIsNone(self.convert())
|
||||
self.assertFalse(drafting.converted_path(self.data).exists())
|
||||
|
||||
def test_the_engine_must_read_the_pack_before_it_is_used(self):
|
||||
out = self.convert()
|
||||
old = self.dir / "old-engine"
|
||||
old.write_bytes(b"\0" * 256)
|
||||
new = self.dir / "new-engine"
|
||||
new.write_bytes(b"\0" * 64 + drafting._MARK_SELQ4 + b"\0" * 64)
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]):
|
||||
self.assertEqual(drafting.pack_path(self.data), self.q4, "no engine named: the pack every engine reads")
|
||||
self.assertEqual(drafting.pack_path(self.data, old), self.q4, "an engine without the text would read the rows as garbage")
|
||||
self.assertEqual(drafting.pack_path(self.data, self.dir / "none"), self.q4)
|
||||
self.assertEqual(drafting.pack_path(self.data, new), out)
|
||||
|
||||
def test_auto_arguments_point_at_the_converted_pack(self):
|
||||
self.convert()
|
||||
new = self.dir / "new-engine"
|
||||
new.write_bytes(b"\0" * 64 + drafting._MARK_SELQ4 + b"\0" * 64 + b"--draft-fallback" + b"--mtp-draft-vocab FILE (or --mtp DIR")
|
||||
vocab = self.data / drafting.FOLDER / drafting.VOCAB_NAME
|
||||
vocab.write_bytes(b"\1\0\0\0" * 8)
|
||||
cfg = {"model_name": "qwen3.8-flash-next-iq3_s", "drafting": "dflash2", "args": ["--spec", "4", "--expert-cache", "1000"]}
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]), \
|
||||
mock.patch.object(drafting, "ENGINE_SERVES_DFLASH2", True), mock.patch.object(drafting, "pack_ready", return_value=True):
|
||||
drafting.apply(cfg, klass="blackwell", data_dir=self.data, backend="cuda", archs=["sm_120"], engine=new)
|
||||
args = cfg["args"]
|
||||
self.assertEqual(args[args.index("--draft-model") + 1], str(drafting.converted_path(self.data)))
|
||||
cfg2 = {"model_name": "qwen3.8-flash-next-iq3_s", "drafting": "dflash2", "args": ["--spec", "4", "--expert-cache", "1000"]}
|
||||
drafting.apply(cfg2, klass="blackwell", data_dir=self.data, backend="cuda", archs=["sm_120"], engine=None)
|
||||
self.assertEqual(cfg2["args"][cfg2["args"].index("--draft-model") + 1], str(self.q4), "an engine that is not known gets the pack every engine reads")
|
||||
|
||||
def test_needs_conversion_only_when_dflash2_is_chosen_and_the_q4_pack_is_here(self):
|
||||
want = dict(mode="dflash2", model_name="qwen3.8-flash-next-iq3_s", data_dir=self.data, backend="cuda", archs=["sm_120"])
|
||||
with mock.patch.object(drafting, "ENGINE_SERVES_DFLASH2", True), mock.patch.object(drafting, "_here", lambda p: Path(p).is_file()):
|
||||
self.assertTrue(drafting.needs_conversion(**want))
|
||||
self.assertFalse(drafting.needs_conversion(**{**want, "mode": "mtp"}))
|
||||
self.assertFalse(drafting.needs_conversion(**{**want, "data_dir": self.dir / "empty"}))
|
||||
self.convert()
|
||||
with mock.patch.object(drafting, "CONVERTED_BYTES", self.pins["size"]), mock.patch.object(drafting, "CONVERTED_SHA256", self.pins["sha256"]):
|
||||
self.assertFalse(drafting.needs_conversion(**want), "already made")
|
||||
|
||||
def test_setup_and_load_convert_a_start_does_not(self):
|
||||
"""ensure() (strata drafting fetch, setup) converts a pack that is already here, with no network; the serving side only reads."""
|
||||
with mock.patch.object(drafting_fetch, "convert_selector") as convert, mock.patch.object(drafting, "pack_ready", return_value=True):
|
||||
drafting_fetch.ensure(self.data)
|
||||
convert.assert_called_once()
|
||||
source = (ROOT / "serve" / "server.py").read_text(encoding="utf-8")
|
||||
self.assertNotIn("convert_selector", source)
|
||||
self.assertNotIn("convert_selector", (ROOT / "strata_cli" / "launch" / "plan.py").read_text(encoding="utf-8") if (ROOT / "strata_cli" / "launch" / "plan.py").exists() else "")
|
||||
|
||||
|
||||
class TheTrueGraphicsMemory(unittest.TestCase):
|
||||
"""drafting.vram_gb: 0.72 GB with the converted pack (an engine that reads it), 0.89 GB otherwise; the plan and its text say the same number."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.dir = Path(self.tmp.name)
|
||||
self.new = self.dir / "new-engine"
|
||||
self.new.write_bytes(b"\0" * 64 + drafting._MARK_SELQ4 + b"\0" * 64)
|
||||
self.old = self.dir / "old-engine"
|
||||
self.old.write_bytes(b"\0" * 256)
|
||||
self.data = self.dir / "data"
|
||||
(self.data / drafting.FOLDER).mkdir(parents=True)
|
||||
|
||||
def pack(self, name):
|
||||
(self.data / drafting.FOLDER / name).write_bytes(b"x" * ((1 << 20) + 1))
|
||||
|
||||
def test_the_table(self):
|
||||
self.assertEqual((drafting.DRAFTER_MIB["dflash2"], drafting.DRAFTER_MIB["dflash2_selq4"], drafting.DRAFTER_MIB["mtp"]), (890, 715, 1082))
|
||||
self.assertEqual((drafting.DRAFTER_GB["dflash2"], drafting.DRAFTER_GB["dflash2_selq4"], drafting.DRAFTER_GB["mtp"], drafting.DRAFTER_GB["off"]), (0.89, 0.72, 1.08, 0.0))
|
||||
self.assertEqual(drafting.VRAM_GB, 0.89, "the bf16-selector figure stays the module constant")
|
||||
|
||||
def test_the_figure_follows_the_engine_and_the_pack(self):
|
||||
self.pack(drafting.PACK_NAME)
|
||||
self.assertEqual(drafting.vram_gb(self.data, self.new), 0.72)
|
||||
self.assertEqual(drafting.vram_gb(self.data, self.old), 0.89, "an engine without the text keeps the bf16 selector")
|
||||
self.assertEqual(drafting.vram_gb(self.data, None), 0.89, "an engine that is not known: the larger figure")
|
||||
self.assertEqual(drafting.vram_gb(None, self.new), 0.72, "no data folder yet: the pack will be the Q4_0 one")
|
||||
|
||||
def test_a_q8_pack_of_an_earlier_strata_is_never_converted_so_the_figure_stays(self):
|
||||
self.pack(drafting.LEGACY_PACK_NAME)
|
||||
self.assertEqual(drafting.vram_gb(self.data, self.new), 0.89)
|
||||
self.pack(drafting.PACK_NAME) # the Q4_0 pack beside it wins (pack_path): the converted figure
|
||||
self.assertEqual(drafting.vram_gb(self.data, self.new), 0.72)
|
||||
|
||||
def test_the_plan_says_the_same_number_in_words_and_in_the_fields(self):
|
||||
plan = drafting_fetch.plan(self.data, self.new)
|
||||
self.assertEqual((plan["vram_gb"], plan["selq4"]), (0.72, True))
|
||||
self.assertIn("about 0.7 GB of graphics memory", plan["text"])
|
||||
plan = drafting_fetch.plan(self.data, self.old)
|
||||
self.assertEqual((plan["vram_gb"], plan["selq4"]), (0.89, False))
|
||||
self.assertIn("about 0.9 GB of graphics memory", plan["text"])
|
||||
plan = drafting_fetch.plan(self.data)
|
||||
self.assertEqual((plan["vram_gb"], plan["selq4"]), (0.89, False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user