From 40c605f7bc192bf5bd517b0486ea9b415997b340 Mon Sep 17 00:00:00 2001 From: Niko1221 Date: Mon, 19 Oct 2026 23:38:49 +0200 Subject: [PATCH] Windows with a non-UTF-8 code page (Russian): PowerShell output is read as UTF-8 (Documents folder, save dialog, fonts, GPU names), Python runs in UTF-8 mode from the launchers and the host's children, and the engine and picture reader get a UTF-8 manifest so a Cyrillic model folder arrives intact (#3202) --- CMakeLists.txt | 13 +++++ START-HERE.bat | 2 + apps/host/server/agent/harness-runtime.mjs | 3 +- apps/host/server/cloudflared-find.mjs | 3 +- apps/host/server/folder-dialog.mjs | 1 + apps/host/server/fonts.mjs | 3 +- apps/host/server/hardware.mjs | 5 +- apps/host/server/index.mjs | 2 +- apps/host/server/model-directory-default.mjs | 3 +- apps/host/server/native-control.mjs | 8 +-- apps/host/server/powershell-utf8.mjs | 9 ++++ apps/host/server/snapshot.mjs | 3 +- apps/host/server/timeskip.mjs | 4 +- apps/host/tests/powershell-utf8.test.mjs | 26 ++++++++++ bin/strata.cmd | 2 + bin/strata.ps1 | 2 + cmake/strata-utf8.manifest | 8 +++ tools/test_cyrillic_paths.py | 54 ++++++++++++++++++++ tools/vision/CMakeLists.txt | 3 ++ tools/vision/strata-utf8.manifest | 8 +++ 20 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 apps/host/server/powershell-utf8.mjs create mode 100644 apps/host/tests/powershell-utf8.test.mjs create mode 100644 cmake/strata-utf8.manifest create mode 100644 tools/test_cyrillic_paths.py create mode 100644 tools/vision/strata-utf8.manifest diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ba7ee63..dc109350 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,17 @@ if(STRATA_PORTABLE AND MSVC) set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded") endif() +# #3202: Windows hands a program its command line in the system's ANSI code page (cp1251 / cp866 on a Russian Windows, cp1252 on an +# English one), but the engine reads every path as UTF-8 (the run configs and the server write UTF-8). A model folder with +# Cyrillic letters arrived as other bytes, or as "?" where the page has no such letter, and the model did not open. This manifest +# makes the process's ANSI code page UTF-8 (Windows 10 1903 and newer; older Windows ignores it and nothing changes), so the +# command line, fopen and std::ifstream all take UTF-8. +function(strata_utf8_manifest target) + if(MSVC) + target_sources(${target} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/cmake/strata-utf8.manifest) + endif() +endfunction() + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -722,6 +733,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) endif() add_executable(strata src/program/generate.cpp) + strata_utf8_manifest(strata) target_include_directories(strata PRIVATE ${_strata_gpu_include_directories}) target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec ${_strata_gpu_runtime_target}) @@ -742,6 +754,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP) target_compile_definitions(strata_qwen35 PRIVATE STRATA_Q36_CPU_POOL=1) endif() add_executable(strata-q36 src/qwen35/q36_main.cpp) + strata_utf8_manifest(strata-q36) target_link_libraries(strata-q36 PRIVATE strata_qwen35 ${_strata_gpu_runtime_target}) target_link_libraries(strata PRIVATE strata_qwen35) target_compile_definitions(strata PRIVATE STRATA_HAVE_QWEN35=1) diff --git a/START-HERE.bat b/START-HERE.bat index e8c7a6fb..5f255c07 100644 --- a/START-HERE.bat +++ b/START-HERE.bat @@ -2,6 +2,8 @@ rem Strata for Windows: the first run installs everything and starts the model; later runs just start it. rem Needs only an NVIDIA or AMD graphics driver. Python is installed for your user account if it is missing (no admin needed). setlocal +rem Python reads and writes text as UTF-8, not in the Windows code page (cp1251, cp866, ...): a folder or user name with Cyrillic or other letters stays intact (#3202). +set "PYTHONUTF8=1" title Strata cd /d "%~dp0" rem A bundled Python runtime (the python\ folder of the terminal archive) is used when it is there: nothing to find, diff --git a/apps/host/server/agent/harness-runtime.mjs b/apps/host/server/agent/harness-runtime.mjs index 5093e54c..078c6cb4 100644 --- a/apps/host/server/agent/harness-runtime.mjs +++ b/apps/host/server/agent/harness-runtime.mjs @@ -13,6 +13,7 @@ import { spawn, spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; import { externalEnvironment } from '../external-env.mjs'; +import { utf8Script } from '../powershell-utf8.mjs'; const URL_PATTERN = /http:\/\/127\.0\.0\.1:(\d+)\/\?token=([A-Za-z0-9_-]+)/; @@ -127,7 +128,7 @@ export class HarnessProcess { function commandLineOf(pid) { if (!Number.isInteger(pid) || pid <= 0) return null; if (process.platform === 'win32') { - const r = spawnSync('powershell', ['-NoProfile', '-NonInteractive', '-Command', `(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`], { encoding: 'utf8', windowsHide: true, timeout: 15000 }); + const r = spawnSync('powershell', ['-NoProfile', '-NonInteractive', '-Command', utf8Script(`(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`)], { encoding: 'utf8', windowsHide: true, timeout: 15000 }); return r.status === 0 && r.stdout.trim() ? r.stdout.trim() : null; } const r = spawnSync('ps', ['-p', String(pid), '-o', 'args='], { encoding: 'utf8' }); diff --git a/apps/host/server/cloudflared-find.mjs b/apps/host/server/cloudflared-find.mjs index cd076047..ed269722 100644 --- a/apps/host/server/cloudflared-find.mjs +++ b/apps/host/server/cloudflared-find.mjs @@ -10,6 +10,7 @@ * The first one that runs `cloudflared --version` and is new enough is used; the answer says which one and from where. * Nothing here downloads or installs anything. */ +import { utf8Script } from './powershell-utf8.mjs'; import { execFile } from 'node:child_process'; import { access, stat } from 'node:fs/promises'; import { constants } from 'node:fs'; @@ -134,7 +135,7 @@ export async function findRunningConnector({ platform = process.platform, exclud const skip = new Set(exclude.filter(Number.isInteger)); let procs = []; if (platform === 'win32') { - const out = await run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', "Get-CimInstance Win32_Process -Filter \"Name='cloudflared.exe'\" | ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }"]); + const out = await run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', utf8Script("Get-CimInstance Win32_Process -Filter \"Name='cloudflared.exe'\" | ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }")]); procs = out.split(/\r?\n/).map(l => /^(\d+)\t(.*)$/.exec(l)).filter(Boolean).map(m => ({ pid: Number(m[1]), command: m[2] })); if (!procs.length) { // no PowerShell answer: at least the pids const list = await run('tasklist', ['/FI', 'IMAGENAME eq cloudflared.exe', '/NH', '/FO', 'CSV']); diff --git a/apps/host/server/folder-dialog.mjs b/apps/host/server/folder-dialog.mjs index 8821bd19..2d07594d 100644 --- a/apps/host/server/folder-dialog.mjs +++ b/apps/host/server/folder-dialog.mjs @@ -56,6 +56,7 @@ export function watchedWindowsDialog(shell, args, { readyMs = 30_000, openMs = 2 return new Promise((resolve, reject) => { const child = spawnProcess(shell, args, { windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'] }); let stdout = '', ready = false, settled = false; + child.stdout?.setEncoding?.('utf8'); // a multi-byte letter of a Cyrillic folder must not be cut between two chunks const end = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); try { child.kill(); } catch { /* gone */ } fn(value); }; const timer = setTimeout(() => end(resolve, ready ? { stdout: '' } : { noDialog: true }), readyMs); const later = () => { clearTimeout(timer); ready = true; setTimeout(() => end(resolve, { stdout: '' }), openMs).unref?.(); }; diff --git a/apps/host/server/fonts.mjs b/apps/host/server/fonts.mjs index 7f65168f..d424fe89 100644 --- a/apps/host/server/fonts.mjs +++ b/apps/host/server/fonts.mjs @@ -1,5 +1,6 @@ import {execFile} from 'node:child_process'; import {promisify} from 'node:util'; +import {utf8Script} from './powershell-utf8.mjs'; const execute=promisify(execFile); let cached=null; export async function systemFonts({platform=process.platform,run=execute}={}){ @@ -11,7 +12,7 @@ export async function systemFonts({platform=process.platform,run=execute}={}){ const {stdout}=await read('/usr/sbin/system_profiler',['SPFontsDataType','-json','-detailLevel','mini']); const collect=value=>{if(!value||typeof value!=='object')return;if(typeof value.family==='string')names.push(value.family);for(const nested of Object.values(value))if(typeof nested==='object')collect(nested);};collect(JSON.parse(stdout)); }else if(platform==='win32'){ - const {stdout}=await read('powershell.exe',['-NoProfile','-NonInteractive','-Command','Add-Type -AssemblyName System.Drawing; (New-Object System.Drawing.Text.InstalledFontCollection).Families.Name | ConvertTo-Json -Compress']); + const {stdout}=await read('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script('Add-Type -AssemblyName System.Drawing; (New-Object System.Drawing.Text.InstalledFontCollection).Families.Name | ConvertTo-Json -Compress')]); const result=JSON.parse(stdout.replace(/^\uFEFF/,''));names=Array.isArray(result)?result:[result]; }else{ const {stdout}=await read('fc-list',['-f','%{family}\n']);names=stdout.split(/[\n,]/); diff --git a/apps/host/server/hardware.mjs b/apps/host/server/hardware.mjs index e747ead7..0a42b729 100644 --- a/apps/host/server/hardware.mjs +++ b/apps/host/server/hardware.mjs @@ -11,6 +11,7 @@ import { parseCpuTopology, parseWindowsProcessors, parseNvidiaDevices, detectDrm import { CONTEXT_TOKENS, VARIANTS, footprint } from './catalog.mjs'; import { MODELS_DIR, diskFreeGb, diskTotalGb } from './store.mjs'; import { probe, round } from './util.mjs'; +import { utf8Script } from './powershell-utf8.mjs'; let cached = null; let cachedAt = 0; @@ -126,7 +127,7 @@ async function detectCpu(platform) { }}catch{} processors=parseCpuTopology(raw,model,process.arch,ghz,names); } else if(platform==='win32') { - const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command','Get-CimInstance Win32_Processor | Select-Object DeviceID,SocketDesignation,Name,NumberOfCores,NumberOfLogicalProcessors,MaxClockSpeed | ConvertTo-Json -Compress']); + const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script('Get-CimInstance Win32_Processor | Select-Object DeviceID,SocketDesignation,Name,NumberOfCores,NumberOfLogicalProcessors,MaxClockSpeed | ConvertTo-Json -Compress')]); processors=parseWindowsProcessors(raw,process.arch); } const names=[...new Set(processors.map(p=>p.model))]; @@ -185,7 +186,7 @@ async function detectGpus(platform, totalGb) { } } if(platform==='win32') { - const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command','Get-CimInstance Win32_VideoController | Select-Object Name,PNPDeviceID | ConvertTo-Json -Compress']); + const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script('Get-CimInstance Win32_VideoController | Select-Object Name,PNPDeviceID | ConvertTo-Json -Compress')]); try {const parsed=JSON.parse(raw);for(const row of Array.isArray(parsed)?parsed:[parsed]) { if(!row?.Name||(/nvidia/i.test(row.Name)&&gpus.some(g=>g.vendor==='nvidia')))continue; const vendor=/amd|radeon/i.test(row.Name)?'amd':/intel/i.test(row.Name)?'intel':'unknown'; diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs index 0b2766a0..fc3b1e86 100644 --- a/apps/host/server/index.mjs +++ b/apps/host/server/index.mjs @@ -806,7 +806,7 @@ function createTuning({ repoRoot, python, pythonArgs = [], env = process.env } = function runCli(args) { return new Promise((resolve) => { const child = spawn(python, [...pythonArgs, cli, ...args, '--output', 'json'], { - cwd: repoRoot, windowsHide: true, env: { ...env, STRATA_TUNE_DIR: stateDir() }, + cwd: repoRoot, windowsHide: true, env: { ...env, PYTHONUTF8: '1', PYTHONIOENCODING: 'utf-8', STRATA_TUNE_DIR: stateDir() }, stdio: ['ignore', 'pipe', 'pipe'], }); let out = '', err = ''; diff --git a/apps/host/server/model-directory-default.mjs b/apps/host/server/model-directory-default.mjs index cfc35ca2..2eba2222 100644 --- a/apps/host/server/model-directory-default.mjs +++ b/apps/host/server/model-directory-default.mjs @@ -2,9 +2,10 @@ import { execFileSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { posix, win32 } from 'node:path'; +import { utf8Script } from './powershell-utf8.mjs'; /** Resolve the user's Documents folder without evaluating shell configuration. */ -export function defaultModelDirectory({platform=process.platform,home=homedir(),env=process.env,read=readFileSync,windowsDocuments=()=>execFileSync('powershell.exe',['-NoProfile','-NonInteractive','-Command',"[Environment]::GetFolderPath('MyDocuments')"],{encoding:'utf8',timeout:5000,windowsHide:true}).trim()}={}) { +export function defaultModelDirectory({platform=process.platform,home=homedir(),env=process.env,read=readFileSync,windowsDocuments=()=>execFileSync('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script("[Environment]::GetFolderPath('MyDocuments')")],{encoding:'utf8',timeout:5000,windowsHide:true}).trim()}={}) { const path=platform==='win32'?win32:posix; if(env.STRATA_MODELS_DIR)return path.resolve(env.STRATA_MODELS_DIR); let documents=env.STRATA_DOCUMENTS_DIR; diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjs index ae4ec648..4210c6ba 100644 --- a/apps/host/server/native-control.mjs +++ b/apps/host/server/native-control.mjs @@ -40,7 +40,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp updated or fetched from here. Without `control` (a source checkout) every path below is the original one. */ const packaged=!!control; /* The auto-tuner and the saved tunings are Strata Control's (`strata tune auto`, `strata presets`): the app runs them and follows their status file. */ - const autotuner=createAutoTune({spawnProcess,workDir:join(directory,'native-tune'),launcher:()=>{if(packaged)return {command:control.control.python,prefix:[control.control.cli],cwd:control.control.dir,env:controlEnvironment(control.control)};const py=python();return {command:py.command,prefix:[...py.args,join(repo(),'strata.py')],cwd:repo(),env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONIOENCODING:'utf-8'}};}}); + const autotuner=createAutoTune({spawnProcess,workDir:join(directory,'native-tune'),launcher:()=>{if(packaged)return {command:control.control.python,prefix:[control.control.cli],cwd:control.control.dir,env:controlEnvironment(control.control)};const py=python();return {command:py.command,prefix:[...py.args,join(repo(),'strata.py')],cwd:repo(),env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8'}};}}); /* In the packaged app the model catalog, the GPU probe and the install job are Control's bundled tools/strata_mcp.py and setup.py, run on Control's own Python in the installer's mode (STRATA_CONTROL_INSTALL: no pip, no engine, no compiler). The install only runs when the user clicks Download; starting and serving (control.start) keep STRATA_NO_ACQUIRE. */ @@ -80,7 +80,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp const folder=repo(); buffer='';const py0=packaged?{command:control.control.python,args:[]}:python(); const script=packaged?join(control.control.dir,'tools/strata_mcp.py'):join(folder,'tools/strata_mcp.py'); - const client=child=spawnProcess(py0.command,[...py0.args,script,...(packaged?['--root',folder,'--src',control.control.dir]:[])],{cwd:packaged?control.control.dir:folder,windowsHide:true,stdio:['pipe','pipe','pipe'],env:packaged?installerEnvironment(control.control,{installDir:folder}):{...process.env,PYTHONUNBUFFERED:'1',PYTHONIOENCODING:'utf-8'}}); + const client=child=spawnProcess(py0.command,[...py0.args,script,...(packaged?['--root',folder,'--src',control.control.dir]:[])],{cwd:packaged?control.control.dir:folder,windowsHide:true,stdio:['pipe','pipe','pipe'],env:packaged?installerEnvironment(control.control,{installDir:folder}):{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8'}}); client.stdout.on('data',chunk=>{if(child!==client)return;buffer+=chunk.toString();let end;while((end=buffer.indexOf('\n'))>=0){const line=buffer.slice(0,end);buffer=buffer.slice(end+1);let message;try{message=JSON.parse(line);}catch{continue;}const p=pending.get(message.id);if(p){clearTimeout(p.alarm);pending.delete(message.id);message.error?p.reject(new Error(message.error.message)):p.resolve(message.result);}}}); child.stderr.on('data',chunk=>append('control',chunk.toString())); client.on('error',error=>{if(child===client){snapshot.error=error.message;disconnect();}}); @@ -261,7 +261,7 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};} try{await startWith(retryConfig);append(id,'Strata Control reports the engine is ready (second try).');} catch(second){const said2=humanStartError(second);instance.status='error';instance.error=said2;/* the first stop and the second try stay in job.details; the pop-up shows the last stop once */emit();throw new Error(instance.error);}}} } else { - proc=spawnProcess(py,[join(repo(),'serve/server.py'),'--engine','strata','--config',configPath,'--port',String(cfg.port),'--host','127.0.0.1'],{cwd:repo(),detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONIOENCODING:'utf-8',STRATA_API_KEY:''}}); + proc=spawnProcess(py,[join(repo(),'serve/server.py'),'--engine','strata','--config',configPath,'--port',String(cfg.port),'--host','127.0.0.1'],{cwd:repo(),detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8',STRATA_API_KEY:''}}); instances.set(id,instance);processes.set(id,proc);if(!defaultId||!args.multiple||wasDefault)defaultId=id; for(const stream of [proc.stdout,proc.stderr])stream.on('data',chunk=>{const lines=chunk.toString().split('\n').filter(Boolean);lines.forEach(line=>append(id,line.replaceAll(key,'[redacted]')));job.summary=lines.at(-1)??job.summary;emit();}); proc.on('error',error=>{instance.status='error';instance.error=error.message;emit();});proc.on('exit',code=>{if(instances.has(id)&&instance.status!=='unloading'){instance.status='error';instance.error=`Engine stopped${code===null?'':` (exit ${code})`}. Open diagnostics or reload.`;emit();}});emit(); @@ -288,7 +288,7 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};} else if(job.kind==='update-engine'){if(!snapshot.installed?.models?.length)throw new Error('Prepare a model first. Its installer downloads the matching engine.');argv=[join(folder,'setup.py'),'--update'];} else if(job.kind==='update-strata'){const status=await execute('git',['status','--porcelain'],folder);if(status.trim())throw new Error('This Strata checkout has local edits. Save or commit them before updating.');const remote=await execute('git',['remote','get-url','origin'],folder);if(!/^https:\/\/github\.com\/Niko1221\/Strata(?:\.git)?\s*$/.test(remote))throw new Error('The native installation must use the official Strata repository to update here.');await execute('git',['pull','--ff-only'],folder);argv=[join(folder,'setup.py'),'--update'];} else throw new Error('Unsupported native operation.'); - return new Promise((resolve,reject)=>{const proc=spawnProcess(command,[...prefix,...argv],{cwd:folder,windowsHide:true,detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1'}});job.process=proc;for(const stream of [proc.stdout,proc.stderr])stream.on('data',chunk=>{chunk.toString().split('\n').filter(Boolean).forEach(line=>append(job.id,line));job.summary=chunk.toString().trim().slice(-500);emit();});proc.on('error',reject);proc.on('exit',code=>{delete job.process;code===0?resolve({summary:job.kind==='calibrate'?'Calibration complete. Native settings were saved.':'Native installation updated.'}):reject(new Error(`Operation stopped (exit ${code}). Open diagnostics for details.`));});}); + return new Promise((resolve,reject)=>{const proc=spawnProcess(command,[...prefix,...argv],{cwd:folder,windowsHide:true,detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8'}});job.process=proc;for(const stream of [proc.stdout,proc.stderr])stream.on('data',chunk=>{chunk.toString().split('\n').filter(Boolean).forEach(line=>append(job.id,line));job.summary=chunk.toString().trim().slice(-500);emit();});proc.on('error',reject);proc.on('exit',code=>{delete job.process;code===0?resolve({summary:job.kind==='calibrate'?'Calibration complete. Native settings were saved.':'Native installation updated.'}):reject(new Error(`Operation stopped (exit ${code}). Open diagnostics for details.`));});}); } /* Packaged app: engine and Control updates and the engine rollback go through Control's explicit update step (`strata update`, `strata components rollback`), never through git, setup.py or a download in this process. The app itself is updated by the diff --git a/apps/host/server/powershell-utf8.mjs b/apps/host/server/powershell-utf8.mjs new file mode 100644 index 00000000..1546002b --- /dev/null +++ b/apps/host/server/powershell-utf8.mjs @@ -0,0 +1,9 @@ +/** + * Windows PowerShell writes to a pipe in the console's OEM code page (cp866 on a Russian Windows, cp437 on an English one), + * not in UTF-8, while Node reads the pipe as UTF-8 - so a Cyrillic folder name came back as mojibake or as question marks (#3202). + * Every script whose output can hold a folder, file or font name starts with this line, before anything is written. + */ +export const POWERSHELL_UTF8 = '[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false; $OutputEncoding = [Console]::OutputEncoding; '; + +/** The script with the UTF-8 line in front of it. */ +export const utf8Script = script => POWERSHELL_UTF8 + script; diff --git a/apps/host/server/snapshot.mjs b/apps/host/server/snapshot.mjs index d365b597..678b29b1 100644 --- a/apps/host/server/snapshot.mjs +++ b/apps/host/server/snapshot.mjs @@ -1,6 +1,7 @@ /** Native Save As for a rendered Monitor PNG. The client cannot choose a host path. */ import { writeFile } from 'node:fs/promises'; import { exec } from './util.mjs'; +import { utf8Script } from './powershell-utf8.mjs'; export function snapshotBytes(data) { if (typeof data !== 'string' || data.length > 24 * 1024 * 1024 || !/^data:image\/png;base64,[A-Za-z0-9+/=]+$/.test(data)) throw new Error('Expected a PNG screenshot smaller than 18 MB.'); @@ -14,7 +15,7 @@ export async function saveSnapshot(data, { platform = process.platform, run = ex let result; try { if (platform === 'darwin') result = await run('osascript', ['-e', `POSIX path of (choose file name with prompt "Save Monitor screenshot" default name "${name}")`]); - else if (platform === 'win32') result = await run('powershell.exe', ['-NoProfile', '-STA', '-Command', `Add-Type -AssemblyName System.Windows.Forms; $dialog = New-Object System.Windows.Forms.SaveFileDialog; $dialog.Filter = 'PNG image (*.png)|*.png'; $dialog.FileName = '${name}'; if ($dialog.ShowDialog() -eq 'OK') { $dialog.FileName }`]); + else if (platform === 'win32') result = await run('powershell.exe', ['-NoProfile', '-STA', '-Command', utf8Script(`Add-Type -AssemblyName System.Windows.Forms; $dialog = New-Object System.Windows.Forms.SaveFileDialog; $dialog.Filter = 'PNG image (*.png)|*.png'; $dialog.FileName = '${name}'; if ($dialog.ShowDialog() -eq 'OK') { $dialog.FileName }`)]); else result = await run('zenity', ['--file-selection', '--save', '--confirm-overwrite', '--title=Save Monitor screenshot', `--filename=${name}`, '--file-filter=PNG images | *.png']); } catch (e) { if (/User canceled|\(-128\)/i.test(e.stderr ?? '') || (platform === 'linux' && e.code === 1)) return { cancelled: true }; diff --git a/apps/host/server/timeskip.mjs b/apps/host/server/timeskip.mjs index 959a402b..d9288723 100644 --- a/apps/host/server/timeskip.mjs +++ b/apps/host/server/timeskip.mjs @@ -305,7 +305,7 @@ export function createTimeSkip({ repoRoot, python, pythonArgs = [], timeoutMs = const child = spawn(python, [...pythonArgs, cli, ...args, '--output', 'json'], { cwd: repoRoot, windowsHide: true, - env: { ...env, STRATA_TIMESKIP_DIR: root }, + env: { ...env, PYTHONUTF8: '1', PYTHONIOENCODING: 'utf-8', STRATA_TIMESKIP_DIR: root }, stdio: ['ignore', 'pipe', 'pipe'], }); let stdout = ''; @@ -452,7 +452,7 @@ export function createTimeSkip({ repoRoot, python, pythonArgs = [], timeoutMs = '--attempts', String(Math.max(0, Math.min(10, Number(attempts) || 0))), '--output', 'json']; if (reviseCommand) args.push('--revise-command', reviseCommand); const child = spawn(python, [...pythonArgs, cli, ...args], { - cwd: repoRoot, windowsHide: true, env: { ...env, ...extraEnv, STRATA_TIMESKIP_DIR: root }, stdio: ['ignore', 'pipe', 'pipe'], + cwd: repoRoot, windowsHide: true, env: { ...env, PYTHONUTF8: '1', PYTHONIOENCODING: 'utf-8', ...extraEnv, STRATA_TIMESKIP_DIR: root }, stdio: ['ignore', 'pipe', 'pipe'], }); let out = '', err = '', closed = null; child.stdout.on('data', b => { out += b.toString(); }); diff --git a/apps/host/tests/powershell-utf8.test.mjs b/apps/host/tests/powershell-utf8.test.mjs new file mode 100644 index 00000000..c4118677 --- /dev/null +++ b/apps/host/tests/powershell-utf8.test.mjs @@ -0,0 +1,26 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { POWERSHELL_UTF8, utf8Script } from '../server/powershell-utf8.mjs'; + +test('the UTF-8 line comes first and leaves the script alone', () => { + assert.equal(utf8Script('Get-Date'), POWERSHELL_UTF8 + 'Get-Date'); + assert.match(POWERSHELL_UTF8, /^\[Console\]::OutputEncoding = New-Object System\.Text\.UTF8Encoding \$false;/); +}); + +// #3202: PowerShell writes to a pipe in the OEM code page (cp866 on a Russian Windows, cp437 elsewhere); a Cyrillic folder came back +// as mojibake or "?". The same happens on any non-UTF-8 OEM page, so this fails on this machine without the line, too. +test('a Cyrillic temp folder name survives PowerShell through a pipe', { skip: process.platform !== 'win32' }, t => { + const word = String.fromCharCode(0x414, 0x43e, 0x43a, 0x443, 0x43c, 0x435, 0x43d, 0x442, 0x44b); // "Документы" + const dir = mkdtempSync(join(tmpdir(), word + '-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const read = script => execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], + { encoding: 'utf8', windowsHide: true, env: { ...process.env, STRATA_UTF8_TEST: dir } }).trim(); + const ask = '(Get-Item -LiteralPath $env:STRATA_UTF8_TEST).FullName'; + const name = path => basename(path.replaceAll('\\', '/')).toLowerCase(); // the temp folder may come back as a short 8.3 path + assert.equal(name(read(utf8Script(ask))), name(dir)); + assert.equal(name(read(utf8Script(`[Console]::Out.Write((${ask}))`))), name(dir)); +}); diff --git a/bin/strata.cmd b/bin/strata.cmd index 673c83e6..e7a2baf0 100644 --- a/bin/strata.cmd +++ b/bin/strata.cmd @@ -6,6 +6,8 @@ rem 2. \python\python.exe the private runtime shipped in th rem 3. \.venv\Scripts\python.exe a source checkout set up by SETUP.bat rem 4. py -3 / python on PATH, 3.10 or newer a developer convenience, never a requirement setlocal +rem UTF-8 text in Python, not the Windows code page: folders with Cyrillic or other letters stay intact (#3202) +set "PYTHONUTF8=1" set "HERE=%~dp0" for %%I in ("%HERE%..") do set "ROOT=%%~fI" if exist "%HERE%strata.py" set "ROOT=%HERE%" diff --git a/bin/strata.ps1 b/bin/strata.ps1 index 369328aa..885650be 100644 --- a/bin/strata.ps1 +++ b/bin/strata.ps1 @@ -50,6 +50,8 @@ if (-not $py) { [Console]::Error.WriteLine(" Reinstall the archive, or set STRATA_PYTHON to a Python 3.10 or newer.") exit 5 } +# UTF-8 text in Python, not the Windows code page: folders with Cyrillic or other letters stay intact (#3202) +$env:PYTHONUTF8 = '1' $rest = @($py | Select-Object -Skip 1) & $py[0] @rest $app @args exit $LASTEXITCODE diff --git a/cmake/strata-utf8.manifest b/cmake/strata-utf8.manifest new file mode 100644 index 00000000..f2fff698 --- /dev/null +++ b/cmake/strata-utf8.manifest @@ -0,0 +1,8 @@ + + + + + UTF-8 + + + diff --git a/tools/test_cyrillic_paths.py b/tools/test_cyrillic_paths.py new file mode 100644 index 00000000..1be52b0d --- /dev/null +++ b/tools/test_cyrillic_paths.py @@ -0,0 +1,54 @@ +"""#3202: a Windows 11 with Russian localization (cp1251 / cp866 code pages): a Cyrillic folder name must reach the engine, +the run config and the tools unchanged.""" +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +WORD = "".join(chr(c) for c in (0x414, 0x43e, 0x43a, 0x443, 0x43c, 0x435, 0x43d, 0x442, 0x44b)) # "Документы" + + +class CyrillicFolder(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix=WORD + "-") + self.dir = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def test_run_config_in_a_cyrillic_folder_is_found_and_read(self): + from strata_control import discovery + cfg = self.dir / "strata-q2_0.json" + cfg.write_text(json.dumps({"exe": "x", "args": ["--native", str(self.dir / "m.gguf")]}, ensure_ascii=False), encoding="utf-8") + place = discovery.places(explicit=self.dir)[0] + self.assertEqual(place.configs, 1) + back = json.loads(discovery.config_files(self.dir)[0].read_text(encoding="utf-8")) + self.assertIn(WORD, back["args"][1]) + + def test_a_child_python_gets_the_path_as_an_argument_and_prints_it_back(self): + # the server starts the engine through serve.server.popen; the child reads the argument and its own file listing + from serve import server + (self.dir / "m.gguf").write_bytes(b"x") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + code = "import sys, os; print(sys.argv[1]); print(os.path.exists(os.path.join(sys.argv[1], 'm.gguf')))" + proc = server.popen("test child", [sys.executable, "-c", code, str(self.dir)], role="test", + stdout=subprocess.PIPE, env=env) + out = proc.stdout.read().decode("utf-8").splitlines() + proc.wait(timeout=30) + self.assertEqual(out[0], str(self.dir)) + self.assertEqual(out[1], "True") + + def test_windows_launchers_set_utf8_mode(self): + for name in ("START-HERE.bat", "bin/strata.cmd", "bin/strata.ps1"): + text = (ROOT / name).read_text(encoding="utf-8") + self.assertIn("PYTHONUTF8", text, name) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/vision/CMakeLists.txt b/tools/vision/CMakeLists.txt index 3b104dc0..7d8a3479 100644 --- a/tools/vision/CMakeLists.txt +++ b/tools/vision/CMakeLists.txt @@ -91,6 +91,9 @@ endif() add_subdirectory(${LLAMA_DIR} llama EXCLUDE_FROM_ALL) add_executable(strata-vision strata_vision.cpp) +if(MSVC) # #3202: the ANSI code page of the process is UTF-8, so a Cyrillic model folder reaches the reader intact (see ../../CMakeLists.txt) + target_sources(strata-vision PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/strata-utf8.manifest) +endif() target_link_libraries(strata-vision PRIVATE mtmd llama) target_compile_features(strata-vision PRIVATE cxx_std_17) set_target_properties(strata-vision PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) diff --git a/tools/vision/strata-utf8.manifest b/tools/vision/strata-utf8.manifest new file mode 100644 index 00000000..f2fff698 --- /dev/null +++ b/tools/vision/strata-utf8.manifest @@ -0,0 +1,8 @@ + + + + + UTF-8 + + +