mirror of
https://github.com/Heretek-AI/Strata.git
synced 2026-11-07 10:05:58 -10:00
Windows with a non-UTF-8 code page (Russian): PowerShell output is read as UTF-8 (Documents folder, save dialog, fonts, GPU names), Python runs in UTF-8 mode from the launchers and the host's children, and the engine and picture reader get a UTF-8 manifest so a Cyrillic model folder arrives intact (#3202)
This commit is contained in:
Niko1221 committed
1 parent
9cc2a9b6aa
commit
40c605f7bc
20 files changed
+148
-14
No files matched your search
@@ -48,6 +48,17 @@ if(STRATA_PORTABLE AND MSVC)
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded")
|
||||
endif()
|
||||
|
||||
# #3202: Windows hands a program its command line in the system's ANSI code page (cp1251 / cp866 on a Russian Windows, cp1252 on an
|
||||
# English one), but the engine reads every path as UTF-8 (the run configs and the server write UTF-8). A model folder with
|
||||
# Cyrillic letters arrived as other bytes, or as "?" where the page has no such letter, and the model did not open. This manifest
|
||||
# makes the process's ANSI code page UTF-8 (Windows 10 1903 and newer; older Windows ignores it and nothing changes), so the
|
||||
# command line, fopen and std::ifstream all take UTF-8.
|
||||
function(strata_utf8_manifest target)
|
||||
if(MSVC)
|
||||
target_sources(${target} PRIVATE ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/cmake/strata-utf8.manifest)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
@@ -722,6 +733,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
|
||||
endif()
|
||||
|
||||
add_executable(strata src/program/generate.cpp)
|
||||
strata_utf8_manifest(strata)
|
||||
target_include_directories(strata PRIVATE ${_strata_gpu_include_directories})
|
||||
target_link_libraries(strata PRIVATE strata_engine strata_prefill strata_spec ${_strata_gpu_runtime_target})
|
||||
|
||||
@@ -742,6 +754,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
|
||||
target_compile_definitions(strata_qwen35 PRIVATE STRATA_Q36_CPU_POOL=1)
|
||||
endif()
|
||||
add_executable(strata-q36 src/qwen35/q36_main.cpp)
|
||||
strata_utf8_manifest(strata-q36)
|
||||
target_link_libraries(strata-q36 PRIVATE strata_qwen35 ${_strata_gpu_runtime_target})
|
||||
target_link_libraries(strata PRIVATE strata_qwen35)
|
||||
target_compile_definitions(strata PRIVATE STRATA_HAVE_QWEN35=1)
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
rem Strata for Windows: the first run installs everything and starts the model; later runs just start it.
|
||||
rem Needs only an NVIDIA or AMD graphics driver. Python is installed for your user account if it is missing (no admin needed).
|
||||
setlocal
|
||||
rem Python reads and writes text as UTF-8, not in the Windows code page (cp1251, cp866, ...): a folder or user name with Cyrillic or other letters stays intact (#3202).
|
||||
set "PYTHONUTF8=1"
|
||||
title Strata
|
||||
cd /d "%~dp0"
|
||||
rem A bundled Python runtime (the python\ folder of the terminal archive) is used when it is there: nothing to find,
|
||||
|
||||
@@ -13,6 +13,7 @@ import { spawn, spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { externalEnvironment } from '../external-env.mjs';
|
||||
import { utf8Script } from '../powershell-utf8.mjs';
|
||||
|
||||
const URL_PATTERN = /http:\/\/127\.0\.0\.1:(\d+)\/\?token=([A-Za-z0-9_-]+)/;
|
||||
|
||||
@@ -127,7 +128,7 @@ export class HarnessProcess {
|
||||
function commandLineOf(pid) {
|
||||
if (!Number.isInteger(pid) || pid <= 0) return null;
|
||||
if (process.platform === 'win32') {
|
||||
const r = spawnSync('powershell', ['-NoProfile', '-NonInteractive', '-Command', `(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`], { encoding: 'utf8', windowsHide: true, timeout: 15000 });
|
||||
const r = spawnSync('powershell', ['-NoProfile', '-NonInteractive', '-Command', utf8Script(`(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`)], { encoding: 'utf8', windowsHide: true, timeout: 15000 });
|
||||
return r.status === 0 && r.stdout.trim() ? r.stdout.trim() : null;
|
||||
}
|
||||
const r = spawnSync('ps', ['-p', String(pid), '-o', 'args='], { encoding: 'utf8' });
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
* The first one that runs `cloudflared --version` and is new enough is used; the answer says which one and from where.
|
||||
* Nothing here downloads or installs anything.
|
||||
*/
|
||||
import { utf8Script } from './powershell-utf8.mjs';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { access, stat } from 'node:fs/promises';
|
||||
import { constants } from 'node:fs';
|
||||
@@ -134,7 +135,7 @@ export async function findRunningConnector({ platform = process.platform, exclud
|
||||
const skip = new Set(exclude.filter(Number.isInteger));
|
||||
let procs = [];
|
||||
if (platform === 'win32') {
|
||||
const out = await run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', "Get-CimInstance Win32_Process -Filter \"Name='cloudflared.exe'\" | ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }"]);
|
||||
const out = await run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', utf8Script("Get-CimInstance Win32_Process -Filter \"Name='cloudflared.exe'\" | ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }")]);
|
||||
procs = out.split(/\r?\n/).map(l => /^(\d+)\t(.*)$/.exec(l)).filter(Boolean).map(m => ({ pid: Number(m[1]), command: m[2] }));
|
||||
if (!procs.length) { // no PowerShell answer: at least the pids
|
||||
const list = await run('tasklist', ['/FI', 'IMAGENAME eq cloudflared.exe', '/NH', '/FO', 'CSV']);
|
||||
|
||||
@@ -56,6 +56,7 @@ export function watchedWindowsDialog(shell, args, { readyMs = 30_000, openMs = 2
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawnProcess(shell, args, { windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
let stdout = '', ready = false, settled = false;
|
||||
child.stdout?.setEncoding?.('utf8'); // a multi-byte letter of a Cyrillic folder must not be cut between two chunks
|
||||
const end = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); try { child.kill(); } catch { /* gone */ } fn(value); };
|
||||
const timer = setTimeout(() => end(resolve, ready ? { stdout: '' } : { noDialog: true }), readyMs);
|
||||
const later = () => { clearTimeout(timer); ready = true; setTimeout(() => end(resolve, { stdout: '' }), openMs).unref?.(); };
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {execFile} from 'node:child_process';
|
||||
import {promisify} from 'node:util';
|
||||
import {utf8Script} from './powershell-utf8.mjs';
|
||||
const execute=promisify(execFile);
|
||||
let cached=null;
|
||||
export async function systemFonts({platform=process.platform,run=execute}={}){
|
||||
@@ -11,7 +12,7 @@ export async function systemFonts({platform=process.platform,run=execute}={}){
|
||||
const {stdout}=await read('/usr/sbin/system_profiler',['SPFontsDataType','-json','-detailLevel','mini']);
|
||||
const collect=value=>{if(!value||typeof value!=='object')return;if(typeof value.family==='string')names.push(value.family);for(const nested of Object.values(value))if(typeof nested==='object')collect(nested);};collect(JSON.parse(stdout));
|
||||
}else if(platform==='win32'){
|
||||
const {stdout}=await read('powershell.exe',['-NoProfile','-NonInteractive','-Command','Add-Type -AssemblyName System.Drawing; (New-Object System.Drawing.Text.InstalledFontCollection).Families.Name | ConvertTo-Json -Compress']);
|
||||
const {stdout}=await read('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script('Add-Type -AssemblyName System.Drawing; (New-Object System.Drawing.Text.InstalledFontCollection).Families.Name | ConvertTo-Json -Compress')]);
|
||||
const result=JSON.parse(stdout.replace(/^\uFEFF/,''));names=Array.isArray(result)?result:[result];
|
||||
}else{
|
||||
const {stdout}=await read('fc-list',['-f','%{family}\n']);names=stdout.split(/[\n,]/);
|
||||
|
||||
@@ -11,6 +11,7 @@ import { parseCpuTopology, parseWindowsProcessors, parseNvidiaDevices, detectDrm
|
||||
import { CONTEXT_TOKENS, VARIANTS, footprint } from './catalog.mjs';
|
||||
import { MODELS_DIR, diskFreeGb, diskTotalGb } from './store.mjs';
|
||||
import { probe, round } from './util.mjs';
|
||||
import { utf8Script } from './powershell-utf8.mjs';
|
||||
|
||||
let cached = null;
|
||||
let cachedAt = 0;
|
||||
@@ -126,7 +127,7 @@ async function detectCpu(platform) {
|
||||
}}catch{}
|
||||
processors=parseCpuTopology(raw,model,process.arch,ghz,names);
|
||||
} else if(platform==='win32') {
|
||||
const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command','Get-CimInstance Win32_Processor | Select-Object DeviceID,SocketDesignation,Name,NumberOfCores,NumberOfLogicalProcessors,MaxClockSpeed | ConvertTo-Json -Compress']);
|
||||
const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script('Get-CimInstance Win32_Processor | Select-Object DeviceID,SocketDesignation,Name,NumberOfCores,NumberOfLogicalProcessors,MaxClockSpeed | ConvertTo-Json -Compress')]);
|
||||
processors=parseWindowsProcessors(raw,process.arch);
|
||||
}
|
||||
const names=[...new Set(processors.map(p=>p.model))];
|
||||
@@ -185,7 +186,7 @@ async function detectGpus(platform, totalGb) {
|
||||
}
|
||||
}
|
||||
if(platform==='win32') {
|
||||
const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command','Get-CimInstance Win32_VideoController | Select-Object Name,PNPDeviceID | ConvertTo-Json -Compress']);
|
||||
const raw=await probe('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script('Get-CimInstance Win32_VideoController | Select-Object Name,PNPDeviceID | ConvertTo-Json -Compress')]);
|
||||
try {const parsed=JSON.parse(raw);for(const row of Array.isArray(parsed)?parsed:[parsed]) {
|
||||
if(!row?.Name||(/nvidia/i.test(row.Name)&&gpus.some(g=>g.vendor==='nvidia')))continue;
|
||||
const vendor=/amd|radeon/i.test(row.Name)?'amd':/intel/i.test(row.Name)?'intel':'unknown';
|
||||
|
||||
@@ -806,7 +806,7 @@ function createTuning({ repoRoot, python, pythonArgs = [], env = process.env } =
|
||||
function runCli(args) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(python, [...pythonArgs, cli, ...args, '--output', 'json'], {
|
||||
cwd: repoRoot, windowsHide: true, env: { ...env, STRATA_TUNE_DIR: stateDir() },
|
||||
cwd: repoRoot, windowsHide: true, env: { ...env, PYTHONUTF8: '1', PYTHONIOENCODING: 'utf-8', STRATA_TUNE_DIR: stateDir() },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let out = '', err = '';
|
||||
|
||||
@@ -2,9 +2,10 @@ import { execFileSync } from 'node:child_process';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { posix, win32 } from 'node:path';
|
||||
import { utf8Script } from './powershell-utf8.mjs';
|
||||
|
||||
/** Resolve the user's Documents folder without evaluating shell configuration. */
|
||||
export function defaultModelDirectory({platform=process.platform,home=homedir(),env=process.env,read=readFileSync,windowsDocuments=()=>execFileSync('powershell.exe',['-NoProfile','-NonInteractive','-Command',"[Environment]::GetFolderPath('MyDocuments')"],{encoding:'utf8',timeout:5000,windowsHide:true}).trim()}={}) {
|
||||
export function defaultModelDirectory({platform=process.platform,home=homedir(),env=process.env,read=readFileSync,windowsDocuments=()=>execFileSync('powershell.exe',['-NoProfile','-NonInteractive','-Command',utf8Script("[Environment]::GetFolderPath('MyDocuments')")],{encoding:'utf8',timeout:5000,windowsHide:true}).trim()}={}) {
|
||||
const path=platform==='win32'?win32:posix;
|
||||
if(env.STRATA_MODELS_DIR)return path.resolve(env.STRATA_MODELS_DIR);
|
||||
let documents=env.STRATA_DOCUMENTS_DIR;
|
||||
|
||||
@@ -40,7 +40,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
|
||||
updated or fetched from here. Without `control` (a source checkout) every path below is the original one. */
|
||||
const packaged=!!control;
|
||||
/* The auto-tuner and the saved tunings are Strata Control's (`strata tune auto`, `strata presets`): the app runs them and follows their status file. */
|
||||
const autotuner=createAutoTune({spawnProcess,workDir:join(directory,'native-tune'),launcher:()=>{if(packaged)return {command:control.control.python,prefix:[control.control.cli],cwd:control.control.dir,env:controlEnvironment(control.control)};const py=python();return {command:py.command,prefix:[...py.args,join(repo(),'strata.py')],cwd:repo(),env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONIOENCODING:'utf-8'}};}});
|
||||
const autotuner=createAutoTune({spawnProcess,workDir:join(directory,'native-tune'),launcher:()=>{if(packaged)return {command:control.control.python,prefix:[control.control.cli],cwd:control.control.dir,env:controlEnvironment(control.control)};const py=python();return {command:py.command,prefix:[...py.args,join(repo(),'strata.py')],cwd:repo(),env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8'}};}});
|
||||
/* In the packaged app the model catalog, the GPU probe and the install job are Control's bundled tools/strata_mcp.py and
|
||||
setup.py, run on Control's own Python in the installer's mode (STRATA_CONTROL_INSTALL: no pip, no engine, no compiler).
|
||||
The install only runs when the user clicks Download; starting and serving (control.start) keep STRATA_NO_ACQUIRE. */
|
||||
@@ -80,7 +80,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
|
||||
const folder=repo();
|
||||
buffer='';const py0=packaged?{command:control.control.python,args:[]}:python();
|
||||
const script=packaged?join(control.control.dir,'tools/strata_mcp.py'):join(folder,'tools/strata_mcp.py');
|
||||
const client=child=spawnProcess(py0.command,[...py0.args,script,...(packaged?['--root',folder,'--src',control.control.dir]:[])],{cwd:packaged?control.control.dir:folder,windowsHide:true,stdio:['pipe','pipe','pipe'],env:packaged?installerEnvironment(control.control,{installDir:folder}):{...process.env,PYTHONUNBUFFERED:'1',PYTHONIOENCODING:'utf-8'}});
|
||||
const client=child=spawnProcess(py0.command,[...py0.args,script,...(packaged?['--root',folder,'--src',control.control.dir]:[])],{cwd:packaged?control.control.dir:folder,windowsHide:true,stdio:['pipe','pipe','pipe'],env:packaged?installerEnvironment(control.control,{installDir:folder}):{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8'}});
|
||||
client.stdout.on('data',chunk=>{if(child!==client)return;buffer+=chunk.toString();let end;while((end=buffer.indexOf('\n'))>=0){const line=buffer.slice(0,end);buffer=buffer.slice(end+1);let message;try{message=JSON.parse(line);}catch{continue;}const p=pending.get(message.id);if(p){clearTimeout(p.alarm);pending.delete(message.id);message.error?p.reject(new Error(message.error.message)):p.resolve(message.result);}}});
|
||||
child.stderr.on('data',chunk=>append('control',chunk.toString()));
|
||||
client.on('error',error=>{if(child===client){snapshot.error=error.message;disconnect();}});
|
||||
@@ -261,7 +261,7 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};}
|
||||
try{await startWith(retryConfig);append(id,'Strata Control reports the engine is ready (second try).');}
|
||||
catch(second){const said2=humanStartError(second);instance.status='error';instance.error=said2;/* the first stop and the second try stay in job.details; the pop-up shows the last stop once */emit();throw new Error(instance.error);}}}
|
||||
} else {
|
||||
proc=spawnProcess(py,[join(repo(),'serve/server.py'),'--engine','strata','--config',configPath,'--port',String(cfg.port),'--host','127.0.0.1'],{cwd:repo(),detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONIOENCODING:'utf-8',STRATA_API_KEY:''}});
|
||||
proc=spawnProcess(py,[join(repo(),'serve/server.py'),'--engine','strata','--config',configPath,'--port',String(cfg.port),'--host','127.0.0.1'],{cwd:repo(),detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8',STRATA_API_KEY:''}});
|
||||
instances.set(id,instance);processes.set(id,proc);if(!defaultId||!args.multiple||wasDefault)defaultId=id;
|
||||
for(const stream of [proc.stdout,proc.stderr])stream.on('data',chunk=>{const lines=chunk.toString().split('\n').filter(Boolean);lines.forEach(line=>append(id,line.replaceAll(key,'[redacted]')));job.summary=lines.at(-1)??job.summary;emit();});
|
||||
proc.on('error',error=>{instance.status='error';instance.error=error.message;emit();});proc.on('exit',code=>{if(instances.has(id)&&instance.status!=='unloading'){instance.status='error';instance.error=`Engine stopped${code===null?'':` (exit ${code})`}. Open diagnostics or reload.`;emit();}});emit();
|
||||
@@ -288,7 +288,7 @@ return {summary:result.summary??'Deleted',removed:result.plan?.gb??null};}
|
||||
else if(job.kind==='update-engine'){if(!snapshot.installed?.models?.length)throw new Error('Prepare a model first. Its installer downloads the matching engine.');argv=[join(folder,'setup.py'),'--update'];}
|
||||
else if(job.kind==='update-strata'){const status=await execute('git',['status','--porcelain'],folder);if(status.trim())throw new Error('This Strata checkout has local edits. Save or commit them before updating.');const remote=await execute('git',['remote','get-url','origin'],folder);if(!/^https:\/\/github\.com\/Niko1221\/Strata(?:\.git)?\s*$/.test(remote))throw new Error('The native installation must use the official Strata repository to update here.');await execute('git',['pull','--ff-only'],folder);argv=[join(folder,'setup.py'),'--update'];}
|
||||
else throw new Error('Unsupported native operation.');
|
||||
return new Promise((resolve,reject)=>{const proc=spawnProcess(command,[...prefix,...argv],{cwd:folder,windowsHide:true,detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1'}});job.process=proc;for(const stream of [proc.stdout,proc.stderr])stream.on('data',chunk=>{chunk.toString().split('\n').filter(Boolean).forEach(line=>append(job.id,line));job.summary=chunk.toString().trim().slice(-500);emit();});proc.on('error',reject);proc.on('exit',code=>{delete job.process;code===0?resolve({summary:job.kind==='calibrate'?'Calibration complete. Native settings were saved.':'Native installation updated.'}):reject(new Error(`Operation stopped (exit ${code}). Open diagnostics for details.`));});});
|
||||
return new Promise((resolve,reject)=>{const proc=spawnProcess(command,[...prefix,...argv],{cwd:folder,windowsHide:true,detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONUTF8:'1',PYTHONIOENCODING:'utf-8'}});job.process=proc;for(const stream of [proc.stdout,proc.stderr])stream.on('data',chunk=>{chunk.toString().split('\n').filter(Boolean).forEach(line=>append(job.id,line));job.summary=chunk.toString().trim().slice(-500);emit();});proc.on('error',reject);proc.on('exit',code=>{delete job.process;code===0?resolve({summary:job.kind==='calibrate'?'Calibration complete. Native settings were saved.':'Native installation updated.'}):reject(new Error(`Operation stopped (exit ${code}). Open diagnostics for details.`));});});
|
||||
}
|
||||
/* Packaged app: engine and Control updates and the engine rollback go through Control's explicit update step (`strata update`,
|
||||
`strata components rollback`), never through git, setup.py or a download in this process. The app itself is updated by the
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Windows PowerShell writes to a pipe in the console's OEM code page (cp866 on a Russian Windows, cp437 on an English one),
|
||||
* not in UTF-8, while Node reads the pipe as UTF-8 - so a Cyrillic folder name came back as mojibake or as question marks (#3202).
|
||||
* Every script whose output can hold a folder, file or font name starts with this line, before anything is written.
|
||||
*/
|
||||
export const POWERSHELL_UTF8 = '[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false; $OutputEncoding = [Console]::OutputEncoding; ';
|
||||
|
||||
/** The script with the UTF-8 line in front of it. */
|
||||
export const utf8Script = script => POWERSHELL_UTF8 + script;
|
||||
@@ -1,6 +1,7 @@
|
||||
/** Native Save As for a rendered Monitor PNG. The client cannot choose a host path. */
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { exec } from './util.mjs';
|
||||
import { utf8Script } from './powershell-utf8.mjs';
|
||||
|
||||
export function snapshotBytes(data) {
|
||||
if (typeof data !== 'string' || data.length > 24 * 1024 * 1024 || !/^data:image\/png;base64,[A-Za-z0-9+/=]+$/.test(data)) throw new Error('Expected a PNG screenshot smaller than 18 MB.');
|
||||
@@ -14,7 +15,7 @@ export async function saveSnapshot(data, { platform = process.platform, run = ex
|
||||
let result;
|
||||
try {
|
||||
if (platform === 'darwin') result = await run('osascript', ['-e', `POSIX path of (choose file name with prompt "Save Monitor screenshot" default name "${name}")`]);
|
||||
else if (platform === 'win32') result = await run('powershell.exe', ['-NoProfile', '-STA', '-Command', `Add-Type -AssemblyName System.Windows.Forms; $dialog = New-Object System.Windows.Forms.SaveFileDialog; $dialog.Filter = 'PNG image (*.png)|*.png'; $dialog.FileName = '${name}'; if ($dialog.ShowDialog() -eq 'OK') { $dialog.FileName }`]);
|
||||
else if (platform === 'win32') result = await run('powershell.exe', ['-NoProfile', '-STA', '-Command', utf8Script(`Add-Type -AssemblyName System.Windows.Forms; $dialog = New-Object System.Windows.Forms.SaveFileDialog; $dialog.Filter = 'PNG image (*.png)|*.png'; $dialog.FileName = '${name}'; if ($dialog.ShowDialog() -eq 'OK') { $dialog.FileName }`)]);
|
||||
else result = await run('zenity', ['--file-selection', '--save', '--confirm-overwrite', '--title=Save Monitor screenshot', `--filename=${name}`, '--file-filter=PNG images | *.png']);
|
||||
} catch (e) {
|
||||
if (/User canceled|\(-128\)/i.test(e.stderr ?? '') || (platform === 'linux' && e.code === 1)) return { cancelled: true };
|
||||
|
||||
@@ -305,7 +305,7 @@ export function createTimeSkip({ repoRoot, python, pythonArgs = [], timeoutMs =
|
||||
const child = spawn(python, [...pythonArgs, cli, ...args, '--output', 'json'], {
|
||||
cwd: repoRoot,
|
||||
windowsHide: true,
|
||||
env: { ...env, STRATA_TIMESKIP_DIR: root },
|
||||
env: { ...env, PYTHONUTF8: '1', PYTHONIOENCODING: 'utf-8', STRATA_TIMESKIP_DIR: root },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let stdout = '';
|
||||
@@ -452,7 +452,7 @@ export function createTimeSkip({ repoRoot, python, pythonArgs = [], timeoutMs =
|
||||
'--attempts', String(Math.max(0, Math.min(10, Number(attempts) || 0))), '--output', 'json'];
|
||||
if (reviseCommand) args.push('--revise-command', reviseCommand);
|
||||
const child = spawn(python, [...pythonArgs, cli, ...args], {
|
||||
cwd: repoRoot, windowsHide: true, env: { ...env, ...extraEnv, STRATA_TIMESKIP_DIR: root }, stdio: ['ignore', 'pipe', 'pipe'],
|
||||
cwd: repoRoot, windowsHide: true, env: { ...env, PYTHONUTF8: '1', PYTHONIOENCODING: 'utf-8', ...extraEnv, STRATA_TIMESKIP_DIR: root }, stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let out = '', err = '', closed = null;
|
||||
child.stdout.on('data', b => { out += b.toString(); });
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { basename, join } from 'node:path';
|
||||
import { POWERSHELL_UTF8, utf8Script } from '../server/powershell-utf8.mjs';
|
||||
|
||||
test('the UTF-8 line comes first and leaves the script alone', () => {
|
||||
assert.equal(utf8Script('Get-Date'), POWERSHELL_UTF8 + 'Get-Date');
|
||||
assert.match(POWERSHELL_UTF8, /^\[Console\]::OutputEncoding = New-Object System\.Text\.UTF8Encoding \$false;/);
|
||||
});
|
||||
|
||||
// #3202: PowerShell writes to a pipe in the OEM code page (cp866 on a Russian Windows, cp437 elsewhere); a Cyrillic folder came back
|
||||
// as mojibake or "?". The same happens on any non-UTF-8 OEM page, so this fails on this machine without the line, too.
|
||||
test('a Cyrillic temp folder name survives PowerShell through a pipe', { skip: process.platform !== 'win32' }, t => {
|
||||
const word = String.fromCharCode(0x414, 0x43e, 0x43a, 0x443, 0x43c, 0x435, 0x43d, 0x442, 0x44b); // "Документы"
|
||||
const dir = mkdtempSync(join(tmpdir(), word + '-'));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
const read = script => execFileSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script],
|
||||
{ encoding: 'utf8', windowsHide: true, env: { ...process.env, STRATA_UTF8_TEST: dir } }).trim();
|
||||
const ask = '(Get-Item -LiteralPath $env:STRATA_UTF8_TEST).FullName';
|
||||
const name = path => basename(path.replaceAll('\\', '/')).toLowerCase(); // the temp folder may come back as a short 8.3 path
|
||||
assert.equal(name(read(utf8Script(ask))), name(dir));
|
||||
assert.equal(name(read(utf8Script(`[Console]::Out.Write((${ask}))`))), name(dir));
|
||||
});
|
||||
@@ -6,6 +6,8 @@ rem 2. <root>\python\python.exe the private runtime shipped in th
|
||||
rem 3. <root>\.venv\Scripts\python.exe a source checkout set up by SETUP.bat
|
||||
rem 4. py -3 / python on PATH, 3.10 or newer a developer convenience, never a requirement
|
||||
setlocal
|
||||
rem UTF-8 text in Python, not the Windows code page: folders with Cyrillic or other letters stay intact (#3202)
|
||||
set "PYTHONUTF8=1"
|
||||
set "HERE=%~dp0"
|
||||
for %%I in ("%HERE%..") do set "ROOT=%%~fI"
|
||||
if exist "%HERE%strata.py" set "ROOT=%HERE%"
|
||||
|
||||
@@ -50,6 +50,8 @@ if (-not $py) {
|
||||
[Console]::Error.WriteLine(" Reinstall the archive, or set STRATA_PYTHON to a Python 3.10 or newer.")
|
||||
exit 5
|
||||
}
|
||||
# UTF-8 text in Python, not the Windows code page: folders with Cyrillic or other letters stay intact (#3202)
|
||||
$env:PYTHONUTF8 = '1'
|
||||
$rest = @($py | Select-Object -Skip 1)
|
||||
& $py[0] @rest $app @args
|
||||
exit $LASTEXITCODE
|
||||
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
|
||||
<application xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
<windowsSettings>
|
||||
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
|
||||
</windowsSettings>
|
||||
</application>
|
||||
</assembly>
|
||||
@@ -0,0 +1,54 @@
|
||||
"""#3202: a Windows 11 with Russian localization (cp1251 / cp866 code pages): a Cyrillic folder name must reach the engine,
|
||||
the run config and the tools unchanged."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
WORD = "".join(chr(c) for c in (0x414, 0x43e, 0x43a, 0x443, 0x43c, 0x435, 0x43d, 0x442, 0x44b)) # "Документы"
|
||||
|
||||
|
||||
class CyrillicFolder(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(prefix=WORD + "-")
|
||||
self.dir = Path(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_run_config_in_a_cyrillic_folder_is_found_and_read(self):
|
||||
from strata_control import discovery
|
||||
cfg = self.dir / "strata-q2_0.json"
|
||||
cfg.write_text(json.dumps({"exe": "x", "args": ["--native", str(self.dir / "m.gguf")]}, ensure_ascii=False), encoding="utf-8")
|
||||
place = discovery.places(explicit=self.dir)[0]
|
||||
self.assertEqual(place.configs, 1)
|
||||
back = json.loads(discovery.config_files(self.dir)[0].read_text(encoding="utf-8"))
|
||||
self.assertIn(WORD, back["args"][1])
|
||||
|
||||
def test_a_child_python_gets_the_path_as_an_argument_and_prints_it_back(self):
|
||||
# the server starts the engine through serve.server.popen; the child reads the argument and its own file listing
|
||||
from serve import server
|
||||
(self.dir / "m.gguf").write_bytes(b"x")
|
||||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
|
||||
code = "import sys, os; print(sys.argv[1]); print(os.path.exists(os.path.join(sys.argv[1], 'm.gguf')))"
|
||||
proc = server.popen("test child", [sys.executable, "-c", code, str(self.dir)], role="test",
|
||||
stdout=subprocess.PIPE, env=env)
|
||||
out = proc.stdout.read().decode("utf-8").splitlines()
|
||||
proc.wait(timeout=30)
|
||||
self.assertEqual(out[0], str(self.dir))
|
||||
self.assertEqual(out[1], "True")
|
||||
|
||||
def test_windows_launchers_set_utf8_mode(self):
|
||||
for name in ("START-HERE.bat", "bin/strata.cmd", "bin/strata.ps1"):
|
||||
text = (ROOT / name).read_text(encoding="utf-8")
|
||||
self.assertIn("PYTHONUTF8", text, name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -91,6 +91,9 @@ endif()
|
||||
add_subdirectory(${LLAMA_DIR} llama EXCLUDE_FROM_ALL)
|
||||
|
||||
add_executable(strata-vision strata_vision.cpp)
|
||||
if(MSVC) # #3202: the ANSI code page of the process is UTF-8, so a Cyrillic model folder reaches the reader intact (see ../../CMakeLists.txt)
|
||||
target_sources(strata-vision PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/strata-utf8.manifest)
|
||||
endif()
|
||||
target_link_libraries(strata-vision PRIVATE mtmd llama)
|
||||
target_compile_features(strata-vision PRIVATE cxx_std_17)
|
||||
set_target_properties(strata-vision PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
|
||||
<application xmlns="urn:schemas-microsoft-com:asm.v3">
|
||||
<windowsSettings>
|
||||
<activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
|
||||
</windowsSettings>
|
||||
</application>
|
||||
</assembly>
|
||||
Reference in new issue
Block a user