@echo off SETLOCAL ENABLEDELAYEDEXPANSION title Llama.cpp Configurable Launcher REM ========================================================= REM GLOBAL DEFAULTS REM ========================================================= set "BASE=%~dp0" set "BIN_DIR=%BASE%bin" set "LOGS_DIR=%BASE%logs" set "EXE_NAME=llama-server.exe" set "LOGFILE=%LOGS_DIR%\launcher_debug.log" REM --- Create log file --- echo [%date% %time%] Script started >> "%LOGFILE%" echo [%date% %time%] BASE=%BASE% >> "%LOGFILE%" REM --- Model Directory Setup --- set "MODELS_DIR=%BASE%models" if exist "%BASE%MODELS_PATH.txt" ( set /p MODELS_DIR=<"%BASE%MODELS_PATH.txt" ) REM --- Environment Sanitation --- REM Prioritize local folder for DLLs and backends to avoid conflicts with system-wide installations (like WinGet) set "PATH=%BIN_DIR%;%PATH%" set "GGML_BACKEND_PATH=%BIN_DIR%" REM --- Connection Settings --- set "PORT=8080" set "HOST=127.0.0.1" REM --- Hardware Settings (Global) --- set "GPU_LAYERS=999" REM Set to 999 to offload all layers to GPU set "THREADS=8" REM CPU threads to use for generation set "PARALLEL_SLOTS=1" REM Number of simultaneous requests supported REM --- Browser Settings --- set "AUTO_OPEN_BROWSER=1" REM Set to 1 to open browser on launch set "BROWSER_DELAY=2" REM Delay in seconds before opening browser (obsolete if port check works) REM --- Model Categorization Keywords --- set "KEYWORDS=NSFW Uncensored Amoral Dark Fallen Rivermind Snowpiercer Abyss Erebus Lewd Anubis Anubus Dusk Vox" REM NOTE: Models like Anubis require the <|im_end|> stop token. REM However, characters like <, |, and > are special Batch operators. REM To add them back, they must be hyper-escaped or passed via environment variables carefully. REM ========================================================= REM SYSTEM STATS REM ========================================================= :GET_STATS echo [%date% %time%] GET_STATS starting >> "%LOGFILE%" set "CPU_LOAD=0" set "RAM_INFO=0/0" set "GPU_NAME=N/A" set "GPU_INFO=0/0" set "GPU_LOAD=0" set "DRIVER_VER=N/A" set "CUDA_VER=N/A" set "PYTHON_VER=N/A" set "LLAMA_VER=N/A" REM --- CPU and RAM Stats --- for /f "skip=1" %%p in ('wmic cpu get loadpercentage') do if not defined CPU_LOAD set "CPU_LOAD=%%p" for /f "skip=1" %%p in ('wmic os get FreePhysicalMemory') do if not defined RAM_FREE set "RAM_FREE=%%p" for /f "skip=1" %%p in ('wmic os get TotalVisibleMemorySize') do if not defined RAM_TOTAL set "RAM_TOTAL=%%p" set /a RAM_TOTAL_GB=%RAM_TOTAL%/1024/1024 set /a RAM_USED_GB=(%RAM_TOTAL% - %RAM_FREE%)/1024/1024 set "RAM_INFO=%RAM_USED_GB%GB / %RAM_TOTAL_GB%GB" REM --- GPU Stats (NVIDIA-SMI) --- nvidia-smi --query-gpu=name,utilization.gpu,memory.used,memory.total,driver_version --format=csv,noheader,nounits > "%LOGS_DIR%\gpu.tmp" 2>nul if %errorlevel%==0 ( for /f "tokens=1,2,3,4,5 delims=," %%a in ('type "%LOGS_DIR%\gpu.tmp"') do ( set "GPU_NAME=%%a" set "GPU_LOAD=%%b" set "GPU_INFO=%%cMB / %%dMB" set "DRIVER_VER=%%e" ) del "%LOGS_DIR%\gpu.tmp" REM Extract CUDA version nvidia-smi | findstr /C:"CUDA Version" > "%LOGS_DIR%\cuda.tmp" set /p CUDA_RAW=<"%LOGS_DIR%\cuda.tmp" for /f "tokens=9" %%x in ("!CUDA_RAW!") do set "CUDA_VER=%%x" del "%LOGS_DIR%\cuda.tmp" ) REM --- Software Versions --- if exist "%BASE%venv\Scripts\python.exe" ( "%BASE%venv\Scripts\python.exe" --version > "%LOGS_DIR%\py.tmp" 2>&1 ) else ( python --version > "%LOGS_DIR%\py.tmp" 2>&1 ) set /p PY_RAW=<"%LOGS_DIR%\py.tmp" if not "%PY_RAW%"=="" set "PYTHON_VER=%PY_RAW%" del "%LOGS_DIR%\py.tmp" if exist "%BIN_DIR%\%EXE_NAME%" ( "%BIN_DIR%\%EXE_NAME%" --version 2>&1 | findstr "build:" > "%LOGS_DIR%\llama.tmp" set /p LLAMA_RAW=<"%LOGS_DIR%\llama.tmp" if not "!LLAMA_RAW!"=="" set "LLAMA_VER=!LLAMA_RAW!" del "%LOGS_DIR%\llama.tmp" ) REM ========================================================= REM MAIN MENU REM ========================================================= :MAIN_MENU echo [%date% %time%] MAIN_MENU reached >> "%LOGFILE%" cls echo ======================================================== echo SYSTEM DASHBOARD echo ======================================================== echo HARDWARE: echo CPU: %CPU_LOAD%%% Load RAM: %RAM_INFO% echo GPU: %GPU_NAME% echo MEM: %GPU_INFO% Load: %GPU_LOAD%%% echo. echo SOFTWARE: echo Llama: %LLAMA_VER% echo Python: %PYTHON_VER% echo NVIDIA: Driver %DRIVER_VER% / CUDA %CUDA_VER% echo ======================================================== echo. echo [1] Select Model and Launch echo [2] Refresh Stats echo [3] Edit Global Defaults (Not implemented) echo [4] Exit echo. echo ======================================================== set /p "CHOICE=Enter selection (1-4): " if "%CHOICE%"=="1" goto :MODEL_SELECT if "%CHOICE%"=="2" goto :GET_STATS if "%CHOICE%"=="3" goto :EDIT_CONFIG if "%CHOICE%"=="4" exit goto :MAIN_MENU REM ========================================================= REM MODEL SELECTION REM ========================================================= :MODEL_SELECT cls echo ======================================================== echo AVAILABLE MODELS echo ======================================================== echo Scanning %MODELS_DIR%... echo. set idx=0 set unc_count=0 set std_count=0 REM Clean array variables for /f "tokens=1 delims==" %%v in ('set f_path[ 2^>nul') do set "%%v=" for /f "tokens=1 delims==" %%v in ('set f_name[ 2^>nul') do set "%%v=" if not exist "%MODELS_DIR%" mkdir "%MODELS_DIR%" for %%f in ("%MODELS_DIR%\*.gguf") do ( set "fname=%%~nxf" set "fpath=%%f" REM Skip mmproj files in the main list echo !fname! | findstr /I "mmproj" >nul if !errorlevel! NEQ 0 ( set "is_unc=0" echo !fname! | findstr /I "%KEYWORDS%" >nul if !errorlevel! EQU 0 set "is_unc=1" if !is_unc! EQU 1 ( set /a unc_count+=1 set "unc_n[!unc_count!]=!fname!" set "unc_p[!unc_count!]=!fpath!" ) else ( set /a std_count+=1 set "std_n[!std_count!]=!fname!" set "std_p[!std_count!]=!fpath!" ) ) ) if %unc_count% EQU 0 if %std_count% EQU 0 ( echo No models found in %MODELS_DIR%. & pause & goto :MAIN_MENU ) if %unc_count% GTR 0 ( echo --- SPECIALIZED / UNCENSORED --- for /L %%i in (1,1,%unc_count%) do ( set /a idx+=1 set "f_name[!idx!]=!unc_n[%%i]!" set "f_path[!idx!]=!unc_p[%%i]!" echo [!idx!] !unc_n[%%i]! ) echo. ) if %std_count% GTR 0 ( echo --- STANDARD / GENERAL --- for /L %%i in (1,1,%std_count%) do ( set /a idx+=1 set "f_name[!idx!]=!std_n[%%i]!" set "f_path[!idx!]=!std_p[%%i]!" echo [!idx!] !std_n[%%i]! ) ) echo. echo [B] Back to Dashboard echo ======================================================== set /p "M_SEL=Select model (1-%idx% or B): " if /I "%M_SEL%"=="B" goto :MAIN_MENU if "%M_SEL%"=="" goto :MODEL_SELECT if %M_SEL% GTR %idx% goto :MODEL_SELECT if %M_SEL% LSS 1 goto :MODEL_SELECT set "SEL_PATH=!f_path[%M_SEL%]!" set "SEL_NAME=!f_name[%M_SEL%]!" echo [%date% %time%] Model selected: !SEL_NAME! >> "%LOGFILE%" echo [%date% %time%] Path: !SEL_PATH! >> "%LOGFILE%" REM ========================================================= REM CHECK FOR SAVED CONFIG (JSON) REM ========================================================= :CHECK_CONFIG echo [%date% %time%] CHECK_CONFIG starting >> "%LOGFILE%" REM Initial default values (will be overwritten if config exists) set "CFG_CONTEXT=8192" set "CFG_TEMPLATE_FLAG=" set "CFG_TEMPLATE_NAME=Auto" set "CFG_MMPROJ_FLAG=" set "CFG_VISION_NAME=None" set "CFG_FLASH_ATTN=1" set "CFG_DIRECT_IO=0" set "CFG_REASON_BUDGET=0" set "CFG_REASON_FORMAT=" set "CFG_SAMPLER_FLAGS=" set "CFG_SAMPLER_NAME=Default" set "CFG_CACHE_K=f16" set "CFG_CACHE_V=f16" REM Check if JSON config exists using Python (temp file avoids for /f quoting issues with spaced paths) set "HAS_JSON_CONFIG=false" "%BASE%venv\Scripts\python.exe" "%BASE%src\config_manager.py" has-config "!SEL_NAME!" > "%LOGS_DIR%\has_config.tmp" 2>nul if exist "%LOGS_DIR%\has_config.tmp" ( set /p HAS_JSON_CONFIG=<"%LOGS_DIR%\has_config.tmp" del "%LOGS_DIR%\has_config.tmp" 2>nul ) echo [%date% %time%] Has JSON config: !HAS_JSON_CONFIG! >> "%LOGFILE%" if "!HAS_JSON_CONFIG!"=="true" ( echo [%date% %time%] Loading JSON config via Python >> "%LOGFILE%" REM Use Python to load config into batch-friendly key=value pairs "%BASE%venv\Scripts\python.exe" "%BASE%src\config_manager.py" get-batch-vars "!SEL_NAME!" > "%LOGS_DIR%\config_vars.tmp" for /f "usebackq tokens=1,* delims==" %%a in ("%LOGS_DIR%\config_vars.tmp") do ( set "%%a=%%b" ) del "%LOGS_DIR%\config_vars.tmp" 2>nul set "USE_JSON_CONFIG=1" goto :AUTO_LAUNCH_COUNTDOWN ) REM Also check for legacy .bat config (for backwards compatibility) set "CONFIG_FILE=%SEL_PATH%.llama_config.bat" if exist "%CONFIG_FILE%" ( echo [%date% %time%] Found legacy .bat config >> "%LOGFILE%" call "%CONFIG_FILE%" set "USE_JSON_CONFIG=0" goto :AUTO_LAUNCH_COUNTDOWN ) echo [%date% %time%] No config, going to MANUAL_SETUP >> "%LOGFILE%" goto :MANUAL_SETUP REM ========================================================= REM AUTO LAUNCH COUNTDOWN REM ========================================================= :AUTO_LAUNCH_COUNTDOWN cls echo ======================================================== echo SAVED CONFIG LOADED echo ======================================================== echo Model: %SEL_NAME% echo Template: %CFG_TEMPLATE_NAME% echo Context: %CFG_CONTEXT% echo Vision: %CFG_VISION_NAME% echo Thinking: %CFG_REASON_BUDGET% (%CFG_REASON_FORMAT%) echo Sampler: %CFG_SAMPLER_NAME% echo KV Cache: %CFG_CACHE_K% / %CFG_CACHE_V% echo ======================================================== echo. echo Launching in 3 seconds... echo. echo [ Press 'E' to Edit/Reset settings ] echo [ Press 'SPACE' to Launch immediately ] echo. choice /C ES /T 3 /D S /N if %errorlevel%==1 goto :RESET_CONFIG goto :PREPARE_LAUNCH :RESET_CONFIG cls echo Resetting configuration for this model... REM Delete JSON config if exists "%BASE%venv\Scripts\python.exe" "%BASE%src\config_manager.py" delete-config "!SEL_NAME!" 2>nul REM Also delete legacy .bat config if exists if exist "%CONFIG_FILE%" del "%CONFIG_FILE%" set "USE_JSON_CONFIG=0" timeout /t 1 >nul goto :MANUAL_SETUP REM ========================================================= REM MANUAL SETUP WIZARD REM ========================================================= :MANUAL_SETUP echo [%date% %time%] MANUAL_SETUP entered >> "%LOGFILE%" REM --- 1. Vision --- :SETUP_VISION echo [%date% %time%] SETUP_VISION >> "%LOGFILE%" cls echo [SETUP 1/6] Vision / Multimodal echo -------------------------------- echo Does %SEL_NAME% need a projector file? (Visual LLMs) echo. echo [1] No (Standard) echo [2] Yes (Select mmproj file) echo. echo [B] Back to Model List echo. set /p "V_CHOICE=Select (1-2 or B): " if /I "%V_CHOICE%"=="B" goto :MODEL_SELECT if "%V_CHOICE%"=="2" ( cls echo Scanning for *mmproj*... set v_idx=0 for /f "tokens=1 delims==" %%v in ('set v_path[ 2^>nul') do set "%%v=" for %%f in ("%MODELS_DIR%\*mmproj*.gguf") do ( set /a v_idx+=1 set "v_path[!v_idx!]=%%f" echo [!v_idx!] %%~nxf ) if !v_idx!==0 ( echo No mmproj files found. Skipping. timeout /t 2 >nul ) else ( echo. echo [B] Back echo. set /p "PROJ_SEL=Select file (1-!v_idx! or B): " if /I "!PROJ_SEL!"=="B" goto :SETUP_VISION if defined v_path[!PROJ_SEL!] ( set "CFG_MMPROJ_FLAG=--mmproj "!v_path[%PROJ_SEL%]!"" set "CFG_VISION_NAME=Active" ) ) ) REM --- 2. Template --- :SETUP_TEMPLATE echo [%date% %time%] SETUP_TEMPLATE >> "%LOGFILE%" cls echo [SETUP 2/6] Chat Template echo -------------------------------- echo [1] Auto-Detect (Standard) echo [2] ChatML (Alice, Uncensored, Dolphin, Qwen, Anubis) echo [3] Llama 3 (Meta Llama 3) echo [4] Llama 3.1 (Meta Llama 3.1) echo [5] Mistral (Mistral v0.1/0.3 / Devstral) echo [6] DeepSeek (V2 / V3) echo [7] Alpaca (Older Merges) echo [8] Mistral V3 Tekken (Brother Dusk, Mistral3 RP) echo [9] Mistral V7 Tekken (Nemo RP Finetunes) echo. echo [B] Back echo. set /p "T_SEL=Select (1-9 or B): " if /I "%T_SEL%"=="B" goto :SETUP_VISION if "%T_SEL%"=="1" (set "CFG_TEMPLATE_FLAG=" & set "CFG_TEMPLATE_NAME=Auto") if "%T_SEL%"=="2" (set "CFG_TEMPLATE_FLAG=--chat-template chatml" & set "CFG_TEMPLATE_NAME=ChatML") if "%T_SEL%"=="3" (set "CFG_TEMPLATE_FLAG=--chat-template llama3" & set "CFG_TEMPLATE_NAME=Llama3") if "%T_SEL%"=="4" (set "CFG_TEMPLATE_FLAG=--chat-template llama3.1" & set "CFG_TEMPLATE_NAME=Llama3.1") if "%T_SEL%"=="5" (set "CFG_TEMPLATE_FLAG=--chat-template mistral" & set "CFG_TEMPLATE_NAME=Mistral") if "%T_SEL%"=="6" (set "CFG_TEMPLATE_FLAG=--chat-template deepseek3" & set "CFG_TEMPLATE_NAME=DeepSeek") if "%T_SEL%"=="7" (set "CFG_TEMPLATE_FLAG=--chat-template alpaca" & set "CFG_TEMPLATE_NAME=Alpaca") if "%T_SEL%"=="8" (set "CFG_TEMPLATE_FLAG=--chat-template mistral-v3-tekken" & set "CFG_TEMPLATE_NAME=MistralV3Tekken") if "%T_SEL%"=="9" (set "CFG_TEMPLATE_FLAG=--chat-template mistral-v7-tekken" & set "CFG_TEMPLATE_NAME=MistralV7Tekken") REM --- 3. Reasoning --- :SETUP_REASONING echo [%date% %time%] SETUP_REASONING >> "%LOGFILE%" REM Check if this is a reasoning model set "IS_REASONING=0" echo !SEL_NAME! | findstr /I /R "\ \ \ \" >nul if !errorlevel! EQU 0 set "IS_REASONING=1" if "!IS_REASONING!"=="0" ( set "CFG_REASON_BUDGET=0" set "CFG_REASON_FORMAT=" goto :SETUP_SAMPLER ) cls echo [SETUP 3/6] Reasoning / Thinking echo -------------------------------- echo Detected a 'Reasoning' model (R1/Thinking). echo. echo [1] Budget: 0 (Disabled) echo [2] Budget: -1 (Unrestricted) echo. echo [3] Format: Default echo [4] Format: DeepSeek (Recommended for R1 models) echo. echo [C] Continue echo [B] Back echo. set /p "R_SEL=Select (1-4, C or B): " if /I "!R_SEL!"=="B" goto :SETUP_TEMPLATE if /I "!R_SEL!"=="C" goto :SETUP_SAMPLER if "!R_SEL!"=="1" set "CFG_REASON_BUDGET=0" if "!R_SEL!"=="2" set "CFG_REASON_BUDGET=-1" if "!R_SEL!"=="3" set "CFG_REASON_FORMAT=" if "!R_SEL!"=="4" set "CFG_REASON_FORMAT=deepseek" REM Loop back to allow setting both budget and format if needed goto :SETUP_REASONING REM --- 4. Sampler --- :SETUP_SAMPLER echo [%date% %time%] SETUP_SAMPLER >> "%LOGFILE%" cls echo [SETUP 4/6] Sampler Presets echo -------------------------------- echo [1] Default (llama.cpp standard) echo [2] Creative (Min-P + DRY Sampler - High Quality RP) echo [3] Precise (Low Temp - Best for Logic/Coding) echo [4] Extreme (High Temp + High Penalty - Chaotic) echo. echo [B] Back echo. set /p "S_SEL=Select (1-4 or B): " if /I "%S_SEL%"=="B" ( echo %SEL_NAME% | findstr /I /R "\ \ \ \" >nul if !errorlevel! EQU 0 goto :SETUP_REASONING goto :SETUP_TEMPLATE ) if "%S_SEL%"=="1" (set "CFG_SAMPLER_FLAGS=" & set "CFG_SAMPLER_NAME=Default") if "%S_SEL%"=="2" ( set "CFG_SAMPLER_FLAGS=--temp 1.0 --min-p 0.02 --dry-multiplier 0.8 --dry-base 1.75 --dry-allowed-length 4" set "CFG_SAMPLER_NAME=Creative (DRY+MinP)" ) if "%S_SEL%"=="3" ( set "CFG_SAMPLER_FLAGS=--temp 0.2 --min-p 0.1" set "CFG_SAMPLER_NAME=Precise" ) if "%S_SEL%"=="4" ( set "CFG_SAMPLER_FLAGS=--temp 1.3 --min-p 0.015 --dry-multiplier 1.0 --dry-base 2.0" set "CFG_SAMPLER_NAME=Extreme" ) REM --- 5. Context --- :SETUP_CONTEXT echo [%date% %time%] SETUP_CONTEXT >> "%LOGFILE%" cls echo [SETUP 5/6] Context Size (Memory) echo -------------------------------- echo [1] 4096 (Rescue Mode) echo [2] 8192 (Standard) echo [3] 16384 (Mid) echo [4] 32768 (Large) echo [5] 65536 (Extra Large) echo [6] 131072 (Full Llama3/8B) echo. echo [B] Back echo. set /p "C_SEL=Select (1-6 or B): " if /I "%C_SEL%"=="B" goto :SETUP_SAMPLER if "%C_SEL%"=="1" set "CFG_CONTEXT=4096" if "%C_SEL%"=="2" set "CFG_CONTEXT=8192" if "%C_SEL%"=="3" set "CFG_CONTEXT=16384" if "%C_SEL%"=="4" set "CFG_CONTEXT=32768" if "%C_SEL%"=="5" set "CFG_CONTEXT=65536" if "%C_SEL%"=="6" set "CFG_CONTEXT=131072" REM --- 6. Performance --- :SETUP_PERF echo [%date% %time%] SETUP_PERF >> "%LOGFILE%" cls echo [SETUP 6/7] Performance echo -------------------------------- echo Enable Flash Attention? (Faster processing) echo. echo [1] Auto (Recommended) echo [2] On (Force Enabled) echo [3] Off (Force Disabled) echo. echo [B] Back echo. set /p "P_SEL=Select (1-3 or B): " if /I "%P_SEL%"=="B" goto :SETUP_CONTEXT set "CFG_FLASH_ATTN=auto" set "CFG_DIRECT_IO=0" if "%P_SEL%"=="2" set "CFG_FLASH_ATTN=1" if "%P_SEL%"=="3" set "CFG_FLASH_ATTN=0" :SETUP_DIRECTIO cls echo [SETUP 6b/7] Performance - Direct I/O echo ------------------------------------------ echo Enable Direct I/O? (May improve performance on some SSDs) echo. echo [1] No (Default) echo [2] Yes (Experimental) echo. echo [B] Back echo. set /p "D_SEL=Select (1-2 or B): " if /I "%D_SEL%"=="B" goto :SETUP_PERF if "%D_SEL%"=="2" set "CFG_DIRECT_IO=1" REM --- 7. KV Cache --- :SETUP_KV cls echo [SETUP 7/7] KV Cache Quantization echo -------------------------------- echo Reduce VRAM usage. K/V Cache can be compressed. echo f16 = Lossless (High VRAM), q8_0 = Good, q4_0 = Smallest echo. echo [1] f16 (Standard) echo [2] q8_0 (Balanced) echo [3] q4_0 (Max Context) echo. echo [B] Back echo. set /p "KV_SEL=Select (1-3 or B): " if /I "%KV_SEL%"=="B" goto :SETUP_DIRECTIO if "%KV_SEL%"=="1" (set "CFG_CACHE_K=f16" & set "CFG_CACHE_V=f16") if "%KV_SEL%"=="2" (set "CFG_CACHE_K=q8_0" & set "CFG_CACHE_V=q8_0") if "%KV_SEL%"=="3" (set "CFG_CACHE_K=q4_0" & set "CFG_CACHE_V=q4_0") REM --- SAVE CONFIG? --- cls echo ======================================================== echo CONFIGURATION READY echo ======================================================== echo Model: %SEL_NAME% echo Template: %CFG_TEMPLATE_NAME% echo Context: %CFG_CONTEXT% echo Vision: %CFG_VISION_NAME% echo Thinking: %CFG_REASON_BUDGET% echo Sampler: %CFG_SAMPLER_NAME% echo Flash-Attn: !CFG_FLASH_ATTN! echo Direct-IO: !CFG_DIRECT_IO! echo KV Cache: !CFG_CACHE_K!/!CFG_CACHE_V! echo ======================================================== echo. set /p "SAVE_CONF=Save as default for this model? (Y/N): " if /I "%SAVE_CONF%"=="Y" ( REM Build JSON config string set "TEMPLATE_ID=auto" if "!CFG_TEMPLATE_NAME!"=="ChatML" set "TEMPLATE_ID=chatml" if "!CFG_TEMPLATE_NAME!"=="Llama3" set "TEMPLATE_ID=llama3" if "!CFG_TEMPLATE_NAME!"=="Llama3.1" set "TEMPLATE_ID=llama3.1" if "!CFG_TEMPLATE_NAME!"=="Mistral" set "TEMPLATE_ID=mistral" if "!CFG_TEMPLATE_NAME!"=="DeepSeek" set "TEMPLATE_ID=deepseek" if "!CFG_TEMPLATE_NAME!"=="Alpaca" set "TEMPLATE_ID=alpaca" if "!CFG_TEMPLATE_NAME!"=="MistralV3Tekken" set "TEMPLATE_ID=mistral-v3-tekken" if "!CFG_TEMPLATE_NAME!"=="MistralV7Tekken" set "TEMPLATE_ID=mistral-v7-tekken" set "SAMPLER_ID=default" echo !CFG_SAMPLER_NAME! | findstr /C:"Creative" >nul && set "SAMPLER_ID=creative" echo !CFG_SAMPLER_NAME! | findstr /C:"Precise" >nul && set "SAMPLER_ID=precise" echo !CFG_SAMPLER_NAME! | findstr /C:"Extreme" >nul && set "SAMPLER_ID=extreme" set "FLASH_BOOL=true" if "!CFG_FLASH_ATTN!"=="0" set "FLASH_BOOL=false" set "DIO_BOOL=false" if "!CFG_DIRECT_IO!"=="1" set "DIO_BOOL=true" set "REASON_FMT_JSON=null" if not "!CFG_REASON_FORMAT!"=="" set "REASON_FMT_JSON="!CFG_REASON_FORMAT!"" set "JSON_CFG={"model":{"context_size":!CFG_CONTEXT!,"chat_template":"!TEMPLATE_ID!"},"hardware":{"flash_attention":!FLASH_BOOL!,"direct_io":!DIO_BOOL!,"cache_type_k":"!CFG_CACHE_K!","cache_type_v":"!CFG_CACHE_V!"},"reasoning":{"budget":!CFG_REASON_BUDGET!,"format":!REASON_FMT_JSON!},"sampler":{"preset":"!SAMPLER_ID!"}}" "%BASE%venv\Scripts\python.exe" "%BASE%src\config_manager.py" save "!SEL_NAME!" "!JSON_CFG!" echo Config saved to JSON! timeout /t 1 >nul ) :PREPARE_LAUNCH echo [%date% %time%] PREPARE_LAUNCH >> "%LOGFILE%" REM If using JSON config, get flags from Python if "!USE_JSON_CONFIG!"=="1" ( echo [%date% %time%] Using JSON config for launch >> "%LOGFILE%" REM Get all flags from Python config manager and store in temp file "%BASE%venv\Scripts\python.exe" "%BASE%src\config_manager.py" get-flags "!SEL_NAME!" > "%LOGS_DIR%\json_flags.tmp" REM Read flags into a single variable set "JSON_FLAGS=" for /f "usebackq delims=" %%f in ("%LOGS_DIR%\json_flags.tmp") do ( set "JSON_FLAGS=!JSON_FLAGS! %%f" ) del "%LOGS_DIR%\json_flags.tmp" 2>nul goto :LAUNCH_JSON ) REM Legacy batch variable flag preparation (for old .bat configs) set "REASON_BUDGET_FLAG=" if "%CFG_REASON_BUDGET%"=="-1" set "REASON_BUDGET_FLAG=--reasoning-budget -1" if "%CFG_REASON_BUDGET%"=="0" set "REASON_BUDGET_FLAG=--reasoning-budget 0" set "REASON_FORMAT_FLAG=" if NOT "%CFG_REASON_FORMAT%"=="" set "REASON_FORMAT_FLAG=--reasoning-format %CFG_REASON_FORMAT%" set "FLASH_ATTN_FLAG=--flash-attn on" if "%CFG_FLASH_ATTN%"=="0" set "FLASH_ATTN_FLAG=--flash-attn off" set "DIRECT_IO_FLAG=" if "%CFG_DIRECT_IO%"=="1" set "DIRECT_IO_FLAG=--direct-io" set "KV_CACHE_FLAG=-ctk %CFG_CACHE_K% -ctv %CFG_CACHE_V%" REM ========================================================= REM LAUNCH (Legacy batch vars) REM ========================================================= :LAUNCH echo [%date% %time%] LAUNCH (legacy) >> "%LOGFILE%" cls echo ======================================================== echo Launching... echo Model: %SEL_NAME% echo Mode: %CFG_CONTEXT% Context / %CFG_TEMPLATE_NAME% echo ======================================================== echo. REM Start a separate process to wait for the server and open the browser if "%AUTO_OPEN_BROWSER%"=="1" ( echo [Browser Watcher Started] start /min "LlamaBrowser" cmd /c "for /L %%i in (1,1,60) do (netstat -ano | findstr :%PORT% | findstr LISTENING >nul && (start http://%HOST%:%PORT% & exit) || timeout /t 1 >nul)" ) REM The main execution call (legacy mode) "%BIN_DIR%\%EXE_NAME%" ^ -m "%SEL_PATH%" ^ --host %HOST% ^ --port %PORT% ^ -c %CFG_CONTEXT% ^ -ngl %GPU_LAYERS% ^ -t %THREADS% ^ -np %PARALLEL_SLOTS% ^ %CFG_TEMPLATE_FLAG% ^ %CFG_MMPROJ_FLAG% ^ %FLASH_ATTN_FLAG% ^ %DIRECT_IO_FLAG% ^ %KV_CACHE_FLAG% ^ %REASON_BUDGET_FLAG% ^ %REASON_FORMAT_FLAG% ^ %CFG_SAMPLER_FLAGS% ^ --no-warmup goto :POST_LAUNCH REM ========================================================= REM LAUNCH (JSON Config) REM ========================================================= :LAUNCH_JSON echo [%date% %time%] LAUNCH (JSON) >> "%LOGFILE%" cls echo ======================================================== echo Launching... echo Model: %SEL_NAME% echo Mode: %CFG_CONTEXT% Context / %CFG_TEMPLATE_NAME% echo ======================================================== echo. REM Start a separate process to wait for the server and open the browser if "%AUTO_OPEN_BROWSER%"=="1" ( echo [Browser Watcher Started] start /min "LlamaBrowser" cmd /c "for /L %%i in (1,1,60) do (netstat -ano | findstr :%PORT% | findstr LISTENING >nul && (start http://%HOST%:%PORT% & exit) || timeout /t 1 >nul)" ) REM The main execution call (JSON config mode - flags from Python) "%BIN_DIR%\%EXE_NAME%" -m "%SEL_PATH%" !JSON_FLAGS! :POST_LAUNCH echo. echo Server closed. pause goto :GET_STATS :EDIT_CONFIG cls echo. echo Opening global configuration is not directly supported yet. echo Please edit the defaults at the top of this batch file. echo. pause goto :MAIN_MENU