The issue is more with libx265 than ffmpeg itself. For an example if you use libsvtav1 you can use /start /affinity xxx and it works fine.
What you can do is first limit amount of cores used by libx265 by adding option -x265-params pools=
Then when ffmpeg is running, set the right affinity
My complete set of 2 scripts to do that, i use 3 cores and keep core 0 unused to keep the system responsive :
pack.bat
@set fname=%*
@set fname=%fname:"=%
@rem define various constants
@set scale1440=-vf "scale=-1:1440:flags=lanczos"
@set scale1080=-vf "scale=-1:1080:flags=lanczos"
@set scale720=-vf "scale=-1:720:flags=lanczos"
@set scale540=-vf "scale=-1:540:flags=lanczos"
@set scale360=-vf "scale=-1:360:flags=lanczos"
@set b10=yuv420p10le
@set b8=yuv420p
@set a_copy=-acodec copy
@set a_aac128=-acodec aac -b:a 128k
@set a_aac160=-acodec aac -b:a 160k
@set svt_params=-svtav1-params tune=0:enable-overlays=1:scd=1:lp=3:film-grain=
@set hevc_params=-x265-params strong-intra-smoothing=0:rect=0:aq-mode=1:pools=3
@rem define output here
@set pix_fmt=%b8%
@set audio=%a_copy%
@set scale=%scale1080%
@set grain=0
@set preset=slow
@set crf=20
@rem some calculations
@set av1=-vcodec libsvtav1 -preset %preset% -crf %crf% %svt_params%%grain%
@set hevc=-vcodec libx265 -preset %preset% -crf %crf% %hevc_params%
@rem pick encoder here
@set video=%hevc%
%ComSpec% /c start /b "ffmpeg fix" powershell.exe .\fix_affinity.ps1
%ComSpec% /c start "title" /b /Low /wait /Affinity E /d "." ffmpeg.exe -n -i "%fname%" %scale% %audio% %video% -pix_fmt %pix_fmt% "%fname%_p%preset%c%crf%g%grain%_%pix_fmt%.mkv"
and fix_affinity.ps1
start-sleep -seconds 5
$ffmpeg_process = get-process ffmpeg
$ffmpeg_process.processoraffinity=14