To determine if a solution exists, it suffices to look at convergents of a quadratic irrational.
SolutionQ[-1|1, _] = True;
SolutionQ[0, _] = False;
SolutionQ[_, 0] = False;
SolutionQ[a_, b_] /; !CoprimeQ[a, 2b] = False;
SolutionQ[a_, b_] :=
Or[
AnyTrue[Most[Convergents[(-b + Sqrt[b^2+a^2])/a]], solQ[a, b]],
AnyTrue[Most[Convergents[(-b - Sqrt[b^2+a^2])/a]], solQ[a, b]]
]
solQ[a_, b_][r_] :=
With[{x = Numerator[r], y = Denominator[r]},
a(x^2 - y^2) + 2 b x y == 1
]
Some comparisons:
SolutionQ[9, 10] // AbsoluteTiming
SolutionQ[13, 9] // AbsoluteTiming
SolutionQ[99, 100] // AbsoluteTiming
SolutionQ[997, 1100] // AbsoluteTiming
SolutionQ[99711, 11011] // AbsoluteTiming
{0.00199, True}
{0.002114, False}
{0.009289, True}
{0.03192, True}
{0.743921, True}
FindInstance[9 (x^2 - y^2) + 2*10 x y == 1, {x, y}, Integers] // AbsoluteTiming
FindInstance[13 (x^2 - y^2) + 2*9 x y == 1, {x, y}, Integers] // AbsoluteTiming
TimeConstrained[FindInstance[99 (x^2 - y^2) + 2*100 x y == 1, {x, y}, Integers], 300]
TimeConstrained[FindInstance[997 (x^2 - y^2) + 2*1100 x y == 1, {x, y}, Integers], 300]
{7.4759, {{x -> -8485, y -> 3256}}}
{0.324242, {}}
$Aborted
$Aborted
{9.85082,True}
seconds. $\endgroup$