based on your idea, I simplify the code to below:

public boolean isSelfCrossing(int[] x) {
if(x == null || x.length <= 3) return false;
return !isSpinIn(x, 0, x.length - 1) && !isSpinOut(x, 0, x.length - 1);
}
// spin-in, if cross, return false.
private boolean isSpinIn(int[] x, int from, int end) {
for(int i = from, j = from + 2; j <= end; i++, j++)
if(i != from && x[i] <= x[j])
return false;
return true;
}
// spin-out / spin-out transit to spin in.
private boolean isSpinOut(int[] x, int from, int end) {
for(int i = from, j = from + 2; j < end; i++, j++)
if(x[i] >= x[j])
// Not crossing during transition from outside spiral into inside spiral and the violated part is a valid spin-in
return (i - 2 >= 0 && x[j] < (x[i] - x[i-2]) || i - 1 >= 0 && x[j+1] < (x[i + 1] - x[i-1])) && isSpinIn(x, i, end);
return true;
}