First part is my original code, second part is speedupped by @StefanPochmann.
He speed it up

from 62 ms to 27 ms by using a HashMap<vis, count>

from 27ms to 5ms by 1).using
int[] map
instead ofhashmap
; 2).using going downwards strategy; 3).usingint[][] arr
instead ofList<Integer>[] arr
.
He just made this code almost perfect.
public class Solution {
Map<Integer, Integer> map = new HashMap();
public int countArrangement(int N) {
List<Integer>[] arr = new List[N + 1]; // idx : possible vals
// construct all possible choices
for (int i = 1; i <= N; i++) {
arr[i] = new ArrayList();
for (int j = 1; j <= N; j++) {
if (i % j == 0  j % i == 0) {
arr[i].add(j);
}
}
}
return helper(arr, 1, 0);
}
private int helper(List<Integer>[] arr, int i, int vis) {
if (map.containsKey(vis)) return map.get(vis);
if (i == arr.length) return 1;
int count = 0;
for (int v : arr[i]) {
if ((vis >> v & 1) == 1) continue;
count += helper(arr, i + 1, vis ^ (1 << v));
}
map.put(vis, count);
return count;
}
}
Below is Stefan's optimized code. Really impressive.
public class Solution {
int[] map = new int[1 << 15];
public int countArrangement(int N) {
int[][] arr = new int[N + 1][]; // idx : possible vals
// construct all possible choices
for (int i = 1; i <= N; i++) {
List<Integer> tmp = new ArrayList();
for (int j = 1; j <= N; j++) {
if (i % j == 0  j % i == 0) {
tmp.add(1 << (j1));
}
}
arr[i] = new int[tmp.size()];
for (int k = 0; k < arr[i].length; k++)
arr[i][k] = tmp.get(k);
}
Arrays.fill(map, 0, 1 << N, 1);
return helper(arr, N, 0);
}
private int helper(int[][] arr, int i, int vis) {
if (map[vis] >= 0) return map[vis];
if (i == 1) return 1;
int count = 0;
for (int v : arr[i])
if ((vis & v) == 0)
count += helper(arr, i  1, vis ^ v);
map[vis] = count;
return count;
}
}