Avoid using visited matrix altogether, same time avoid using special characters. Note that adding 800 helps us later recover the original char since all chars are <256.

public void dfs(char[][] board, String str, int x, int y, Trie trie) {
if (x < 0 || x >= board.length || y < 0 || y >= board[0].length) return;
if (board[x][y] > 800) return;
str += board[x][y];
if (!trie.startsWith(str)) return;
if (trie.search(str)) {
res.add(str);
}
board[x][y] += 800;
dfs(board, str, x - 1, y, trie);
dfs(board, str, x + 1, y, trie);
dfs(board, str, x, y - 1, trie);
dfs(board, str, x, y + 1, trie);
board[x][y] -= 800;
}