Inspired by a neat technique for finding selfrepeating pattern (https://discuss.leetcode.com/topic/71442/shortpython).

DP for substring. For convenience, just use substring as key of memoization. strEncoding[str] = shortest encoding of str.

Base case (in fact this 'base' is still recursive):
selfrepeating: repeatingIdx = (s + s).find(s, 1), repeatingIdx < len(s).
Find index of s in s + s starting from index 1.
E.g., 'abbbabbbcabbbabbbc' return 9, return 2[strEncoding(abbbabbbc)]. 
Recursion by Iterating string splitting point:
E.g., 'abbbabbbc' > strEncoding(abbbabbb) + strEncoding(c)
class Solution(object):
def encode(self, s):
strEncoding = dict()
return self.getEncoding(s, strEncoding)
def getEncoding(self, s, strEncoding):
encoding = s
# Base case: selfrepeating, such as 'abbbabbbcabbbabbbc' > 2[strEncoding(abbbabbbc)]
# Find index of s in s + s starting from index 1.
# E.g., 'abbbabbbcabbbabbbc' return 9. 'abcabcabcabc' return 3.
repeatingIdx = (s + s).find(s, 1)
if repeatingIdx < len(s):
repeatingSubstr = s[: repeatingIdx]
numRepeating = len(s) / len(repeatingSubstr)
if repeatingSubstr not in strEncoding:
strEncoding[repeatingSubstr] = self.getEncoding(repeatingSubstr, strEncoding)
repeatingSubstrEncoding = strEncoding[repeatingSubstr]
candidateEncoding = str(numRepeating) + '[' + repeatingSubstrEncoding + ']'
if len(candidateEncoding) < len(encoding):
encoding = candidateEncoding
# If not selfrepating, split string into substrings,
# such as 'abbbabbbc' > strEncoding(abbbabbb) + strEncoding(c)
else:
for i in range(1, len(s)):
leftSubstr = s[: i]
rightSubstr = s[i :]
if leftSubstr not in strEncoding:
strEncoding[leftSubstr] = self.getEncoding(leftSubstr, strEncoding)
leftSubstrEncoding = strEncoding[leftSubstr]
if rightSubstr not in strEncoding:
strEncoding[rightSubstr] = self.getEncoding(rightSubstr, strEncoding)
rightSubstrEncoding = strEncoding[rightSubstr]
if len(leftSubstrEncoding) + len(rightSubstrEncoding) < len(encoding):
encoding = leftSubstrEncoding + rightSubstrEncoding
return encoding