fix: handle paths with non-utf-8 bytes

Signed-off-by: Adrian Braemer <[email protected]>

Author: abraemer

src/commoncode/resource.py

@@ -389,7 +389,7 @@ def _get_resource_cache_location(self, path, create_dirs=False):
         path = clean_path(path)
 
         # for the cached file name, we use an md5 of the path to avoid things being too long
-        resid = str(md5(path.encode("utf-8")).hexdigest())
+        resid = str(md5(path.encode("utf-8", "surrogateescape")).hexdigest())
         cache_sub_dir, cache_file_name = resid[-2:], resid
 
         parent = join(self.cache_dir, cache_sub_dir)

tests/test_resource.py

@@ -694,6 +694,11 @@ def test_codebase_cache_default(self):
         codebase.save_resource(child)
         child_2 = codebase.get_resource(path=child.path)
         assert child_2 == child
+    
+    def test_codebase_cache_handles_non_utf8_path(self):
+        test_codebase = self.get_test_loc("resource/cache2")
+        codebase = Codebase(test_codebase)
+        codebase._get_resource_cache_location('resource/cache2/\udce9', create_dirs=True)
 
     def test_codebase_cache_all_in_memory(self):
         test_codebase = self.get_test_loc("resource/cache2")