Fix slow push (#1392)

To determine which files are part of a filtered subset,
regardless of their potentially changed path a trick is used:
The filter is applied and then the inverted filter is applied
to the result.
The implementation of that trick was to simply chain the filter
with the inverted one. However this caused flatten() to create
a hughe intermediate representation (something like a cartesian product)
that then has to be simplified down again by optimize()/simplify().
While that worked, it was very computationally expensive making
push extremly slow in some cases.
Now instead of chaining apply() is called twice, avoiding the
huge intermediate representation and thus expensive optimization.

Change: fix-slow-push

Author: christian-schilling

josh-core/src/filter/mod.rs

@@ -1105,11 +1105,8 @@ pub fn unapply<'a>(
     parent_tree: git2::Tree<'a>,
 ) -> JoshResult<git2::Tree<'a>> {
     if let Ok(inverted) = invert(filter) {
-        let matching = apply(
-            transaction,
-            chain(invert(inverted)?, inverted),
-            parent_tree.clone(),
-        )?;
+        let filtered = apply(transaction, invert(inverted)?, parent_tree.clone())?;
+        let matching = apply(transaction, inverted, filtered)?;
         let stripped = tree::subtract(transaction, parent_tree.id(), matching.id())?;
         let new_tree = apply(transaction, inverted, tree)?;
 
@@ -1166,11 +1163,8 @@ fn unapply_workspace<'a>(
             let wsj_file = chain(root, wsj_file);
             let filter = compose(wsj_file, compose(workspace, root));
             let original_filter = compose(wsj_file, compose(original_workspace, root));
-            let matching = apply(
-                transaction,
-                chain(original_filter, invert(original_filter)?),
-                parent_tree.clone(),
-            )?;
+            let filtered = apply(transaction, original_filter, parent_tree.clone())?;
+            let matching = apply(transaction, invert(original_filter)?, filtered)?;
             let stripped = tree::subtract(transaction, parent_tree.id(), matching.id())?;
             let new_tree = apply(transaction, invert(filter)?, tree)?;
 

tests/proxy/workspace.t

@@ -335,8 +335,6 @@
       |   |   `-- f4045367114a7584eefa64b95bb69d7f840aef
       |   |-- 64
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 69
       |   |   `-- d753308940fb9a6c60bf04d0b1c7421fd91aba
       |   |-- 77
@@ -365,8 +363,6 @@
       |   |   `-- 76696a87dd29d841caa7f0d48e2f4e5359d369
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- be
       |   |   |-- 06ec33a91b8a108bd135b5a42a84292b035d6b
       |   |   `-- 1afe91719c4fb58e09207874905f09858cfb66
@@ -404,6 +400,6 @@
           |-- namespaces
           `-- tags
 
-  96 directories, 84 files
+  94 directories, 82 files
 
 $ cat ${TESTTMP}/josh-proxy.out

tests/proxy/workspace_create.t

@@ -611,8 +611,7 @@ Note that ws/d/ is now present in the ws
       |   |-- 40
       |   |   `-- c389b6b248e13f3cb88dcd79467d7396a4489e
       |   |-- 43
-      |   |   |-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
-      |   |   `-- c475ca897b62fd739409aee5db69b0c480aa0d
+      |   |   `-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
       |   |-- 46
       |   |   `-- bc1eabe4b2029b9fcb661f0d447d8389d17337
       |   |-- 47
@@ -622,8 +621,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
       |   |-- 4c
       |   |   `-- a9fbf65ee12663ac24db4be4cab10e53d6d6e7
-      |   |-- 4d
-      |   |   `-- dbf4f812dbd547947a2511b01985fb7a460eae
       |   |-- 5b
       |   |   `-- 560252b0c3c6abc5e0327596a49efb15e494cb
       |   |-- 5e
@@ -634,8 +631,6 @@ Note that ws/d/ is now present in the ws
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6c
       |   |   `-- 68dd37602c8e2036362ab81b12829c4d6c0867
       |   |-- 6d
@@ -684,8 +679,6 @@ Note that ws/d/ is now present in the ws
       |   |   |-- 269dc50ffcdc1b87664a061926bf9a072a3456
       |   |   |-- a7760c60ac08ecdcac395823637989a4d681a6
       |   |   `-- c31372c5de4fb705ffdcbf5a4ec5c5103231d9
-      |   |-- a9
-      |   |   `-- 954d28dd1313fb2d8f20fc11a1106376c2ffae
       |   |-- b1
       |   |   `-- c3c9b731fea98c82ce8e9286e213fcb033f70d
       |   |-- b2
@@ -700,8 +693,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- f9dcf1394d5152de67b115f55f25e4dc0a2398
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- bf
       |   |   `-- 7841c0db704465dda4b387d5da09694647d188
       |   |-- c1
@@ -755,6 +746,6 @@ Note that ws/d/ is now present in the ws
           |-- namespaces
           `-- tags
 
-  154 directories, 160 files
+  150 directories, 155 files
 
 $ cat ${TESTTMP}/josh-proxy.out

tests/proxy/workspace_edit_commit.t

@@ -295,6 +295,8 @@
       |   |   `-- cff5ef0fce401a1a33c2ac2713d6356cbc1b15
       |   |-- 1b
       |   |   `-- 46698f32d1d1db1eaeb34f8c9037778d65f3a9
+      |   |-- 1f
+      |   |   `-- 8abd6f6494fc2f715b3c1a39ac2e01ba806938
       |   |-- 20
       |   |   `-- 31777de79cd7c74834915674377e96d6864cc9
       |   |-- 22
@@ -337,8 +339,6 @@
       |   |   `-- 7ff045529989036cbd11bc32b2eaf5a8311aea
       |   |-- 60
       |   |   `-- 5066c26f66fca5a424aa32bd042ae71c7c8705
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6a
       |   |   `-- 80a5b3af9023d11cb7f37bc1f80d1d1805bfdb
       |   |-- 6c
@@ -387,7 +387,6 @@
       |   |   |-- c8440fe2cd36638ddb6b3505c1e8f2202f6191
       |   |   `-- cfe79e25ecd337b379e7ec06d7939dbcb2f6a0
       |   |-- bd
-      |   |   |-- 495daf53fe6fd641cc91e8208674050f602955
       |   |   `-- 56f16bf42ff74e68cfb7a59869c81920b02b87
       |   |-- be
       |   |   `-- c9383652a22b8a07acb86d5357a75f988286dc
@@ -433,4 +432,4 @@
           |-- namespaces
           `-- tags
 
-  129 directories, 124 files
+  129 directories, 123 files

tests/proxy/workspace_in_workspace.t

@@ -397,8 +397,6 @@
       |   |   `-- 5066c26f66fca5a424aa32bd042ae71c7c8705
       |   |-- 64
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 68
       |   |   `-- b1430cedd477c8117976bdb8aba3cee8b9aa9e
       |   |-- 6b
@@ -439,8 +437,6 @@
       |   |   `-- bd62ec41c785d12270e69b9d49f9babe62fcd6
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c2
       |   |   `-- d86319b61f31a7f4f1bc89b8ea4356b60c4658
       |   |-- c5
@@ -474,5 +470,5 @@
           |-- namespaces
           `-- tags
 
-  119 directories, 111 files
+  117 directories, 109 files
 

tests/proxy/workspace_modify.t

@@ -616,8 +616,7 @@ Note that ws/d/ is now present in the ws
       |   |-- 40
       |   |   `-- c389b6b248e13f3cb88dcd79467d7396a4489e
       |   |-- 43
-      |   |   |-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
-      |   |   `-- c475ca897b62fd739409aee5db69b0c480aa0d
+      |   |   `-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
       |   |-- 44
       |   |   `-- 625a9b34b1c6747c29903c3e641a4b2e580673
       |   |-- 46
@@ -628,8 +627,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
       |   |-- 4c
       |   |   `-- a9fbf65ee12663ac24db4be4cab10e53d6d6e7
-      |   |-- 4d
-      |   |   `-- dbf4f812dbd547947a2511b01985fb7a460eae
       |   |-- 59
       |   |   `-- 632d8d838ce9390679767c02c6bfe6c0d244a9
       |   |-- 5a
@@ -643,8 +640,6 @@ Note that ws/d/ is now present in the ws
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6c
       |   |   `-- 68dd37602c8e2036362ab81b12829c4d6c0867
       |   |-- 6d
@@ -694,8 +689,6 @@ Note that ws/d/ is now present in the ws
       |   |   |-- 1e8a91058875f157ca1246bdc403b88e93cd94
       |   |   |-- 269dc50ffcdc1b87664a061926bf9a072a3456
       |   |   `-- c31372c5de4fb705ffdcbf5a4ec5c5103231d9
-      |   |-- a9
-      |   |   `-- 954d28dd1313fb2d8f20fc11a1106376c2ffae
       |   |-- ab
       |   |   `-- a295fbe181a47f04650542b7d5582fbd983b98
       |   |-- b1
@@ -709,8 +702,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- f9dcf1394d5152de67b115f55f25e4dc0a2398
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c1
       |   |   `-- 489fc8fd6ae9ac08c0168d7cabaf5645b922fa
       |   |-- c2
@@ -766,6 +757,6 @@ Note that ws/d/ is now present in the ws
           |-- namespaces
           `-- tags
 
-  163 directories, 163 files
+  159 directories, 158 files
 
 $ cat ${TESTTMP}/josh-proxy.out

tests/proxy/workspace_modify_chain.t

@@ -396,8 +396,6 @@
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- ba3cf39148efcb72db4e7b663e4f42247e2273
       |   |-- 67
       |   |   `-- 48a16f7d9b29607fcd4df93681361a6105a14a
       |   |-- 69
@@ -440,8 +438,6 @@
       |   |   `-- 0969c65843463b2d0e86fd4c6efcae33012332
       |   |-- b0
       |   |   `-- 3b5fbc9a12109cd3f5308929e4812b3c998da6
-      |   |-- b7
-      |   |   `-- dfc3aee5bd47bcfcf5a40b7964164a2b1e838b
       |   |-- b8
       |   |   `-- 54e8ec3db62174179b215404936a58f1bb6a79
       |   |-- b9
@@ -460,8 +456,7 @@
       |   |   |-- 0544cd5f702a9b3418205ec0425c6ae77f9e3e
       |   |   `-- 7c94c08f59302a2b1587a01d4fd1680d7378c9
       |   |-- d7
-      |   |   |-- 330ea337031af43ba1cf6982a873a40b9170ac
-      |   |   `-- 94016e24d97a10d550de2384527f80f9fecfc8
+      |   |   `-- 330ea337031af43ba1cf6982a873a40b9170ac
       |   |-- e1
       |   |   `-- 25e6d9f8f9acca5ffd25ee3c97d09748ad2a8b
       |   |-- e5
@@ -492,6 +487,6 @@
           |-- namespaces
           `-- tags
 
-  124 directories, 120 files
+  122 directories, 117 files
 
 $ cat ${TESTTMP}/josh-proxy.out | grep VIEW

tests/proxy/workspace_modify_chain_prefix_subtree.t

@@ -469,8 +469,7 @@ Note that ws/d/ is now present in the ws
       |   |-- 40
       |   |   `-- c389b6b248e13f3cb88dcd79467d7396a4489e
       |   |-- 43
-      |   |   |-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
-      |   |   `-- c475ca897b62fd739409aee5db69b0c480aa0d
+      |   |   `-- 52611a9e7c56dfdfeadec043ced6d6ef7a5c33
       |   |-- 44
       |   |   |-- 625a9b34b1c6747c29903c3e641a4b2e580673
       |   |   `-- edc62d506b9805a3edfc74db15b1cc0bfc6871
@@ -482,8 +481,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- 825dc642cb6eb9a060e54bf8d69288fbee4904
       |   |-- 4c
       |   |   `-- a9fbf65ee12663ac24db4be4cab10e53d6d6e7
-      |   |-- 4d
-      |   |   `-- dbf4f812dbd547947a2511b01985fb7a460eae
       |   |-- 5a
       |   |   `-- 8a4b8c1452d54f1ca88454067369112809a4d2
       |   |-- 5b
@@ -493,8 +490,6 @@ Note that ws/d/ is now present in the ws
       |   |-- 64
       |   |   |-- 6fd2c5bfe156d57ba03f62f2fe735ddbb74e22
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 67
       |   |   `-- 12cb1b8c89e3b2272182f140c81aef3b718671
       |   |-- 6c
@@ -547,8 +542,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- c31372c5de4fb705ffdcbf5a4ec5c5103231d9
       |   |-- a4
       |   |   `-- b68220bdf7fb846eb9780f7846a2f4bf7cbcc3
-      |   |-- a9
-      |   |   `-- 954d28dd1313fb2d8f20fc11a1106376c2ffae
       |   |-- ab
       |   |   `-- a295fbe181a47f04650542b7d5582fbd983b98
       |   |-- b1
@@ -561,8 +554,6 @@ Note that ws/d/ is now present in the ws
       |   |   `-- f9dcf1394d5152de67b115f55f25e4dc0a2398
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c1
       |   |   `-- 489fc8fd6ae9ac08c0168d7cabaf5645b922fa
       |   |-- c2
@@ -606,6 +597,6 @@ Note that ws/d/ is now present in the ws
           |-- namespaces
           `-- tags
 
-  151 directories, 152 files
+  147 directories, 147 files
 
 $ cat ${TESTTMP}/josh-proxy.out | grep VIEW

tests/proxy/workspace_tags.t

@@ -371,8 +371,6 @@
       |   |   `-- 5066c26f66fca5a424aa32bd042ae71c7c8705
       |   |-- 64
       |   |   `-- d1f8d32b274d8c1eeb69891931f52b6ade9417
-      |   |-- 66
-      |   |   `-- b81c71c0ad10acdb2b4df3b04eef8abd79e64b
       |   |-- 6b
       |   |   `-- e0d68b8e87001c8b91281636e21d6387010332
       |   |-- 78
@@ -408,8 +406,6 @@
       |   |   `-- bd62ec41c785d12270e69b9d49f9babe62fcd6
       |   |-- bc
       |   |   `-- 665856e841c4ae4a956483dc57b2ea4cc20116
-      |   |-- bd
-      |   |   `-- 495daf53fe6fd641cc91e8208674050f602955
       |   |-- c2
       |   |   `-- d86319b61f31a7f4f1bc89b8ea4356b60c4658
       |   |-- c5
@@ -440,6 +436,6 @@
           |-- namespaces
           `-- tags
 
-  109 directories, 101 files
+  107 directories, 99 files
 
 $ cat ${TESTTMP}/josh-proxy.out