Skip to content

Commit 566eee1

Browse files
author
Joseph Sawaya
committed
Add rollback
This commit adds simple rollback by trying to undo the changes made during a failed Apply by Applying in the opposite direction. Signed-off-by: Joseph Sawaya <[email protected]>
1 parent 94d4b29 commit 566eee1

File tree

10 files changed

+211
-17
lines changed

10 files changed

+211
-17
lines changed

‎.github/workflows/docker-image.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1130,6 +1130,103 @@ jobs:
11301130
- name: Check the capabilities of cap1
11311131
run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi
11321132

1133+
rollback-validate:
1134+
runs-on: ubuntu-latest
1135+
needs: [ build , pull-and-archive ]
1136+
steps:
1137+
- name: pull in podman
1138+
uses: actions/download-artifact@v1
1139+
with:
1140+
name: podman-bins
1141+
path: bin
1142+
1143+
- name: replace
1144+
run: |
1145+
chmod +x bin/podman
1146+
sudo mv bin/podman /usr/bin/podman
1147+
1148+
- name: Enable the podman socket
1149+
run: sudo systemctl enable --now podman.socket
1150+
1151+
- uses: actions/checkout@v3
1152+
with:
1153+
path: main
1154+
1155+
- name: checkout with token
1156+
uses: actions/checkout@v3
1157+
with:
1158+
path: ci-rollback
1159+
ref: ci-rollback
1160+
1161+
- name: Enable the podman socket
1162+
run: sudo systemctl enable --now podman.socket
1163+
1164+
- name: reset and commit working
1165+
run: |
1166+
cd ci-rollback
1167+
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
1168+
git config --local user.name "github-actions[bot]"
1169+
rm -rf ./examples/ci-rollback || true
1170+
mkdir ./examples/ci-rollback
1171+
cp ./examples/rollback/working.yaml ./examples/ci-rollback/working.yaml
1172+
git add ./examples
1173+
git commit -m "add working" -a || true
1174+
git push -f || true
1175+
1176+
- name: pull artifact
1177+
uses: actions/download-artifact@v1
1178+
with:
1179+
name: fetchit-image
1180+
path: /tmp
1181+
1182+
- name: Load the image
1183+
run: sudo podman load -i /tmp/fetchit.tar
1184+
1185+
- name: pull artifact
1186+
uses: actions/download-artifact@v1
1187+
with:
1188+
name: colors
1189+
path: /tmp
1190+
1191+
- name: Load the image
1192+
run: sudo podman load -i /tmp/colors.tar
1193+
1194+
- name: tag the image
1195+
run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest
1196+
1197+
- name: set values relating to the current env
1198+
run: |
1199+
cd ci-rollback
1200+
sed -i 's| url: http://github.com/containers/fetchit| url: http://github.com/${{ github.repository }}|g' ./examples/ci-rollback.yaml
1201+
sed -i 's| branch: ci-rollback| branch: "{{ github.ref }}"|g' ./examples/ci-rollback.yaml
1202+
- name: Start fetchit
1203+
run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /home/runner/work/fetchit/fetchit/ci-rollback/examples/ci-rollback.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest
1204+
1205+
- name: identify rollback1 container
1206+
run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do sudo podman ps; c=$(sudo podman ps | grep rollback | wc -l); done'
1207+
1208+
- name: Logs
1209+
if: always()
1210+
run: sudo podman logs fetchit
1211+
1212+
- name: commit working
1213+
run: |
1214+
cd ci-rollback
1215+
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
1216+
git config --local user.name "github-actions[bot]"
1217+
ls
1218+
cp ./examples/rollback/1-working.yaml ./examples/ci-rollback/1-working.yaml
1219+
cp ./examples/rollback/2-broken.yaml ./examples/ci-rollback/2-broken.yaml
1220+
git add ./examples
1221+
git commit -m "add 1-working and 2-broken" -a
1222+
git push -f
1223+
1224+
- name: Wait for 2 pods
1225+
run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do sudo podman ps; c=$(sudo podman ps | grep rollback | wc -l); done'
1226+
1227+
- name: Check rollback succeeded
1228+
run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman ps | grep rollback | wc -l); done'
1229+
11331230
multi-engine-validate:
11341231
runs-on: ubuntu-latest
11351232
needs: [ build , pull-and-archive ]

‎examples/ci-rollback.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
targetConfigs:
2+
- name: fetchit
3+
url: http://github.com/josephsawaya/fetchit
4+
raw:
5+
- name: raw
6+
targetPath: examples/ci-rollback
7+
schedule: "*/1 * * * *"
8+
trackBadCommits: true
9+
branch: ci-rollback

‎examples/raw/cap.yaml‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
1-
{
2-
"Image":"docker.io/mmumshad/simple-webapp-color:latest",
3-
"Name": "cap2",
4-
"Env": {"APP_COLOR": "blue", "tree": "trunk"},
5-
"Mounts": [],
6-
"Volumes": [],
7-
"CapDrop": ["all"],
8-
"Ports": [{
9-
"host_ip": "",
10-
"container_port": 8080,
11-
"host_port": 9090,
12-
"range": 0,
13-
"protocol": ""}]
14-
}
1+
Image: docker.io/mmumshad/simple-webapp-color:latest
2+
Name: cap2
3+
Env:
4+
APP_COLOR": blue
5+
tree: trunk
6+
CapDrop:
7+
- all
8+
Ports:
9+
- container_port: 8080
10+
host_port: 9091
11+
range: 0

‎examples/rollback/1-working.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Image: "docker.io/mmumshad/simple-webapp-color:latest"
2+
Name: "rollback2"
3+
Env:
4+
APP_COLOR: "pink"
5+
tree: "trunk"
6+
Ports:
7+
- container_port: 8080
8+
host_port: 8081
9+
range: 0

‎examples/rollback/2-broken.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Typo below
2+
Image: "docker.io/mmumshad/simple-webapp-clor:latest"
3+
Name: "rollback3"
4+
Env:
5+
APP_COLOR: "pink"
6+
tree: "trunk"
7+
Ports:
8+
- container_port: 8080
9+
host_port: 8080
10+
range: 0

‎examples/rollback/working.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Image: "docker.io/mmumshad/simple-webapp-color:latest"
2+
Name: "rollback1"
3+
Env:
4+
APP_COLOR: "pink"
5+
tree: "trunk"
6+
Ports:
7+
- container_port: 8080
8+
host_port: 8080
9+
range: 0

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ require (
1919
gopkg.in/yaml.v3 v3.0.1
2020
k8s.io/api v0.23.5
2121
k8s.io/apimachinery v0.23.5
22+
k8s.io/klog/v2 v2.60.1-0.20220317184644-43cc75f9ae89
2223
sigs.k8s.io/yaml v1.3.0
2324
)
2425

@@ -250,7 +251,6 @@ require (
250251
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
251252
gopkg.in/warnings.v0 v0.1.2 // indirect
252253
gopkg.in/yaml.v2 v2.4.0 // indirect
253-
k8s.io/klog/v2 v2.60.1-0.20220317184644-43cc75f9ae89 // indirect
254254
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
255255
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
256256
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect

‎pkg/engine/apply.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,31 @@ func certHexFingerprint(cert *x509.Certificate) string {
153153
return hex.EncodeToString(fpr[:])
154154
}
155155

156+
func checkout(target *Target, hash plumbing.Hash) error {
157+
if hash == plumbing.ZeroHash {
158+
return nil
159+
}
160+
161+
directory := getDirectory(target)
162+
163+
repo, err := git.PlainOpen(directory)
164+
if err != nil {
165+
return utils.WrapErr(err, "Error opening repository: %s", directory)
166+
}
167+
168+
wt, err := repo.Worktree()
169+
if err != nil {
170+
return utils.WrapErr(err, "Error getting reference to worktree for repository", directory)
171+
}
172+
173+
err = wt.Checkout(&git.CheckoutOptions{Hash: hash})
174+
if err != nil {
175+
return utils.WrapErr(err, "Error checking out %s on branch %s", hash, target.branch)
176+
}
177+
178+
return nil
179+
}
180+
156181
func getCurrent(target *Target, methodType, methodName string) (plumbing.Hash, error) {
157182
directory := getDirectory(target)
158183
tagName := fmt.Sprintf("current-%s-%s", methodType, methodName)

‎pkg/engine/common.go‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ import (
77
"path/filepath"
88
"strings"
99

10+
"github.com/containers/fetchit/pkg/engine/utils"
1011
"github.com/go-git/go-git/v5/plumbing"
1112
"github.com/go-git/go-git/v5/plumbing/object"
13+
"k8s.io/klog/v2"
1214
)
1315

1416
type CommonMethod struct {
@@ -70,6 +72,11 @@ func getDirectory(target *Target) string {
7072
return filepath.Base(trimDir)
7173
}
7274

75+
type empty struct {
76+
}
77+
78+
var BadCommitList map[string]map[string]map[plumbing.Hash]empty = make(map[string]map[string]map[plumbing.Hash]empty)
79+
7380
func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[]string) error {
7481
directory := getDirectory(target)
7582
if target.disconnected {
@@ -79,6 +86,7 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[
7986
localDevicePull(directory, target.device, "", false)
8087
}
8188
}
89+
8290
latest, err := getLatest(target)
8391
if err != nil {
8492
return fmt.Errorf("Failed to get latest commit: %v", err)
@@ -89,10 +97,38 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[
8997
return fmt.Errorf("Failed to get current commit: %v", err)
9098
}
9199

100+
if target.trackBadCommits {
101+
if _, ok := BadCommitList[directory]; !ok {
102+
BadCommitList[directory] = make(map[string]map[plumbing.Hash]empty)
103+
}
104+
105+
if _, ok := BadCommitList[directory][m.GetKind()]; !ok {
106+
BadCommitList[directory][m.GetKind()] = make(map[plumbing.Hash]empty)
107+
}
108+
109+
if _, ok := BadCommitList[directory][m.GetKind()][latest]; ok {
110+
klog.Infof("No changes applied to target %s this run, %s currently at %s", directory, m.GetKind(), current)
111+
return nil
112+
}
113+
}
114+
92115
if latest != current {
93-
if err := m.Apply(ctx, conn, current, latest, tag); err != nil {
94-
return fmt.Errorf("Failed to apply changes: %v", err)
116+
if err = m.Apply(ctx, conn, current, latest, tag); err != nil {
117+
// Roll back automatically
118+
klog.Errorf("Failed to apply changes, rolling back to %v: %v", current, err)
119+
if err = checkout(target, current); err != nil {
120+
return utils.WrapErr(err, "Failed to checkout %s", current)
121+
}
122+
if err = m.Apply(ctx, conn, latest, current, tag); err != nil {
123+
// Roll back failed
124+
return fmt.Errorf("Roll back failed, state between %s and %s: %v", current, latest, err)
125+
}
126+
if target.trackBadCommits {
127+
BadCommitList[directory][m.GetKind()][latest] = empty{}
128+
}
129+
return fmt.Errorf("Rolled back to %v: %v", current, err)
95130
}
131+
96132
updateCurrent(ctx, target, latest, m.GetKind(), m.GetName())
97133
logger.Infof("Moved %s from %s to %s for git target %s", m.GetName(), current.String()[:hashReportLen], latest, target.url)
98134
} else {

‎pkg/engine/types.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ type TargetConfig struct {
3636
Device string `mapstructure:"device"`
3737
Disconnected bool `mapstructure:"disconnected"`
3838
VerifyCommitsInfo *VerifyCommitsInfo `mapstructure:"verifyCommitsInfo"`
39+
TrackBadCommits bool `mapstructure:"trackBadCommits"`
3940
Branch string `mapstructure:"branch"`
4041
Ansible []*Ansible `mapstructure:"ansible"`
4142
FileTransfer []*FileTransfer `mapstructure:"filetransfer"`
@@ -58,6 +59,7 @@ type Target struct {
5859
disconnected bool
5960
gitsignVerify bool
6061
gitsignRekorURL string
62+
trackBadCommits bool
6163
}
6264

6365
type SchedInfo struct {

0 commit comments

Comments
 (0)