Skip to content

Commit fc2de06

Browse files
authored
Add format and lint to Makefile and CI (#23)
1 parent 02e98c5 commit fc2de06

File tree

16 files changed

+223
-47
lines changed

16 files changed

+223
-47
lines changed

.circleci/config.yml

+9-6
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,15 @@ version: 2
33
jobs:
44
test:
55
docker:
6-
- image: circleci/golang:1.11.5
6+
- image: circleci/python:3.7.2-stretch
77
steps:
88
- checkout
9+
- run: wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz && sudo tar -C /usr/local -xzf go1.11.5.linux-amd64.tar.gz && rm -rf go*.tar.gz && echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV && echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV
910
- run: GO111MODULE=off go get -u -v golang.org/x/lint/golint
11+
- run: sudo pip3 install black
1012
- run:
11-
name: Go Lint
12-
command: make lint-go
13+
name: Lint
14+
command: make lint
1315
- run:
1416
name: License Test
1517
command: make find-missing-license
@@ -18,17 +20,18 @@ jobs:
1820
command: make test-go
1921
test-release:
2022
docker:
21-
- image: circleci/golang:1.11.5
23+
- image: circleci/python:3.7.2-stretch
2224
steps:
2325
- checkout
2426
- run:
2527
name: Version Test
2628
command: make find-missing-version
2729
deploy:
2830
docker:
29-
- image: circleci/golang:1.11.5
31+
- image: circleci/python:3.7.2-stretch
3032
steps:
31-
- run: curl -s "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" && unzip awscli-bundle.zip && sudo ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws && rm -rf awscli-bundle*
33+
- run: wget https://dl.google.com/go/go1.11.5.linux-amd64.tar.gz && sudo tar -C /usr/local -xzf go1.11.5.linux-amd64.tar.gz && rm -rf go*.tar.gz && echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV && echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV
34+
- run: sudo pip3 install awscli
3235
- checkout
3336
- setup_remote_docker
3437
- run: make build-images

.github/pull_request_template.md

-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
---
32
Checklist:
43
- [ ] Run automated tests (`make test`)

LICENSE

-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
Apache License
32
Version 2.0, January 2004
43
http://www.apache.org/licenses/

Makefile

+7-4
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,10 @@ aws-clear-bucket:
9898
tools:
9999
@go get -u -v golang.org/x/lint/golint
100100
@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun
101-
@curl -fLSs https://circle.ci/cli | bash
101+
@pip3 install black
102+
103+
format:
104+
@./dev/format.sh
102105

103106
#########
104107
# Tests #
@@ -110,12 +113,12 @@ test:
110113
test-go:
111114
@./build/test.sh go
112115

113-
lint-go:
114-
@./build/golint.sh go
115-
116116
test-python:
117117
@./build/test.sh python
118118

119+
lint:
120+
@./build/lint.sh
121+
119122
###############
120123
# CI Commands #
121124
###############

build/build-image.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@
1717

1818
set -euo pipefail
1919

20+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
21+
2022
CORTEX_VERSION=master
2123

2224
dir=$1
2325
image=$2
2426

25-
docker build . -f $dir/Dockerfile -t cortexlabs/$image \
26-
-t cortexlabs/$image:$CORTEX_VERSION
27+
docker build "$ROOT" -f $dir/Dockerfile -t cortexlabs/$image \
28+
-t cortexlabs/$image:$CORTEX_VERSION

build/cli.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717

1818
set -euo pipefail
1919

20+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
21+
2022
CORTEX_VERSION=master
2123

2224
function build_and_upload() {
2325
set -euo pipefail
2426

2527
os=$1
26-
GOOS=$os GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go build -o cortex github.com/cortexlabs/cortex/cli
28+
GOOS=$os GOARCH=amd64 CGO_ENABLED=0 GO111MODULE=on go build -o cortex "$ROOT/cli"
2729
aws s3 cp cortex s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex --only-show-errors
2830
rm cortex
2931
echo "Uploaded CLI to s3://$CLI_BUCKET_NAME/$CORTEX_VERSION/cli/$os/cortex"

build/find-missing-license.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
output=$(find . -type f \
17+
18+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
19+
20+
output=$(cd "$ROOT" && find . -type f \
1821
! -path "./vendor/*" \
1922
! -path "./examples/*" \
2023
! -path "./dev/config/*" \

build/find-missing-version.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
output=$(find . -type f \
17+
18+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
19+
20+
output=$(cd "$ROOT" && find . -type f \
1821
! -path "./vendor/*" \
1922
! -path "./bin/*" \
2023
! -path "./build/find-missing-version.sh" \

build/golint.sh

-25
This file was deleted.

build/lint.sh

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/bash
2+
3+
# Copyright 2019 Cortex Labs, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
18+
set -euo pipefail
19+
20+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
21+
22+
if ! command -v golint >/dev/null 2>&1; then
23+
echo "golint must be installed"
24+
exit 1
25+
fi
26+
27+
if ! command -v gofmt >/dev/null 2>&1; then
28+
echo "gofmt must be installed"
29+
exit 1
30+
fi
31+
32+
if ! command -v black >/dev/null 2>&1; then
33+
echo "black must be installed"
34+
exit 1
35+
fi
36+
37+
go vet "$ROOT/..."
38+
39+
output=$(golint "$ROOT/..." | grep -v "comment" || true)
40+
if [[ $output ]]; then
41+
echo "$output"
42+
exit 1
43+
fi
44+
45+
output=$(gofmt -s -l "$ROOT")
46+
if [[ $output ]]; then
47+
echo "go files not properly formatted:"
48+
echo "$output"
49+
exit 1
50+
fi
51+
52+
output=$(black --quiet --diff --line-length=100 "$ROOT")
53+
if [[ $output ]]; then
54+
echo "python files not properly formatted:"
55+
echo "$output"
56+
exit 1
57+
fi
58+
59+
# Check for trailing whitespace
60+
output=$(cd "$ROOT" && find . -type f \
61+
! -path "./vendor/*" \
62+
! -path "./bin/*" \
63+
! -path "./.git/*" \
64+
! -name ".*" \
65+
-exec egrep -l " +$" {} \;)
66+
if [[ $output ]]; then
67+
echo "File(s) have lines with trailing whitespace:"
68+
echo "$output"
69+
exit 1
70+
fi
71+
72+
# Check for missing new line at end of file
73+
output=$(cd "$ROOT" && find . -type f \
74+
! -path "./vendor/*" \
75+
! -path "./bin/*" \
76+
! -path "./.git/*" \
77+
! -name ".*" \
78+
-print0 | \
79+
xargs -0 -L1 bash -c 'test "$(tail -c 1 "$0")" && echo "No new line at end of $0"' || true)
80+
if [[ $output ]]; then
81+
echo "$output"
82+
exit 1
83+
fi
84+
85+
# Check for multiple new lines at end of file
86+
output=$(cd "$ROOT" && find . -type f \
87+
! -path "./vendor/*" \
88+
! -path "./bin/*" \
89+
! -path "./.git/*" \
90+
! -name ".*" \
91+
-print0 | \
92+
xargs -0 -L1 bash -c 'test "$(tail -c 2 "$0")" || echo "Multiple new lines at end of $0"' || true)
93+
if [[ $output ]]; then
94+
echo "$output"
95+
exit 1
96+
fi
97+
98+
# Check for new line(s) at beginning of file
99+
output=$(cd "$ROOT" && find . -type f \
100+
! -path "./vendor/*" \
101+
! -path "./bin/*" \
102+
! -path "./.git/*" \
103+
! -name ".*" \
104+
-print0 | \
105+
xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || echo "New line at beginning of $0"' || true)
106+
if [[ $output ]]; then
107+
echo "$output"
108+
exit 1
109+
fi

build/test.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ set -euo pipefail
2020
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
2121

2222
function run_go_tests() {
23-
(cd $ROOT && GO111MODULE=on go test ./... && echo -e "\ngo vet..." && GO111MODULE=on go vet ./... && echo "go tests passed")
23+
(cd $ROOT && GO111MODULE=on go test ./... && echo "go tests passed")
2424
}
2525

2626
function run_python_tests() {

dev/format.sh

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/bin/bash
2+
3+
# Copyright 2019 Cortex Labs, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -euo pipefail
18+
19+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null && pwd)"
20+
21+
if ! command -v gofmt >/dev/null 2>&1; then
22+
echo "gofmt must be installed"
23+
exit 1
24+
fi
25+
26+
if ! command -v black >/dev/null 2>&1; then
27+
echo "black must be installed"
28+
exit 1
29+
fi
30+
31+
gofmt -s -w "$ROOT"
32+
33+
black --quiet --line-length=100 "$ROOT"
34+
35+
# Trim trailing whitespace
36+
if [[ "$OSTYPE" == "darwin"* ]]; then
37+
output=$(cd "$ROOT" && find . -type f \
38+
! -path "./vendor/*" \
39+
! -path "./bin/*" \
40+
! -path "./.git/*" \
41+
! -name ".*" \
42+
-print0 | \
43+
xargs -0 sed -i '' -e's/[[:space:]]*$//')
44+
else
45+
output=$(cd "$ROOT" && find . -type f \
46+
! -path "./vendor/*" \
47+
! -path "./bin/*" \
48+
! -path "./.git/*" \
49+
! -name ".*" \
50+
-print0 | \
51+
xargs -0 sed -i 's/[[:space:]]*$//')
52+
fi
53+
54+
# Add new line to end of file
55+
(cd "$ROOT" && find . -type f \
56+
! -path "./vendor/*" \
57+
! -path "./bin/*" \
58+
! -path "./.git/*" \
59+
! -name ".*" \
60+
-print0 | \
61+
xargs -0 -L1 bash -c 'test "$(tail -c 1 "$0")" && echo "" >> "$0"' || true)
62+
63+
# Remove repeated new lines at end of file
64+
(cd "$ROOT" && find . -type f \
65+
! -path "./vendor/*" \
66+
! -path "./bin/*" \
67+
! -path "./.git/*" \
68+
! -name ".*" \
69+
-print0 | \
70+
xargs -0 -L1 bash -c 'test "$(tail -c 2 "$0")" || (trimmed=$(printf "%s" "$(< $0)") && echo "$trimmed" > "$0")' || true)
71+
72+
# Remove new lines at beginning of file
73+
(cd "$ROOT" && find . -type f \
74+
! -path "./vendor/*" \
75+
! -path "./bin/*" \
76+
! -path "./.git/*" \
77+
! -name ".*" \
78+
-print0 | \
79+
xargs -0 -L1 bash -c 'test "$(head -c 1 "$0")" || (trimmed=$(sed '"'"'/./,$!d'"'"' "$0") && echo "$trimmed" > "$0")' || true)

docs/applications/implementations/models.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,4 @@ oauthlib==3.0.0
6565
httplib2==0.12.0
6666
```
6767

68-
You can install additional PyPI packages and import your own Python packages. See [Python Packages](../advanced/python-packages.md) for more details.
68+
You can install additional PyPI packages and import your own Python packages. See [Python Packages](../advanced/python-packages.md) for more details.

docs/faq.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,4 @@ Cortex is built on top of Kubernetes to limit cloud provider lock-in.
2626

2727
## Can I use Cortex with my existing codebase?
2828

29-
Yes you can. Your custom packages will be installed and made available for use in the Cortex pipeline.
29+
Yes you can. Your custom packages will be installed and made available for use in the Cortex pipeline.

examples/mnist/requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
pillow==5.4.1
1+
pillow==5.4.1

images/tf-base/Dockerfile

-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,3 @@ FROM tensorflow/tensorflow:1.12.0-py3
33
RUN apt-get update -qq && apt-get install -y -q \
44
zlib1g-dev \
55
&& apt-get clean -qq && rm -rf /var/lib/apt/lists/*
6-

0 commit comments

Comments
 (0)