Upgrade to python 3.9 and cleanup

Author: trent-codecov

.deep-dive.yaml

@@ -1,6 +1,6 @@
 search:
   exclude_files:
-    python3libs: '/usr/lib/python3\.8.*'
+    python3libs: '/usr/lib/python3\.9.*'
   files:
     python: '.*\.pyc?'
     json:  '.*\.json'

Dockerfile.circle-enterprise

@@ -18,16 +18,11 @@ RUN apk --update --no-cache add \
 COPY            . /app
 WORKDIR         /app
 
-# Build bootloader for alpine
-RUN git clone --depth 1 --single-branch --branch v3.5 https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller \
-    && cd /tmp/pyinstaller/bootloader \
-    && CFLAGS="-Wno-stringop-overflow" python ./waf configure --no-lsb all \
-    && pip install .. \
-    && rm -Rf /tmp/pyinstaller
 
-# install cython 
-RUN pip install cython==3.0a1
-RUN pip install PyCrypto==2.6.1
+RUN pip install pyinstaller
+# install cython
+RUN pip install cython==3.0a9
+RUN pip install tinyaes
 
 # set settings module
 ENV DJANGO_SETTINGS_MODULE "codecov.settings_enterprise"
@@ -55,7 +50,6 @@ COPY enterprise/ldd /pyinstaller/ldd
 COPY enterprise/setup.py setup.py
 COPY enterprise/cythonize.sh /pyinstaller/cythonize_everything.sh
 
-COPY enterprise/package.sh /pyinstaller/package.sh
 COPY enterprise/pyi_rth_django.py /pyinstaller/pyi_rth_django.py
 COPY enterprise/generate_pyinstaller_args.py /pyinstaller/generate_pyinstaller_args.py
 
@@ -66,6 +60,7 @@ COPY enterprise/hooks /hooks
 RUN  chmod a+x /pyinstaller/*
 RUN         ["/pyinstaller/cythonize_everything.sh"]
 
+COPY enterprise/package.sh /pyinstaller/package.sh
 RUN  chmod a+x /pyinstaller/*
 RUN         ["/pyinstaller/package.sh"]
 ENV RUN_ENV=ENTERPRISE
@@ -82,7 +77,7 @@ RUN chown codecov:application /home/enterprise.sh && \
     chown codecov:application /home/api
 
 
-FROM alpine:3.13
+FROM alpine:3.14
 
 RUN         addgroup -S application && adduser -S codecov -G application && rm -rf /home/codecov
 USER        codecov

Dockerfile.enterprise

@@ -14,15 +14,15 @@ RUN apk --update --no-cache add \
     && pip install --upgrade pip
 
 # Build bootloader for alpine
-RUN git clone --depth 1 --single-branch --branch v3.5 https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller \
-    && cd /tmp/pyinstaller/bootloader \
-    && CFLAGS="-Wno-stringop-overflow" python ./waf configure --no-lsb all \
-    && pip install .. \
-    && rm -Rf /tmp/pyinstaller
-
+#RUN git clone --depth 1 --single-branch --branch v3.5 https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller \
+#    && cd /tmp/pyinstaller/bootloader \
+#    && CFLAGS="-Wno-stringop-overflow" python ./waf configure --no-lsb all \
+#    && pip install .. \
+#    && rm -Rf /tmp/pyinstaller
+RUN pip install pyinstaller
 # install cython 
-RUN pip install cython==3.0a1
-RUN pip install PyCrypto==2.6.1
+RUN pip install cython==3.0a9
+RUN pip install tinyaes
 
 # set settings module
 ENV DJANGO_SETTINGS_MODULE "codecov.settings_enterprise"
@@ -50,7 +50,7 @@ COPY enterprise/ldd /pyinstaller/ldd
 COPY enterprise/setup.py setup.py
 COPY enterprise/cythonize.sh /pyinstaller/cythonize_everything.sh
 
-COPY enterprise/package.sh /pyinstaller/package.sh
+
 COPY enterprise/pyi_rth_django.py /pyinstaller/pyi_rth_django.py
 COPY enterprise/generate_pyinstaller_args.py /pyinstaller/generate_pyinstaller_args.py
 
@@ -61,6 +61,8 @@ COPY enterprise/hooks /hooks
 RUN  chmod a+x /pyinstaller/*
 RUN         ["/pyinstaller/cythonize_everything.sh"]
 
+RUN pip install tinyaes
+COPY enterprise/package.sh /pyinstaller/package.sh
 RUN  chmod a+x /pyinstaller/*
 RUN         ["/pyinstaller/package.sh"]
 ENV RUN_ENV=ENTERPRISE
@@ -76,7 +78,7 @@ RUN chown codecov:application /home/enterprise.sh && \
     chmod +x /home/enterprise.sh && \
     chown codecov:application /home/api
 
-FROM alpine:3.13
+FROM alpine:3.14
 
 RUN         addgroup -S application && adduser -S codecov -G application
 RUN         rm -rf /home/codecov

docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   api:
-    image: codecov/api:latest
+    image: codecov/enterprise-api:v4.6.2
     networks:
       - codecovapi
     ports:
@@ -14,8 +14,8 @@ services:
     depends_on:
       - postgres
       - redis
-    entrypoint:
-      - ./dev.sh
+#    entrypoint:
+#      - ./api
     volumes:
       - ./:/app
     environment:
@@ -27,7 +27,9 @@ services:
     networks:
       - codecovapi
     ports:
-      - "5432:5432"
+      - "5436:5432"
+    environment:
+      - POSTGRES_PASSWORD=postgres
   redis:
     image: redis:4.0-alpine
     volumes:

enterprise/generate_pyinstaller_args.py

@@ -7,7 +7,7 @@
 
 finder = ModuleFinder()
 
-so_extension = ".cpython-37m-x86_64-linux-gnu.so"
+so_extension = ".cpython-39-x86_64-linux-gnu.so"
 
 
 def get_relevant_paths(path):
@@ -31,9 +31,13 @@ def get_relevant_dirs(path):
 
 
 def find_imported_modules(filename):
-    finder.run_script(filename)
-    for name, mod in finder.modules.items():
-        yield name
+    try:
+        finder.run_script(filename)
+        for name, mod in finder.modules.items():
+            yield name
+    except AttributeError:
+        pass
+
 
 
 def generate_files_to_be_cythonized():
@@ -85,6 +89,7 @@ def main():
             "whitenoise",
             "whitenoise.middleware",
             "graphql_api",
+            "graphql_api.types.enums",
             "legacy_migrations",
             "legacy_migrations.migrations",
             "shared.celery_config",

enterprise/package.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Simple wrapper around pyinstaller
 set -e
-set -x
+#set -x
 
 # Generate a random key for encryption
 random_key=$(pwgen -s 16 1)
@@ -10,13 +10,13 @@ random_key=$(pwgen -s 16 1)
 PATH="/pyinstaller:$PATH"
 
 args_to_use=$(cat /pyinstaller/installer_args.txt)
-echo $(find . -name "*.so")
+#echo $(find . -name "*.so")
 
 mkdir src
 echo 'true' > src/is_enterprise
 
 # # Exclude pycrypto and PyInstaller from built packages
-pyinstaller -F --exclude-module pycrypto --exclude-module PyInstaller --add-data src:/src --name api --clean --key $random_key ${args_to_use} ${pyinstaller_args} /app/manage.py
+pyinstaller -F --exclude-module PyInstaller --add-data src:/src --name api --clean --key ${random_key} ${args_to_use} ${pyinstaller_args} /app/manage.py
 
 cat api.spec
 
@@ -28,4 +28,4 @@ rm -rf /hooks
 mv -v app/dist/* /home
 rm -rf /app
 rm -rf /pyinstaller
-rm -rf /usr/local/lib/python3.8/site-packages
+rm -rf /usr/local/lib/python3.9/site-packages

graphql_api/types/enums/__init__.py

@@ -4,4 +4,6 @@
     OrderingDirection,
     PullRequestState,
     RepositoryOrdering,
+    TypeProjectOnboarding,
+    GoalOnboarding
 )

prod.sh

@@ -8,7 +8,7 @@ if [ -f "/usr/local/bin/berglas" ]; then
 fi
 
 if [ $ELASTIC_APM_ENABLED ]; then
-  $prefix gunicorn codecov.wsgi:application --workers=2 --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT}
+  $prefix gunicorn codecov.wsgi:application --workers=${GUNICORN_WORKERS:-3} --threads=${GUNICORN_THREADS:-6} --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT}
 else
-  $prefix ddtrace-run gunicorn codecov.wsgi:application --workers=2 --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT}
+  $prefix ddtrace-run gunicorn codecov.wsgi:application --workers=${GUNICORN_WORKERS:-3} --threads=${GUNICORN_THREADS:-6} --bind 0.0.0.0:8000 --access-logfile '-' --statsd-host ${STATSD_HOST}:${STATSD_PORT}
 fi

staging.sh

@@ -12,8 +12,5 @@ suffix=""
 if [[ "$STATSD_HOST" ]]; then
   suffix="--statsd-host ${STATSD_HOST}:${STATSD_PORT}"
 fi
-if [ $ELASTIC_APM_ENABLED ]; then
-  $prefix gunicorn codecov.wsgi:application --reload --bind 0.0.0.0:8000 --access-logfile '-' $suffix
-else
-  $prefix ddtrace-run gunicorn codecov.wsgi:application --reload --bind 0.0.0.0:8000 --access-logfile '-' $suffix
-fi
+
+$prefix gunicorn codecov.wsgi:application --reload --workers=${GUNICORN_WORKERS:-3} --threads=${GUNICORN_THREADS:-6} --worker-class=gthread --bind 0.0.0.0:8000 --access-logfile '-' $suffix