Changes during 1284 investigation (#496)

* fix: Refactor downloads - records now removed

* refactor: More lint-based refactoring

* refactor: Refactor readme construction

* refactor: missing-sdfs now missing_sdfs

* style: sshtunnel logging level now CRITICAL (was ERROR)

* fix: Tweak 'no authenticated' message

* style: Log tweaks

* docs: Adds comment to unused fields of DounloadLinks

* fix: Improved DouwnloadLinks removal logging & improved Dockerfile

* refactor: Note about emerging complexity and tweak keeplainve timeout value

---------

Co-authored-by: Alan Christie <[email protected]>

.pylintrc

@@ -12,4 +12,4 @@ disable = C, F, R,
           fixme,
           import-error,
           imported-auth-user,
-	  new-db-field-with-default,
+	      new-db-field-with-default,

Dockerfile

@@ -27,14 +27,13 @@ WORKDIR /srv/logs
 WORKDIR /code/logs
 WORKDIR /code
 
+COPY requirements.txt ./
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir --requirement requirements.txt
+
 COPY nginx.conf /etc/nginx/nginx.conf
 COPY django_nginx.conf /etc/nginx/sites-available/default.conf
 COPY proxy_params /etc/nginx/frag_proxy_params
-COPY requirements.txt ./
-
-
-RUN ln -s /etc/nginx/sites-available/default.conf /etc/nginx/sites-enabled && \
-    pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir --requirement requirements.txt
+RUN ln -s /etc/nginx/sites-available/default.conf /etc/nginx/sites-enabled
 
 COPY . ./

nginx.conf

@@ -27,7 +27,6 @@ http {
         sendfile on;
         tcp_nopush on;
         tcp_nodelay on;
-        keepalive_timeout 650;
         types_hash_max_size 2048;
         # server_tokens off;
 
@@ -51,7 +50,8 @@ http {
         access_log /var/log/nginx/access.log;
         error_log /var/log/nginx/error.log;
 
-        # Tuneout settings
+        # Timeeout settings
+        keepalive_timeout           1000s;
         proxy_connect_timeout       1000s;
         proxy_send_timeout          1000s;
         proxy_read_timeout          1000s;

proxy_params

@@ -2,3 +2,10 @@ proxy_set_header Host $http_host;
 proxy_set_header X-Real-IP $remote_addr;
 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 # proxy_set_header X-Forwarded-Proto $scheme;
+
+# Parameters are also located in nginx.con, and dnago_nginx.xonf
+# In fact the whole ingress configuration fo the satck is now
+# potentially "too complicated" and needs to be simplified.
+#
+# Alan Christie
+# January 2024

viewer/download_structures.py

@@ -819,6 +819,7 @@ def erase_out_of_date_download_records():
     with the file-system the model should continue to reflect the current state
     of the world.
     """
+    num_removed = 0
     out_of_date_dynamic_records = DownloadLinks.objects.filter(
         keep_zip_until__lt=datetime.now(timezone.utc)
     ).filter(static_link=False)
@@ -832,18 +833,22 @@ def erase_out_of_date_download_records():
         if os.path.isdir(dir_name):
             logger.debug('Removing file_url directory (%s)...', dir_name)
             shutil.rmtree(dir_name, ignore_errors=True)
+            logger.debug('Removed (%s)', dir_name)
 
         # Does the file exist now?
         # Hopefully not - but cater for 'cosmic-ray-effect' and
         # only delete the originating record if the file has been removed.
         if os.path.isdir(dir_name):
             logger.warning(
-                '- Failed removal of file_url directory (%s), not removing record',
+                'Failed removal of file_url directory (%s), leaving record',
                 dir_name,
             )
         else:
             logger.info(
-                '- Removed file_url directory (%s), removing DownloadLinks record',
+                'Removed file_url directory (%s), removing DownloadLinks record...',
                 dir_name,
             )
             out_of_date_dynamic_record.delete()
+            num_removed += 1
+
+    logger.info('Erased %d', num_removed)