Dev

.github/workflows/ci-base-image.yml

@@ -20,9 +20,8 @@ jobs:
           docker build . \
             --file docker/Dockerfile \
             --target emfos-base \
-            -t ${{ vars.DOCKER_USERNAME }}/emfos-base:main
-          docker push ${{ vars.DOCKER_USERNAME}}/emfos-base:main
-
+            -t ${{ vars.DOCKER_USERNAME }}/emfos-base:latest
+          docker push ${{ vars.DOCKER_USERNAME}}/emfos-base:latest
 
       - name: Display Success Message
         run: echo "emfos-base image has been built and pushed manually."

config/cgm_worker/post_processing.properties

@@ -1,5 +1,4 @@
 [MAIN]
-NET_INTERCHANGE_THRESHOLD = 200
 FIX_INJECTION_ERRORS = True
 INJECTION_THRESHOLD = 0.1
 SMALL_ISLAND_SIZE = 10

config/cgm_worker/replacement_conf.json

@@ -8,10 +8,10 @@
             "request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"]
         },
         "replacement_length": {
-            "request_list": ["6d", "6d", "6d", "2w", "6d", "1M", "4M"]
+            "request_list": ["2w", "2w", "2w", "2w", "6d", "1M", "4M"]
         },
         "month_ahead": {
-            "hours": ["07:30", "08:30", "09:30", "06:30", "05:30", "04:30"],
+            "hours": ["09:30", "08:30", "10:30", "11:30", "07:30", "12:30"],
             "business_type": ["1D","2D"]
         },
         "hours": [

config/integrations/elastic.properties

@@ -1,3 +1,4 @@
 [MAIN]
 ELK_SERVER = access_url
+ELK_TOKEN = access_token
 BATCH_SIZE = 20000

config/task_generator/manual_task_generator.properties

@@ -15,10 +15,11 @@ TASK_VERSION =
 TASK_MERGING_ENTITY =
 RUN_REPLACEMENT =
 RUN_SCALING =
+OUTAGE_UPDATE =
+FORCE_OUTAGE_FIX =
 UPLOAD_TO_OPDM =
 UPLOAD_TO_MINIO =
 SEND_MERGE_REPORT =
 POST_TEMP_FIXES =
-FORCE_OUTAGE_FIX =
 FIX_NET_INTERCHANGE2 =
 LVL8_REPORTING =

config/task_generator/process_conf.json

@@ -34,13 +34,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "True",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -70,13 +69,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "True",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -106,13 +104,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "True",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -142,13 +139,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "True",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "True",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -178,13 +174,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "True",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       }
@@ -232,13 +227,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -275,13 +269,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -318,13 +311,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -360,13 +352,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -403,13 +394,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -446,13 +436,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -489,13 +478,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },   
@@ -536,13 +524,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       },
@@ -580,13 +567,12 @@
         "replacement_local":"False",
         "pre_temp_fixes":"True",
         "post_temp_fixes":"True",
-        "fix_net_interchange2": "False",
+        "outage_update": "True",
         "force_outage_fix":"False",
         "scaling": "False",
         "upload_to_opdm": "False",
         "upload_to_minio": "True",
         "send_merge_report": "True",
-        "force_outage_fix": "False",
         "lvl8_reporting" : "True"
        }
       }

docker/Dockerfile

@@ -7,16 +7,18 @@ WORKDIR /src
 
 # Install tools to set up python environment
 RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -rf /var/lib/apt/lists/*
-RUN pip install --upgrade setuptools pip uv pipenv
-
-# Install needed modules
-COPY Pipfile .
-COPY Pipfile.lock .
-#RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
-#RUN pipenv install --system --deploy
-RUN pipenv requirements > requirements.txt
-RUN uv pip sync requirements.txt --system
-#RUN uv cache clean
+RUN pip install --upgrade setuptools pip uv
+
+# Sync python modules as dependencies
+COPY pyproject.toml .
+COPY uv.lock .
+RUN uv export -o pylock.toml
+RUN uv pip sync pylock.toml --system  && uv cache clean
+
+# Copy license file and dependencies license reference
+RUN mkdir -p /licenses
+COPY LICENSE /licenses/
+COPY licenses/ /licenses/
 
 # Set to use interpreter from venv (if venv is used)
 #ENV PATH="/.venv/bin:$PATH"

emf/common/integrations/elastic.py

@@ -21,16 +21,17 @@
 
 class Elastic:
 
-    def __init__(self, server: str = ELK_SERVER, debug: bool = False):
+    def __init__(self, server: str = ELK_SERVER, api_key: str = ELK_TOKEN, debug: bool = False):
         self.server = server
         self.debug = debug
-        self.client = Elasticsearch(self.server)
+        self.client = Elasticsearch(self.server, api_key=api_key)
 
     @staticmethod
     def send_to_elastic(index: str,
                         json_message: dict,
                         id: str = None,
                         server: str = ELK_SERVER,
+                        api_key: str = ELK_TOKEN,
                         iso_timestamp: str = None,
                         debug: bool = False):
         """
@@ -64,7 +65,8 @@ def send_to_elastic(index: str,
         if json_message.get('args', None):  # TODO revise if this is proper solution
             json_message.pop('args')
         json_data = json.dumps(json_message, default=str, ensure_ascii=True, skipkeys=True)
-        response = requests.post(url=url, data=json_data.encode(), headers={"Content-Type": "application/json"})
+        headers = {"Authorization": f"ApiKey {api_key}", "Content-Type": "application/json"}
+        response = requests.post(url=url, data=json_data.encode(), headers=headers, verify=False)
         if json.loads(response.content).get('error'):
             logger.error(f"Send to Elasticsearch responded with error: {response.text}")
         if debug:
@@ -79,6 +81,7 @@ def send_to_elastic_bulk(index: str,
                              id_metadata_list: List[str] | None = None,
                              hashing: bool = False,
                              server: str = ELK_SERVER,
+                             api_key: str = ELK_TOKEN,
                              batch_size: int = int(BATCH_SIZE),
                              iso_timestamp: str | None = None,
                              debug: bool = False):
@@ -127,10 +130,12 @@ def __generate_id(element):
             # Executing POST to push messages into ELK
             if debug:
                 logger.debug(f"Sending batch ({batch}-{batch + batch_size})/{len(json_message_list)} to {url}")
+            headers = {"Authorization": f"ApiKey {api_key}", "Content-Type": "application/x-ndjson"}
             response = requests.post(url=url,
                                      data=(ndjson.dumps(json_message_list[batch:batch + batch_size])+"\n").encode(),
                                      timeout=None,
-                                     headers={"Content-Type": "application/x-ndjson"})
+                                     headers=headers,
+                                     verify=False)
             if json.loads(response.content).get('errors'):
                 logger.error(f"Send to Elasticsearch responded with errors: {response.text}")
             if debug:
@@ -259,7 +264,8 @@ def handle(self, message: bytes, properties: dict,  **kwargs):
 
     # Create client
     server = "access_url"
-    service = Elastic(server=server)
+    api_key = "acces_token"
+    service = Elastic(server=server, api_key=api_key)
 
     # Example get documents by query
     # query = {"match": {"scenario_date": "2023-03-21"}}