Myst awareness

api/config/common.yaml

@@ -55,13 +55,9 @@ DATA_ROOT_PATH: "/DATA"
 # this is expected to be under the DATA_ROOT_PATH
 JB_ROOT_FOLDER: "book-artifacts"
 
-# Name of the folder that will contain published MyST websites
-# this is expected to be under the DATA_ROOT_PATH
-MYST_PUBLISH_FOLDER: "myst"
-
 # Name of the folder that will contain MyST websites
 # source code. This is expected to be under the DATA_ROOT_PATH
-MYST_SOURCE_FOLDER: "myst_sources"
+MYST_FOLDER: "myst"
 
 # Container path where the MyST sources will be mounted
 CONTAINER_MYST_SOURCE_PATH: "/home/jovyan"

api/neurolibre_celery_tasks.py

@@ -34,14 +34,15 @@
 
 config_keys = [
     'DOI_PREFIX', 'DOI_SUFFIX', 'JOURNAL_NAME', 'PAPERS_PATH', 'BINDER_REGISTRY',
-    'DATA_ROOT_PATH', 'JB_ROOT_FOLDER', 'GH_ORGANIZATION', 'MYST_SOURCE_FOLDER', 'MYST_PUBLISH_FOLDER',
+    'DATA_ROOT_PATH', 'JB_ROOT_FOLDER', 'GH_ORGANIZATION', 'MYST_FOLDER',
     'CONTAINER_MYST_SOURCE_PATH', 'CONTAINER_MYST_DATA_PATH']
 globals().update({key: common_config[key] for key in config_keys})
 
 JB_INTERFACE_OVERRIDE = preprint_config['JB_INTERFACE_OVERRIDE']
 
 PRODUCTION_BINDERHUB = f"https://{preprint_config['BINDER_NAME']}.{preprint_config['BINDER_DOMAIN']}"
 PREVIEW_SERVER = f"https://{preview_config['SERVER_SLUG']}.{common_config['SERVER_DOMAIN']}"
+PREVIEW2_SERVER = f"https://{preview_config['SERVER_SLUG']}2.{common_config['SERVER_DOMAIN']}"
 
 """
 Configuration START
@@ -125,20 +126,20 @@ def __init__(self, celery_task, screening=None, payload=None):
             raise ValueError("Either screening or payload must be provided.")
 
     def start(self, message=""):
-        self.screening.respond().STARTED(message)
+        self.screening.respond.STARTED(message)
         self.update_state(states.STARTED, {'message': message})
 
     def fail(self, message):
-        self.screening.respond().FAILURE(message)
+        self.screening.respond.FAILURE(message,collapsable=False)
         self.update_state(state=states.FAILURE, meta={
             'exc_type': f"{JOURNAL_NAME} celery exception",
             'exc_message': "Custom",
             'message': message
         })
         raise Ignore()
 
-    def succeed(self, message):
-        self.screening.respond().SUCCESS(message)
+    def succeed(self, message, collapsable=True):
+        self.screening.respond.SUCCESS(message, collapsable=collapsable)
 
     def update_state(self, state, meta):
         self.celery_task.update_state(state=state, meta=meta)
@@ -155,11 +156,8 @@ def path_join(self, *args):
     def join_data_root_path(self, *args):
         return self.path_join(DATA_ROOT_PATH, *args)
 
-    def join_myst_source_path(self, *args):
-        return self.path_join(DATA_ROOT_PATH, MYST_SOURCE_FOLDER, *args)
-
-    def join_myst_publish_path(self, *args):
-        return self.path_join(DATA_ROOT_PATH, MYST_PUBLISH_FOLDER, *args)
+    def join_myst_path(self, *args):
+        return self.path_join(DATA_ROOT_PATH, MYST_FOLDER, *args)
 
     def get_deposit_dir(self, *args):
         return self.path_join(get_deposit_dir(self.payload['issue_id']), *args)
@@ -1308,27 +1306,32 @@ def preview_build_myst_task(self, screening_dict):
                   dotenv = task.get_dotenv_path()))
 
     hub = JupyterHubLocalSpawner(rees_resources,
-                             host_build_source_parent_dir = task.join_myst_source_path(),
+                             host_build_source_parent_dir = task.join_myst_path(),
                              container_build_source_mount_dir = CONTAINER_MYST_SOURCE_PATH, #default
                              host_data_parent_dir = DATA_ROOT_PATH, #optional
                              container_data_mount_dir = CONTAINER_MYST_DATA_PATH)
     # Spawn the JupyterHub
     task.start("Cloning repository, pulling binder image, spawning JupyterHub...")
     hub.spawn_jupyter_hub()
     # hub.rees.
-
+
+    expected_source_path = task.join_myst_path(task.owner_name,task.repo_name,task.screening.commit_hash)
+    if os.path.exists(expected_source_path) and os.listdir(expected_source_path):
+        task.start("Successfully cloned the repository.")
+    else:
+        task.fail(f"Source repository {task.owner_name}/{task.repo_name} at {task.screening.commit_hash} not found.")
+
+    # Initialize the builder
     task.start("Warming up the myst builder...")
+    expected_webpage_path = task.join_myst_path(task.owner_name,task.repo_name,task.screening.commit_hash,"_build","html")
     builder = MystBuilder(hub)
-    expected_publish_path = task.join_myst_publish_path(task.owner_name,task.repo_name,task.screening.commit_hash)
-    builder.setenv('BASE_URL',expected_publish_path)
+    # Set the base url
+    builder.setenv('BASE_URL',expected_webpage_path.split("/DATA")[-1])
+    # Start the build
     task.start("Started MyST build...")
     builder.build()
 
-    expected_build_path = task.join_myst_source_path(task.owner_name,task.repo_name,task.screening.commit_hash,"_build")
-    if os.path.exists(expected_build_path):
-        # Copy myst artifacts to the expected path.
-        task.start("MyST build completed, copying artifacts...")
-        fast_copytree(expected_build_path, expected_publish_path)
-        task.succeed(f"MyST build succeeded: https://{PREVIEW_SERVER}/myst/{task.owner_name}/{task.repo_name}/{task.screening.commit_hash}/_build/html/index.html")
+    if os.path.exists(expected_webpage_path) and os.listdir(expected_webpage_path):
+        task.succeed(f"🌺 MyST build succeeded: \n\n {PREVIEW2_SERVER}/myst/{task.owner_name}/{task.repo_name}/{task.screening.commit_hash}/_build/html/index.html", collapsable=False)
     else:
-        raise FileNotFoundError(f"Expected build path not found: {expected_build_path}")
+        raise FileNotFoundError(f"Expected build path not found: {expected_webpage_path}")

api/screening_client.py

@@ -26,6 +26,7 @@ def __init__(self, task_name, issue_id, target_repo_url = None, task_id=None, co
         self.commit_hash = commit_hash
         self.comment_id = comment_id
         self.__extra_payload = extra_payload
+        self._init_responder()
 
         for key, value in extra_payload.items():
             setattr(self, key, value)
@@ -40,7 +41,37 @@ def __init__(self, task_name, issue_id, target_repo_url = None, task_id=None, co
 
         # If no comment ID is provided, create a new comment with a pending status
         if self.comment_id is None:
-            self.comment_id = self.respond().PENDING("Awaiting task assignment...")
+            self.comment_id = self.respond.PENDING("Awaiting task assignment...")
+
+    def _init_responder(self):
+        class PhaseResponder:
+            def __init__(self, phase, client):
+                self.phase = phase
+                self.client = client
+
+            def __call__(self, message="", collapsable=True):
+                template = self.client.gh_response_template(message=message, collapse=collapsable)
+                if self.phase == "PENDING":
+                    return self.client.gh_create_comment(template['PENDING'])
+                else:
+                    return self.client.gh_update_comment(template[self.phase])
+
+        class ResponderContainer:
+            def __init__(self, client):
+                self.client = client
+                self.PENDING = PhaseResponder("PENDING", client)
+                self.RECEIVED = PhaseResponder("RECEIVED", client)
+                self.STARTED = PhaseResponder("STARTED", client)
+                self.SUCCESS = PhaseResponder("SUCCESS", client)
+                self.FAILURE = PhaseResponder("FAILURE", client)
+                self.EXISTS = PhaseResponder("EXISTS", client)
+
+            def __getattr__(self, phase):
+                if phase in self.__dict__:
+                    return lambda message="", collapsable=True: self.__dict__[phase](message, collapsable)
+                raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{phase}'")
+
+        self.respond = ResponderContainer(self)
 
     def to_dict(self):
         # Convert the object to a dictionary to pass to Celery
@@ -79,11 +110,11 @@ def start_celery_task(self, celery_task_func):
         if task_result.task_id is not None:
             self.task_id = task_result.task_id
             message = f"Celery task assigned successfully. Task ID: {self.task_id}"
-            self.respond().RECEIVED(message)
+            self.respond.RECEIVED(message)
             return make_response(jsonify(message), 200)
         else:
             message = f"Celery task assignment failed. Task Name: {self.task_name}"
-            self.respond().FAILURE(message)
+            self.respond.FAILURE(message, collapsable=False)
             return make_response(jsonify(message), 500)
 
     @staticmethod
@@ -153,30 +184,6 @@ def gh_update_comment(self, comment_body):
         comment = issue.get_comment(self.comment_id)
         comment.edit(comment_body)
 
-    def respond(self):
-        # Just for the sake of semantics.
-        manager = self
-        class PhaseResponder:
-            def __init__(self, phase):
-                self.phase = phase
-
-            def __call__(self, message="", collapsable=True):
-                template = manager.gh_response_template(message=message, collapse=collapsable)
-                if self.phase == "PENDING":
-                    return manager.gh_create_comment(template['PENDING'])
-                else:
-                    return manager.gh_update_comment(template[self.phase])
-
-        # Returning an object where each phase is a method
-        return type("PhaseResponderContainer", (object,), {
-            "PENDING": PhaseResponder("PENDING"),
-            "RECEIVED": PhaseResponder("RECEIVED"),
-            "STARTED": PhaseResponder("STARTED"),
-            "SUCCESS": PhaseResponder("SUCCESS"),
-            "FAILURE": PhaseResponder("FAILURE"),
-            "EXISTS": PhaseResponder("EXISTS"),
-        })()
-
     def gh_get_project_name(self):
         repo = self.github_client.get_repo(self.gh_filter(self.review_repository))
         contents = repo.get_contents("binder/data_requirement.json")