Add admin facing UI for delta lake dump and spec configuration

.gitignore

@@ -116,4 +116,5 @@ credentials.tar.gz
 *.ipynb
 *.pkl
 # ignore sentinel conf outputs
-*running.conf
+*running.conf
+preview_ui.py

materializationengine/app.py

@@ -3,7 +3,7 @@
 from datetime import date, datetime, timedelta
 
 import numpy as np
-from dynamicannotationdb.models import Base, AnalysisVersion
+from dynamicannotationdb.models import AnalysisVersion, Base
 from flask import Blueprint, Flask, current_app, jsonify, redirect, url_for
 from flask_cors import CORS
 from flask_restx import Api
@@ -13,18 +13,19 @@
 from materializationengine.admin import setup_admin
 from materializationengine.blueprints.client.api import client_bp
 from materializationengine.blueprints.client.api2 import client_bp as client_bp2
+from materializationengine.blueprints.deltalake.api import deltalake_bp
 from materializationengine.blueprints.materialize.api import mat_bp
-from materializationengine.blueprints.upload.api import upload_bp, spatial_lookup_bp
-from materializationengine.blueprints.upload.storage import StorageService
+from materializationengine.blueprints.upload.api import spatial_lookup_bp, upload_bp
 from materializationengine.blueprints.upload.models import init_staging_database
+from materializationengine.blueprints.upload.storage import StorageService
 from materializationengine.config import config, configure_app
 from materializationengine.database import db_manager
-from materializationengine.schemas import ma
-from materializationengine.utils import get_instance_folder_path
-from materializationengine.views import views_bp
 from materializationengine.limiter import limiter
 from materializationengine.migrate import migrator
 from materializationengine.request_db import init_request_db_cleanup
+from materializationengine.schemas import ma
+from materializationengine.utils import get_instance_folder_path
+from materializationengine.views import views_bp
 
 db = SQLAlchemy(model_class=Base)
 
@@ -53,7 +54,7 @@ def create_app(config_name: str = None):
         template_folder="../templates",
     )
     CORS(app, expose_headers=["WWW-Authenticate", "column_names"], send_wildcard=True)
-    
+
     app.json_encoder = AEEncoder
     app.config["RESTX_JSON"] = {"cls": AEEncoder}
 
@@ -62,19 +63,19 @@ def create_app(config_name: str = None):
         app.config.from_object(config[config_name])
     else:
         app = configure_app(app)
-    logging.basicConfig(level=app.config['LOGGING_LEVEL'])
-    
+    logging.basicConfig(level=app.config["LOGGING_LEVEL"])
+
     # Suppress noisy debug messages from fsevents
-    logging.getLogger('fsevents').setLevel(app.config['LOGGING_LEVEL'])
-    logging.getLogger('urllib3').setLevel(app.config['LOGGING_LEVEL'])
-    logging.getLogger('google').setLevel(app.config['LOGGING_LEVEL'])
-    logging.getLogger('materializationengine').setLevel(app.config['LOGGING_LEVEL'])
-    logging.getLogger('root').setLevel(app.config['LOGGING_LEVEL'])
-    logging.getLogger('python_jsonschema_objects').setLevel(app.config['LOGGING_LEVEL'])
+    logging.getLogger("fsevents").setLevel(app.config["LOGGING_LEVEL"])
+    logging.getLogger("urllib3").setLevel(app.config["LOGGING_LEVEL"])
+    logging.getLogger("google").setLevel(app.config["LOGGING_LEVEL"])
+    logging.getLogger("materializationengine").setLevel(app.config["LOGGING_LEVEL"])
+    logging.getLogger("root").setLevel(app.config["LOGGING_LEVEL"])
+    logging.getLogger("python_jsonschema_objects").setLevel(app.config["LOGGING_LEVEL"])
 
     # Initialize request-scoped database session cleanup
     init_request_db_cleanup(app)
-    
+
     # register blueprints
     apibp = Blueprint("api", __name__, url_prefix="/materialize")
 
@@ -90,9 +91,9 @@ def version():
     def index():
         return redirect("/materialize/views/")
 
-    app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {
-        'pool_pre_ping': True,
-        'pool_recycle': 3600,
+    app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {
+        "pool_pre_ping": True,
+        "pool_recycle": 3600,
     }
 
     db.init_app(app)
@@ -116,16 +117,17 @@ def index():
         app.register_blueprint(apibp)
         app.register_blueprint(views_bp)
         app.register_blueprint(upload_bp)
+        app.register_blueprint(deltalake_bp)
         limiter.init_app(app)
         try:
             db.create_all()
         except Exception as e:
             app.logger.error(f"Error creating database tables: {e}")
-            
+
         admin = setup_admin(app, db)
         if app.config.get("STAGING_DATABASE_NAME"):
             init_staging_database(app)
-            
+
         # setup cors on upload bucket
         try:
             bucket_name = app.config.get("MATERIALIZATION_UPLOAD_BUCKET_PATH")
@@ -139,10 +141,11 @@ def health():
         aligned_volume = current_app.config.get("TEST_DB_NAME", "annotation")
         with db_manager.session_scope(aligned_volume) as session:
             n_versions = session.query(AnalysisVersion).count()
-        
+
         return jsonify({aligned_volume: n_versions}), 200
 
     @app.teardown_appcontext
     def shutdown_session(exception=None):
         db_manager.cleanup()
+
     return app

materializationengine/blueprints/deltalake/__init__.py

@@ -0,0 +1,3 @@
+from materializationengine.blueprints.deltalake.api import deltalake_bp
+
+__all__ = ["deltalake_bp"]