diff --git a/_gsdocs-proposals/.dockerignore b/_gsdocs-proposals/.dockerignore new file mode 100644 index 00000000..90137660 --- /dev/null +++ b/_gsdocs-proposals/.dockerignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +*.pyo +*.pyd +.env +venv/ +.git/ \ No newline at end of file diff --git a/_gsdocs-proposals/.env b/_gsdocs-proposals/.env new file mode 100644 index 00000000..56e71c55 --- /dev/null +++ b/_gsdocs-proposals/.env @@ -0,0 +1,6 @@ +HF_TOKEN="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" + +SEGMENTATION_MODEL="facebook/mask2former-swin-tiny-ade-semantic" +INPAINTING_MODEL="runwayml/stable-diffusion-inpainting" + +DEVICE=cuda diff --git a/_gsdocs-proposals/Dockerfile b/_gsdocs-proposals/Dockerfile new file mode 100644 index 00000000..1a5929e0 --- /dev/null +++ b/_gsdocs-proposals/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y \ + git \ + ffmpeg \ + libgl1 \ + libglib2.0-0 \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . + +RUN pip install --upgrade pip + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 7860 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"] \ No newline at end of file diff --git a/_gsdocs-proposals/app.py b/_gsdocs-proposals/app.py new file mode 100644 index 00000000..d53e6b64 --- /dev/null +++ b/_gsdocs-proposals/app.py @@ -0,0 +1,71 @@ +import os +import io +import torch +import huggingface_hub +import asyncio +import traceback +from fastapi import FastAPI, UploadFile, File, HTTPException +from starlette.responses import StreamingResponse +from PIL import Image +import uvicorn + +if not hasattr(torch, 'xpu'): + class XPU: + @staticmethod + def is_available(): return False + @staticmethod + def empty_cache(): pass + torch.xpu = XPU + +if not hasattr(huggingface_hub, "cached_download"): + huggingface_hub.cached_download = huggingface_hub.hf_hub_download + +os.environ["DIFFUSERS_VERIFY_COMPATIBILITY"] = "0" +os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + +from pipeline import reconstruct_artifact + +app = FastAPI(title="AI Artifact Reconstruction") + +@app.get("/") +async def home(): + return { + "status": "online", + "device": "CUDA" if torch.cuda.is_available() else "CPU", + "torch_version": torch.__version__ + } + +@app.post("/generate") +async def generate(file: UploadFile = File(...)): + if not file.content_type.startswith("image/"): + raise HTTPException(status_code=400, detail="File must be an image") + + content = await file.read() + + if len(content) > 5 * 1024 * 1024: + raise HTTPException(status_code=400, detail="File too large") + + try: + Image.open(io.BytesIO(content)).verify() + except: + raise HTTPException(status_code=400, detail="Invalid image") + + try: + processed_img = await asyncio.wait_for( + asyncio.to_thread(reconstruct_artifact, content), + timeout=60 + ) + + img_byte_arr = io.BytesIO() + processed_img.save(img_byte_arr, format="PNG") + img_byte_arr.seek(0) + + return StreamingResponse(img_byte_arr, media_type="image/png") + + except Exception as e: + traceback.print_exc() + return {"error": str(e)} + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=7860) diff --git a/_gsdocs-proposals/login_me.py b/_gsdocs-proposals/login_me.py new file mode 100644 index 00000000..7b43a097 --- /dev/null +++ b/_gsdocs-proposals/login_me.py @@ -0,0 +1,2 @@ +from huggingface_hub import login +login() \ No newline at end of file diff --git a/_gsdocs-proposals/pipeline.py b/_gsdocs-proposals/pipeline.py new file mode 100644 index 00000000..1ef88cd3 --- /dev/null +++ b/_gsdocs-proposals/pipeline.py @@ -0,0 +1,101 @@ +import os +import torch +import io +import numpy as np +import cv2 +from PIL import Image +from dotenv import load_dotenv +from transformers import AutoImageProcessor, Mask2FormerForUniversalSegmentation +from diffusers import StableDiffusionInpaintPipeline + +load_dotenv() +HF_TOKEN = os.getenv("HF_TOKEN") + +device = "cuda" if torch.cuda.is_available() else "cpu" + +seg_id = "facebook/mask2former-swin-tiny-ade-semantic" +processor = AutoImageProcessor.from_pretrained(seg_id, token=HF_TOKEN) +seg_model = Mask2FormerForUniversalSegmentation.from_pretrained(seg_id, token=HF_TOKEN).to(device) + +pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", + token=HF_TOKEN, + torch_dtype=torch.float16 if device == "cuda" else torch.float32 +) + +if device == "cpu": + pipe.enable_attention_slicing() +else: + pipe.to(device) + + +def refine_mask(mask): + kernel = np.ones((5, 5), np.uint8) + mask = cv2.GaussianBlur(mask, (5, 5), 0) + mask = cv2.dilate(mask, kernel, iterations=2) + mask = cv2.erode(mask, kernel, iterations=1) + _, mask = cv2.threshold(mask, 127, 255, cv2.THRESH_BINARY) + return mask + + +def detect_missing_regions(init_image): + gray = cv2.cvtColor(np.array(init_image), cv2.COLOR_RGB2GRAY) + edges = cv2.Canny(gray, 80, 160) + + inputs = processor(images=init_image, return_tensors="pt").to(device) + with torch.no_grad(): + outputs = seg_model(**inputs) + + prediction = processor.post_process_semantic_segmentation( + outputs, target_sizes=[init_image.size[::-1]] + )[0].cpu().numpy() + + seg_mask = np.where( + (prediction == 0) | (prediction == 1) | (prediction == 255), + 255, + 0 + ).astype(np.uint8) + + combined = cv2.bitwise_or(edges, seg_mask) + combined = refine_mask(combined) + + if np.sum(combined) < 3000: + combined = np.ones_like(gray) * 255 + + return combined + + +def reconstruct_artifact(image_bytes): + init_image = Image.open(io.BytesIO(image_bytes)).convert("RGB").resize((512, 512)) + + mask_array = detect_missing_regions(init_image) + + coverage = np.sum(mask_array) / (512 * 512 * 255) + if coverage < 0.02: + return init_image + + mask_image = Image.fromarray(mask_array) + + prompt = ( + "infrared underpainting, hidden sketch beneath painting, " + "classical artwork, faded pigments, original composition, " + "historically accurate restoration, museum quality" + ) + + negative_prompt = ( + "modern objects, distorted shapes, unrealistic textures, " + "blurry, oversmoothed, low quality" + ) + + with torch.inference_mode(): + result = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + image=init_image, + mask_image=mask_image, + num_inference_steps=28, + guidance_scale=8.0, + strength=0.65 + ).images[0] + + return result \ No newline at end of file diff --git a/_gsdocs-proposals/readme.md b/_gsdocs-proposals/readme.md new file mode 100644 index 00000000..53b9113b --- /dev/null +++ b/_gsdocs-proposals/readme.md @@ -0,0 +1,258 @@ +title: Painting in a Painting - AI-driven Hidden Image Reconstruction without Multispectral Dependency +layout: gsoc_proposal +project: ArtExtraction(Srishti-1806) +year: 2026 +organization: + +* Alabama + +--- + +## Description + +Traditional approaches to uncover hidden images beneath paintings rely heavily on multispectral and X-ray imaging. While effective, these methods require expensive hardware, controlled environments, and expert handling, making them difficult to scale and inaccessible to many researchers. + +X-ray-based systems are not widely available, especially to smaller institutions and independent historians. This proposal addresses these limitations by introducing a **software-first, AI-driven reconstruction pipeline** that operates solely on standard RGB images. + +The core objective is not just detection, but **probabilistic reconstruction of plausible underlying compositions**, using modern generative AI techniques. + +--- + +## Problem Statement & Proposed Solution + +### 1. Hardware Dependency & Lack of Scalability + +**Challenge:** +Hidden image detection relies on costly imaging techniques (X-ray, multispectral), limiting accessibility. + +**Solution:** +An **RGB-based AI inference pipeline** that leverages deep learning to infer underlying structures directly from standard images. This removes the need for specialized equipment and enables scalable deployment. + +--- + +### 2. Manual Identification of Layered History + +**Challenge:** +Identifying overpaintings and layered compositions requires manual inspection and expert analysis. + +**Solution:** +A **Generative Reconstruction Pipeline** combining: + +* Semantic Segmentation (Mask2Former) +* Diffusion-based Inpainting (Stable Diffusion) + +This allows automatic identification and reconstruction of altered regions. + +--- + +### 3. Data Complexity & Information Extraction + +**Challenge:** +Analyzing damage, pigment variation, and restorations from raw data is complex and time-intensive. + +**Solution:** +An **end-to-end automated pipeline** integrating: + +* Edge detection (Canny) +* Structure guidance (ControlNet) + +This enables structured extraction and reconstruction without manual intervention. + +--- + +## Proposed Approach + +The system leverages **diffusion models, segmentation networks, and structure-preserving conditioning** to reconstruct underlying visual content. + +### Pipeline Overview + +``` +Input Image (RGB) + ↓ +Denoising & Preprocessing + ↓ +Segmentation (Mask2Former) + ↓ +Mask Generation (Altered Regions) + ↓ +Edge Extraction (Canny) + ↓ +ControlNet (Structure Conditioning) + ↓ +Stable Diffusion (Inpainting) + ↓ +Super Resolution (RealESRGAN) + ↓ +Final Reconstruction +``` + +--- + +## Training Strategy (Key Innovation) + +A major challenge is the lack of ground truth hidden images. + +To address this, I propose a **synthetic data generation pipeline**: + +* Start with clean paintings +* Simulate overpainting using: + + * occlusions + * texture overlays + * noise and degradation + +This creates supervised pairs: + +| Input (Modified Painting) | Target (Original Painting) | +| ------------------------- | -------------------------- | + +This enables effective training of reconstruction models while maintaining realism. + +--- + +## Evaluation Strategy + +Since reconstruction is inherently uncertain, evaluation is performed across multiple dimensions: + +### Quantitative Metrics + +* **SSIM (Structural Similarity Index)** +* **PSNR (Peak Signal-to-Noise Ratio)** +* **LPIPS (Perceptual Similarity)** + +### Structural Consistency + +* Edge similarity (Canny overlap) +* Feature similarity using CLIP embeddings + +### Retrieval-based Validation + +* Compare reconstructed outputs with nearest neighbors in embedding space + +### Qualitative Evaluation + +* Visual inspection +* Comparison with known restored artworks + +--- + +## Uncertainty Estimation + +The system models reconstruction as a **probabilistic process**: + +* Generate multiple reconstructions per image +* Measure variance across outputs +* Highlight low-confidence regions + +This ensures transparency and avoids overconfident interpretations. + +--- + +## Why this proposal is an improvement + +### 1. Accessibility and Scalability + +Works entirely on RGB images → eliminates dependency on specialized hardware. + +### 2. Software-first Approach + +Transforms a hardware-intensive problem into a scalable AI solution. + +### 3. End-to-End Automated Pipeline + +Fully integrated pipeline reduces manual effort. + +### 4. Generative Reconstruction + +Moves beyond detection → reconstructs plausible hidden compositions. + +### 5. Deployment-ready Architecture + +Designed for real-world usage using FastAPI + Docker. + +--- + +## Duration + +Total project length: 175 hours + +--- + +## Task ideas + +* Design and implement multi-stage reconstruction pipeline +* Develop segmentation-based masking techniques +* Train synthetic data generation pipeline +* Integrate diffusion-based inpainting +* Apply ControlNet for structure preservation +* Implement enhancement and super-resolution +* Optimize for CPU/GPU environments +* Develop evaluation and uncertainty estimation modules + +--- + +## Expected results + +* Functional AI pipeline for artifact reconstruction +* Ability to generate plausible hidden compositions +* Quantitative + qualitative evaluation framework +* Modular architecture for future extensions +* Deployment-ready API + +--- + +## Tech stack + +* **Programming Language:** Python +* **Framework:** PyTorch +* **Computer Vision:** OpenCV, NumPy + +### Models & Libraries: + +* Hugging Face Transformers (**Mask2Former**) +* Diffusers (**Stable Diffusion Inpainting**) +* **ControlNet** +* **RealESRGAN** + +### Backend & Deployment: + +* FastAPI +* Docker +* Hugging Face Hub + +--- + +## Why my proposal stands out + +This proposal distinguishes itself through: + +* **Bridging research and engineering** +* **Eliminating dependency on expensive imaging hardware** +* **Introducing synthetic supervision for training** +* **Combining multiple SOTA models into a unified pipeline** +* **Providing uncertainty-aware reconstruction** +* **Focusing on deployability and real-world usability** + +Rather than only detecting hidden images, this system provides **interpretable, visual reconstructions**, enabling deeper insights into artistic processes and history. + +--- + +## Requirements + +* Python +* PyTorch +* Computer vision fundamentals +* Understanding of deep learning and generative models + +--- + +## Project difficulty level + +Medium to High + +--- + +## Mentors + +* Emanuele Usai (University of Alabama) +* Sergei Gleyzer (University of Alabama) diff --git a/_gsdocs-proposals/requirements.txt b/_gsdocs-proposals/requirements.txt new file mode 100644 index 00000000..a356be6c --- /dev/null +++ b/_gsdocs-proposals/requirements.txt @@ -0,0 +1,26 @@ +fastapi==0.115.9 +uvicorn==0.29.0 +python-multipart==0.0.9 + +torch==2.1.1 +torchvision==0.16.1 +torchaudio==2.1.1 + +transformers==4.41.0 +diffusers==0.27.2 +accelerate==0.28.0 +safetensors==0.4.3 + +pillow==10.3.0 +numpy==1.26.4 +scipy==1.12.0 +opencv-python-headless==4.10.0.84 + +realesrgan==0.3.0 +basicsr==1.4.2 +facexlib==0.3.0 +gfpgan==1.3.8 + +python-dotenv==1.0.1 +huggingface_hub==0.33.4 +tokenizers==0.19.1 \ No newline at end of file diff --git a/_gsdocs-proposals/run.sh b/_gsdocs-proposals/run.sh new file mode 100644 index 00000000..1441e38a --- /dev/null +++ b/_gsdocs-proposals/run.sh @@ -0,0 +1,8 @@ +python -m venv env +env/scripts/activate +pip install --upgrade pip +pip install -r requirements.txt + + +#on command prompt type the below command +#./run.sh