Merge pull request #100 from singlestore-labs/fs/add-author-to-notebook

fcmsilva · web-flow · commit 13860ec49273 · 2024-06-24T10:43:57.000+01:00
Add author information
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -48,14 +48,18 @@ repos:
         language: system
         files: \.ipynb$
         exclude: notebooks/notebook-style-guide/notebook.ipynb
-      - id: toml-check
-        name: toml-check
-        entry: resources/toml-check.py
+      - id: nb-meta-check
+        name: nb-meta-check
+        entry: resources/nb-meta-check.py
         language: system
-        files: \.toml$
-        exclude: "lessons/.*|^meta.toml"
+        files: notebooks/.*/meta\.toml
       - id: lesson-check
         name: lesson-check
         entry: resources/lesson-check.py
         language: system
         files: lessons/.*\.toml
+      - id: author-check
+        name: author-check
+        entry: resources/author-check.py
+        language: system
+        files: authors/.*\.toml
diff --git a/authors/singlestore.toml b/authors/singlestore.toml
@@ -0,0 +1,4 @@
+name="SingleStore"
+title="Engineering Team"
+image="singlestore"
+external=false
diff --git a/common/images/author-images/singlestore.png b/common/images/author-images/singlestore.png
diff --git a/notebooks/atlas-and-kai/meta.toml b/notebooks/atlas-and-kai/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Mongo Atlas & SingleStore Kai"
 description="""\
     Run your MongoDB queries on SingleStore Kai with no code change. \
diff --git a/notebooks/backup-database-s3/meta.toml b/notebooks/backup-database-s3/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title = "Backup Database to AWS S3"
 description = """\
     Performs backup either single or multiple databases to AWS S3. \
diff --git a/notebooks/building-a-multi-agent-ai-app-with-autogen/meta.toml b/notebooks/building-a-multi-agent-ai-app-with-autogen/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="How to Build a Multi-Agent AI App with AutoGen"
 description="""\
     Learn how to build a multi-agent group chat with RAG using Autogen and SingleStore   \
diff --git a/notebooks/cdc-mongodb-sql-commands/meta.toml b/notebooks/cdc-mongodb-sql-commands/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title = "Ingest data from MONGODB® to SingleStore using SQL commands"
 description = """
     Setup CDC Replication from MongoDB® to SingleStore using SQL commands. """
diff --git a/notebooks/confluent-cloud-integration/meta.toml b/notebooks/confluent-cloud-integration/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title = "Ingest data from Confluent Cloud (Kafka)"
 description = """
     A step-by-step guide on seamlessly ingesting data from Confluent Cloud (Kafka) into your SingleStoreDB database using a robust pipeline. """
diff --git a/notebooks/evaluating-llms-with-uptrain/meta.toml b/notebooks/evaluating-llms-with-uptrain/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Evaluating LLMs with Uptrain"
 description="""\
 Using Uptrain to evaluate LLMs built with SingleStore as the contextual store. This notebook uses OpenAI embedding models and Langchain as a development framework.
diff --git a/notebooks/getting-started-with-dataframes/meta.toml b/notebooks/getting-started-with-dataframes/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Getting Started with DataFrames in SingleStoreDB"
 description="""\
     Data in SingleStoreDB can be queried and modified using a
diff --git a/notebooks/getting-started-with-fusion-sql/meta.toml b/notebooks/getting-started-with-fusion-sql/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Getting Started with Fusion SQL"
 description="""\
     Fusion SQL allows you to manage your SingleStoreDB Cloud
diff --git a/notebooks/getting-started-with-mongocdc/meta.toml b/notebooks/getting-started-with-mongocdc/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Getting Started With CDC Replication from MongoDB"
 description="""\
     Setup Zero ETL data replication from MongoDB to SingleStore \
diff --git a/notebooks/getting-started-with-notebooks/meta.toml b/notebooks/getting-started-with-notebooks/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Getting Started with Notebooks"
 description="""\
     SingleStore Notebooks with an AI Copilot (SQrL) for data prototyping, \
diff --git a/notebooks/how-to-build-llm-apps-that-can-see-hear-speak/meta.toml b/notebooks/how-to-build-llm-apps-that-can-see-hear-speak/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="How to Build LLM Apps that can See Hear Speak"
 description="""\
     Using OpenAI to build an app that can take images, audio, and text data to generate output
diff --git a/notebooks/hybrid-full-text-vector-search/meta.toml b/notebooks/hybrid-full-text-vector-search/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Hybrid Full-text and Vector Search"
 description="Example of similarity search over vector data and a hybrid search that combines full-text search with an indexed vector search."
 difficulty="intermediate"
diff --git a/notebooks/hybrid-search/meta.toml b/notebooks/hybrid-search/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Hybrid Search"
 description="""\
     Hybrid search combines keyword search with semantic search,
diff --git a/notebooks/image-matching-with-sql/meta.toml b/notebooks/image-matching-with-sql/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Image Matching with SQL"
 description="""\
     Facial recognition using dot_product function on vectors stored in SingleStoreDB. \
diff --git a/notebooks/ingest-pdfs-with-unstructured/meta.toml b/notebooks/ingest-pdfs-with-unstructured/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Ask questions of your PDFs with Unstructured"
 description="Ask questions of your unstructured PDFs. In this notebook, Unstructured.io ingests pdfs accurately, then Open AI is used to create embeddings, the vector data is stored in SingleStore and finally ask questions of your PDF data"
 icon="file-export"
diff --git a/notebooks/inserting-embeddings-from-multiple-models-into-singlestore-using-external-functions/meta.toml b/notebooks/inserting-embeddings-from-multiple-models-into-singlestore-using-external-functions/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Inserting embeddings from multiple models into SingleStore Using External Functions"
 description="""\
     Discover the power of SingleStoreDB's external functions to dynamically fetch and store vector embeddings from leading AI models into your database. This demo highlights leveraging SingleStore's robust vector data type and external functions for efficient management and analysis of machine learning embeddings.\
diff --git a/notebooks/insure-gpt-demo/meta.toml b/notebooks/insure-gpt-demo/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="InsureGPT: OpenAI and SingleStore for Insurance"
 description="Creating an insurance analysis with OpenAI and vectorizing the results into SingleStore."
 difficulty="intermediate"
diff --git a/notebooks/integrating-with-pandas/meta.toml b/notebooks/integrating-with-pandas/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Integrating pandas with SingleStoreDB"
 description="""\
     Moving data to and from pandas DataFrames is demonstrated using
diff --git a/notebooks/kebab-case/meta.toml b/notebooks/kebab-case/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Automatically warm up your disk after resuming workspace"
 description="""\
     Runs through the queries to bring data residing in object storage onto disk for a specified database. \
diff --git a/notebooks/launch-open-source-apps-with-langchain/meta.toml b/notebooks/launch-open-source-apps-with-langchain/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Launch Open-Source Apps with LangChain"
 description="""\
     LangChain connector to use SingleStoreDB as your vector database for your apps. \
diff --git a/notebooks/load-json-files-s3/meta.toml b/notebooks/load-json-files-s3/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Load JSON files with Pipeline from S3"
 description="""\
     This notebook will help you load JSON files from a public open AWS S3 bucket.
diff --git a/notebooks/load-s3-files-into-shared-tier/meta.toml b/notebooks/load-s3-files-into-shared-tier/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Load files from S3 into Shared Tier"
 description="""\
     This notebook guides you through data ingestion of CSV files from an AWS S3 location into your shared tier workspace. \
diff --git a/notebooks/managing-stage-files-with-fusion-sql/meta.toml b/notebooks/managing-stage-files-with-fusion-sql/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Managing Stage files with Fusion SQL"
 description="""\
     Learn how to manage your Stage files in SingleStoreDB
diff --git a/notebooks/movie-recommendation/meta.toml b/notebooks/movie-recommendation/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Movie Recommendation"
 description="""\
     Movie recommendation engine using vectors stored in SingleStore to find your next watch. \
diff --git a/notebooks/network-intrusion-detection-part-1/meta.toml b/notebooks/network-intrusion-detection-part-1/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="IT Threat Detection, Part 1"
 description="""
     Part 1 or Real-time threat Detection - This notebook demonstrates the
diff --git a/notebooks/network-intrusion-detection-part-2/meta.toml b/notebooks/network-intrusion-detection-part-2/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="IT Threat Detection, Part 2"
 description="""\
     Part 2 or Real-time threat Detection - Validate the accuracy of the
diff --git a/notebooks/network-intrusion-detection-part-3/meta.toml b/notebooks/network-intrusion-detection-part-3/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="IT Threat Detection, Part 3"
 description="""
     Part 3 of Real-time threat Detection - Integrate with Kafka, run
diff --git a/notebooks/notebook-basics/meta.toml b/notebooks/notebook-basics/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="SingleStoreDB Notebook Basics"
 description="""\
     Foundational guide to notebooks with essential tips and tricks. \
diff --git a/notebooks/notebook-style-guide/meta.toml b/notebooks/notebook-style-guide/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Notebook Style Guide"
 description="Become a pro at formatting and organizing your notebooks."
 icon="link"
diff --git a/notebooks/optimize-performance-with-tpch-100/meta.toml b/notebooks/optimize-performance-with-tpch-100/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Learn how to Optimize Performance with TPCH 100"
 description="""\
     This notebook will help you understand how you can take advantage of SingleStoreDB distributed capability using TPCH-100.
diff --git a/notebooks/performance-troubleshooting/meta.toml b/notebooks/performance-troubleshooting/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Database Performance Troubleshoot Notebook"
 description="""\
     This notebook will help you perform database performance troubleshoot
diff --git a/notebooks/pipelines-query-tuning/meta.toml b/notebooks/pipelines-query-tuning/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Kafka Pipelines and Query Tuning"
 description="""\
     Create a SingleStore pipeline to track the International Space Station and adjust queries & schema to optimize performance.
diff --git a/notebooks/rag-with-bedrock/meta.toml b/notebooks/rag-with-bedrock/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Retrieval Augmented Question & Answering with Amazon Bedrock"
 description="""\
     Infuse domain-specific knowledge from SingleStoreDB into generative AI models on Amazon Bedrock to showcase Question & Answering using RAG (retrieval-agumented generation) to improve the quality of responses.\
diff --git a/notebooks/real-time-anomaly-detection/meta.toml b/notebooks/real-time-anomaly-detection/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Real-Time Anomaly Detection"
 description="""\
     Real-time anomaly detection in IoT sensor data, harnessing the robust capabilities of SingleStoreDB and advanced analytical techniques. \
diff --git a/notebooks/real-time-recommendation-engine/meta.toml b/notebooks/real-time-recommendation-engine/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Real Time Recommendation Engine"
 description="""\
 We demonstrate how to build and host a real-time recommendation engine for free with SingleStore. The notebook also leverages our new SingleStore Job Service to ensure that the latest data is ingested and used in providing recommendations.\
diff --git a/notebooks/representing-unstructured-data-as-vectors/meta.toml b/notebooks/representing-unstructured-data-as-vectors/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Representing Unstructured Data as Vectors"
 description="Transforming unstructured data into vectors and representing them in a 3-dimensional space to find the distance between them using various techniques"
 difficulty="beginner"
diff --git a/notebooks/restore-database-from-s3/meta.toml b/notebooks/restore-database-from-s3/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title = "Restore Database from AWS S3"
 description = """\
     This notebook will help you perform database restore from AWS S3.
diff --git a/notebooks/resume-evaluator-ANN-index-search/meta.toml b/notebooks/resume-evaluator-ANN-index-search/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Resume Evaluator with Vector Index"
 description="""\
     Resume Evaluator using vectors stored in SingleStore to find sutiable resume for your job description, with Indexed ANN search.\
diff --git a/notebooks/resume-evaluator/meta.toml b/notebooks/resume-evaluator/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Resume Evaluator"
 description="""\
     Resume Evaluator using vectors stored in SingleStore to find sutiable resume for your job description. \
diff --git a/notebooks/searching-all-of-wikipedia/meta.toml b/notebooks/searching-all-of-wikipedia/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title = "Searching all of Wikipedia"
 description = """\
     Simulate searching through 10 million paragraphs in Wikipedia for information about video games, and generate answers using RAG! \
diff --git a/notebooks/semantic-search-with-hugging-face/meta.toml b/notebooks/semantic-search-with-hugging-face/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Semantic Search with Hugging Face Models and Datasets"
 description="""\
     Use Hugging Face to create embeddings and run semantic search \
diff --git a/notebooks/semantic-search-with-openai-embedding-creation/meta.toml b/notebooks/semantic-search-with-openai-embedding-creation/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Semantic Search with OpenAI Embedding Creation"
 description="""\
     Generate embeddings and run semantic search in your database in SQL. \
diff --git a/notebooks/semantic-search-with-openai-qa/meta.toml b/notebooks/semantic-search-with-openai-qa/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Semantic Search with OpenAI QA"
 description="""\
     Provide context to chatGPT using data stored in SingleStoreDB. \
diff --git a/notebooks/semantic-visualization-and-vector-datatype/meta.toml b/notebooks/semantic-visualization-and-vector-datatype/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Semantic Visualization and Vector Datatype"
 description="""\
     SingleStoreDB leverages vector database processing to enable storage and search of vector data, allowing for semantic similarity-based searches to find content based on meaning rather than exact matches, such as locating related words like "happy" and "joyful." \
diff --git a/notebooks/similarity-search-on-vector-data/meta.toml b/notebooks/similarity-search-on-vector-data/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Similarity Search on Vector Data"
 description="Example of similarity searches over vector data."
 difficulty="beginner"
diff --git a/notebooks/singlestore-april-challenge-haiku-ascii/meta.toml b/notebooks/singlestore-april-challenge-haiku-ascii/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="SingleStore April Challenge: Haiku ASCII"
 description="""\
     SingleStore is hosting a challenge inviting participants to craft a unique Haiku or create captivating ASCII art using SingleStore Notebooks. The most creative masterpiece wins a set of Meta RayBan Smart sunglasses and a $500 AWS gift card!" \
diff --git a/notebooks/unified-data-analysis-sql-nosql-kai/meta.toml b/notebooks/unified-data-analysis-sql-nosql-kai/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Unified Data Analysis: SQL & NoSQL on a Single Database with Kai"
 description="""\
     Perform both SQL and NoSQL queries on multi-modal relational and JSON data \
diff --git a/notebooks/vector-database-basics/meta.toml b/notebooks/vector-database-basics/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="A Deep Dive Into Vector Databases"
 description="Using SingleStoreDB as a vector database and vector database use cases."
 difficulty="beginner"
diff --git a/notebooks/vector-search-with-kai/meta.toml b/notebooks/vector-search-with-kai/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title = "Vector Search with Kai"
 description = """
     Run Vector Search using MongoDB clients and power GenAI usecases for your MongoDB applications """
diff --git a/notebooks/working-with-vector-data/meta.toml b/notebooks/working-with-vector-data/meta.toml
@@ -1,4 +1,5 @@
 [meta]
+authors=["singlestore"]
 title="Working with Vector Data"
 description="""\
     This example demonstrates the various ways to upload and download
diff --git a/resources/author-check.py b/resources/author-check.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import os
+import re
+import sys
+import tomllib
+
+
+def error(msg):
+    print('ERROR:', msg, file=sys.stderr)
+    sys.exit(1)
+
+
+def check_author(author_path):
+    print(f'Checking {author_path}...')
+
+    with open(author_path, 'rb') as f:
+        meta = tomllib.load(f)
+
+        if 'name' not in meta:
+            error(f'No `name` in `meta` section of {author_path}')
+
+        if 'title' not in meta:
+            error(f'No `title` in `meta` section of {author_path}')
+
+        if 'external' not in meta:
+            error(f'No `external` in `meta` section of {author_path}')
+
+        # Image is optional, but if defined a corresponding image must exist
+        # Image can either be a URL or a filename in common/images/author-images
+        if 'image' in meta:
+            img_reference = meta['image']
+            is_url = bool(re.match(r'^https?://', img_reference))
+
+            if (not is_url):
+                img_filename = f'{img_reference}.png'
+                img_path = os.path.join('common/images/author-images', img_filename)
+                if not os.path.isfile(img_path):
+                    error(f'Author image does not exist at {img_path} for {author_path}')
+
+
+if __name__ == '__main__':
+    for f in sys.argv[1:]:
+        check_author(f)
diff --git a/resources/nb-meta-check.py b/resources/nb-meta-check.py
@@ -32,6 +32,14 @@ def error(msg):
     if 'description' not in meta:
         error(f'No `description` in `meta` section of {f}')
 
+    # Authors must be a non-empty list
+    if (
+        'authors' not in meta
+        or not isinstance(meta['authors'], list)
+        or not meta['authors']
+    ):
+        error(f'No `authors` in `meta` section of {f}')
+
     if 'icon' not in meta:
         error(f'No `icon` in `meta` section of {f}')
 
@@ -85,3 +93,10 @@ def error(msg):
             error(f'Lesson area {lesson_area} not found in card icons')
         if expected_icon_name not in preview_icons:
             error(f'Lesson area {lesson_area} not found in preview icons')
+
+    # Authors must have a corresponding author entry
+    author_meta_files = os.listdir('authors')
+    for author in meta['authors']:
+        author_filename = f'{author}.toml'
+        if author_filename not in author_meta_files:
+            error(f'Author {author} does not have a corresponding author entry in {f}')

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`[meta]`
	`2`	`+authors=["singlestore"]`
`2`	`3`	`title="Mongo Atlas & SingleStore Kai"`
`3`	`4`	`description="""\`
`4`	`5`	`Run your MongoDB queries on SingleStore Kai with no code change. \`