refinements

shiffman · shiffman · commit 46b4c296e00b · 2024-10-30T11:02:26.000-04:00
diff --git a/embeddings/clustering/sketch.js b/embeddings/clustering/sketch.js
@@ -1,3 +1,6 @@
+// Programming A to Z
+// https://github.com/Programming-from-A-to-Z/A2Z-F24
+
 // Function to load transformers.js dynamically
 async function loadTransformers() {
   try {
diff --git a/embeddings/comparison-matrix/sketch.js b/embeddings/comparison-matrix/sketch.js
@@ -1,3 +1,6 @@
+// Programming A to Z
+// https://github.com/Programming-from-A-to-Z/A2Z-F24
+
 // Function to load transformers.js dynamically
 async function loadTransformers() {
   try {
diff --git a/llms/chat-conversation/sketch.js b/llms/chat-conversation/sketch.js
@@ -1,5 +1,5 @@
-// Introduction to Machine Learning for the Arts
-// https://github.com/ml5js/Intro-ML-Arts-IMA-F24
+// Programming A to Z, Fall 2024
+// https://github.com/Programming-from-A-to-Z/A2Z-F24
 
 let conversationHistory = [];
 let inputBox;
@@ -39,6 +39,7 @@ async function sendMessage() {
     try {
       // Generate a response based on the input prompt
       const output = await generator(conversationHistory, { max_new_tokens: 128 });
+      // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/at
       const reply = output[0].generated_text.at(-1).content;
       conversationHistory.push({ role: 'assistant', content: reply });
       chatLog = `Chatbot: ${reply}</br></br>` + chatLog;
diff --git a/llms/chat-prompt/sketch.js b/llms/chat-prompt/sketch.js
@@ -1,5 +1,5 @@
-// Introduction to Machine Learning for the Arts
-// https://github.com/ml5js/Intro-ML-Arts-IMA-F24
+// Programming A to Z
+// https://github.com/Programming-from-A-to-Z/A2Z-F24
 
 let generator;
 let inputText;
@@ -15,8 +15,11 @@ async function setup() {
 
   // Create a text generation pipeline with specific model and options
   generator = await pipeline('text-generation', 'onnx-community/Llama-3.2-1B-Instruct-q4f16', {
-    dtype: 'q4',
+    dtype: 'q4f16',
     device: 'webgpu',
+    progress_callback: (x) => {
+      console.log(x);
+    },
   });
 
   // Create a button after model is loaded
@@ -36,8 +39,10 @@ async function generateText() {
 
     // Generate a response based on the input prompt
     const output = await generator(messages, { max_new_tokens: 128 });
+    console.log(output);
 
     // Extract and display the generated text
+    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/at
     let outputText = output[0].generated_text.at(-1).content;
     background(240);
     text(outputText, 10, 10, width - 20, height - 20);
diff --git a/llms/text-completion/index.html b/llms/text-completion/index.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.10.0/p5.js"></script>
+    <meta charset="utf-8" />
+  </head>
+  <body>
+    <main></main>
+    <script src="tf-helper.js"></script>
+    <script src="sketch.js"></script>
+  </body>
+</html>
diff --git a/llms/text-completion/sketch.js b/llms/text-completion/sketch.js
@@ -0,0 +1,49 @@
+// Programming A to Z
+// https://github.com/Programming-from-A-to-Z/A2Z-F24
+
+let generator;
+let inputText;
+let outputText;
+
+async function setup() {
+  // Create a canvas and text input field
+  createCanvas(400, 200);
+  inputText = createInput('Type a prompt here...');
+
+  // Load the Transformers.js model pipeline with async/await
+  let pipeline = await loadTransformers();
+
+  // Try
+  // https://huggingface.co/HuggingFaceTB/SmolLM-135M
+  // https://huggingface.co/HuggingFaceTB/SmolLM-360M
+
+  // Create a text generation pipeline with specific model and options
+  generator = await pipeline('text-generation', 'HuggingFaceTB/SmolLM-135M', {
+    dtype: 'q4',
+    device: 'webgpu',
+    progress_callback: (x) => {
+      console.log(x);
+    },
+  });
+
+  // Create a button after model is loaded
+  let button = createButton('Generate Text');
+  button.mousePressed(generateText);
+}
+
+// Asynchronous function to generate text based on user input
+async function generateText() {
+  // Ensure the model is loaded
+  if (generator) {
+    // Complete the user's text
+    const output = await generator(inputText.value(), { max_new_tokens: 128 });
+    console.log(output);
+    // Extract and display the generated text
+    let outputText = output[0].generated_text;
+    background(240);
+    text(outputText, 10, 10, width - 20, height - 20);
+  } else {
+    // Log a message if the model is not yet loaded
+    console.log('Model not loaded yet, try again in a minute.');
+  }
+}
diff --git a/llms/text-completion/tf-helper.js b/llms/text-completion/tf-helper.js
@@ -0,0 +1,11 @@
+async function loadTransformers() {
+  try {
+    const module = await import(
+      "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0"
+    );
+    const { pipeline } = module;
+    return pipeline;
+  } catch (error) {
+    console.error("Failed to load transformers.js", error);
+  }
+}
diff --git a/whisper-demo/sketch.js b/whisper-demo/sketch.js
@@ -1,3 +1,7 @@
+// Programming A to Z
+// https://github.com/Programming-from-A-to-Z/A2Z-F24
+
+// Import the Transformers.js pipeline for speech recognition
 import { pipeline } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2';
 
 let mediaRecorder;
@@ -6,69 +10,90 @@ let isRecording = false;
 let transcriptDiv = document.getElementById('transcript');
 let recordButton = document.getElementById('recordButton');
 
-// Downsample audio to 16000Hz
+// Function to downsample audio buffer to 16000Hz for Whisper model
 function downsampleAudioBuffer(buffer, targetSampleRate) {
   const sampleRate = buffer.sampleRate;
+
+  // If sample rate matches target, return original buffer
   if (sampleRate === targetSampleRate) {
     return buffer;
   }
+
+  // Calculate downsample ratio and new buffer length
   const ratio = sampleRate / targetSampleRate;
   const newLength = Math.round(buffer.length / ratio);
   const newBuffer = new Float32Array(newLength);
+
+  // Populate new buffer with downsampled audio data
   for (let i = 0; i < newLength; i++) {
     newBuffer[i] = buffer.getChannelData(0)[Math.round(i * ratio)];
   }
   return newBuffer;
 }
 
+// Asynchronous function to transcribe audio using Whisper
 async function transcribeAudio(blob) {
+  // Show transcribing status
   transcriptDiv.textContent = 'transcribing...';
+
+  // Load Whisper model from Transformers.js
   const model = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
-  // Convert Blob to ArrayBuffer
+
+  // Convert Blob to ArrayBuffer for audio decoding
   const arrayBuffer = await blob.arrayBuffer();
-  // Use the Web Audio API to decode the ArrayBuffer into audio data
+
+  // Decode ArrayBuffer to audio data using Web Audio API
   const audioContext = new (window.AudioContext || window.webkitAudioContext)();
   const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
 
-  // Downsample audio to 16000Hz, as required by Whisper
+  // Downsample the audio to 16000Hz as required by the model
   const downsampledAudio = downsampleAudioBuffer(audioBuffer, 16000);
 
-  // Perform transcription with Whisper
+  // Perform transcription with Whisper model
   const result = await model(downsampledAudio);
 
-  // Display the transcription result
+  // Display transcription result in the DOM
   transcriptDiv.textContent = result.text;
 }
 
+// Function to start audio recording
 function startRecording() {
+  // Request microphone access from the user
   navigator.mediaDevices
     .getUserMedia({ audio: true })
     .then((stream) => {
+      // Initialize MediaRecorder with the audio stream
       mediaRecorder = new MediaRecorder(stream);
       mediaRecorder.start();
       isRecording = true;
       recordButton.textContent = 'stop recording';
+
+      // Collect audio data while recording
       mediaRecorder.ondataavailable = (event) => {
         audioChunks.push(event.data);
       };
-      // Stop recording and transcribe the audio
+
+      // Stop recording and start transcription
       mediaRecorder.onstop = () => {
         const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
         transcribeAudio(audioBlob);
         audioChunks = [];
       };
     })
     .catch((error) => {
+      // Handle errors when accessing the microphone
       console.error('Error accessing microphone: ', error);
     });
 }
 
+// Function to stop audio recording
 function stopRecording() {
   mediaRecorder.stop();
   isRecording = false;
   recordButton.textContent = 'start recording';
 }
 
+// Event listener to toggle recording state
 recordButton.addEventListener('click', () => {
   if (isRecording) {
     stopRecording();
