Integrating Browser + Android + Web Hosting features. (#419)

.github/workflows/build-docker-container.yml

@@ -0,0 +1,65 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# This workflow builds a Docker container and deploys it to the TypeAgent container repository
+
+name: Build and deploy Node.js app to Azure Web App - typeagent
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+
+  deploy:
+    runs-on: ubuntu-latest # pnpm deploy does not work currently on Windows. Fails with EPERM. 
+    steps: 
+      - name: Setup Git LF
+        run: |
+          git config --global core.autocrlf false
+          
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            ts:
+              - "ts/**"
+              - ".github/workflows/build-ts.yml"
+              
+      - uses: pnpm/action-setup@v4
+        if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts == 'true' }}
+        name: Install pnpm
+        with:
+          version: 9
+          run_install: false
+
+      - uses: actions/setup-node@v4
+        if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts == 'true' }}
+        with:
+          node-version: ${{ matrix.version }}
+          #cache: "pnpm"
+          #cache-dependency-path: ts/pnpm-lock.yaml
+
+      - name: Login to Azure
+        uses: azure/[email protected]
+        with:
+          client-id: ${{ secrets.AZUREAPPSERVICE_CLIENTID_5B0D2D6BA40F4710B45721D2112356DD }}
+          tenant-id: ${{ secrets.AZUREAPPSERVICE_TENANTID_39BB903136F14B6EAD8F53A8AB78E3AA }}
+          subscription-id: ${{ secrets.AZUREAPPSERVICE_SUBSCRIPTIONID_F36C1F2C4B2C49CA8DD5C52FAB98FA30 }}           
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3    
+
+      - run: az acr build -t typeagent:latest -r typeagentContainerRegistry --file ${{ github.workspace }}/ts/Dockerfile --subscription b64471de-f2ac-4075-a3cb-7656bca768d0 ${{ github.workspace }}/ts
+
+

.gitignore

@@ -437,3 +437,4 @@ FodyWeavers.xsd
 
 # Python virtual env
 .venv
+android/samples/mobile/.idea/deploymentTargetSelector.xml

ts/examples/android/app/build.gradle.kts → android/samples/mobile/app/build.gradle.kts

@@ -5,12 +5,12 @@ plugins {
 
 android {
     namespace = "com.microsoft.typeagent.sample"
-    compileSdk = 34
+    compileSdk = 35
 
     defaultConfig {
         applicationId = "com.microsoft.typeagent.sample"
         minSdk = 33
-        targetSdk = 34
+        targetSdk = 35
         versionCode = 1
         versionName = "1.0"
 
@@ -50,7 +50,6 @@ android {
 }
 
 dependencies {
-
     implementation(libs.androidx.core.ktx)
     implementation(libs.androidx.lifecycle.runtime.ktx)
     implementation(libs.androidx.activity.compose)
@@ -59,6 +58,8 @@ dependencies {
     implementation(libs.androidx.ui.graphics)
     implementation(libs.androidx.ui.tooling.preview)
     implementation(libs.androidx.material3)
+    implementation(libs.androidx.webkit)
+    implementation(libs.simple.android.bridge)
     testImplementation(libs.junit)
     androidTestImplementation(libs.androidx.junit)
     androidTestImplementation(libs.androidx.espresso.core)

android/samples/mobile/app/src/main/AndroidManifest.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.CAMERA" />
+    <uses-permission android:name="android.hardware.camera" />
+    <uses-permission android:name="com.android.alarm.permission.SET_ALARM" />
+    <uses-permission android:name="android.permission.CALL_PHONE" />
+    <uses-permission android:name="android.permission.SEND_SMS" />
+    <uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
+    <uses-permission android:name="android.permission.ACCESS_COARSE_LOCATION" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+<!--    <uses-permission android:name="android.webkit.PermissionRequest" />-->
+<!--    <uses-permission android:name="android.webkit.resource.AUDIO_CAPTURE" />-->
+<!--    <uses-permission android:name="android.webkit.resource.VIDEO_CAPTURE" />-->
+    <uses-permission android:name="android.permission.READ_MEDIA_IMAGES" />
+    <uses-permission android:name="android.permission.READ_MEDIA_VISUAL_USER_SELECTED" />
+
+    <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
+    <uses-feature android:name="android.hardware.camera.autofocus" android:required="true" />
+    <uses-feature android:name="android.hardware.camera.front" android:required="true" />
+    <uses-feature android:name="android.hardware.camera" android:required="true" />
+    <uses-feature android:name="android.hardware.camera.level.full" android:required="true" />
+    <uses-feature android:name="android.hardware.camera.capability.raw" android:required="true" />
+    <uses-feature android:name="android.hardware.camera.any" android:required="true" />
+    <uses-feature android:name="android.hardware.microphone" android:required="true" />
+    <uses-feature android:name="android.hardware.camera2" android:required="true" />
+
+
+
+    <uses-feature
+        android:name="android.hardware.telephony"
+        android:required="false" />
+
+    <application
+        android:allowBackup="true"
+        android:dataExtractionRules="@xml/data_extraction_rules"
+        android:fullBackupContent="@xml/backup_rules"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.TypeAgentAndroidSample"
+        android:usesCleartextTraffic="true">
+        <activity
+            android:name=".MainActivity"
+            android:exported="true"
+            android:theme="@style/Theme.TypeAgentAndroidSample">
+
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+
+            <!-- Intent filter for launching the from other apps via intents -->
+            <intent-filter>
+                <action android:name="android.intent.action.VIEW" />
+
+                <data android:scheme="typeagent" android:host="main" />
+
+                <category android:name="android.intent.category.DEFAULT" />
+                <category android:name="android.intent.category.BROWSABLE" />
+            </intent-filter>
+
+
+        </activity>
+    </application>
+
+</manifest>

android/samples/mobile/app/src/main/java/com/microsoft/typeagent/sample/JavaScriptInterface.kt

@@ -0,0 +1,135 @@
+package com.microsoft.typeagent.sample
+
+import android.content.Context
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.net.Uri
+import android.provider.AlarmClock
+import android.util.Log
+import android.webkit.JavascriptInterface
+import android.widget.Toast
+import androidx.core.content.ContextCompat
+import androidx.core.content.ContextCompat.startActivities
+import androidx.core.content.ContextCompat.startActivity
+import de.andycandy.android.bridge.CallType
+import de.andycandy.android.bridge.DefaultJSInterface
+import de.andycandy.android.bridge.JSFunctionWithArg
+import de.andycandy.android.bridge.NativeCall
+import de.andycandy.android.bridge.Promise
+import java.time.LocalDateTime
+import java.util.Locale
+import java.util.concurrent.locks.Condition
+
+typealias SpeechRecognitionCallback = (String) -> Void;
+
+
+class JavaScriptInterface(var context: Context) : DefaultJSInterface("Android") {
+
+    public var speechCallback: SpeechRecognitionCallback? = null
+    public var speechPromise: Promise<String?>? = null
+    private var recoId: Int = 0
+    private val recoLocks: MutableMap<Int, Condition> = mutableMapOf()
+    private val recoCallbacks: MutableMap<Int, JSFunctionWithArg<String?>> = mutableMapOf()
+
+    @JavascriptInterface
+    fun showToast(message: String) {
+        Toast.makeText(context, message, Toast.LENGTH_SHORT).show()
+    }
+
+    //@JavascriptInterface
+    fun startIntent() {
+        val latitude = 45.03923F
+        val longitude = 122.12343F
+        val uri = String.format(Locale.ROOT, "geo:%f,%f", latitude, longitude)
+        val intent = Intent(Intent.ACTION_VIEW, Uri.parse(uri))
+        startActivity(context, intent, null)
+    }
+
+    @JavascriptInterface
+    fun setAlarm(time: String) {
+        Log.i("javascript", "setAlarm")
+        val t: LocalDateTime = LocalDateTime.parse(time);
+        val intent = Intent(AlarmClock.ACTION_SET_ALARM)
+            .putExtra(AlarmClock.EXTRA_HOUR, t.hour)
+            .putExtra(AlarmClock.EXTRA_MINUTES, t.minute)
+        startActivity(context, intent, null)
+    }
+
+    @JavascriptInterface
+    fun callPhoneNumber(phoneNumber: String) {
+        Log.i("javascript", "callPhoneNumber")
+        val uri = String.format(Locale.ROOT, "tel:%s", phoneNumber)
+        val intent = Intent(Intent.ACTION_CALL, Uri.parse(uri))
+        startActivity(context, intent, null);
+    }
+
+    @JavascriptInterface
+     fun sendSMS(phoneNumber: String, message: String) {
+         Log.i("javascript", "sendSMS")
+        val uri = String.format(Locale.ROOT, "smsto:%s", phoneNumber)
+        val intent = Intent(Intent.ACTION_SENDTO, Uri.parse(uri))
+            .putExtra("sms_body", message);
+        startActivity(context, intent, null)
+    }
+
+    @JavascriptInterface
+    fun searchNearby(searchTerm: String) {
+        Log.i("javascript", "searchNearby")
+        val uri = Uri.parse("geo:0,0?q=$searchTerm")
+        val intent = Intent(Intent.ACTION_VIEW, uri)
+        intent.setPackage("com.google.android.apps.maps")
+        startActivity(context, intent, null)
+    }
+
+    @JavascriptInterface
+    fun isSpeechRecognitionSupported(): Boolean {
+        return PackageManager.PERMISSION_GRANTED == ContextCompat.checkSelfPermission(context, android.Manifest.permission.RECORD_AUDIO)
+    }
+
+    @JavascriptInterface
+    fun automateUI(prompt: String) {
+        Log.i("javascript", "automateUI")
+        val uri = Uri.parse("maia://main?execute=true&prompt=${prompt}")
+        val intent = Intent(Intent.ACTION_VIEW)
+            .setData(uri)
+            .putExtra("prompt", prompt)
+            .addCategory(Intent.CATEGORY_BROWSABLE)
+        context.startActivity(intent)
+    }
+
+    /**
+     * Called by the client once the app has fully loaded
+     * TypeScript types in lib.android.d.ts as Bridge.interfaces.Android.domReady()
+     */
+    @NativeCall(CallType.FULL_SYNC)
+    fun domReady(callback: JSFunctionWithArg<String>?) {
+        // if there's an initial prompt we should send that here
+        if (callback != null) {
+            MainActivity.currentActivity!!.prompt?.let { callback.call(it) }
+        }
+    }
+
+    /**
+     * Starts a recognition request, awaits the result and returns the result as a promise
+     * TypeScript types in lib.android.d.ts as Bridge.interfaces.Android.recognize()
+     */
+    @NativeCall(CallType.FULL_SYNC)
+    fun recognize(callback: JSFunctionWithArg<String?>) {
+        // set speech callback
+        val id = ++recoId;
+        recoCallbacks[id] = callback
+
+        // start speech reco
+        MainActivity.currentActivity!!.speechToText(id)
+    }
+
+    /**
+     * Calls recognition callbacks
+     */
+    fun recognitionComplete(id: Int, text: String?) {
+        if (recoCallbacks.containsKey(id)) {
+            recoCallbacks[id]!!.call(text)
+            recoLocks.remove(id)
+        }
+    }
+}