diff --git a/api/build.gradle b/api/build.gradle index 3d64f9ce8d6..6ea64ae59b1 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -1029,6 +1029,84 @@ dependencies { ) ) + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-starter-mcp-server-webmvc:${springAiVersion}", + "spring-ai-starter-mcp-server-webmvc", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "MPC Servlet" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-bom:${springAiVersion}", + "spring-ai-bom", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "MPC Servlet" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-starter-model-google-genai:${springAiVersion}", + "spring-ai-starter-google-genai", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "LLM Chat Client integration for Gemini" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-client-chat:${springAiVersion}", + "spring-ai-client-chat", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "LLM Chat Client" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-advisors-vector-store:${springAiVersion}", + "spring-ai-advisors-vector-store", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "Vector store integration" + ) + ) + + BuildUtils.addExternalDependency( + project, + new ExternalDependency( + "org.springframework.ai:spring-ai-starter-model-google-genai-embedding:${springAiVersion}", + "spring-ai-starter-model-google-genai-embedding", + "spring-ai", + "https://github.com/spring-projects/spring-ai", + ExternalDependency.APACHE_2_LICENSE_NAME, + ExternalDependency.APACHE_2_LICENSE_URL, + "Vector store integration" + ) + ) + jspImplementation files(project.tasks.jar) jspImplementation apache, jackson, spring } diff --git a/api/src/org/labkey/api/mcp/AbstractAgentAction.java b/api/src/org/labkey/api/mcp/AbstractAgentAction.java new file mode 100644 index 00000000000..91dddcc7eaf --- /dev/null +++ b/api/src/org/labkey/api/mcp/AbstractAgentAction.java @@ -0,0 +1,83 @@ +package org.labkey.api.mcp; + +import com.google.genai.errors.ClientException; +import com.google.genai.errors.ServerException; +import jakarta.servlet.http.HttpSession; +import org.json.JSONObject; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.util.HtmlString; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.validation.BindException; + +import java.util.Map; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +/** + * "agent" it is too strong a word, but if you want to create a tools specific chat endpoint then + * start here. + * First implement getServicePrompt() to tell your "agent its mission. You can also listen in on the + * conversation to help you user get the right results. + */ +public abstract class AbstractAgentAction extends ReadOnlyApiAction +{ + protected abstract String getAgentName(); + + protected abstract String getServicePrompt(); + + protected ChatClient getChat() + { + HttpSession session = getViewContext().getRequest().getSession(true); + ChatClient chatSession = McpService.get().getChat(session, getAgentName(), this::getServicePrompt); + return chatSession; + } + + @Override + public Object execute(PromptForm form, BindException errors) throws Exception + { + try (var mcpPush = McpContext.withContext(getViewContext())) + { + ChatClient chatSession = getChat(); + if (null == chatSession) + return new JSONObject(Map.of( + "contentType", "text/plain", + "response", "Service is not ready yet", + "success", Boolean.FALSE)); + + String prompt = form.getPrompt(); + McpService.MessageResponse response = McpService.get().sendMessage(chatSession, prompt); + var ret = new JSONObject(Map.of("success", Boolean.TRUE)); + if (!HtmlString.isBlank(response.html())) + { + ret.put("contentType", "text/html"); + ret.put("response", response.html()); + } + else if (isNotBlank(response.text())) + { + ret.put("contentType", response.contentType()); + ret.put("response", response.text()); + } + else + { + ret.put("contentType", "text/plain"); + ret.put("response", "I got nothing"); + } + return ret; + } + catch (ServerException x) + { + return new JSONObject(Map.of( + "error", x.getMessage(), + "text", "ERROR: " + x.getMessage(), + "success", Boolean.FALSE)); + } + catch (ClientException ex) + { + var ret = new JSONObject(Map.of( + "text", ex.getMessage(), + "user", getViewContext().getUser().getName(), + "success", Boolean.FALSE)); + return ret; + } + } +} diff --git a/api/src/org/labkey/api/mcp/McpContext.java b/api/src/org/labkey/api/mcp/McpContext.java new file mode 100644 index 00000000000..d1b0975f46f --- /dev/null +++ b/api/src/org/labkey/api/mcp/McpContext.java @@ -0,0 +1,86 @@ +package org.labkey.api.mcp; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.ContainerUser; +import org.springframework.ai.chat.model.ToolContext; +import java.util.Map; + +/** + * TODO MCP tool calling supports passing along a ToolContext. And most all + * interesting tools probably need a User and Container. This is not all hooked-up + * yet. This is an area for further investiation. + */ +public class McpContext implements ContainerUser +{ + final User user; + final Container container; + + public McpContext(ContainerUser ctx) + { + this.container = ctx.getContainer(); + this.user = ctx.getUser(); + } + + public McpContext(Container container, User user) + { + if (!container.hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException(); + this.container = container; + this.user = user; + } + + public ToolContext getToolContext() + { + return new ToolContext(Map.of("container", getContainer(), "user", getUser())); + } + + + @Override + public Container getContainer() + { + return container; + } + + @Override + public User getUser() + { + return user; + } + + + // + // I'd like to get away from using ThreadLocal, but I haven't + // researched if there are other ways to pass context around to Tools registerd by McpService + // + + private static final ThreadLocal contexts = new ThreadLocal(); + + public static @NotNull McpContext get() + { + var ret = contexts.get(); + if (null == ret) + throw new IllegalStateException("McpContext is not set"); + return ret; + } + + public static AutoCloseable withContext(ContainerUser ctx) + { + return with(new McpContext(ctx)); + } + + public static AutoCloseable withContext(Container container, User user) + { + return with(new McpContext(container, user)); + } + + private static AutoCloseable with(McpContext ctx) + { + final McpContext prev = contexts.get(); + contexts.set(ctx); + return () -> contexts.set(prev); + } +} diff --git a/api/src/org/labkey/api/mcp/McpService.java b/api/src/org/labkey/api/mcp/McpService.java new file mode 100644 index 00000000000..ba22c53d545 --- /dev/null +++ b/api/src/org/labkey/api/mcp/McpService.java @@ -0,0 +1,96 @@ +package org.labkey.api.mcp; + + +import io.modelcontextprotocol.server.McpServerFeatures; +import jakarta.servlet.http.HttpSession; +import org.jetbrains.annotations.NotNull; +import org.jspecify.annotations.NonNull; +import org.labkey.api.module.McpProvider; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.util.HtmlString; +import org.springaicommunity.mcp.provider.resource.SyncMcpResourceProvider; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.support.ToolCallbacks; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.ToolCallbackProvider; +import org.springframework.ai.vectorstore.VectorStore; + +import java.util.Arrays; +import java.util.List; +import java.util.function.Supplier; + +/** + * This service lets you expose functionality over the MCP protocol (only simple http for now). This allows + * external chat sessions to pull information from LabKey Server. These methods are also made available + * to chat session shosted by LabKey (see AbstractAgentAction). + *

+ * These calls are not security checked. Any tools registered here must check user permissions. Maybe that + * will come as we get further along. Note that the LLM may make callbacks concerning containers other than the + * current container. This is an area for investigation. + */ +public interface McpService extends ToolCallbackProvider +{ + // marker interface for classes that we will "ingest" using Spring annotations + interface McpImpl {}; + + static @NotNull McpService get() + { + return ServiceRegistry.get().getService(McpService.class); + } + + static void setInstance(McpService service) + { + ServiceRegistry.get().registerService(McpService.class, service); + } + + boolean isReady(); + + + default void register(McpImpl obj) + { + ToolCallback[] tools = ToolCallbacks.from(obj); + if (null != tools && tools.length > 0) + registerTools(Arrays.asList(tools)); + + List ret; + var resources = new SyncMcpResourceProvider(List.of(obj)).getResourceSpecifications(); + if (null != resources && !resources.isEmpty()) + registerResources(resources); + } + + + default void register(McpProvider mcp) + { + registerTools(mcp.getMcpTools()); + registerPrompts(mcp.getMcpPrompts()); + registerResources(mcp.getMcpResources()); + } + + void registerTools(@NotNull List tools); + + void registerPrompts(@NotNull List prompts); + + void registerResources(@NotNull List resources); + + @Override + ToolCallback @NonNull [] getToolCallbacks(); + + ChatClient getChat(HttpSession session, String agentName, Supplier systemPromptSupplier); + + record MessageResponse(String contentType, String text, HtmlString html) {} + + /** get consolidated response (good for many text oriented agents/use-cases) */ + MessageResponse sendMessage(ChatClient chat, String message); + + /** get individual response parts, useful for agents that generate SQL or programmatic responses */ + default List sendMessageEx(ChatClient chat, String message) + { + return List.of(sendMessage(chat, message)); + } + + /** + * return an in-memory Vector store for prototyping RAG features + * CONSIDER: Is it possible to implement VectorStoreRetriever wrapper for SearchService??? + */ + VectorStore getVectorStore(); +} diff --git a/api/src/org/labkey/api/mcp/PromptForm.java b/api/src/org/labkey/api/mcp/PromptForm.java new file mode 100644 index 00000000000..a6017f4d266 --- /dev/null +++ b/api/src/org/labkey/api/mcp/PromptForm.java @@ -0,0 +1,16 @@ +package org.labkey.api.mcp; + +public class PromptForm +{ + public String prompt; + + public void setPrompt(String prompt) + { + this.prompt = prompt; + } + + public String getPrompt() + { + return this.prompt; + } +} diff --git a/api/src/org/labkey/api/module/McpProvider.java b/api/src/org/labkey/api/module/McpProvider.java new file mode 100644 index 00000000000..6ca9061e73f --- /dev/null +++ b/api/src/org/labkey/api/module/McpProvider.java @@ -0,0 +1,24 @@ +package org.labkey.api.module; + +import io.modelcontextprotocol.server.McpServerFeatures; +import org.springframework.ai.tool.ToolCallback; + +import java.util.List; + +public interface McpProvider +{ + default List getMcpTools() + { + return List.of(); + } + + default List getMcpPrompts() + { + return List.of(); + } + + default List getMcpResources() + { + return List.of(); + } +} diff --git a/api/src/org/labkey/api/query/QueryKey.java b/api/src/org/labkey/api/query/QueryKey.java index f8fd48ec2d1..98cc178b914 100644 --- a/api/src/org/labkey/api/query/QueryKey.java +++ b/api/src/org/labkey/api/query/QueryKey.java @@ -329,6 +329,11 @@ public String toDisplayString() * Returns a string appropriate for usage in LabKey SQL. */ public String toSQLString() + { + return toSQLString(false); + } + + public String toSQLString(boolean quoteEverything) { List parts = getParts(); StringBuilder escapedName = new StringBuilder(); @@ -336,7 +341,7 @@ public String toSQLString() for (String part : parts) { escapedName.append(sep); - if (needsQuotes(part)) + if (quoteEverything || needsQuotes(part)) escapedName.append("\"").append(part.replace("\"", "\"\"")).append("\""); else escapedName.append(part); diff --git a/api/src/org/labkey/api/search/NoopSearchService.java b/api/src/org/labkey/api/search/NoopSearchService.java index fbd402582a5..3b91b203d22 100644 --- a/api/src/org/labkey/api/search/NoopSearchService.java +++ b/api/src/org/labkey/api/search/NoopSearchService.java @@ -414,6 +414,12 @@ public void notFound(URLHelper url) { } + @Override + public List getAllCategories() + { + return List.of(); + } + @Override public List getCategories(String categories) { diff --git a/api/src/org/labkey/api/search/SearchService.java b/api/src/org/labkey/api/search/SearchService.java index 4d64b9ce747..9a1fc3687f6 100644 --- a/api/src/org/labkey/api/search/SearchService.java +++ b/api/src/org/labkey/api/search/SearchService.java @@ -37,6 +37,7 @@ import org.labkey.api.security.permissions.Permission; import org.labkey.api.services.ServiceRegistry; import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; import org.labkey.api.util.Pair; import org.labkey.api.util.Path; import org.labkey.api.util.URLHelper; @@ -53,6 +54,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.net.URISyntaxException; import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; @@ -347,6 +349,26 @@ class SearchHit public JSONObject jsonData; public float score; + public String fullHref(Path contextPath) + { + try + { + ActionURL action = new ActionURL(this.url); + Path path = action.getParsedExtraPath(); + if (path.size() == 1 && GUID.isGUID(action.getParsedExtraPath().getName())) + { + Container c = ContainerManager.getForId(action.getParsedExtraPath().getName()); + if (null != c) + action.setContainer(c); + } + return action.getURIString(false); + } + catch (IllegalArgumentException x) + { + return normalizeHref(contextPath); + } + } + public String normalizeHref(Path contextPath) { Container c = ContainerManager.getForId(container); @@ -362,18 +384,18 @@ public String normalizeHref(Path contextPath, Container c) try { - if (null != c && href.startsWith("/")) - { + if (null != c && href.startsWith("/")) + { URLHelper url = new URLHelper(href); Path path = url.getParsedPath(); - if (path.startsWith(contextPath)) + if (path.startsWith(contextPath)) + { + int pos = path.size() - 2; // look to see if second to last path part is GUID + if (pos>=0 && c.getId().equals(path.get(pos))) { - int pos = path.size() - 2; // look to see if second to last path part is GUID - if (pos>=0 && c.getId().equals(path.get(pos))) - { - path = path.subpath(0,pos) - .append(c.getParsedPath()) - .append(path.subpath(pos+1,path.size())); + path = path.subpath(0,pos) + .append(c.getParsedPath()) + .append(path.subpath(pos+1,path.size())); url.setPath(path); return url.getLocalURIString(false); } @@ -458,6 +480,7 @@ public String normalizeHref(Path contextPath, Container c) // void addSearchCategory(SearchCategory category); + List getAllCategories(); List getCategories(String categories); void addResourceResolver(@NotNull String prefix, @NotNull ResourceResolver resolver); @@ -727,6 +750,27 @@ public Builder(SearchOptions options) this.sortField = options.sortField; } + public Builder limit(int limit) + { + this.limit = limit; + return this; + } + + public Builder scope(SearchScope scope) + { + this.scope = scope; + return this; + } + + /** space separated list */ + public Builder categories(String categories) + { + var list = SearchService.get().getCategories(categories); + if (null != list && !list.isEmpty()) + this.categories = list; + return this; + } + public SearchOptions build() { return new SearchOptions(queryString, user, container, categories, scope, sortField, offset, limit, invertResults, fields); diff --git a/api/src/org/labkey/api/wiki/WikiService.java b/api/src/org/labkey/api/wiki/WikiService.java index 533f34d2ffd..58fa73db6d9 100644 --- a/api/src/org/labkey/api/wiki/WikiService.java +++ b/api/src/org/labkey/api/wiki/WikiService.java @@ -46,7 +46,15 @@ static void setInstance(WikiService impl) WebPartView getView(Container c, String name, boolean renderContentOnly); WebPartView getHistoryView(Container c, String name); - HtmlString getHtml(Container c, String name); + record RenderedWiki (String name, String title, HtmlString html, String entityId) {} + + RenderedWiki getRenderedWiki(Container c, String name); + + default HtmlString getHtml(Container c, String name) + { + var wiki = getRenderedWiki(c, name); + return null == wiki ? null : wiki.html(); + } void insertWiki(User user, Container container, String name, String content, WikiRendererType renderType, String title); diff --git a/core/src/org/labkey/core/CoreMcp.java b/core/src/org/labkey/core/CoreMcp.java new file mode 100644 index 00000000000..06482fe5627 --- /dev/null +++ b/core/src/org/labkey/core/CoreMcp.java @@ -0,0 +1,66 @@ +package org.labkey.core; + +import org.json.JSONObject; +import org.labkey.api.data.Container; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.security.User; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.util.HtmlString; +import org.springframework.ai.tool.annotation.Tool; + +import java.util.Map; +import java.util.Objects; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +public class CoreMcp implements McpService.McpImpl +{ + // TODO ChatSessions are currently per session. The McpService should detect change of folder. + @Tool(description = "Call this tool before answering any prompts! This tool provides useful context information about the current user (name, userid), webserver (name, url, description), and current folder (name, path, url, description).") + String whereAmIWhoAmITalkingTo() + { + McpContext context = McpContext.get(); + User user = context.getUser(); + Container folder = context.getContainer(); + AppProps appProps = AppProps.getInstance(); + Study study = null != StudyService.get() ? Objects.requireNonNull(StudyService.get()).getStudy(folder) : null; + LookAndFeelProperties laf = LookAndFeelProperties.getInstance(folder); + + JSONObject userObj = new JSONObject(); + userObj.put("userId", user.getUserId()); + userObj.put("displayName", user.getDisplayName(user)); + if (isNotBlank(user.getFirstName())) + userObj.put("firstName", user.getFirstName()); + + JSONObject folderObj = new JSONObject(); + folderObj.put("name", folder.getName()); + folderObj.put("path", folder.getPath()); + folderObj.put("startUrl", folder.getStartURL(user).getURIString()); + if (isNotBlank(folder.getDescription())) + folderObj.put("description", folder.getDescription()); + if (null != study) + { + var studyDescription = study.getDescriptionHtml(); + if (!HtmlString.isBlank(studyDescription)) + { + folderObj.put("studyDescription", new JSONObject(Map.of("contentType", "text/html", "content", studyDescription.toString()))); + } + } + + JSONObject siteObj = new JSONObject(); + siteObj.put("name", appProps.getServerName()); + siteObj.put("baseServerUrl", appProps.getBaseServerUrl()); + siteObj.put("description", laf.getDescription()); + siteObj.put("homePageUrl", appProps.getHomePageUrl()); + + return new JSONObject(Map.of( + "user", userObj, + "currentFolder", folderObj, + "site", siteObj + )).toString(); + } +} diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index f959bf5aa1e..ce6b69dab9c 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -97,6 +97,7 @@ import org.labkey.api.module.SchemaUpdateType; import org.labkey.api.module.SpringModule; import org.labkey.api.module.Summary; +import org.labkey.api.mcp.McpService; import org.labkey.api.notification.EmailMessage; import org.labkey.api.notification.EmailService; import org.labkey.api.notification.NotificationMenuView; @@ -252,6 +253,7 @@ import org.labkey.core.login.LoginController; import org.labkey.core.metrics.SimpleMetricsServiceImpl; import org.labkey.core.metrics.WebSocketConnectionManager; +import org.labkey.core.mpc.McpServiceImpl; import org.labkey.core.notification.EmailPreferenceConfigServiceImpl; import org.labkey.core.notification.EmailPreferenceContainerListener; import org.labkey.core.notification.EmailPreferenceUserListener; @@ -559,9 +561,12 @@ public QuerySchema createSchema(DefaultSchema schema, Module module) ScriptEngineManagerImpl.registerEncryptionMigrationHandler(); + McpService.get().register(new CoreMcp()); + deleteTempFiles(); } + private void deleteTempFiles() { try @@ -1280,6 +1285,8 @@ public void moduleStartupComplete(ServletContext servletContext) CoreMigrationSchemaHandler.register(); Encryption.checkMigration(); + + McpServiceImpl.get().startMpcServer(); } // Issue 7527: Auto-detect missing SQL views and attempt to recreate @@ -1308,6 +1315,9 @@ public void registerServlets(ServletContext servletCtx) _webdavServletDynamic = servletCtx.addServlet("static", new WebdavServlet(true)); _webdavServletDynamic.setMultipartConfig(SpringActionController.getMultiPartConfigElement()); _webdavServletDynamic.addMapping("/_webdav/*"); + + McpService.setInstance(new McpServiceImpl()); + McpServiceImpl.get().registerServlets(servletCtx); } @Override diff --git a/core/src/org/labkey/core/mpc/McpServiceImpl.java b/core/src/org/labkey/core/mpc/McpServiceImpl.java new file mode 100644 index 00000000000..b752e808032 --- /dev/null +++ b/core/src/org/labkey/core/mpc/McpServiceImpl.java @@ -0,0 +1,485 @@ +package org.labkey.core.mpc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.genai.Client; +import com.google.genai.types.ClientOptions; +import io.modelcontextprotocol.json.McpJsonMapper; +import io.modelcontextprotocol.server.McpServer; +import io.modelcontextprotocol.server.McpServerFeatures; +import io.modelcontextprotocol.server.McpSyncServer; +import io.modelcontextprotocol.server.transport.HttpServletStreamableServerTransportProvider; +import io.modelcontextprotocol.spec.McpSchema; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jspecify.annotations.NonNull; +import org.labkey.api.collections.CopyOnWriteHashMap; +import org.labkey.api.markdown.MarkdownService; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.util.ContextListener; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.SessionHelper; +import org.labkey.api.util.ShutdownListener; +import org.labkey.api.util.logging.LogHelper; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.client.advisor.MessageChatMemoryAdvisor; +import org.springframework.ai.chat.client.advisor.vectorstore.QuestionAnswerAdvisor; +import org.springframework.ai.chat.memory.ChatMemory; +import org.springframework.ai.chat.memory.ChatMemoryRepository; +import org.springframework.ai.chat.memory.InMemoryChatMemoryRepository; +import org.springframework.ai.chat.memory.MessageWindowChatMemory; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.google.genai.GoogleGenAiChatModel; +import org.springframework.ai.google.genai.GoogleGenAiChatOptions; +import org.springframework.ai.google.genai.GoogleGenAiEmbeddingConnectionDetails; +import org.springframework.ai.google.genai.text.GoogleGenAiTextEmbeddingModel; +import org.springframework.ai.google.genai.text.GoogleGenAiTextEmbeddingOptions; +import org.springframework.ai.mcp.McpToolUtils; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.vectorstore.SimpleVectorStore; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.springframework.ai.chat.messages.MessageType.ASSISTANT; + + +public class McpServiceImpl implements McpService +{ + public static final String MESSAGE_ENDPOINT = "/_mcp/message"; + public static final String SSE_ENDPOINT = "/_mcp/sse"; + + private final CopyOnWriteHashMap toolMap = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap promptMap = new CopyOnWriteHashMap<>(); + private final CopyOnWriteHashMap resourceMap = new CopyOnWriteHashMap<>(); + + private final ObjectMapper objectMapper = JsonUtil.DEFAULT_MAPPER; + private final _McpServlet mcpServlet = new _McpServlet(JsonUtil.DEFAULT_MAPPER, MESSAGE_ENDPOINT, SSE_ENDPOINT); + private final ChatMemoryRepository chatMemoryRepository = new InMemoryChatMemoryRepository(); + + private boolean serverReady = false; + + public static McpServiceImpl get() + { + return (McpServiceImpl) McpService.get(); + } + + + /** + * Called by CoreModule.registerServlets() + * The servlet will return SC_SERVICE_UNAVAILABLE until startMcpServer() is called + */ + public void registerServlets(ServletContext servletCtx) + { + var mcpServletDynamic = servletCtx.addServlet("mcpServlet", mcpServlet); + mcpServletDynamic.setAsyncSupported(true); + mcpServletDynamic.addMapping(MESSAGE_ENDPOINT + "/*"); + mcpServletDynamic.addMapping(SSE_ENDPOINT + "/*"); + } + + + public static class HelloWorld + { + @Tool(description = "Call this tool when starting a new conversation") + String hello() + { + return "hello world!"; + } + + @Tool(description = "Call this tool when ending a conversation") + String bye() + { + return "bye now"; + } + } + + + public void startMpcServer() + { + /* For now the presense of GEMINI_API_KEY will enable/disable the McpServer */ + if (isBlank(System.getenv("GEMINI_API_KEY"))) + return; + vectorStore = createVectorStore(); + mcpServlet.startMcpServer(); + serverReady = true; + } + + + @Override + public boolean isReady() + { + return serverReady; + } + + + @Override + public void registerTools(@NotNull List tools) + { + tools.forEach(tool -> toolMap.put(tool.getToolDefinition().name(), tool)); + } + + @Override + public void registerPrompts(@NotNull List prompts) + { + prompts.forEach(prompt -> promptMap.put(prompt.prompt().name(), prompt)); + } + + @Override + public void registerResources(@NotNull List resources) + { + resources.forEach(resource -> resourceMap.put(resource.resource().name(), resource)); + } + + + @Override + public ToolCallback @NonNull [] getToolCallbacks() + { + return toolMap.values().toArray(new ToolCallback[0]); + } + + + public List tools() + { + McpJsonMapper mapper = McpJsonMapper.getDefault(); + return toolMap.values().stream().map(ToolCallback::getToolDefinition).map(td -> + McpSchema.Tool.builder() + .name(td.name()) + .description(td.description()) + .inputSchema(mapper, td.inputSchema()) + .build() + ).toList(); + } + + private class _McpServlet extends HttpServlet // wraps HttpServletSseServerTransportProvider + { + HttpServletStreamableServerTransportProvider transportProvider = null; + McpSyncServer mcpServer = null; + + _McpServlet(ObjectMapper objectMapper, String messageEndpoint, String sseEndpoint) + { +// transportProvider = HttpServletSseServerTransportProvider.builder() +// .jsonMapper(McpJsonMapper.getDefault()) +// .messageEndpoint(messageEndpoint) +// .sseEndpoint(sseEndpoint) +// .build(); + + transportProvider = HttpServletStreamableServerTransportProvider.builder() + .jsonMapper(McpJsonMapper.getDefault()) + .mcpEndpoint(messageEndpoint) + .build(); + } + + void startMcpServer() + { + List tools = Arrays.stream(getToolCallbacks()).map(McpToolUtils::toSyncToolSpecification).toList(); + List resources = new ArrayList<>(resourceMap.values()); + + mcpServer = McpServer.sync(transportProvider) + .tools(tools) + .resources(resources) +// .capabilities(new McpSchema.ServerCapabilities()) + .build(); + ContextListener.addShutdownListener(new _ShutdownListener()); + } + + @Override + public void service(ServletRequest sreq, ServletResponse sres) throws ServletException, IOException + { + if (!(sreq instanceof HttpServletRequest req) || !(sres instanceof HttpServletResponse res)) + { + // how to set error??? + throw new ServletException("non-HTTP request"); + } + + if (null == mcpServer) + { + res.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + return; + } + + + if ("POST".equals(req.getMethod())) + { + if (null == req.getParameter("sessionId") && null == req.getSession(true).getAttribute("McpServiceImpl#mcpSessionId")) + { + // USE SSE endpoint to get a sessionId + MockHttpServletRequest mockRequest = new MockHttpServletRequest(req.getServletContext(), "GET", SSE_ENDPOINT); + mockRequest.setAsyncSupported(true); + MockHttpServletResponse mockResponse = new MockHttpServletResponse(); + transportProvider.service(mockRequest, mockResponse); + String body = new String(mockResponse.getContentAsByteArray(), StandardCharsets.UTF_8); + String mcpSessionId = StringUtils.substringBetween(body, "sessionId=", "\n"); + req.getSession(true).setAttribute("McpServiceImpl#mcpSessionId", mcpSessionId); + mockRequest.close(); + mockResponse.getOutputStream().close(); + } + + req = new HttpServletRequestWrapper(req) + { + @Override + public String getParameter(String name) + { + var ret = super.getParameter(name); + if (null == ret && "sessionId".equals(name)) + return String.valueOf(Objects.requireNonNull(((HttpServletRequest) getRequest()).getSession(true).getAttribute("McpServiceImpl#mcpSessionId"))); + return ret; + } + }; + } + transportProvider.service(req, res); + } + + public Mono closeGracefully() + { + if (null != transportProvider) + return transportProvider.closeGracefully(); + return Mono.empty(); + } + } + + + // ShutdownListener + + class _ShutdownListener implements ShutdownListener + { + @Override + public String getName() + { + return "MpcService"; + } + + + Mono closing = null; + + @Override + public void shutdownPre() + { + closing = mcpServlet.closeGracefully(); + saveVectorDatabase(); + } + + @Override + public void shutdownStarted() + { + if (null == closing) + closing = mcpServlet.closeGracefully(); + closing.block(Duration.ofSeconds(5)); + } + } + + + /* GEMINI CHAT SERVICE */ + + String getModel() + { + return "gemini-2.5-flash"; + // gemini-2.5-flash + // gemini-2.5-pro + // gemini-3-flash-preview + } + + String getEmbeddingModel() + { + return "gemini-embedding-001"; + } + + + Object _initClientLock = new Object(); + Client _genAiClient = null; + + Client getLlmClient() + { + synchronized (_initClientLock) + { + if (null == _genAiClient) + { + ClientOptions clientOptions = ClientOptions.builder() + .build(); + _genAiClient = Client.builder() + .clientOptions(clientOptions) + .build(); + } + + return _genAiClient; + } + } + + + // SPRING AI CHAT SERVICE + @Override + public ChatClient getChat(HttpSession session, String agentName, Supplier systemPromptSupplier) + { + if (!serverReady) + return null; + + return SessionHelper.getAttribute(session, ChatClient.class.getName() + "#" + agentName, () -> + { + String systemPrompt = systemPromptSupplier.get(); + String conversationId = session.getId() + ":" + agentName; + + Client genAiClient = getLlmClient(); + + GoogleGenAiChatOptions chatOptions = GoogleGenAiChatOptions.builder() + .model(getModel()) + .toolCallbacks(getToolCallbacks()) + .build(); + ChatModel chatModel = GoogleGenAiChatModel.builder() + .genAiClient(genAiClient) + .defaultOptions(chatOptions) + .build(); + ChatMemory chatMemory = MessageWindowChatMemory.builder() + .maxMessages(100) + .chatMemoryRepository(chatMemoryRepository) + .build(); + return ChatClient.builder(chatModel) + .defaultOptions(chatOptions) + .defaultAdvisors(MessageChatMemoryAdvisor.builder(chatMemory) + .conversationId(conversationId) + .build()) + .defaultAdvisors(QuestionAnswerAdvisor.builder(getVectorStore()).build()) + .defaultSystem(systemPrompt) + .build(); + }); + } + + + @Override + public MessageResponse sendMessage(ChatClient chatSession, String message) + { + var callResponse = chatSession + .prompt(message) + .toolContext(McpContext.get().getToolContext().getContext()) + .call(); + StringBuilder sb = new StringBuilder(); + for (Generation result : callResponse.chatResponse().getResults()) + { + var output = result.getOutput(); + if (ASSISTANT == output.getMessageType()) + { + sb.append(output.getText()); + sb.append("\n\n"); + } + } + String md = sb.toString().strip(); + HtmlString html = HtmlString.unsafe(MarkdownService.get().toHtml(md)); + return new MessageResponse("text/markdown", md, html); + } + + @Override + public List sendMessageEx(ChatClient chatSession, String message) + { + if (isBlank(message)) + return List.of(); + var callResponse = chatSession + .prompt(message) + .toolContext(McpContext.get().getToolContext().getContext()) + .call(); + List ret = new ArrayList<>(); + for (Generation result : callResponse.chatResponse().getResults()) + { + var output = result.getOutput(); + if (ASSISTANT == output.getMessageType()) + { + String md = output.getText(); + HtmlString html = HtmlString.unsafe(MarkdownService.get().toHtml(md)); + ret.add(new MessageResponse("text/markdown", md, html)); + } + } + return ret; + } + + + VectorStore vectorStore = null; + + private VectorStore createVectorStore() + { + SimpleVectorStore ret = null; + + try + { + ClientOptions clientOptions = ClientOptions.builder() + .build(); + Client client = Client.builder() // not shared with getLlmClient() ??? maybe causing problems? + .clientOptions(clientOptions) + .build(); + GoogleGenAiEmbeddingConnectionDetails connectionDetails = GoogleGenAiEmbeddingConnectionDetails.builder() + .genAiClient(client) + .build(); + GoogleGenAiTextEmbeddingOptions embeddingOptions = GoogleGenAiTextEmbeddingOptions.builder() + .model(getEmbeddingModel()) + .build(); + EmbeddingModel embeddingModel = new GoogleGenAiTextEmbeddingModel(connectionDetails, embeddingOptions); + ret = SimpleVectorStore.builder(embeddingModel).build(); + + var savedFile = FileUtil.getTempDirectoryFileLike().resolveChild("VectorStore.database"); + if (savedFile.exists()) + { + try + { + ret.load(savedFile.toNioPathForRead().toFile()); + } + catch (Exception x) + { + LogHelper.getLogger(McpServiceImpl.class,"mcp service") + .error("error restoring saved vectordb: " + savedFile.toNioPathForRead(), x); + } + } + } + catch (Exception x) + { + System.err.println(x.getMessage()); + } + + return ret; + } + + + @Override + public VectorStore getVectorStore() + { + return !serverReady ? null : vectorStore; + } + + + void saveVectorDatabase() + { + SimpleVectorStore vectorStore = (SimpleVectorStore)getVectorStore(); + if (null == vectorStore) + return; + + var db = FileUtil.getTempDirectoryFileLike().resolveChild("VectorStore.database"); + try + { + vectorStore.save(db.toNioPathForRead().toFile()); + } + catch (Exception x) + { + System.err.println(x.getMessage()); + } + } +} diff --git a/devtools/src/org/labkey/devtools/TestController.java b/devtools/src/org/labkey/devtools/TestController.java index be8d2cff08a..09f7fe0ef9b 100644 --- a/devtools/src/org/labkey/devtools/TestController.java +++ b/devtools/src/org/labkey/devtools/TestController.java @@ -18,14 +18,21 @@ import jakarta.servlet.http.HttpServletResponse; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.labkey.api.action.ApiResponse; import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ConfirmAction; import org.labkey.api.action.FormArrayList; import org.labkey.api.action.FormViewAction; import org.labkey.api.action.ReadOnlyApiAction; import org.labkey.api.action.SimpleResponse; import org.labkey.api.action.SimpleViewAction; import org.labkey.api.action.SpringActionController; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.mcp.AbstractAgentAction; +import org.labkey.api.mcp.McpService; import org.labkey.api.security.CSRF; import org.labkey.api.security.MethodsAllowed; import org.labkey.api.security.RequiresLogin; @@ -42,7 +49,9 @@ import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.DOM; import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.URLHelper; import org.labkey.api.view.ActionURL; @@ -54,9 +63,15 @@ import org.labkey.api.view.UnauthorizedException; import org.labkey.api.view.ViewContext; import org.labkey.api.view.template.ClientDependency; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.wiki.WikiService; +import org.springframework.ai.document.Document; +import org.springframework.ai.vectorstore.SimpleVectorStore; +import org.springframework.ai.vectorstore.VectorStore; import org.springframework.dao.PessimisticLockingFailureException; import org.springframework.validation.BindException; import org.springframework.validation.Errors; +import org.springframework.validation.ObjectError; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.mvc.Controller; @@ -66,6 +81,9 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Gatherers; import static org.labkey.api.util.DOM.Attribute.name; import static org.labkey.api.util.DOM.Attribute.src; @@ -1269,4 +1287,121 @@ public void addNavTrail(NavTree root) { } } + + @RequiresLogin + public static class ChatAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + if (null == McpService.get() || !McpService.get().isReady()) + return HtmlView.of("Service is not ready yet."); + getPageConfig().setTemplate(PageConfig.Template.Dialog); + return new JspView<>("/org/labkey/devtools/view/chat.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("chat"); + } + } + + + @RequiresLogin + public static class ChatEndpointAction extends AbstractAgentAction + { + @Override + protected String getAgentName() + { + return "TestController.chat"; + } + + @Override + protected String getServicePrompt() + { + return "You are the generic LabKey agent. Good luck."; + } + } + + + @RequiresLogin + public static class PopulateVectorStoreAction extends ConfirmAction + { + AtomicInteger count = new AtomicInteger(); + + @Override + public ModelAndView getConfirmView(Object o, BindException errors) throws Exception + { + var db = FileUtil.getTempDirectoryFileLike().resolveChild("VectorStore.database"); + HtmlStringBuilder message = HtmlStringBuilder.of(); + message.append("This will add the contents of /Documention wikis to the vector store.").append(HtmlString.BR); + message.append("This may take a few minutes."); + if (db.exists()) + message.unsafeAppend("

").append("I see a vector store file already exists. Just FYI."); + return new HtmlView(message); + } + + @Override + public void validateCommand(Object o, Errors errors) + { + } + + @Override + public @NotNull URLHelper getSuccessURL(Object o) + { + return null; + } + + + // not usually used but some actions return views that close the current window etc... + public ModelAndView getSuccessView(Object form) + { + return HtmlView.of(count.get() + " documents added to vector store"); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + Container documentsContainer = ContainerManager.getForPath("/Documentation"); + if (null == documentsContainer) + throw new NotFoundException(); + VectorStore vs = McpService.get().getVectorStore(); + if (null == vs) + throw new NotFoundException(); + + ActionURL wikiBase = new ActionURL("wiki","page",documentsContainer); + + WikiService service = Objects.requireNonNull(WikiService.get()); + List all = service.getNames(documentsContainer); + all.stream() + .map(name -> service.getRenderedWiki(documentsContainer, name)) + .filter(Objects::nonNull) + .map(wiki -> + { + count.incrementAndGet(); + var metadata = Map.of( + "Content-Type", "text/html", + "filename", wiki.name() + ".html", + "title", (Object)wiki.title(), + "source", wikiBase.clone().addParameter("name",wiki.name()).getURIString() + ); + return new Document(wiki.entityId(), wiki.html().toString(), metadata); + }) + .gather(Gatherers.windowFixed(50)) + .forEach(vs); + + var db = FileUtil.getTempDirectoryFileLike().resolveChild("VectorStore.database"); + try + { + ((SimpleVectorStore)vs).save(db.toNioPathForRead().toFile()); + return true; + } + catch (Exception x) + { + errors.addError(new ObjectError("form", "error saving vectordb: " + x.getMessage())); + return false; + } + } + } } diff --git a/devtools/src/org/labkey/devtools/view/chat.jsp b/devtools/src/org/labkey/devtools/view/chat.jsp new file mode 100644 index 00000000000..82e001cca21 --- /dev/null +++ b/devtools/src/org/labkey/devtools/view/chat.jsp @@ -0,0 +1,123 @@ +<%@ page import="org.labkey.api.util.DOM" %> +<%@ page import="java.util.stream.Stream" %> +<%@ page import="static org.labkey.api.util.DOM.*" %> +<%@ page import="static org.labkey.api.util.DOM.Attribute.*" %> +<%@ page extends="org.labkey.api.jsp.JspBase" %> +<%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %> + + +function startChatting(chatEndpoint) +{ + var elPrompt = document.getElementById('chatPrompt'); + + function scrollToBottom() + { + const div = document.getElementById("chatHistory"); + div.scrollTo({ top: div.scrollHeight, behavior: "smooth" }); + } + + function appendUserPrompt(text) + { + const chatItem = document.createElement('div'); + chatItem.className = 'chatItem userPrompt'; + chatItem.innerText = text; + document.getElementById('chatHistory').appendChild(chatItem); + scrollToBottom(); + } + + function appendTextResponse(text) + { + appendResponse({contentType:"text/plain", response:text}); + } + + function appendResponse(res) + { + const chatItem = document.createElement('div'); + chatItem.className = 'chatItem genaiResponse'; + + if ("text/html" === res['contentType']) + chatItem.innerHTML = res.response; + else + chatItem.innerText = res.response; + + document.getElementById('chatHistory').appendChild(chatItem); + scrollToBottom(); + } + + function handleChatResponse(event) + { + const req = event.target; + if (req.readyState === 4) { + if (req.status >= 200 && req.status < 300) + { + appendResponse(JSON.parse(req.responseText)); + } + else + { + appendTextResponse('Request failed: ' + req.status + ' ' + (req.statusText || '')); + } + } + } + + function sendMessage(prompt) + { + var url = new URL(chatEndpoint); + url.searchParams.set('prompt', prompt); + var req = new XMLHttpRequest(); + req.open('GET', url.toString(), true); + req.onreadystatechange = handleChatResponse; + req.send(); + } + + let firstChat = true; + elPrompt.addEventListener('keydown', function (ev) + { + var isEnter = (ev.key === 'Enter') || (ev.keyCode === 13); + if (ev.shiftKey && isEnter) + { + const prompt = elPrompt.value; + appendUserPrompt(prompt); + elPrompt.value = ''; + sendMessage(prompt); + ev.preventDefault(); + ev.stopPropagation(); + return false; + } + return true; + }); + + sendMessage("Please introduce yourself"); +} + +LABKEY.Utils.onReady(function() + { + startChatting(new URL("./test-chatEndpoint.api", window.location.href)); + } +); + + +

+ diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 570345cd737..b9a63544ada 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -35,6 +35,7 @@ import org.labkey.api.module.DefaultModule; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; +import org.labkey.api.mcp.McpService; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.JavaExportScriptFactory; @@ -93,6 +94,7 @@ import org.labkey.query.audit.QueryUpdateAuditProvider; import org.labkey.query.controllers.OlapController; import org.labkey.query.controllers.QueryController; +import org.labkey.query.controllers.QueryMcp; import org.labkey.query.controllers.SqlController; import org.labkey.query.jdbc.QueryDriver; import org.labkey.query.olap.MemberSet; @@ -237,6 +239,8 @@ public QuerySchema createSchema(DefaultSchema schema, Module module) "Allow for lookup fields in product folders to query across all folders within the top-level folder.", false); OptionalFeatureService.get().addExperimentalFeatureFlag(QueryServiceImpl.EXPERIMENTAL_PRODUCT_PROJECT_DATA_LISTING_SCOPED, "Product folders display folder-specific data", "Only list folder-specific data within product folders.", false); + + McpService.get().register(new QueryMcp()); } diff --git a/query/src/org/labkey/query/controllers/LabKeySql.md b/query/src/org/labkey/query/controllers/LabKeySql.md new file mode 100644 index 00000000000..9beef68e462 --- /dev/null +++ b/query/src/org/labkey/query/controllers/LabKeySql.md @@ -0,0 +1,212 @@ +### **LabKey SQL Documentation** + +LabKey SQL is a unique SQL dialect that extends standard SQL functionality with features tailored for the LabKey Server platform, particularly for scientific data management. + +----- + +### **1. Lookups and Joins** + +LabKey SQL simplifies data joining by providing an intuitive **lookup syntax** that often eliminates the need for explicit `JOIN` statements. + +* **Syntax:** + `SELECT parent_table.lookup_column.target_column FROM parent_table` +* **Functionality:** + This syntax allows you to access columns from a foreign table by following the lookup relationship with dot notation. This is a powerful feature for simplifying queries that involve related data. +* **Examples:** + * **Simple Lookup:** To retrieve a participant's gender from the `Demographics` table using a lookup from the `PhysicalExam` table: + ```sql + SELECT PhysicalExam.ParticipantId, PhysicalExam.weight_kg, Demographics.Gender, Demographics.Height + FROM PhysicalExam + ``` + * **Joining Across Folders:** To join data from a `Demographics` table in one folder with a `Languages` list in a different folder: + ```sql + JOIN "/Other/Folder".lists.Languages ON Demographics.Language=Languages.Language + ``` + +----- + +### **2. Calculated Columns** + +LabKey SQL allows you to create virtual columns within a query by using SQL expressions. These calculated columns are not stored in the database but are computed on the fly. + +* **Syntax:** + `SELECT expression AS column_name FROM table` +* **Functionality:** + The syntax involves performing a calculation and then aliasing the result with a new column name using the `as` keyword. +* **Examples:** + * **Pulse Pressure:** To calculate pulse pressure from systolic and diastolic blood pressure values: + ```sql + PhysicalExam.systolicBP-PhysicalExam.diastolicBP as PulsePressure + ``` + * **BMI (Body Mass Index):** A more complex example that uses an intermediate query to calculate BMI from height and weight data: + ```sql + ROUND(weight_kg / (height_m * height_m), 2) AS BMI + ``` + +----- + +### **3. Pivoting** + +A `PIVOT` query helps you summarize and re-visualize data by transforming rows into columns. + +* **Syntax for PIVOT...BY Query:** + A pivot query is a `SELECT` statement specifying how to pivot and group columns. The basic syntax is `PIVOT [aggregating_column] BY [pivoting_column]`. +* **PIVOT...BY...IN Syntax:** + You can use an `IN` clause to specify a fixed set of column names to pivot. This is more efficient. + ```sql + PIVOT new_column_name BY pivoting_column IN ('value1', 'value2') + ``` + Note that pivot column names are case-sensitive. You may need to use `LOWER()` or `UPPER()` in your query to work around this issue. +* **Pivoting by Two Columns:** + Two levels of `PIVOT` are not directly supported. However, you can achieve a similar result by concatenating the two values together and pivoting on that "calculated" column. + ```sql + SELECT + Run.SampleCondition || ' ' || PeakLabel AS ConditionPeak, + AVG(Data.PercTimeCorrArea) AS AvgPercTimeCorrArea + FROM Data + GROUP BY Run.SampleCondition || ' ' || PeakLabel + PIVOT AvgPercTimeCorrArea BY ConditionPeak + ``` + +----- + +### **4. Cross-Folder Queries** + +You can write queries that access data from different folders within the LabKey Server instance, allowing for data integration across projects. + +* **Syntax:** + `SELECT * FROM Project."folder_path".schema.table` +* **Functionality:** + The folder path is a dot-delimited string that specifies the location of the table, including the project name. The user must have "Reader" permissions in each folder referenced in the query. +* **Example:** + ```sql + SELECT + p.ParticipantID, + ROUND(AVG(p.Temp_C), 1) AS AverageTemp + FROM Project."Tutorials/Demo/".study."Physical Exam" p + GROUP BY p.ParticipantID + ``` + +----- + +### **5. Parameterized Queries** + +LabKey SQL supports **parameterized queries** to improve security and reusability. + +* **Syntax:** + `PARAMETERS(param1, param2) SELECT * FROM table WHERE column = param1` +* **Functionality:** + The `PARAMETERS` keyword declares parameters that can be passed into the query. +* **Example:** A query with two parameters, `MinTemp` and `MinWeight`: + ```sql + PARAMETERS(MinTemp double, MinWeight double) + SELECT + ParticipantID, + temperature_C, + weight_kg + FROM PhysicalExam + WHERE temperature_C >= MinTemp AND weight_kg >= MinWeight + ``` + +----- + +### **6. Metadata Annotations** + +LabKey SQL allows you to directly annotate your SQL statements to override how column metadata is displayed in the LabKey interface. + +* **Syntax:** + `SELECT column_name @annotation FROM table` +* **Functionality:** + Annotations control the display of a column without changing the underlying data. +* **Examples:** + * **Hiding a Column:** + ```sql + SELECT ratio @hidden, log(ratio) as log_ratio + ``` + * **Setting a Title and Format:** + ```sql + SELECT 10/7.0 AS Num @title='Calculated Number' @Format='0.00' + ``` + +----- + +### **7. Available Methods** + +Here is a summary of the available functions and methods in LabKey SQL. + +#### **Mathematical Functions** + +* `abs(value)`: Returns the absolute value. +* `acos(value)`: Returns the arc cosine. +* `asin(value)`: Returns the arc sine. +* `atan(value)`: Returns the arc tangent. +* `atan2(value1, value2)`: Returns the arctangent of the quotient. +* `ceiling(value)`: Rounds the value up. +* `cos(radians)`: Returns the cosine. +* `cot(radians)`: Returns the cotangent. +* `degrees(radians)`: Returns degrees. +* `exp(n)`: Returns Euler's number 'e' raised to the nth power. +* `floor(value)`: Rounds down. +* `log(n)`: Returns the natural logarithm. +* `log10(n)`: Returns the base 10 logarithm. +* `mod(dividend, divider)`: Returns the remainder. +* `pi()`: Returns the value of pi. +* `power(base, exponent)`: Returns the base raised to the power of the exponent. +* `radians(degrees)`: Returns the radians. +* `rand()`, `rand(seed)`: Returns a random number. +* `round(value, precision)`: Rounds to the specified decimal places. +* `sign(value)`: Returns the sign of the value. +* `sin(value)`: Returns the sine. +* `sqrt(value)`: Returns the square root. +* `tan(value)`: Returns the tangent. +* `truncate(numeric value, precision)`: Truncates the numeric value. + +#### **String Functions** + +* `concat(value1, value2)`: Concatenates two values. +* `lcase(string)`, `lower(string)`: Converts to lower case. +* `left(string, integer)`: Returns the left side of the string. +* `length(string)`: Returns the length. +* `locate(substring, string, [startIndex])`: Returns the location of a substring. +* `ltrim(string)`: Trims white space from the left. +* `repeat(string, count)`: Repeats the string. +* `rtrim(string)`: Trims white space from the right. +* `startswith(string, prefix)`: Tests if a string starts with a prefix. +* `substring(string, start, length)`: Returns a portion of the string. +* `ucase(string)`, `upper(string)`: Converts to upper case. + +#### **Date and Time Functions** + +* `age(date1, date2, [interval])`: Supplies the difference in age. +* `age_in_months(date1, date2)`: Returns age in months. +* `age_in_years(date1, date2)`: Returns age in years. +* `curdate()`, `curtime()`: Returns the current date/time. +* `dayofmonth(date)`: Returns the day of the month. +* `dayofweek(date)`: Returns the day of the week. +* `dayofyear(date)`: Returns the day of the year. +* `hour(time)`, `minute(time)`, `second(time)`: Return time components. +* `month(date)`, `monthname(date)`: Return month values. +* `now()`: Returns the system date and time. +* `quarter(date)`: Returns the yearly quarter. +* `timestampadd(interval, number, timestamp)`: Adds an interval. +* `timestampdiff(interval, ts1, ts2)`: Finds the difference between timestamps. +* `week(date)`, `year(date)`: Return week and year values. + +#### **Conditional and Utility Functions** + +* `coalesce(v1,...,vN)`: Returns the first non-null value. +* `greatest(a, b, c, ...)`: Returns the greatest value. +* `ifdefined(column_name)`: References columns that may not exist. +* `ifnull(testValue, defaultValue)`: Returns a default value if the test value is null. +* `isequal(a,b)`: Returns true if `a` equals `b` or if both are `NULL`. +* `least(a, b, c, ...)`: Returns the smallest value. + +#### **LabKey SQL Extensions** + +* `contextPath()`, `folderName()`, `folderPath()`: Return path information. +* `ismemberof(groupid)`: Checks if a user is a member of a group. +* `javaConstant(fieldName)`: Provides access to Java static final variables. +* `moduleProperty(module name, property name)`: Returns a module property. +* `overlaps(START1, END1, START2, END2)`: Tests for overlapping time intervals (PostgreSQL only). +* `userid()`, `username()`: Return user information. +* `version()`: Returns the current schema version. \ No newline at end of file diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 17eab5ba6aa..96ddc095e92 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -19,6 +19,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.genai.Chat; +import com.google.genai.errors.ClientException; +import com.google.genai.errors.ServerException; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -29,6 +32,7 @@ import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Strings; import org.apache.commons.lang3.mutable.MutableInt; @@ -154,6 +158,10 @@ import org.labkey.api.files.FileContentService; import org.labkey.api.gwt.client.AuditBehaviorType; import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.mcp.AbstractAgentAction; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.mcp.PromptForm; import org.labkey.api.module.ModuleHtmlView; import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.RecordedAction; @@ -175,6 +183,7 @@ import org.labkey.api.query.QueryForm; import org.labkey.api.query.QueryParam; import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.QueryParseWarning; import org.labkey.api.query.QuerySchema; import org.labkey.api.query.QueryService; import org.labkey.api.query.QuerySettings; @@ -198,6 +207,7 @@ import org.labkey.api.security.MutableSecurityPolicy; import org.labkey.api.security.RequiresAllOf; import org.labkey.api.security.RequiresAnyOf; +import org.labkey.api.security.RequiresLogin; import org.labkey.api.security.RequiresNoPermission; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.SecurityManager; @@ -223,10 +233,12 @@ import org.labkey.api.stats.BaseAggregatesAnalyticsProvider; import org.labkey.api.stats.ColumnAnalyticsProvider; import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.DOM; import org.labkey.api.util.ExceptionUtil; import org.labkey.api.util.FileUtil; import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; import org.labkey.api.util.JavaScriptFragment; import org.labkey.api.util.JsonUtil; import org.labkey.api.util.LinkBuilder; @@ -298,6 +310,7 @@ import org.labkey.remoteapi.SelectRowsStreamHack; import org.labkey.remoteapi.query.SelectRowsCommand; import org.labkey.vfs.FileLike; +import org.springframework.ai.chat.client.ChatClient; import org.springframework.beans.MutablePropertyValues; import org.springframework.beans.PropertyValue; import org.springframework.beans.PropertyValues; @@ -340,6 +353,7 @@ import java.util.stream.Stream; import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trimToEmpty; import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; @@ -6629,7 +6643,7 @@ public ApiResponse execute(final SetCheckForm form, BindException errors) throws { for (String id : ids) { - if (StringUtils.isNotBlank(id)) + if (isNotBlank(id)) selection.add(id); } } @@ -6703,7 +6717,7 @@ public ApiResponse execute(final SetCheckForm form, BindException errors) { for (String id : ids) { - if (StringUtils.isNotBlank(id)) + if (isNotBlank(id)) selection.add(id); } } @@ -6729,7 +6743,7 @@ public ApiResponse execute(final SetCheckForm form, BindException errors) { for (String id : ids) { - if (StringUtils.isNotBlank(id)) + if (isNotBlank(id)) selection.add(id); } } @@ -8789,4 +8803,195 @@ private JSONArray getTestRows(String val) return rows; } } + + + public static class SqlPromptForm extends PromptForm + { + public String schemaName; + + public String getSchemaName() + { + return schemaName; + } + + public void setSchemaName(String schemaName) + { + this.schemaName = schemaName; + } + } + + + @RequiresPermission(ReadPermission.class) + @RequiresLogin + public static class QueryAgentAction extends AbstractAgentAction + { + SqlPromptForm _form; + + @Override + public void validateForm(SqlPromptForm sqlPromptForm, Errors errors) + { + _form = sqlPromptForm; + } + + @Override + protected String getAgentName() + { + return QueryAgentAction.class.getName(); + } + + @Override + protected String getServicePrompt() + { + StringBuilder serviceMessage = new StringBuilder(); + serviceMessage.append("Your job is to generate SQL statements. Here is some reference material formatted as markdown:\n").append(getSQLHelp()).append("\n\n"); + serviceMessage.append("NOTE: Prefer using lookup syntax rather than JOIN where possible.\n"); + serviceMessage.append("NOTE: When helping generate SQL please don't use names of tables and columns from documentation examples. Always refer to the available tools for retrieving database metadata.\n"); + + DefaultSchema defaultSchema = DefaultSchema.get(getUser(), getContainer()); + + if (!isBlank(_form.getSchemaName())) + { + var schema = defaultSchema.getSchema(_form.getSchemaName()); + if (null != schema) + { + serviceMessage.append("\n\nCurrent default schema is " + schema.getSchemaPath().toSQLString() + "."); + } + } + return serviceMessage.toString(); + } + + String getSQLHelp() + { + try + { + return IOUtils.resourceToString("org/labkey/query/controllers/LabKeySql.md", null, QueryController.class.getClassLoader()); + } + catch (IOException x) + { + throw new ConfigurationException("error loading resource", x); + } + } + + @Override + public Object execute(SqlPromptForm form, BindException errors) throws Exception + { + // save form here for context in getServicePrompt() + _form = form; + + try (var mcpPush = McpContext.withContext(getViewContext())) + { + // TODO when/how to do we reset or isolate different chat sessions, e.g. if two SQL windows are open concurrently? + ChatClient chatSession = getChat(); + String prompt = form.getPrompt(); + List responses; + SqlResponse sqlResponse; + + if (isBlank(prompt)) + { + return new JSONObject(Map.of( + "text", "🤷", + "success", Boolean.TRUE)); + } + + try + { + responses = McpService.get().sendMessageEx(chatSession, prompt); + sqlResponse = extractSql(responses); + } + catch (ServerException x) + { + return new JSONObject(Map.of( + "error", x.getMessage(), + "text", "ERROR: " + x.getMessage(), + "success", Boolean.FALSE)); + } + + /* VALIDATE SQL */ + if (null != sqlResponse.sql()) + { + QuerySchema schema = DefaultSchema.get(getUser(), getContainer()).getSchema("study"); + try + { + TableInfo ti = QueryService.get().createTable(schema, sqlResponse.sql(), null, true); + var warnings = ti.getWarnings(); + if (null != warnings) + { + var warning = warnings.stream().findFirst(); + if (warning.isPresent()) + throw warning.get(); + } + } + catch (QueryException x) + { + String validationPrompt = "That SQL caused the " + (x instanceof QueryParseWarning ? "warning" : "error") + " below, can you attempt to fix this?\n```" + x.getMessage() + "```"; + responses = McpService.get().sendMessageEx(chatSession, validationPrompt); + var newSqlResponse = extractSql(responses); + if (isNotBlank(newSqlResponse.sql())) + sqlResponse = newSqlResponse; + } + } + + var ret = new JSONObject(Map.of( + "success", Boolean.TRUE)); + if (null != sqlResponse.sql()) + ret.put("sql", sqlResponse.sql()); + if (null != sqlResponse.html()) + ret.put("html", sqlResponse.html()); + return ret; + } + catch (ClientException ex) + { + var ret = new JSONObject(Map.of( + "text", ex.getMessage(), + "user", getViewContext().getUser().getName(), + "success", Boolean.FALSE)); + return ret; + } + } + } + + record SqlResponse(HtmlString html, String sql) + { + } + + static SqlResponse extractSql(List responses) + { + HtmlStringBuilder html = HtmlStringBuilder.of(); + String sql = null; + + for (var response : responses) + { + if (null == sql) + { + var text = response.text(); + String sqlFind = extractSql(text); + if (null != sqlFind) + { + sql = sqlFind; + if (sql.equals(text) || text.startsWith("```sql")) + continue; // Don't append this to the html response + } + } + html.append(response.html()); + } + return new SqlResponse(html.getHtmlString(), sql); + } + + static String extractSql(String text) + { + if (text.startsWith("SELECT ")) + return text; + if (text.startsWith("WITH ") && text.contains("SELECT ")) + return text; + if (text.startsWith("PARAMETERS ") && text.contains("SELECT ")) + return text; + var sql = text.indexOf("```sql\n"); + if (sql >= 0) + { + var end = text.indexOf("```", sql+7); + if (end >= 0) + return text.substring(sql+7,end); + } + return null; + } } diff --git a/query/src/org/labkey/query/controllers/QueryMcp.java b/query/src/org/labkey/query/controllers/QueryMcp.java new file mode 100644 index 00000000000..99bb2df99c0 --- /dev/null +++ b/query/src/org/labkey/query/controllers/QueryMcp.java @@ -0,0 +1,386 @@ +package org.labkey.query.controllers; + +import io.modelcontextprotocol.spec.McpSchema; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.TableDescription; +import org.labkey.api.data.TableInfo; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.QueryKey; +import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleSchemaTreeVisitor; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.UserManager; +import org.labkey.query.sql.SqlParser; +import org.springaicommunity.mcp.annotation.McpResource; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; + +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +/* TODO: integrate ToolContext support */ + +public class QueryMcp implements McpService.McpImpl +{ + @McpResource( + uri = "resource://org/labkey/query/controllers/LabKeySql.md", + mimeType = "application/markdown", + name = "LabKey SQL", + description = "Provide documentation for LabKey SQL specific syntax") + public McpSchema.ReadResourceResult getLabKeySQLDocumentation() throws IOException + { + String markdown = IOUtils.resourceToString("org/labkey/query/controllers/LabKeySql.md", null, QueryController.class.getClassLoader()); + return new McpSchema.ReadResourceResult(List.of( + new McpSchema.TextResourceContents( + "resource://org/labkey/query/controllers/LabKeySql.md", + "application/markdown", + markdown) + )); + } + + + @Tool(description = "Provide column metadata for a sql table. This tool will also return SQL source for saved queries.") + String listColumnMetaData(@ToolParam(description = "Fully qualified table name as it would appear in SQL e.g. \"schema\".\"table\"") String fullQuotedTableName) + { + var json = _listColumnsForTable(fullQuotedTableName); + // can I just return a JSONObject + return json.toString(); + } + + @Tool(description = "Provide list of tables within the provided schema.") + String listTablesForSchema(@ToolParam(description = "Fully qualified schema name as it would appear in SQL e.g. \"schema\"") String quotedSchemaName) + { + var json = _listTablesForSchema(quotedSchemaName); + // can I just return a JSONObject + return json.toString(); + } + + @Tool(description = "Provide list of database schemas") + String listSchemas() + { + McpContext context = getContext(); + var map = _listAllSchemas(DefaultSchema.get(context.getUser(), context.getContainer())); + var array = new JSONArray(); + for (var entry : map.entrySet()) + { + array.put(new JSONObject(Map.of( + "name", entry.getKey().getName(), + "quotedName", entry.getKey().toSQLString(), + "description", StringUtils.trimToEmpty(entry.getValue().getDescription()) + ))); + } + return new JSONObject(Map.of("success", "true", "schemas", array)).toString(); + } + + + @Tool(description = "Provide the SQL source for a saved query.") + String getSourceForSavedQuery(@ToolParam(description = "Fully qualified query name as it would appear in SQL e.g. \"schema\".\"table or query\"") String fullQuotedTableName) + { + var json = _listTablesForSchema(fullQuotedTableName); + if (json.has("sql")) + return "```sql\n" + json.getString("sql") + "\n```\n"; + else + return "I could not find the source for " + fullQuotedTableName; + } + + + @Tool(description = """ + Save addition information for database columns. If additional metadata is gathered via + chat, it can be saved to improve further interactions. + """) + String saveColumnDescription( + @ToolParam(description = "Fully qualified table or query name as it would appear in SQL e.g. \"schema\".\"table or query\"") + String fullQuotedTableName, + @ToolParam(description = "Quoted column name as it would appear in SQL e.g. \"column name\"") + String quotedColumnName, + @ToolParam(description = "Additional metadata to remember for future use. This will replace any currently saved value") + String columnMetadata + ) + { + McpContext context = McpContext.get(); + var map = PropertyManager.getWritableProperties(context.getContainer(), "QueryMCP.annotations", true); + String fullPath = normalizeIdentifier(fullQuotedTableName + "." + quotedColumnName); + map.put(fullPath, columnMetadata); + try (var ignore = SpringActionController.ignoreSqlUpdates()) + { + map.save(); + } + return new JSONObject(Map.of("success",Boolean.TRUE)).toString(); + } + + /* TODO McpContext setup */ + + static McpContext getContext() + { + try + { + return McpContext.get(); + } + catch (Exception x) + { + return new McpContext(ContainerManager.getHomeContainer(), UserManager.getGuestUser()); + } + } + + /* For now, list all schemas. CONSIDER support incremental querying. */ + public static Map _listAllSchemas(DefaultSchema root) + { + SimpleSchemaTreeVisitor, Void> visitor = new SimpleSchemaTreeVisitor<>(false) + { + @Override + public Map visitUserSchema(UserSchema schema, Path path, Void v) + { + Map r = Map.of(schema.getSchemaPath(),schema); + return visitAndReduce(schema.getUserSchemas(false), path, null, r); + } + + @Override + public Map reduce(Map r1, Map r2) + { + if (null == r1 || null == r2) + return null==r1 && null==r2 ? Map.of() : null==r1 ? r2 : r1; + var ret = new TreeMap(); + ret.putAll(r1); + ret.putAll(r2); + return ret; + } + }; + + // DefaultSchema does not implement UserSchema which is inconvenient. + TreeMap ret = new TreeMap<>(); + for (String name : root.getUserSchemaNames(false)) + { + UserSchema s = root.getUserSchema(name); + if (null != s) + { + var res = visitor.visit(s, null, null); + ret.putAll(res); + } + } + return ret; + } + + + public static JSONObject _listTablesForSchema(String fullQuotedName) + { + SchemaKey fullKey; + + // TODO : correct method for parsing quoted identifier + if (fullQuotedName.startsWith("\"") && fullQuotedName.endsWith("\"")) + { + String[] parts = StringUtils.strip(fullQuotedName, "\"").split("\"\\.\""); + fullKey = SchemaKey.fromParts(parts); + } + else + { + String[] parts = StringUtils.split(fullQuotedName, "."); + fullKey = SchemaKey.fromParts(parts); + } + + McpContext context = getContext(); + var defaultSchema = DefaultSchema.get(context.getUser(), context.getContainer()); + var schema = DefaultSchema.resolve(defaultSchema, fullKey); + if (!(schema instanceof UserSchema userSchema)) + return new JSONObject("error", "could not find schema for : " + fullQuotedName); + + JSONArray array = new JSONArray(); + CaseInsensitiveHashSet names = new CaseInsensitiveHashSet(schema.getTableNames()); + var qds = userSchema.getQueryDefs(); + names.addAll(qds.keySet()); + + for (String tableName : names) + { + // CONSIDER schema.getTableDescription()??? + TableInfo td; + try + { + td = schema.getTable(tableName, null); + if (null == td) + continue; + } + catch (QueryParseException qpe) + { + continue; + } + QueryDefinition qd = ((UserSchema)schema).getQueryDef(tableName); + JSONObject table = new JSONObject(); + table.put("schemaName", schema.getName()); + table.put("tableName", td.getName()); + table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); + table.put("description", td.getDescription()); + table.put("type", null==qd ? "TABLE" : "QUERY"); + array.put(table); + } + + var ret = new JSONObject(); + ret.put("schemaName", schema.getName()); + ret.put("fullQuotedName", schema.getSchemaPath().toSQLString()); + if (isNotBlank(schema.getDescription())) + ret.put("description", schema.getDescription()); + ret.put("tables", array); + return ret; + } + + public static JSONObject _listColumnsForTable(String fullQuotedName) + { + McpContext context = McpContext.get(); + QueryKey fullKey = dottedIdentifier(fullQuotedName); + SchemaKey schemaKey; + + var props = PropertyManager.getProperties(context.getContainer(), "QueryMCP.annotations"); + + String tableName; + if (fullKey.size() > 1) + { + schemaKey = SchemaKey.fromParts(fullKey.getParent().getParts()); + tableName = fullKey.getName(); + } + else if (fullKey.size() == 1) + { + schemaKey = SchemaKey.fromParts("study"); + tableName = fullKey.getName(); + } + else + { + return new JSONObject("error", "could not find table"); + } + + SchemaKey tableKey = new SchemaKey(schemaKey, tableName); + + var defaultSchema = DefaultSchema.get(context.getUser(), context.getContainer()); + + var schema = DefaultSchema.resolve(defaultSchema, schemaKey); + if (null == schema) + return new JSONObject("error", "could not find table"); + + TableInfo td = schema.getTable(tableName, null); + if (null == td) + return new JSONObject("error", "could not find table"); + + String sourceSQL = null; + if (schema instanceof UserSchema userSchema) + { + QueryDefinition d = userSchema.getQueryDef(tableName); + if (null != d) + sourceSQL = d.getSql(); + } + + JSONObject table = new JSONObject(); + table.put("schemaName", schema.getName()); + table.put("tableName", td.getName()); + table.put("fullQuotedName", new SchemaKey(schema.getSchemaPath(), td.getName()).toSQLString()); + if (isNotBlank(td.getDescription())) + table.put("description", td.getDescription()); + if (isNotBlank(sourceSQL)) + table.put("sql", sourceSQL); + + var pkColumns = td.getPkColumns(); + var pk = pkColumns.size() == 1 ? pkColumns.get(0).getFieldKey() : null; + JSONArray columns = new JSONArray(); + for (ColumnInfo col : td.getColumns()) + { + String columnPropsKey = new SchemaKey(tableKey, col.getName()).toSQLString(true).toLowerCase(); + String extra = props.get(columnPropsKey); + + JSONObject md = new JSONObject(); + md.put("name", col.getName()); + md.put("label", col.getLabel()); + md.put("type", col.getJdbcType().name()); + String description = ""; + description += Objects.toString(col.getDescription(), col.getLabel()); + description += "\n" + Objects.toString(extra, ""); + if (null != col.getDescription()) + md.put("description", description.strip()); + if (col.getFieldKey().equals(pk)) + md.put("is_primary_key", Boolean.TRUE); + var fk = col.getFk(); + if (null != fk) + { + if (fk instanceof QueryForeignKey qfk) + { + SchemaKey qfkSchema = qfk.getLookupSchemaKey(); + SchemaKey qfkTable = new SchemaKey(qfkSchema, qfk.getLookupTableName()); + SchemaKey qfkColumn = new SchemaKey(qfkTable, qfk.getLookupColumnName()); + md.put("is_foreign_key", Boolean.TRUE); + md.put("references", qfkColumn.toSQLString()); + } + else + { + TableDescription references = fk.getLookupTableDescription(); + if (null != references && references.isPublic()) + { + SchemaKey qfkTable = SchemaKey.fromParts(td.getSchema().getQuerySchemaName(), td.getName()); + SchemaKey qfkColumn = new SchemaKey(qfkTable, fk.getLookupColumnName()); + md.put("is_foreign_key", Boolean.TRUE); + md.put("references", qfkColumn.toSQLString()); + } + } + } + columns.put(md); + } + table.put("columns",columns); + + return table; + } + + + static QueryKey dottedIdentifier(String compoundIdentifier) + { + return new SqlParser().parseIdentifier(compoundIdentifier); + } + + + // QueryKey supports toSQLString(), but not parseSQLString()? Does parsing code for this exist outside of SqlBase.g? + static String normalizeIdentifier(String compoundIdentifier) + { + return new SqlParser().parseIdentifier(compoundIdentifier).toSQLString(true).toLowerCase(); + } + + + /** JSON schema example provided by GEMINI, using triple tick-marks to delimit the machine-readable structured data + * + * Here is the database schema in JSON format: + * ```{ + * "database": "ecommerce", + * "tables": [ + * { + * "name": "customers", + * "description": "Stores customer details, including their contact information and location.", + * "columns": [ + * {"name": "customer_id", "type": "INTEGER", "description": "Unique identifier for each customer.", "is_primary_key": true}, + * {"name": "first_name", "type": "VARCHAR(50)", "description": "The customer's first name."}, + * {"name": "email", "type": "VARCHAR(100)", "description": "The customer's email address.", "is_unique": true} + * ] + * }, + * { + * "name": "orders", + * "description": "Tracks customer orders.", + * "columns": [ + * {"name": "order_id", "type": "INTEGER", "description": "Unique identifier for each order.", "is_primary_key": true}, + * {"name": "customer_id", "type": "INTEGER", "description": "Foreign key linking to the customers table.", "is_foreign_key": true, "references": "customers.customer_id"}, + * {"name": "order_date", "type": "DATE", "description": "The date the order was placed."} + * ] + * } + * ] + * }``` + */ +} + diff --git a/query/src/org/labkey/query/sql/SqlParser.java b/query/src/org/labkey/query/sql/SqlParser.java index 9174a6e2207..446131a6155 100644 --- a/query/src/org/labkey/query/sql/SqlParser.java +++ b/query/src/org/labkey/query/sql/SqlParser.java @@ -43,10 +43,12 @@ import org.labkey.api.query.AliasManager; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryKey; import org.labkey.api.query.QueryParseException; import org.labkey.api.query.QueryParseWarning; import org.labkey.api.query.QuerySchema; import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; import org.labkey.api.sql.LabKeySql; import org.labkey.api.util.DateUtil; import org.labkey.api.util.JunitUtil; @@ -416,6 +418,52 @@ public QExpr parseExpr(String str, boolean constExpression, List(); + try (var parser = getAntlrParser()) + { + parser.reset(str, _parseErrors); + try + { + ParserRuleReturnScope scope = parser.dottedIdentifier(); + + int last = parser.getTokenStream().LA(1); + if (EOF != last || !_parseErrors.isEmpty()) + return SchemaKey.fromParts(str); + + LinkedList q = new LinkedList<>(); + ArrayList parts = new ArrayList<>(); + q.add((CommonTree) scope.getTree()); + while (!q.isEmpty()) + { + CommonTree t = q.removeFirst(); + if (t.getType() == IDENT) + parts.add(t.getText()); + else if (t.getType() == QUOTED_IDENTIFIER) + parts.add(LabKeySql.unquoteIdentifier(t.getText())); + else if (t.getType() == DOT) + { + q.addFirst((CommonTree)t.getChildren().get(1)); + q.addFirst((CommonTree)t.getChildren().get(0)); + } + else + return SchemaKey.fromParts(str); + } + return SchemaKey.fromParts(parts); + } + catch (Exception x) + { + return SchemaKey.fromParts(str); + } + } + catch (Exception x) + { + return SchemaKey.fromParts(str); + } + } + + public static String toPrefixString(Tree tree) { @@ -2220,6 +2268,18 @@ public void testPrecedence() throws SQLException assertEquals(2, evalInt("2&1+1")); } + @Test + public void testParseIdentifier() + { + assertEquals("\"a\"", new SqlParser().parseIdentifier("a").toSQLString(true)); + assertEquals("\"a\"", new SqlParser().parseIdentifier("\"a\"").toSQLString(true)); + assertEquals("\"a\".\"b\"", new SqlParser().parseIdentifier("a.b").toSQLString(true)); + assertEquals("\"a\".\"b\"", new SqlParser().parseIdentifier("a.\"b\"").toSQLString(true)); + assertEquals("\"a\".\"b\"", new SqlParser().parseIdentifier("\"a\".b").toSQLString(true)); + assertEquals("\"a\".\"b\"", new SqlParser().parseIdentifier("\"a\".\"b\"").toSQLString(true)); + assertEquals("\"a\".\"b\".\"c\"", new SqlParser().parseIdentifier("a.\"b\".c").toSQLString(true)); + } + QuerySchema core = null; diff --git a/query/src/org/labkey/query/view/sourceQuery.jsp b/query/src/org/labkey/query/view/sourceQuery.jsp index 0c87691e053..c12fa995bcd 100644 --- a/query/src/org/labkey/query/view/sourceQuery.jsp +++ b/query/src/org/labkey/query/view/sourceQuery.jsp @@ -22,6 +22,8 @@ <%@ page import="org.labkey.api.view.HttpView" %> <%@ page import="org.labkey.api.view.template.ClientDependencies" %> <%@ page import="org.labkey.query.controllers.QueryController" %> +<%@ page import="org.labkey.api.mcp.McpService" %> +<%@ page import="org.labkey.api.util.JavaScriptFragment" %> <%@ taglib prefix="labkey" uri="http://www.labkey.org/taglib" %> <%@ page extends="org.labkey.api.jsp.JspBase" %> <%! @@ -51,6 +53,7 @@ boolean canEdit = queryDef.canEdit(getUser()); boolean canEditMetadata = queryDef.canEditMetadata(getUser()); boolean canDelete = queryDef.canDelete(getUser()); + boolean isChatReady = McpService.get().isReady(); %> + +<% if (isChatReady) { %> +<%-- should use Ext4 Panel for layout, but this is just a prototype anyway --%> + + + +
+ +
+ 		+
+ +
+<% } else { %>
+<% } %> + diff --git a/query/webapp/query/QueryEditorPanel.js b/query/webapp/query/QueryEditorPanel.js index 319c2a48587..2f6f2a4ea65 100644 --- a/query/webapp/query/QueryEditorPanel.js +++ b/query/webapp/query/QueryEditorPanel.js @@ -304,6 +304,15 @@ Ext4.define('LABKEY.query.SourceEditorPanel', { if (this.codeMirror) queryText = this.codeMirror.getValue(); return queryText; + }, + + setValue : function(queryText) + { + this.query.queryText = queryText; + if (this.codeMirror) + { + return this.codeMirror.setValue(queryText); + } } }); diff --git a/search/src/org/labkey/search/SearchMcp.java b/search/src/org/labkey/search/SearchMcp.java new file mode 100644 index 00000000000..d896830d8e0 --- /dev/null +++ b/search/src/org/labkey/search/SearchMcp.java @@ -0,0 +1,132 @@ +package org.labkey.search; + +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.search.SearchScope; +import org.labkey.api.search.SearchService; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; + +public class SearchMcp implements McpService.McpImpl +{ + final static String mdSearchHelp = """ + The search functionality is implmeneted by Lucene. The query syntax is + + Core Syntax Elements + + * **Terms and Phrases**: + * A **term** is a single word (e.g., `error`). + * A **phrase** is a group of words surrounded by double quotes (e.g., `"network error"`), which searches for all words in the specified order. + * **Fields**: You can search specific fields using the format `fieldName:searchValue` (e.g., `title:malaria` or `user:"John Doe"`). + * **Grouping**: Use parentheses `()` to group clauses and control Boolean logic (e.g., `title:(apple OR pie) AND description:apple`). + + Boolean Operators + + Boolean operators must be in **ALL CAPS**. + + * `AND` (`&&`, `+`): Requires both terms to be present (e.g., `wifi AND luxury` or `+wifi +luxury`). + * `OR` (`||`): Requires at least one term to be present (default operator if none is specified) (e.g., `wifi OR luxury`). + * `NOT` (`!`, `-`): Excludes documents that contain the term after the operator (e.g., `wifi NOT luxury` or `wifi -luxury`). + + Term Modifiers + + * **Wildcard Searches**: + * `?` for a single character (e.g., `te?t` matches "test" or "text"). + * `*` for multiple characters (zero or more) (e.g., `test*` matches "test", "tests", "tester"). + * _Note_: You cannot use `*` or `?` as the first character of a search term. + * **Fuzzy Searches**: Use the tilde `~` symbol at the end of a single word to find terms with a similar spelling (e.g., `roam~` finds "foam" and "roams"). An optional number between 0 and 2 can specify the required similarity (default is 0.5). + * **Proximity Searches**: Use the tilde `~` at the end of a phrase to find words within a specific distance (e.g., `"jakarta apache"~10` finds "jakarta" and "apache" within 10 words of each other). + * **Range Searches**: Match documents whose field values are between a lower and upper bound. + * Inclusive (square brackets): `mod_date:[20020101 TO 20030101]`. + * Exclusive (curly brackets): `title:{Aida TO Carmen}`. + * One-sided range: `score:[2.5 TO *]`. + * **Boosting Terms**: Use the caret `^` symbol with a numerical boost factor to increase the relevance of a term (e.g., `jakarta^4 apache` makes "jakarta" more relevant). + + Escaping Special Characters + + To use a special character as part of your search text, escape it with a single backslash `\\`. Special characters include: + `+ - && || ! ( ) { } [ ] ^ " ~ * ? : \\ /` + """; + + @Tool(description = "Search this LabKey server. This may be useful for site navigation purposes. When rendering results that use this tool present full URLs whenever relevant.") + String siteSearch( + @ToolParam(description = mdSearchHelp) String query, + @ToolParam(required=false, description="comma separated list of categories, use category=navigation to find folders/projects/studies. use the listSearchCategories tool to find other options.") String categories + ) + { + SearchService ss = SearchService.get(); + if (isBlank(query)) + { + return new JSONObject(Map.of( + "success", Boolean.TRUE, + "hits", new JSONArray())).toString(); + } + + var list = PageFlowUtil.splitStringToValuesForImport(trimToEmpty(categories)); + + McpContext context = McpContext.get(); + Path contextPath = AppProps.getInstance().getParsedContextPath(); + var options = new SearchService.SearchOptions.Builder(query,context.getUser(),context.getContainer()) + .limit(20) + .scope(SearchScope.All); + if (!list.isEmpty()) + options.categories(StringUtils.join(list,' ')); + + try + { + JSONArray hits = new JSONArray(); + var searchResult = ss.search(options.build()); + for (var hit : searchResult.hits) + { + JSONObject o = new JSONObject(); + o.put("title", hit.title); + o.put("container", hit.container); + o.put("url", hit.fullHref(contextPath)); + o.put("summary", trimToEmpty(hit.summary)); + o.put("score", hit.score); + o.put("identifiers", hit.identifiers); + o.put("category", trimToEmpty(hit.category)); + hits.put(o); + } + return new JSONObject(Map.of( + "success", Boolean.TRUE, + "baseServerUrl", AppProps.getInstance().getBaseServerUrl(), + "hits", hits, + "totalHits", searchResult.totalHits + )).toString(); + } + catch (IOException io) + { + return new JSONObject(Map.of( + "success", Boolean.FALSE, + "error", io.getMessage() + )).toString(); + } + } + + @Tool(description = "Return list of valid categories for the siteSearch tool") + String listSearchCategories() + { + JSONArray list = new JSONArray(); + for (var cat : SearchService.get().getAllCategories()) + { + list.put(cat.getName()); + } + return new JSONObject(Map.of( + "success", Boolean.TRUE, + "categories", list + )).toString(); + } +} diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index 1a93547b925..6305b9dae18 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -29,6 +29,7 @@ import org.labkey.api.data.UpgradeCode; import org.labkey.api.mbean.LabKeyManagement; import org.labkey.api.mbean.SearchMXBean; +import org.labkey.api.mcp.McpService; import org.labkey.api.migration.DatabaseMigrationConfiguration; import org.labkey.api.migration.DatabaseMigrationService; import org.labkey.api.migration.DefaultMigrationSchemaHandler; @@ -132,6 +133,12 @@ public WebdavResource resolve(@NotNull String path) return WebdavService.get().lookup(path); } }); + + var mcp = McpService.get(); + if (null != mcp) + { + mcp.register(new SearchMcp()); + } } @Override diff --git a/search/src/org/labkey/search/model/AbstractSearchService.java b/search/src/org/labkey/search/model/AbstractSearchService.java index eadde97fbd9..92dc036e571 100644 --- a/search/src/org/labkey/search/model/AbstractSearchService.java +++ b/search/src/org/labkey/search/model/AbstractSearchService.java @@ -1376,6 +1376,12 @@ public void addSearchCategory(SearchCategory category) } } + @Override + public List getAllCategories() + { + return _readonlyCategories; + } + @Override public List getCategories(String categories) { diff --git a/wiki/src/org/labkey/wiki/WikiManager.java b/wiki/src/org/labkey/wiki/WikiManager.java index dd191607a75..c487e0e1259 100644 --- a/wiki/src/org/labkey/wiki/WikiManager.java +++ b/wiki/src/org/labkey/wiki/WikiManager.java @@ -849,7 +849,7 @@ private void indexWikiContainerFast(SearchService.TaskIndexingQueue queue, @Null /** Note: Does not handle the client dependencies declared by the wiki or any of its embedded webparts! */ @Override - public HtmlString getHtml(Container c, String name) + public RenderedWiki getRenderedWiki(Container c, String name) { if (null == c || null == name) return null; @@ -860,7 +860,8 @@ public HtmlString getHtml(Container c, String name) if (null == wiki) return null; WikiVersion version = wiki.getLatestVersion(); - return version.getHtml(c, wiki); + HtmlString html = version.getHtml(c, wiki); + return new RenderedWiki(name, version.getTitle(), html, wiki.getEntityId()); } catch (Exception x) {