diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c8243933..a4fa9fea 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -54,7 +54,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: 3.x - - run: pip install mkdocs mkdocs-mermaid2-plugin + - run: pip install mkdocs mkdocs-material mkdocs-minify-plugin - run: | cd docs mkdocs gh-deploy --force --clean --verbose diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..b694934f --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +.venv \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 9cf5f215..22cdab32 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,33 +1,120 @@ +# Project information site_name: OSDU Developer site_author: Microsoft Azure Energy +site_name: OSDU Developer +site_url: https://azure.github.io/osdu-developer/ +site_author: Daniel Scholl +site_description: >- + OSDU Developer - Personal Instances : Simplified + docs_dir: src -theme: readthedocs +# Repository +repo_name: azure/osdu-developer +repo_url: https://github.com/azure/osdu-developer -extra_css: - - css/override.css +# Copyright +copyright: Copyright © Microsoft Corporation. + +extra: + generator: false + +# Configuration +theme: + name: material + logo: images/favicon.png + favicon: images/favicon.ico + # custom_dir: overrides + font: + text: Inter + code: Fira Code + features: + - content.code.copy + - navigation.collapsible + - navigation.instant + - navigation.instant.preview + - navigation.tabs + - navigation.sections + - search.highlight + - search.share + - search.suggest + palette: + # Palette toggle for automatic mode + - media: "(prefers-color-scheme)" + toggle: + icon: material/toggle-switch + name: Switch to light mode + + # # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: custom + toggle: + icon: material/toggle-switch + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: custom + toggle: + icon: material/toggle-switch-off + name: Switch to system preference markdown_extensions: + - attr_list + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true - admonition - def_list - - attr_list +# Plugins plugins: - - search - - mermaid2 + - search: + separator: '[\s\u200b\-_,:!=\[\]()"`/]+|\.(?!\d)|&[lg]t;|(?!\b)(?=[A-Z][a-z])' + - minify: + minify_html: true + +extra_javascript: + - https://unpkg.com/tablesort@5.3.0/dist/tablesort.min.js + - javascripts/tablesort.js +extra_css: + - stylesheets/extra.css nav: - - index.md - - before_you_start.md - - architecture.md - - software.md - - feature_flags.md + - Concepts: + - concepts_overview.md + - before_you_start.md + - feature_flags.md + - Architecture: + - architecture.md + - software.md - Tutorials: - - tutorial_cli.md - - tutorial_click.md - - tutorial_rest.md - - tutorial_vnet.md - - OSDU Services: - - osdu_source.md - - osdu_core.md - - osdu_reference.md \ No newline at end of file + - tutorials_overview.md + - Deployment: + - tutorial_cli.md + - tutorial_click.md + - tutorial_rest.md + - Debugging: + - tutorial_logs.md + - tutorial_metrics.md + - Advanced Scenarios: + - tutorial_vnet.md + - Services: + - services_overview.md + - services_core.md + - services_reference.md \ No newline at end of file diff --git a/docs/src/index.md b/docs/src/concepts_overview.md similarity index 99% rename from docs/src/index.md rename to docs/src/concepts_overview.md index 77a7aa5b..f23f681e 100644 --- a/docs/src/index.md +++ b/docs/src/concepts_overview.md @@ -1,6 +1,5 @@ -# OSDU Developer +# Overview -## Overview OSDU Developer is an open-source solution designed to enable the creation of lightweight, personal instances of [OSDU™](https://osduforum.org/osdu-data-platform-primer-1/) running on the Azure Public Cloud. These personal instances are tailored specifically for developers and work with the [Azure Developer CLI](https://learn.microsoft.com/en-us/azure/developer/azure-developer-cli/overview). This allows engineers to explore and author applications or work directly on technology prior to the transition to a fully managed service offering. A simplified one-click deployment capability with less development features is also available. diff --git a/docs/src/images/browser.png b/docs/src/images/browser.png new file mode 100644 index 00000000..aa8f6d13 Binary files /dev/null and b/docs/src/images/browser.png differ diff --git a/docs/src/images/favicon.png b/docs/src/images/favicon.png new file mode 100644 index 00000000..9c16f539 Binary files /dev/null and b/docs/src/images/favicon.png differ diff --git a/docs/src/javascripts/tablesort.js b/docs/src/javascripts/tablesort.js new file mode 100644 index 00000000..c916015f --- /dev/null +++ b/docs/src/javascripts/tablesort.js @@ -0,0 +1,6 @@ +document$.subscribe(function() { + var tables = document.querySelectorAll("article table:not([class])") + tables.forEach(function(table) { + new Tablesort(table) + }) + }) \ No newline at end of file diff --git a/docs/src/osdu_reference.md b/docs/src/osdu_reference.md deleted file mode 100644 index 84a51390..00000000 --- a/docs/src/osdu_reference.md +++ /dev/null @@ -1 +0,0 @@ -# Reference Services \ No newline at end of file diff --git a/docs/src/osdu_core.md b/docs/src/services_core.md similarity index 99% rename from docs/src/osdu_core.md rename to docs/src/services_core.md index 7d336657..11d0ab47 100644 --- a/docs/src/osdu_core.md +++ b/docs/src/services_core.md @@ -1,4 +1,4 @@ -# Core Services +# OSDU Core This repository can be used to hold the source code for the OSDU Core Services. diff --git a/docs/src/osdu_source.md b/docs/src/services_overview.md similarity index 99% rename from docs/src/osdu_source.md rename to docs/src/services_overview.md index a84ca22e..933c9993 100644 --- a/docs/src/osdu_source.md +++ b/docs/src/services_overview.md @@ -1,4 +1,4 @@ -# Gitlab Source +# Overview Welcome to the OSDU Source Code directory! This directory is structured to help you easily clone down the OSDU services and related repositories using the `gita` command. diff --git a/docs/src/services_reference.md b/docs/src/services_reference.md new file mode 100644 index 00000000..aec3b1bb --- /dev/null +++ b/docs/src/services_reference.md @@ -0,0 +1 @@ +# OSDU Reference \ No newline at end of file diff --git a/docs/src/stylesheets/extra.css b/docs/src/stylesheets/extra.css new file mode 100644 index 00000000..a1284c77 --- /dev/null +++ b/docs/src/stylesheets/extra.css @@ -0,0 +1,24 @@ +:root > * { + --md-typeset-a-color: #0199a1; + --md-footer-fg-color: hsla(0, 0%, 100%, 1); + --md-footer-fg-color--light: hsla(0, 0%, 100%, 0.7); + --md-footer-fg-color--lighter: hsla(0, 0%, 100%, 0.3); + --md-footer-bg-color: #01393b;; + --md-footer-bg-color--dark: #01393b; + --md-typeset-a-color: #0199a1; +} + +[data-md-color-scheme="default"] { + --md-primary-fg-color: #01696e; + --md-accent-fg-color: #02c9d3; +} + +[data-md-color-scheme="slate"] { + --md-primary-fg-color: #01393b; + --md-accent-fg-color: #02c9d3; +} + +li.md-nav__item--section > label.md-nav__link > span.md-ellipsis { + color: #01696e; +} + diff --git a/docs/src/tutorial._airflow.md b/docs/src/tutorial._airflow.md new file mode 100644 index 00000000..7c82a9f9 --- /dev/null +++ b/docs/src/tutorial._airflow.md @@ -0,0 +1,3 @@ +# Working with Airflow + +Coming soon. \ No newline at end of file diff --git a/docs/src/tutorial_vnet.md b/docs/src/tutorial_vnet.md index 66f5a93c..c2868fec 100644 --- a/docs/src/tutorial_vnet.md +++ b/docs/src/tutorial_vnet.md @@ -75,111 +75,209 @@ __Resource Group__ Use the following command to create a new resource group: -```bash -NETWORK_GROUP='operations' -AZURE_LOCATION='eastus2' -# resource_group -az group create --name $NETWORK_GROUP \ ---location $AZURE_LOCATION -``` +=== "Bash" -__Network Security Group__ - -Network Security Groups (NSGs) are essential for securing virtual network resources. NSGs control inbound and outbound traffic to network interfaces (NIC), VMs, and subnets. - -Use the following commands set up an NSG with rules to allow HTTP and HTTPS traffic. + ```bash + NETWORK_GROUP='operations' + AZURE_LOCATION='eastus2' + # resource_group + az group create --name $NETWORK_GROUP \ + --location $AZURE_LOCATION + ``` -```bash -NSG_NAME='custom-vnet-nsg' +=== "Powershell" -# network_security_group -az network nsg create --name $NSG_NAME \ ---resource-group $NETWORK_GROUP \ ---location $AZURE_LOCATION + ```pwsh + $NETWORK_GROUP = 'operations' + $AZURE_LOCATION = 'eastus2' + # resource_group + az group create --name $NETWORK_GROUP ` + --location $AZURE_LOCATION + ``` -# http_inbound_rule -az network nsg rule create --name AllowHttpInbound \ ---nsg-name $NSG_NAME --resource-group $NETWORK_GROUP \ ---priority 200 --access Allow --direction Inbound \ ---protocol 'Tcp' --source-address-prefixes 'VirtualNetwork' --source-port-ranges '*' \ ---destination-address-prefixes '*' --destination-port-ranges '80' -# https_inbound_rule -az network nsg rule create --name AllowHttpsInbound \ ---nsg-name $NSG_NAME --resource-group $NETWORK_GROUP \ ---priority 210 --access Allow --direction Inbound \ ---protocol 'Tcp' --source-address-prefixes 'VirtualNetwork' --source-port-ranges '*' \ ---destination-address-prefixes '*' --destination-port-ranges '443' -``` +__Network Security Group__ +Network Security Groups (NSGs) are essential for securing virtual network resources. NSGs control inbound and outbound traffic to network interfaces (NIC), VMs, and subnets. -__Virtual Network__ +Use the following commands set up an NSG with rules to allow HTTP and HTTPS traffic. -The virtual network is a critical component that enables Azure resources like AKS to communicate effectively. This step involves setting up the required 'ClusterSubnet' and an optional 'PodSubnet'. +=== "Bash" -Use the following commands set up the network with a required subnet for the cluster and an optional subnet for the pods. + ```shell + NSG_NAME='custom-vnet-nsg' -```bash -NETWORK_NAME='custom-vnet' -VNET_PREFIX='172.20.0.0/22' + # network_security_group + az network nsg create --name $NSG_NAME \ + --resource-group $NETWORK_GROUP \ + --location $AZURE_LOCATION -CLUSTER_SUBNET_NAME='cluster' -CLUSTER_SUBNET_PREFIX='172.20.0.0/24' -POD_SUBNET_NAME='pods' -POD_SUBNET_PREFIX='172.20.1.0/24' + # http_inbound_rule + az network nsg rule create --name AllowHttpInbound \ + --nsg-name $NSG_NAME --resource-group $NETWORK_GROUP \ + --priority 200 --access Allow --direction Inbound \ + --protocol 'Tcp' --source-address-prefixes 'VirtualNetwork' --source-port-ranges '*' \ + --destination-address-prefixes '*' --destination-port-ranges '80' + # https_inbound_rule + az network nsg rule create --name AllowHttpsInbound \ + --nsg-name $NSG_NAME --resource-group $NETWORK_GROUP \ + --priority 210 --access Allow --direction Inbound \ + --protocol 'Tcp' --source-address-prefixes 'VirtualNetwork' --source-port-ranges '*' \ + --destination-address-prefixes '*' --destination-port-ranges '443' + ``` +=== "Powershell" -# virtual_network -az network vnet create --name $NETWORK_NAME \ ---resource-group $NETWORK_GROUP \ ---location $AZURE_LOCATION \ ---address-prefix $VNET_PREFIX + ```shell + $NSG_NAME = 'custom-vnet-nsg' -# virtual_network_subnet_cluster -az network vnet subnet create --name $CLUSTER_SUBNET_NAME \ ---resource-group $NETWORK_GROUP \ ---vnet-name $NETWORK_NAME \ ---address-prefix $CLUSTER_SUBNET_PREFIX \ ---network-security-group $NSG_NAME + # network_security_group + az network nsg create --name $NSG_NAME ` + --resource-group $NETWORK_GROUP ` + --location $AZURE_LOCATION -# virtual_network_subnet_pods -az network vnet subnet create --name $POD_SUBNET_NAME \ ---resource-group $NETWORK_GROUP \ ---vnet-name $NETWORK_NAME \ ---address-prefix $POD_SUBNET_PREFIX \ ---network-security-group $NSG_NAME -# managed_identity -az identity create --name $NETWORK_NAME \ ---resource-group $NETWORK_GROUP \ ---location $AZURE_LOCATION + # http_inbound_rule + az network nsg rule create --name AllowHttpInbound ` + --nsg-name $NSG_NAME --resource-group $NETWORK_GROUP ` + --priority 200 --access Allow --direction Inbound ` + --protocol 'Tcp' --source-address-prefixes 'VirtualNetwork' --source-port-ranges '*' ` + --destination-address-prefixes '*' --destination-port-ranges '80' -# managed_identity_principal_id -IDENTITY_PID=$(az identity show --name $NETWORK_NAME \ ---resource-group $NETWORK_GROUP \ ---query "principalId" --output tsv) + # https_inbound_rule + az network nsg rule create --name AllowHttpsInbound ` + --nsg-name $NSG_NAME --resource-group $NETWORK_GROUP ` + --priority 210 --access Allow --direction Inbound ` + --protocol 'Tcp' --source-address-prefixes 'VirtualNetwork' --source-port-ranges '*' ` + --destination-address-prefixes '*' --destination-port-ranges '443' + ``` -# managed_identity_id -NETWORK_IDENTITY=$(az identity show --name $NETWORK_NAME \ ---resource-group $NETWORK_GROUP \ ---query "id" --output tsv) +__Virtual Network__ -# network_id -NETWORK_ID=$(az network vnet show --name $NETWORK_NAME \ ---resource-group $NETWORK_GROUP \ ---query "id" -o tsv) +The virtual network is a critical component that enables Azure resources like AKS to communicate effectively. This step involves setting up the required 'ClusterSubnet' and an optional 'PodSubnet'. -# role_assignment -az role assignment create --assignee $IDENTITY_ID \ ---role "Network Contributor" \ ---scope $NETWORK_ID -``` +Use the following commands set up the network with a required subnet for the cluster and an optional subnet for the pods. +=== "Bash" + + ```shell + NETWORK_NAME='custom-vnet' + VNET_PREFIX='172.20.0.0/22' + + CLUSTER_SUBNET_NAME='cluster' + CLUSTER_SUBNET_PREFIX='172.20.0.0/24' + + POD_SUBNET_NAME='pods' + POD_SUBNET_PREFIX='172.20.1.0/24' + + # virtual_network + az network vnet create --name $NETWORK_NAME \ + --resource-group $NETWORK_GROUP \ + --location $AZURE_LOCATION \ + --address-prefix $VNET_PREFIX + + # virtual_network_subnet_cluster + az network vnet subnet create --name $CLUSTER_SUBNET_NAME \ + --resource-group $NETWORK_GROUP \ + --vnet-name $NETWORK_NAME \ + --address-prefix $CLUSTER_SUBNET_PREFIX \ + --network-security-group $NSG_NAME + + # virtual_network_subnet_pods + az network vnet subnet create --name $POD_SUBNET_NAME \ + --resource-group $NETWORK_GROUP \ + --vnet-name $NETWORK_NAME \ + --address-prefix $POD_SUBNET_PREFIX \ + --network-security-group $NSG_NAME + + # managed_identity + az identity create --name $NETWORK_NAME \ + --resource-group $NETWORK_GROUP \ + --location $AZURE_LOCATION + + # managed_identity_principal_id + IDENTITY_PID=$(az identity show --name $NETWORK_NAME \ + --resource-group $NETWORK_GROUP \ + --query "principalId" --output tsv) + + # managed_identity_id + NETWORK_IDENTITY=$(az identity show --name $NETWORK_NAME \ + --resource-group $NETWORK_GROUP \ + --query "id" --output tsv) + + # network_id + NETWORK_ID=$(az network vnet show --name $NETWORK_NAME \ + --resource-group $NETWORK_GROUP \ + --query "id" -o tsv) + + # role_assignment + az role assignment create --assignee $IDENTITY_ID \ + --role "Network Contributor" \ + --scope $NETWORK_ID + ``` + +=== "Powershell" + + ```powershell + $NETWORK_NAME = 'custom-vnet' + $VNET_PREFIX = '172.20.0.0/22' + + $CLUSTER_SUBNET_NAME = 'cluster' + $CLUSTER_SUBNET_PREFIX = '172.20.0.0/24' + + $POD_SUBNET_NAME = 'pods' + $POD_SUBNET_PREFIX = '172.20.1.0/24' + + # virtual_network + az network vnet create --name $NETWORK_NAME ` + --resource-group $NETWORK_GROUP ` + --location $AZURE_LOCATION ` + --address-prefix $VNET_PREFIX + + # virtual_network_subnet_cluster + az network vnet subnet create --name $CLUSTER_SUBNET_NAME ` + --resource-group $NETWORK_GROUP ` + --vnet-name $NETWORK_NAME ` + --address-prefix $CLUSTER_SUBNET_PREFIX ` + --network-security-group $NSG_NAME + + # virtual_network_subnet_pods + az network vnet subnet create --name $POD_SUBNET_NAME ` + --resource-group $NETWORK_GROUP ` + --vnet-name $NETWORK_NAME ` + --address-prefix $POD_SUBNET_PREFIX ` + --network-security-group $NSG_NAME + + # managed_identity + az identity create --name $NETWORK_NAME ` + --resource-group $NETWORK_GROUP ` + --location $AZURE_LOCATION + + # managed_identity_principal_id + $IDENTITY_PID = az identity show --name $NETWORK_NAME ` + --resource-group $NETWORK_GROUP ` + --query "principalId" --output tsv + + # managed_identity_id + $NETWORK_IDENTITY = az identity show --name $NETWORK_NAME ` + --resource-group $NETWORK_GROUP ` + --query "id" --output tsv + + # network_id + $NETWORK_ID = az network vnet show --name $NETWORK_NAME ` + --resource-group $NETWORK_GROUP ` + --query "id" -o tsv + + # role_assignment + az role assignment create --assignee $IDENTITY_ID ` + --role "Network Contributor" ` + --scope $NETWORK_ID + ``` ## Initialize and Configure Solution @@ -190,48 +288,100 @@ __Authenticate and Initialize__ First, authenticate your session and then initialize a custom environment: -```bash -# authenticate_session -azd auth login -# create_new_environment -azd env new custom -``` +=== "Bash" + + ```bash + # authenticate_session + azd auth login + + # create_new_environment + azd env new custom + ``` + +=== "Powershell" + + ```pwsh + # authenticate_session + azd auth login + + # create_new_environment + azd env new custom + ``` + + __Configure Environment Variables__ Set the necessary environment variables for your deployment: -```bash -# define_application_id -APP_NAME= -azd env set AZURE_CLIENT_ID $(az ad app list --display-name $APP_NAME --query "[].appId" -otsv) - -# identify_software_repository -azd env set SOFTWARE_REPOSITORY https://github.com/azure/osdu-developer -azd env set SOFTWARE_BRANCH main - -# enable_feature_toggles -azd env set ENABLE_POD_SUBNET true - -# define_network_configuration -azd env set VIRTUAL_NETWORK_GROUP $NETWORK_GROUP -azd env set VIRTUAL_NETWORK_NAME $NETWORK_NAME -azd env set VIRTUAL_NETWORK_PREFIX $VNET_PREFIX -azd env set AKS_SUBNET_NAME $CLUSTER_SUBNET_NAME -azd env set AKS_SUBNET_PREFIX $CLUSTER_SUBNET_PREFIX -azd env set POD_SUBNET_NAME $POD_SUBNET_NAME -azd env set POD_SUBNET_PREFIX $POD_SUBNET_PREFIX -azd env set VIRTUAL_NETWORK_IDENTITY $NETWORK_IDENTITY -``` +=== "Bash" + + ```shell + # define_application_id + APP_NAME= + azd env set AZURE_CLIENT_ID $(az ad app list --display-name $APP_NAME --query "[].appId" -otsv) + + # identify_software_repository + azd env set SOFTWARE_REPOSITORY https://github.com/azure/osdu-developer + azd env set SOFTWARE_BRANCH main + + # enable_feature_toggles + azd env set ENABLE_POD_SUBNET true + + # define_network_configuration + azd env set VIRTUAL_NETWORK_GROUP $NETWORK_GROUP + azd env set VIRTUAL_NETWORK_NAME $NETWORK_NAME + azd env set VIRTUAL_NETWORK_PREFIX $VNET_PREFIX + azd env set AKS_SUBNET_NAME $CLUSTER_SUBNET_NAME + azd env set AKS_SUBNET_PREFIX $CLUSTER_SUBNET_PREFIX + azd env set POD_SUBNET_NAME $POD_SUBNET_NAME + azd env set POD_SUBNET_PREFIX $POD_SUBNET_PREFIX + azd env set VIRTUAL_NETWORK_IDENTITY $NETWORK_IDENTITY + ``` + + +=== "Powershell" + + ```shell + # define_application_id + $APP_NAME = '' + azd env set AZURE_CLIENT_ID (az ad app list --display-name $APP_NAME --query "[].appId" -otsv) + + # identify_software_repository + azd env set SOFTWARE_REPOSITORY 'https://github.com/azure/osdu-developer' + azd env set SOFTWARE_BRANCH 'main' + + # enable_feature_toggles + azd env set ENABLE_POD_SUBNET 'true' + + # define_network_configuration + azd env set VIRTUAL_NETWORK_GROUP $NETWORK_GROUP + azd env set VIRTUAL_NETWORK_NAME $NETWORK_NAME + azd env set VIRTUAL_NETWORK_PREFIX $VNET_PREFIX + azd env set AKS_SUBNET_NAME $CLUSTER_SUBNET_NAME + azd env set AKS_SUBNET_PREFIX $CLUSTER_SUBNET_PREFIX + azd env set POD_SUBNET_NAME $POD_SUBNET_NAME + azd env set POD_SUBNET_PREFIX $POD_SUBNET_PREFIX + azd env set VIRTUAL_NETWORK_IDENTITY $NETWORK_IDENTITY + ``` __Start the Deployment__ Initiate the deployment using the following command: -```bash -# provision_solution -azd provision -``` +=== "Bash" + + ```bash + # provision_solution + azd provision + ``` + +=== "Powershell" + + ```shell + # provision_solution + azd provision + ``` [0]: images/network.png "Network Diagram" \ No newline at end of file diff --git a/docs/src/tutorials_overview.md b/docs/src/tutorials_overview.md new file mode 100644 index 00000000..b452d7ec --- /dev/null +++ b/docs/src/tutorials_overview.md @@ -0,0 +1,3 @@ +# Overview + +Generic overview of tutorials. \ No newline at end of file