diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index efdcea614c..0000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM mcr.microsoft.com/devcontainers/java:11 diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index cfee9a4374..6fa1ec67df 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,34 +1,96 @@ { "name": "kafka-docker-playground", "build": { - "dockerfile": "Dockerfile" + "dockerfile": "../.gitpod.Dockerfile" }, - "settings": { - "terminal.integrated.defaultProfile.linux": "zsh", - "terminal.integrated.profiles.linux": { - "bash": { - "path": "bash", - "icon": "terminal-bash" - }, - "zsh": { - "path": "zsh" - } - } + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, - "remoteEnv": { - "PATH": "${containerEnv:PATH}:/workspaces/kafka-docker-playground/scripts/cli", - "C3_PORT": "9023" + "remoteUser": "gitpod", + "containerUser": "gitpod", + "workspaceFolder": "/workspace/kafka-docker-playground", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace/kafka-docker-playground,type=bind", + "forwardPorts": [ + 9999, + 9092, + 10000, + 29092, + 8083, + 5005, + 10002, + 9021, + 8081, + 10001, + 8088, + 10003 + ], + "portsAttributes": { + "10000": { + "label": "Port 10000", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "10001": { + "label": "Port 10001", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "10002": { + "label": "Port 10002", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "10003": { + "label": "Port 10003", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "29092": { + "label": "Port 29092", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "5005": { + "label": "Port 5005", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "8081": { + "label": "Port 8081", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "8083": { + "label": "Port 8083", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "8088": { + "label": "Port 8088", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "9021": { + "label": "Port 9021", + "onAutoForward": "notify", + "elevateIfNeeded": true + }, + "9092": { + "label": "Port 9092", + "onAutoForward": "ignore", + "elevateIfNeeded": true + }, + "9999": { + "label": "Port 9999", + "onAutoForward": "ignore", + "elevateIfNeeded": true + } }, - "forwardPorts": [9023], - "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "ghcr.io/mikaello/devcontainer-features/modern-shell-utils:1": {}, - "ghcr.io/devcontainers-contrib/features/fzf:1": {}, - "ghcr.io/devcontainers-contrib/features/starship:1": {}, - "ghcr.io/devcontainers/features/java:1": { - "version": "none", - "installGradle": "false", - "installMaven": "true" - } + "customizations": { + "vscode": { + "extensions": [ + "ms-azuretools.vscode-docker" + ] + } } } \ No newline at end of file diff --git a/.github/release-notes.yml b/.github/release-notes.yml new file mode 100644 index 0000000000..724509efdd --- /dev/null +++ b/.github/release-notes.yml @@ -0,0 +1,6 @@ +changelog: + sections: + - title: "šŸŒŸ Enhancements" + labels: ["enhancement"] + - title: "šŸ› Bugs" + labels: ["bug"] diff --git a/.github/workflows/ci-environments.yml b/.github/workflows/ci-environments.yml deleted file mode 100644 index 523190de89..0000000000 --- a/.github/workflows/ci-environments.yml +++ /dev/null @@ -1,385 +0,0 @@ -name: CI with environments - -on: - # push: - # branches: - # - master - - # schedule: - # - cron: '0 18 * * *' # every day at 6 pm - - workflow_dispatch: - inputs: - test_name: - description: 'test(s) to run (example connect/connect-jms-weblogic-sink connect/connect-http-sink)' - required: false - default: '' - -jobs: - pre-build: - if: ${{ github.event.inputs.test_name == '' }} - name: Cleanup resources - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - repository: vdesabou/kafka-docker-playground - fetch-depth: 0 - - build: - if: ${{ github.event.inputs.test_name == '' }} - runs-on: ubuntu-latest - needs: pre-build - name: ${{ matrix.environment }} ${{ matrix.test_list }} - strategy: - fail-fast: false - matrix: - tag: ["7.5.3"] - #environment: ["ccloud", "sasl-ssl", "sasl-plain", "2way-ssl", "sasl-scram", "kraft-external-plaintext", "kraft-plaintext", "kerberos", "ssl_kerberos", "ldap-authorizer-sasl-plain", "ldap-sasl-plain"] - environment: [ "2way-ssl", "ssl_kerberos"] - test_list : [ - "šŸš€ connect/connect-active-mq-sink connect/connect-debezium-sqlserver-source connect/connect-http-sink" - ] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - repository: vdesabou/kafka-docker-playground - fetch-depth: 0 - - - name: "Free up disk space" - run: | - df -h - sudo apt-get -qq purge build-essential ghc* - sudo apt-get clean - sudo apt-get install expect fzf coreutils -y - docker system prune -af - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - sudo rm -rf \ - /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ - /usr/lib/jvm || true - echo "some directories deleted" - sudo apt install aptitude -y >/dev/null 2>&1 - sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ - esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ - google-cloud-sdk imagemagick \ - libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ - mercurial apt-transport-https mono-complete libmysqlclient \ - unixodbc-dev yarn chrpath libssl-dev libxft-dev \ - libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ - snmp pollinate libpq-dev postgresql-client powershell ruby-full \ - sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ - -y -f >/dev/null 2>&1 - sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 - sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true - sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true - sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 - sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 - sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 - sudo apt-get autoremove -y >/dev/null 2>&1 - sudo apt-get autoclean -y >/dev/null 2>&1 - echo "some packages purged" - - df -h - - - name: "Install confluent CLI" - run: | - curl -L --http1.1 https://cnfl.io/cli | sudo sh -s -- -b /usr/local/bin - export PATH=$PATH:/usr/local/bin - - - name: Decrypt secrets.tar - run: | - ./.github/scripts/decrypt_secret.sh - tar xvf secrets.tar - rm secrets.tar - mkdir -p $HOME/.aws - mv aws_credentials_with_assuming_iam_role $HOME/.aws/credentials-with-assuming-iam-role - mv aws_credentials_aws_account_with_assume_role $HOME/.aws/credentials_aws_account_with_assume_role - chmod -R a+rw $HOME/.aws - mkdir -p $HOME/.confluent - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - env: - SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - - - name: Build and Test - run: bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}" "${{ matrix.environment }}" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} - AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID}} - AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY}} - AWS_STS_ROLE_ARN: ${{ secrets.AWS_STS_ROLE_ARN}} - AZ_USER: ${{ secrets.AZ_USER}} - AZ_PASS: ${{ secrets.AZ_PASS}} - AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}} - CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}} - CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}} - ENVIRONMENT: ${{ secrets.ENVIRONMENT}} - CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}} - CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}} - CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}} - CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}} - AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}} - AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}} - AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}} - AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}} - GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}} - GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}} - GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}} - GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}} - AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}} - AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}} - AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}} - AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}} - SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}} - CONFLUENT_LICENSE: ${{ secrets.CONFLUENT_LICENSE}} - SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME}} - SALESFORCE_PASSWORD: ${{ secrets.SALESFORCE_PASSWORD}} - SALESFORCE_CONSUMER_KEY: ${{ secrets.SALESFORCE_CONSUMER_KEY}} - SALESFORCE_CONSUMER_PASSWORD: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD}} - SALESFORCE_SECURITY_TOKEN: ${{ secrets.SALESFORCE_SECURITY_TOKEN}} - SALESFORCE_INSTANCE: ${{ secrets.SALESFORCE_INSTANCE}} - SALESFORCE_USERNAME_ACCOUNT2: ${{ secrets.SALESFORCE_USERNAME_ACCOUNT2}} - SALESFORCE_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_PASSWORD_ACCOUNT2}} - SALESFORCE_SECURITY_TOKEN_ACCOUNT2: ${{ secrets.SALESFORCE_SECURITY_TOKEN_ACCOUNT2}} - SALESFORCE_CONSUMER_KEY_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_KEY_ACCOUNT2}} - SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2}} - SALESFORCE_CONSUMER_KEY_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_KEY_WITH_JWT}} - SALESFORCE_CONSUMER_PASSWORD_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_WITH_JWT}} - DD_API_KEY: ${{ secrets.DD_API_KEY}} - DD_APP_KEY: ${{ secrets.DD_APP_KEY}} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - JIRA_URL: ${{ secrets.JIRA_URL}} - JIRA_USERNAME: ${{ secrets.JIRA_USERNAME}} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN}} - MARKETO_ENDPOINT_URL: ${{ secrets.MARKETO_ENDPOINT_URL}} - MARKETO_CLIENT_ID: ${{ secrets.MARKETO_CLIENT_ID}} - MARKETO_CLIENT_SECRET: ${{ secrets.MARKETO_CLIENT_SECRET}} - PAGERDUTY_USER_EMAIL: ${{ secrets.PAGERDUTY_USER_EMAIL}} - PAGERDUTY_API_KEY: ${{ secrets.PAGERDUTY_API_KEY}} - PAGERDUTY_SERVICE_ID: ${{ secrets.PAGERDUTY_SERVICE_ID}} - CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY}} - CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET}} - SERVICENOW_URL: ${{ secrets.SERVICENOW_URL}} - SERVICENOW_PASSWORD: ${{ secrets.SERVICENOW_PASSWORD}} - SERVICENOW_DEVELOPER_USERNAME: ${{ secrets.SERVICENOW_DEVELOPER_USERNAME}} - SERVICENOW_DEVELOPER_PASSWORD: ${{ secrets.SERVICENOW_DEVELOPER_PASSWORD}} - SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_ACCOUNT_NAME}} - SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME}} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD}} - ZENDESK_URL: ${{ secrets.ZENDESK_URL}} - ZENDESK_USERNAME: ${{ secrets.ZENDESK_USERNAME}} - ZENDESK_PASSWORD: ${{ secrets.ZENDESK_PASSWORD}} - CONNECTOR_GITHUB_ACCESS_TOKEN: ${{ secrets.CONNECTOR_GITHUB_ACCESS_TOKEN}} - CI_GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN}} - AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS: ${{ secrets.AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS}} - AUDIT_LOG_CLUSTER_API_KEY: ${{ secrets.AUDIT_LOG_CLUSTER_API_KEY}} - AUDIT_LOG_CLUSTER_API_SECRET: ${{ secrets.AUDIT_LOG_CLUSTER_API_SECRET}} - NGROK_AUTH_TOKEN: ${{ secrets.NGROK_CI_AUTH_TOKEN}} - DATABRICKS_AWS_BUCKET_NAME: ${{ secrets.DATABRICKS_AWS_BUCKET_NAME}} - DATABRICKS_AWS_BUCKET_REGION: ${{ secrets.DATABRICKS_AWS_BUCKET_REGION}} - DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID}} - DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY}} - DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME}} - DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH}} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN}} - ORACLE_CONTAINER_REGISTRY_USERNAME: ${{ secrets.ORACLE_CONTAINER_REGISTRY_USERNAME}} - ORACLE_CONTAINER_REGISTRY_PASSWORD: ${{ secrets.ORACLE_CONTAINER_REGISTRY_PASSWORD}} - GCP_KEYFILE_CONTENT: ${{ secrets.GCP_KEYFILE_CONTENT}} - GCP_PROJECT: ${{ secrets.GCP_PROJECT}} - HPE_MAPR_EMAIL: ${{ secrets.HPE_MAPR_EMAIL}} - HPE_MAPR_TOKEN: ${{ secrets.HPE_MAPR_TOKEN}} - - execute_one_test: - if: ${{ github.event.inputs.test_name != '' }} - runs-on: ubuntu-latest - name: ${{ matrix.environment }} ${{ matrix.test_list }} - strategy: - fail-fast: false - matrix: - tag: ["7.5.3"] - environment: ["ccloud", "sasl-ssl", "sasl-plain", "2way-ssl", "sasl-scram", "kraft-external-plaintext", "kraft-plaintext", "kerberos", "ssl_kerberos", "ldap-authorizer-sasl-plain", "ldap-sasl-plain"] - test_list : [ - "šŸš€ ${{ github.event.inputs.test_name }}" - ] - steps: - - # - name: Maximize build space - # uses: easimon/maximize-build-space@master - # with: - # root-reserve-mb: 512 - # swap-size-mb: 1024 - # remove-dotnet: 'true' - # remove-android: 'true' - # remove-haskell: 'true' - # remove-codeql: 'true' - - - name: Checkout code - uses: actions/checkout@v4 - with: - repository: vdesabou/kafka-docker-playground - fetch-depth: 0 - - - name: "Free up disk space" - run: | - df -h - sudo apt-get -qq purge build-essential ghc* - sudo apt-get clean - sudo apt-get install expect fzf coreutils -y - docker system prune -af - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - sudo rm -rf \ - /usr/share/dotnet /usr/local/lib/android /opt/ghc \ - /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ - /usr/lib/jvm || true - echo "some directories deleted" - sudo apt install aptitude -y >/dev/null 2>&1 - sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \ - esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \ - google-cloud-sdk imagemagick \ - libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \ - mercurial apt-transport-https mono-complete libmysqlclient \ - unixodbc-dev yarn chrpath libssl-dev libxft-dev \ - libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \ - snmp pollinate libpq-dev postgresql-client powershell ruby-full \ - sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \ - -y -f >/dev/null 2>&1 - sudo aptitude purge google-cloud-sdk -f -y >/dev/null 2>&1 - sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true - sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true - sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1 - sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1 - sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1 - sudo apt-get autoremove -y >/dev/null 2>&1 - sudo apt-get autoclean -y >/dev/null 2>&1 - echo "some packages purged" - - df -h - - - name: "Install confluent CLI" - run: | - curl -L --http1.1 https://cnfl.io/cli | sudo sh -s -- -b /usr/local/bin - export PATH=$PATH:/usr/local/bin - - - name: Decrypt secrets.tar - run: | - ./.github/scripts/decrypt_secret.sh - tar xvf secrets.tar - rm secrets.tar - mkdir -p $HOME/.aws - mv aws_credentials_with_assuming_iam_role $HOME/.aws/credentials-with-assuming-iam-role - mv aws_credentials_aws_account_with_assume_role $HOME/.aws/credentials_aws_account_with_assume_role - chmod -R a+rw $HOME/.aws - mkdir -p $HOME/.confluent - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - env: - SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - - - name: Build and Test - run: bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}" "${{ matrix.environment }}" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} - AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID}} - AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY}} - AWS_STS_ROLE_ARN: ${{ secrets.AWS_STS_ROLE_ARN}} - AZ_USER: ${{ secrets.AZ_USER}} - AZ_PASS: ${{ secrets.AZ_PASS}} - AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}} - CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}} - CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}} - ENVIRONMENT: ${{ secrets.ENVIRONMENT}} - CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}} - CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}} - CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}} - CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}} - AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}} - AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}} - AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}} - AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}} - GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}} - GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}} - GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}} - GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}} - AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}} - AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}} - AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}} - AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}} - SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}} - CONFLUENT_LICENSE: ${{ secrets.CONFLUENT_LICENSE}} - SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME}} - SALESFORCE_PASSWORD: ${{ secrets.SALESFORCE_PASSWORD}} - SALESFORCE_CONSUMER_KEY: ${{ secrets.SALESFORCE_CONSUMER_KEY}} - SALESFORCE_CONSUMER_PASSWORD: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD}} - SALESFORCE_SECURITY_TOKEN: ${{ secrets.SALESFORCE_SECURITY_TOKEN}} - SALESFORCE_INSTANCE: ${{ secrets.SALESFORCE_INSTANCE}} - SALESFORCE_USERNAME_ACCOUNT2: ${{ secrets.SALESFORCE_USERNAME_ACCOUNT2}} - SALESFORCE_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_PASSWORD_ACCOUNT2}} - SALESFORCE_SECURITY_TOKEN_ACCOUNT2: ${{ secrets.SALESFORCE_SECURITY_TOKEN_ACCOUNT2}} - SALESFORCE_CONSUMER_KEY_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_KEY_ACCOUNT2}} - SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2}} - SALESFORCE_CONSUMER_KEY_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_KEY_WITH_JWT}} - SALESFORCE_CONSUMER_PASSWORD_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_WITH_JWT}} - DD_API_KEY: ${{ secrets.DD_API_KEY}} - DD_APP_KEY: ${{ secrets.DD_APP_KEY}} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - JIRA_URL: ${{ secrets.JIRA_URL}} - JIRA_USERNAME: ${{ secrets.JIRA_USERNAME}} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN}} - MARKETO_ENDPOINT_URL: ${{ secrets.MARKETO_ENDPOINT_URL}} - MARKETO_CLIENT_ID: ${{ secrets.MARKETO_CLIENT_ID}} - MARKETO_CLIENT_SECRET: ${{ secrets.MARKETO_CLIENT_SECRET}} - PAGERDUTY_USER_EMAIL: ${{ secrets.PAGERDUTY_USER_EMAIL}} - PAGERDUTY_API_KEY: ${{ secrets.PAGERDUTY_API_KEY}} - PAGERDUTY_SERVICE_ID: ${{ secrets.PAGERDUTY_SERVICE_ID}} - CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY}} - CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET}} - SERVICENOW_URL: ${{ secrets.SERVICENOW_URL}} - SERVICENOW_PASSWORD: ${{ secrets.SERVICENOW_PASSWORD}} - SERVICENOW_DEVELOPER_USERNAME: ${{ secrets.SERVICENOW_DEVELOPER_USERNAME}} - SERVICENOW_DEVELOPER_PASSWORD: ${{ secrets.SERVICENOW_DEVELOPER_PASSWORD}} - SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_ACCOUNT_NAME}} - SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME}} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD}} - ZENDESK_URL: ${{ secrets.ZENDESK_URL}} - ZENDESK_USERNAME: ${{ secrets.ZENDESK_USERNAME}} - ZENDESK_PASSWORD: ${{ secrets.ZENDESK_PASSWORD}} - CONNECTOR_GITHUB_ACCESS_TOKEN: ${{ secrets.CONNECTOR_GITHUB_ACCESS_TOKEN}} - CI_GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN}} - AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS: ${{ secrets.AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS}} - AUDIT_LOG_CLUSTER_API_KEY: ${{ secrets.AUDIT_LOG_CLUSTER_API_KEY}} - AUDIT_LOG_CLUSTER_API_SECRET: ${{ secrets.AUDIT_LOG_CLUSTER_API_SECRET}} - NGROK_AUTH_TOKEN: ${{ secrets.NGROK_CI_AUTH_TOKEN}} - DATABRICKS_AWS_BUCKET_NAME: ${{ secrets.DATABRICKS_AWS_BUCKET_NAME}} - DATABRICKS_AWS_BUCKET_REGION: ${{ secrets.DATABRICKS_AWS_BUCKET_REGION}} - DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID}} - DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY}} - DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME}} - DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH}} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN}} - ORACLE_CONTAINER_REGISTRY_USERNAME: ${{ secrets.ORACLE_CONTAINER_REGISTRY_USERNAME}} - ORACLE_CONTAINER_REGISTRY_PASSWORD: ${{ secrets.ORACLE_CONTAINER_REGISTRY_PASSWORD}} - GCP_KEYFILE_CONTENT: ${{ secrets.GCP_KEYFILE_CONTENT}} - GCP_PROJECT: ${{ secrets.GCP_PROJECT}} - HPE_MAPR_EMAIL: ${{ secrets.HPE_MAPR_EMAIL}} - HPE_MAPR_TOKEN: ${{ secrets.HPE_MAPR_TOKEN}} diff --git a/.github/workflows/ci-self-managed.yml b/.github/workflows/ci-self-managed.yml deleted file mode 100644 index 52f84b0519..0000000000 --- a/.github/workflows/ci-self-managed.yml +++ /dev/null @@ -1,212 +0,0 @@ -name: CI Self-Managed - -on: - # push: - # branches: - # - master - - # schedule: - # - cron: '0 0 * * 0' # every week on sunday at 0 am - - workflow_dispatch: - inputs: - test_name: - description: 'test(s) to run (example connect/connect-jms-weblogic-sink connect/connect-http-sink)' - required: false - default: '' - -jobs: - start-runner: - name: Start Github self-hosted runner - runs-on: ubuntu-latest - steps: - - - name: Checkout code - uses: actions/checkout@v4 - with: - repository: vdesabou/kafka-docker-playground - fetch-depth: 0 - - - name: Decrypt secrets.tar - run: | - ./.github/scripts/decrypt_secret.sh - tar xvf secrets.tar - rm secrets.tar - mkdir -p $HOME/.aws - mv aws_credentials $HOME/.aws/credentials - chmod -R a+rw $HOME/.aws - mkdir -p $HOME/.confluent - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - env: - SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - - - name: Start EC2 instance github-actions-runner-vsaboulin - run: | - aws ec2 start-instances --instance-ids i-089ef31a75cb3f0e6 - - build: - runs-on: self-hosted - needs: start-runner # required to start the main job when the runner is read - name: ${{ matrix.tag }} ${{ matrix.test_list }} - strategy: - fail-fast: false - matrix: - tag: ["7.6.1"] - test_list : [ - "šŸš€ connect/connect-mapr-sink", - ] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - repository: vdesabou/kafka-docker-playground - fetch-depth: 0 - - - name: Decrypt secrets.tar - run: | - ./.github/scripts/decrypt_secret.sh - tar xvf secrets.tar - rm secrets.tar - mkdir -p $HOME/.aws - mv aws_credentials_with_assuming_iam_role $HOME/.aws/credentials-with-assuming-iam-role - chmod -R a+rw $HOME/.aws - mkdir -p $HOME/.confluent - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - env: - SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - - - name: "Install confluent CLI" - run: | - curl -L --http1.1 https://cnfl.io/cli | sudo sh -s -- -b /usr/local/bin - export PATH=$PATH:/usr/local/bin - - - name: Build and Test - run: bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID}} - AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY}} - AWS_STS_ROLE_ARN: ${{ secrets.AWS_STS_ROLE_ARN}} - AZ_USER: ${{ secrets.AZ_USER}} - AZ_PASS: ${{ secrets.AZ_PASS}} - AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}} - CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}} - CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}} - ENVIRONMENT: ${{ secrets.ENVIRONMENT}} - CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}} - CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}} - CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}} - CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}} - AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}} - AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}} - AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}} - AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}} - GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}} - GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}} - GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}} - GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}} - AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}} - AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}} - AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}} - AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}} - SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}} - CONFLUENT_LICENSE: ${{ secrets.CONFLUENT_LICENSE}} - SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME}} - SALESFORCE_PASSWORD: ${{ secrets.SALESFORCE_PASSWORD}} - SALESFORCE_CONSUMER_KEY: ${{ secrets.SALESFORCE_CONSUMER_KEY}} - SALESFORCE_CONSUMER_PASSWORD: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD}} - SALESFORCE_SECURITY_TOKEN: ${{ secrets.SALESFORCE_SECURITY_TOKEN}} - SALESFORCE_INSTANCE: ${{ secrets.SALESFORCE_INSTANCE}} - SALESFORCE_USERNAME_ACCOUNT2: ${{ secrets.SALESFORCE_USERNAME_ACCOUNT2}} - SALESFORCE_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_PASSWORD_ACCOUNT2}} - SALESFORCE_SECURITY_TOKEN_ACCOUNT2: ${{ secrets.SALESFORCE_SECURITY_TOKEN_ACCOUNT2}} - SALESFORCE_CONSUMER_KEY_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_KEY_ACCOUNT2}} - SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2}} - SALESFORCE_CONSUMER_KEY_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_KEY_WITH_JWT}} - SALESFORCE_CONSUMER_PASSWORD_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_WITH_JWT}} - DD_API_KEY: ${{ secrets.DD_API_KEY}} - DD_APP_KEY: ${{ secrets.DD_APP_KEY}} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - JIRA_URL: ${{ secrets.JIRA_URL}} - JIRA_USERNAME: ${{ secrets.JIRA_USERNAME}} - JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN}} - MARKETO_ENDPOINT_URL: ${{ secrets.MARKETO_ENDPOINT_URL}} - MARKETO_CLIENT_ID: ${{ secrets.MARKETO_CLIENT_ID}} - MARKETO_CLIENT_SECRET: ${{ secrets.MARKETO_CLIENT_SECRET}} - PAGERDUTY_USER_EMAIL: ${{ secrets.PAGERDUTY_USER_EMAIL}} - PAGERDUTY_API_KEY: ${{ secrets.PAGERDUTY_API_KEY}} - PAGERDUTY_SERVICE_ID: ${{ secrets.PAGERDUTY_SERVICE_ID}} - CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_KEY}} - CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET: ${{ secrets.CCLOUD_REST_PROXY_SECURITY_PLUGIN_API_SECRET}} - SERVICENOW_URL: ${{ secrets.SERVICENOW_URL}} - SERVICENOW_PASSWORD: ${{ secrets.SERVICENOW_PASSWORD}} - SERVICENOW_DEVELOPER_USERNAME: ${{ secrets.SERVICENOW_DEVELOPER_USERNAME}} - SERVICENOW_DEVELOPER_PASSWORD: ${{ secrets.SERVICENOW_DEVELOPER_PASSWORD}} - SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_ACCOUNT_NAME}} - SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME}} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD}} - ZENDESK_URL: ${{ secrets.ZENDESK_URL}} - ZENDESK_USERNAME: ${{ secrets.ZENDESK_USERNAME}} - ZENDESK_PASSWORD: ${{ secrets.ZENDESK_PASSWORD}} - CONNECTOR_GITHUB_ACCESS_TOKEN: ${{ secrets.CONNECTOR_GITHUB_ACCESS_TOKEN}} - CI_GITHUB_TOKEN: ${{ secrets.CI_GITHUB_TOKEN}} - AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS: ${{ secrets.AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS}} - AUDIT_LOG_CLUSTER_API_KEY: ${{ secrets.AUDIT_LOG_CLUSTER_API_KEY}} - AUDIT_LOG_CLUSTER_API_SECRET: ${{ secrets.AUDIT_LOG_CLUSTER_API_SECRET}} - NGROK_AUTH_TOKEN: ${{ secrets.NGROK_CI_AUTH_TOKEN}} - DATABRICKS_AWS_BUCKET_NAME: ${{ secrets.DATABRICKS_AWS_BUCKET_NAME}} - DATABRICKS_AWS_BUCKET_REGION: ${{ secrets.DATABRICKS_AWS_BUCKET_REGION}} - DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.DATABRICKS_AWS_STAGING_S3_ACCESS_KEY_ID}} - DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.DATABRICKS_AWS_STAGING_S3_SECRET_ACCESS_KEY}} - DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_SERVER_HOSTNAME}} - DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH}} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN}} - ORACLE_CONTAINER_REGISTRY_USERNAME: ${{ secrets.ORACLE_CONTAINER_REGISTRY_USERNAME}} - ORACLE_CONTAINER_REGISTRY_PASSWORD: ${{ secrets.ORACLE_CONTAINER_REGISTRY_PASSWORD}} - GCP_KEYFILE_CONTENT: ${{ secrets.GCP_KEYFILE_CONTENT}} - GCP_PROJECT: ${{ secrets.GCP_PROJECT}} - HPE_MAPR_EMAIL: ${{ secrets.HPE_MAPR_EMAIL}} - HPE_MAPR_TOKEN: ${{ secrets.HPE_MAPR_TOKEN}} - MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST}} - MONGODB_ATLAS_USER: ${{ secrets.MONGODB_ATLAS_USER}} - MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD}} - - stop-runner: - name: Stop Github self-hosted runner - needs: - - start-runner # required to get output from the start-runner job - - build # required to wait when the main job is done - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - runs-on: ubuntu-latest - steps: - - - name: Checkout code - uses: actions/checkout@v4 - with: - repository: vdesabou/kafka-docker-playground - fetch-depth: 0 - - - name: Decrypt secrets.tar - run: | - ./.github/scripts/decrypt_secret.sh - tar xvf secrets.tar - rm secrets.tar - mkdir -p $HOME/.aws - mv aws_credentials_with_assuming_iam_role $HOME/.aws/credentials-with-assuming-iam-role - chmod -R a+rw $HOME/.aws - mkdir -p $HOME/.confluent - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - env: - SECRETS_ENCRYPTION_PASSWORD: ${{ secrets.SECRETS_ENCRYPTION_PASSWORD }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - - - name: Stop EC2 instance github-actions-runner-vsaboulin - run: | - aws ec2 stop-instances --instance-ids i-089ef31a75cb3f0e6 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7be12dff4..1609e36b9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,22 +43,22 @@ jobs: AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}} CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}} CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}} - ENVIRONMENT: ${{ secrets.ENVIRONMENT}} - CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}} - CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}} - CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}} + ENVIRONMENT: ${{ vars.ENVIRONMENT}} + CLUSTER_NAME: ${{ vars.CLUSTER_NAME}} + CLUSTER_REGION: ${{ vars.CLUSTER_REGION}} + CLUSTER_CLOUD: ${{ vars.CLUSTER_CLOUD}} CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}} - AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}} - AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}} - AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}} + AWS_DATABRICKS_CLUSTER_NAME: ${{ vars.AWS_DATABRICKS_CLUSTER_NAME}} + AWS_DATABRICKS_CLUSTER_REGION: ${{ vars.AWS_DATABRICKS_CLUSTER_REGION}} + AWS_DATABRICKS_CLUSTER_CLOUD: ${{ vars.AWS_DATABRICKS_CLUSTER_CLOUD}} AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}} - GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}} - GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}} - GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}} + GCP_CLUSTER_NAME: ${{ vars.GCP_CLUSTER_NAME}} + GCP_CLUSTER_REGION: ${{ vars.GCP_CLUSTER_REGION}} + GCP_CLUSTER_CLOUD: ${{ vars.GCP_CLUSTER_CLOUD}} GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}} - AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}} - AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}} - AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}} + AZURE_CLUSTER_NAME: ${{ vars.AZURE_CLUSTER_NAME}} + AZURE_CLUSTER_REGION: ${{ vars.AZURE_CLUSTER_REGION}} + AZURE_CLUSTER_CLOUD: ${{ vars.AZURE_CLUSTER_CLOUD}} AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}} SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}} SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_ACCOUNT_NAME}} @@ -68,7 +68,7 @@ jobs: GCP_PROJECT: ${{ secrets.GCP_PROJECT}} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} + AWS_REGION: ${{ vars.AWS_REGION}} CLOUD_API_KEY: ${{ secrets.CLOUD_API_KEY}} CLOUD_API_SECRET: ${{ secrets.CLOUD_API_SECRET}} SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME}} @@ -91,27 +91,27 @@ jobs: strategy: fail-fast: false matrix: - tag: ["7.7.0"] + tag: ["8.0.0"] test_list : [ # requiring ngrok "šŸš€1ļøāƒ£ ccloud/fm-debezium-mysql-legacy-source ccloud/fm-debezium-postgresql-legacy-source ccloud/fm-debezium-mysql-v2-source ccloud/fm-debezium-sqlserver-legacy-source ccloud/fm-debezium-sqlserver-v2-source ccloud/fm-elasticsearch-sink ccloud/fm-jdbc-sqlserver-sink ccloud/fm-jdbc-sqlserver-source ccloud/fm-ibm-mq-source ccloud/fm-jdbc-mysql-source ccloud/fm-jdbc-postgresql-sink ccloud/fm-mqtt-source ccloud/fm-debezium-postgresql-v2-source ccloud/fm-jdbc-postgresql-source", - "šŸš€ ccloud/fm-aws-cloudwatch-logs-source ccloud/fm-aws-s3-sink ccloud/fm-aws-s3-source ccloud/fm-azure-event-hubs-source ccloud/fm-azure-service-bus-source ccloud/fm-databricks-delta-lake-sink ccloud/fm-gcp-gcs-sink ccloud/fm-gcp-gcs-source ccloud/fm-aws-dynamodb-sink ccloud/fm-azure-data-lake-storage-gen2-sink ccloud/fm-azure-functions-sink ccloud/fm-azure-synapse-analytics-sink ccloud/fm-azure-cosmosdb-sink ccloud/fm-azure-cosmosdb-source ccloud/fully-managed-datadog-metrics-sink", + "šŸš€ ccloud/fm-aws-cloudwatch-logs-source ccloud/fm-aws-s3-sink ccloud/fm-aws-s3-source ccloud/fm-azure-event-hubs-source ccloud/fm-azure-service-bus-source ccloud/fm-databricks-delta-lake-sink ccloud/fm-gcp-gcs-sink ccloud/fm-gcp-gcs-source ccloud/fm-aws-dynamodb-sink ccloud/fm-azure-data-lake-storage-gen2-sink ccloud/fm-azure-functions-sink ccloud/fm-azure-synapse-analytics-sink ccloud/fm-azure-cosmosdb-sink ccloud/fm-azure-cosmosdb-v2-sink ccloud/fm-azure-cosmosdb-source ccloud/fm-azure-cosmosdb-v2-source ccloud/fully-managed-datadog-metrics-sink", - "šŸš€ ccloud/fm-salesforce-cdc-source ccloud/fm-salesforce-platform-events-sink ccloud/fm-salesforce-pushtopics-source ccloud/fm-salesforce-sobject-sink ccloud/fm-snowflake-sink ccloud/fm-gcp-pubsub-source ccloud/fm-gcp-spanner-sink ccloud/fm-aws-cloudwatch-metrics-sink ccloud/fm-gcp-bigquery-legacy-sink ccloud/fm-gcp-bigquery-v2-sink ccloud/fm-github-source ccloud/fm-jira-source ccloud/fm-gcp-cloud-functions-sink ccloud/fm-gcp-bigtable-sink ccloud/fm-pagerduty-sink ccloud/fm-azure-log-analytics-sink", + "šŸš€ ccloud/fm-salesforce-cdc-source ccloud/fm-salesforce-platform-events-sink ccloud/fm-salesforce-pushtopics-source ccloud/fm-salesforce-sobject-sink ccloud/fm-snowflake-sink ccloud/fm-snowflake-source ccloud/fm-gcp-pubsub-source ccloud/fm-gcp-spanner-sink ccloud/fm-aws-cloudwatch-metrics-sink ccloud/fm-gcp-bigquery-legacy-sink ccloud/fm-gcp-bigquery-v2-sink ccloud/fm-github-source ccloud/fm-jira-source ccloud/fm-gcp-cloud-functions-legacy-sink ccloud/fm-gcp-cloud-functions-gen2-sink ccloud/fm-gcp-bigtable-sink ccloud/fm-pagerduty-sink ccloud/fm-azure-log-analytics-sink", "šŸš€ ccloud/fm-aws-kinesis-source ccloud/fm-aws-redshift-sink ccloud/fm-aws-sqs-source ccloud/fm-aws-lambda-sink ccloud/fm-azure-blob-storage-sink ccloud/fm-azure-blob-storage-source ccloud/fm-azure-cognitive-search-sink ccloud/fm-mongodb-atlas-source ccloud/fm-mongodb-atlas-sink ccloud/fm-salesforce-bulkapi-source ccloud/fm-salesforce-bulkapi-2-0-source ccloud/custom-connector-connect-aws-s3-sink", - "šŸš€ ccloud/fm-datadog-metrics-sink ccloud/fm-salesforce-bulkapi-2-0-sink ccloud/fm-salesforce-platform-events-source ccloud/fm-servicenow-source ccloud/fm-servicenow-sink ccloud/fm-aws-dynamodb-cdc-source", + "šŸš€ ccloud/fm-datadog-metrics-sink ccloud/fm-salesforce-bulkapi-2-0-sink ccloud/fm-salesforce-platform-events-source ccloud/fm-servicenow-source ccloud/fm-servicenow-v2-source ccloud/fm-servicenow-sink ccloud/fm-aws-dynamodb-cdc-source ccloud/fm-datagen-source ccloud/fm-couchbase-source ccloud/fm-couchbase-sink", "šŸš€ connect/connect-servicenow-sink connect/connect-servicenow-source connect/connect-datagen-source", "šŸš€ connect/connect-salesforce-bulkapi-sink connect/connect-salesforce-bulkapi-source connect/connect-salesforce-pushtopics-source connect/connect-salesforce-sobject-sink connect/connect-salesforce-cdc-source connect/connect-salesforce-platform-events-sink connect/connect-salesforce-platform-events-source", "šŸš€ connect/connect-splunk-sink connect/connect-splunk-source connect/connect-splunk-s2s-source connect/connect-spool-dir-source connect/connect-syslog-source other/connect-override-policy-sftp-sink other/connect-override-policy-sftp-source", - "šŸš€ connect/connect-minio-s3-sink connect/connect-marketo-source connect/connect-active-mq-sink connect/connect-active-mq-source connect/connect-lenses-active-mq-source connect/connect-cassandra-sink connect/connect-couchbase-sink connect/connect-couchbase-source connect/connect-hbase-sink", - "šŸš€ connect/connect-jms-tibco-sink connect/connect-jms-tibco-source connect/connect-debezium-mongodb-source connect/connect-debezium-mysql-source connect/connect-debezium-postgresql-source connect/connect-debezium-sqlserver-source connect/connect-elasticsearch-sink connect/connect-datadiode-source-sink connect/connect-jms-sag-um-source connect/connect-jms-sag-um-sink", - "šŸš€ connect/connect-hdfs2-sink connect/connect-hdfs2-source connect/connect-hdfs3-sink connect/connect-hdfs3-source connect/connect-ibm-mq-sink connect/connect-ibm-mq-source connect/connect-snmp-source connect/connect-omnisci-sink", + "šŸš€ connect/connect-minio-s3-sink connect/connect-active-mq-sink connect/connect-active-mq-source connect/connect-lenses-active-mq-source connect/connect-cassandra-sink connect/connect-couchbase-sink connect/connect-couchbase-source connect/connect-hbase-sink", + "šŸš€ connect/connect-jms-tibco-sink connect/connect-jms-tibco-source connect/connect-debezium-mongodb-source connect/connect-debezium-mysql-source connect/connect-debezium-postgresql-source connect/connect-debezium-sqlserver-source connect/connect-elasticsearch-sink connect/connect-datadiode-source-sink", + "šŸš€ connect/connect-hdfs2-sink connect/connect-hdfs2-source connect/connect-hdfs3-sink connect/connect-hdfs3-source connect/connect-ibm-mq-sink connect/connect-ibm-mq-source connect/connect-snmp-source connect/connect-heavy-ai-sink", "šŸš€ connect/connect-cdc-oracle11-source connect/connect-jdbc-oracle11-sink connect/connect-jdbc-oracle11-source connect/connect-influxdb-sink connect/connect-influxdb-source connect/connect-jdbc-mysql-sink connect/connect-jdbc-mysql-source connect/connect-jdbc-postgresql-sink connect/connect-jdbc-postgresql-source connect/connect-jdbc-sqlserver-sink", - "šŸš€ connect/connect-jdbc-sqlserver-source connect/connect-jdbc-vertica-sink connect/connect-singlestore-sink connect/connect-jdbc-singlestore-source connect/connect-jms-active-mq-source connect/connect-jms-active-mq-sink connect/connect-jms-solace-sink connect/connect-jms-solace-source connect/connect-mongodb-sink connect/connect-mongodb-source connect/connect-mqtt-sink connect/connect-mqtt-source connect/connect-neo4j-sink connect/connect-tibco-sink connect/connect-tibco-source", + "šŸš€ connect/connect-jdbc-sqlserver-source connect/connect-jdbc-vertica-sink connect/connect-singlestore-sink connect/connect-jdbc-singlestore-source connect/connect-jms-active-mq-source connect/connect-jms-active-mq-sink connect/connect-jms-solace-sink connect/connect-jms-solace-source connect/connect-mongodb-sink connect/connect-mongodb-source connect/connect-mqtt-sink connect/connect-mqtt-source connect/connect-tibco-sink connect/connect-tibco-source", "šŸš€ connect/connect-jdbc-oracle12-source connect/connect-jdbc-oracle12-sink", "šŸš€ connect/connect-jdbc-oracle19-source connect/connect-jdbc-oracle19-sink connect/connect-jms-oracle19-sink connect/connect-jms-oracle19-source", "šŸš€ connect/connect-jdbc-oracle21-source connect/connect-jdbc-oracle21-sink connect/connect-jms-oracle21-sink connect/connect-jms-oracle21-source", @@ -120,27 +120,26 @@ jobs: "šŸš€ connect/connect-gcp-bigquery-sink connect-jdbc-gcp-bigquery-source connect/connect-gcp-cloud-functions-sink connect/connect-vertica-sink connect/connect-prometheus-sink connect/connect-aws-sqs-source connect/connect-aws-s3-sink connect/connect-aws-s3-source connect/connect-databricks-delta-lake-sink", "šŸš€ connect/connect-gcp-pubsub-source connect/connect-gcp-google-pubsub-source connect/connect-gcp-google-pubsub-sink connect/connect-gcp-gcs-sink connect/connect-gcp-gcs-source connect/connect-gcp-bigtable-sink connect/connect-kudu-source connect/connect-kudu-sink", "šŸš€ connect/connect-azure-data-lake-storage-gen2-sink connect/connect-azure-event-hubs-source connect/connect-azure-cognitive-search-sink connect/connect-azure-functions-sink connect/connect-azure-service-bus-source connect/connect-azure-blob-storage-source", - "šŸš€ connect/connect-ftps-source connect/connect-ftps-sink connect/connect-rabbitmq-sink connect/connect-amps-source connect/connect-jira-source connect/connect-github-source connect/connect-pivotal-gemfire-sink connect/connect-azure-blob-storage-sink connect/connect-azure-synapse-analytics-sink connect/connect-jdbc-azure-synapse-analytics-source", + "šŸš€ connect/connect-rabbitmq-sink connect/connect-jira-source connect/connect-github-source connect/connect-pivotal-gemfire-sink connect/connect-azure-blob-storage-sink connect/connect-azure-synapse-analytics-sink connect/connect-jdbc-azure-synapse-analytics-source", "šŸš€ connect/connect-http-sink connect/connect-iceberg-sink", "šŸš€ multi-data-center/replicator-connect", "šŸš€ multi-data-center/replicator-executable", "šŸš€ multi-data-center/mirrormaker2 connect/connect-pagerduty-sink connect/connect-zendesk-source connect/connect-datadog-metrics-sink connect/connect-gcp-spanner-sink connect/connect-gcp-firebase-source connect/connect-gcp-firebase-sink", "šŸš€ other/filebeat-to-kafka other/rest-proxy-security-plugin other/tiered-storage-with-aws other/write-logs-to-files other/audit-logs schema-registry/multiple-event-types-in-topic other/broker-schema-validation other/schema-registry-security-plugin multi-data-center/cluster-linking other/secrets-management other/connect-secret-registry other/schema-format-protobuf other/schema-format-json-schema other/schema-format-avro", - "šŸš€ ccloud/replicator ccloud/rest-proxy-security-plugin ccloud/schema-registry-security-plugin ccloud/audit-log-connector", + "šŸš€ ccloud/rest-proxy-security-plugin ccloud/schema-registry-security-plugin ccloud/audit-log-connector", "šŸš€ connect/connect-cdc-oracle12-source", - "šŸš€ connect/connect-cdc-oracle19-source", + "šŸš€ connect/connect-cdc-oracle19-source connect/connect-cdc-xstream-oracle19-source", "šŸš€ connect/connect-cdc-oracle18-source", "šŸš€ connect/connect-cdc-oracle21-source", - "šŸš€ connect/connect-weblogic-source connect/connect-jms-weblogic-sink connect/connect-jms-weblogic-source", + "šŸš€ connect/connect-jms-weblogic-sink connect/connect-jms-weblogic-source", "šŸš€ connect/connect-azure-cosmosdb-source connect/connect-azure-cosmosdb-sink connect/connect-jdbc-cockroachdb-source connect/connect-jdbc-ibmdb2-source connect/connect-jdbc-ibmdb2-sink connect/connect-jdbc-sybase-source connect/connect-jdbc-sybase-sink", - "šŸš€ environment/plaintext environment/2way-ssl environment/kerberos environment/ldap-authorizer-sasl-plain environment/ldap-sasl-plain environment/mdc-kerberos environment/mdc-plaintext environment/mdc-sasl-plain environment/rbac-sasl-plain environment/sasl-plain environment/sasl-scram environment/sasl-ssl environment/ssl_kerberos environment/kraft-plaintext connect/connect-snowflake-sink", - "šŸš€ connect/connect-mapr-sink", - "šŸš€ other/syslog-logstash-ksqldb other/mqtt-proxy connect/connect-jdbc-snowflake-source connect/connect-jdbc-snowflake-sink connect/connect-filestream-source connect/connect-filestream-sink connect/connect-filepulse-source connect/connect-jdbc-mariadb-source connect/connect-jdbc-mariadb-sink", - "šŸš€ connect/connect-jdbc-sap-hana-sink connect/connect-jdbc-sap-hana-source connect/connect-sap-hana-sink", + "šŸš€ environment/plaintext environment/2way-ssl environment/kerberos environment/ldap-authorizer-sasl-plain environment/ldap-sasl-plain environment/mdc-kerberos environment/mdc-plaintext environment/mdc-sasl-plain environment/rbac-sasl-plain environment/sasl-plain environment/sasl-scram environment/sasl-ssl environment/ssl_kerberos connect/connect-snowflake-sink", + "šŸš€ other/syslog-logstash-ksqldb connect/connect-jdbc-snowflake-sink connect/connect-filestream-source connect/connect-filestream-sink connect/connect-filepulse-source connect/connect-jdbc-mariadb-source connect/connect-jdbc-mariadb-sink", + "šŸš€ connect/connect-jdbc-sap-hana-sink connect/connect-jdbc-sap-hana-source connect/connect-sap-hana-sink other/kafka-connect-jsonata", # requiring ngrok - "šŸš€2ļøāƒ£ ccloud/fm-influxdb2-sink ccloud/fm-influxdb2-source ccloud/fm-jdbc-oracle19-source ccloud/fm-jdbc-oracle19-sink ccloud/fm-cdc-oracle19-source ccloud/fm-rabbitmq-source ccloud/fm-zendesk-source ccloud/fm-splunk-sink ccloud/fm-rabbitmq-sink ccloud/fm-jdbc-mysql-sink ccloud/fm-sftp-source ccloud/fm-http-sink ccloud/fm-http-source", - "šŸš€3ļøāƒ£ ccloud/fm-cdc-oracle11-source ccloud/fm-solace-sink ccloud/fm-opensearch-sink ccloud/fm-sftp-sink ccloud/fm-redis-sink" + "šŸš€2ļøāƒ£ ccloud/fm-influxdb2-sink ccloud/fm-influxdb2-source ccloud/fm-jdbc-oracle19-source ccloud/fm-jdbc-oracle19-sink ccloud/fm-cdc-oracle19-source ccloud/fm-cdc-xstream-oracle19-source ccloud/fm-rabbitmq-source ccloud/fm-zendesk-source ccloud/fm-splunk-sink ccloud/fm-rabbitmq-sink ccloud/fm-jdbc-mysql-sink ccloud/fm-sftp-source ccloud/fm-http-sink ccloud/fm-http-v2-sink ccloud/fm-http-source ccloud/fm-http-v2-source", + "šŸš€3ļøāƒ£ ccloud/fm-cdc-oracle11-source ccloud/fm-solace-sink ccloud/fm-opensearch-sink ccloud/fm-sftp-sink ccloud/fm-redis-sink ccloud/fm-active-mq-source ccloud/fm-clickhouse-sink ccloud/fm-debezium-mariadb-source" ] steps: @@ -165,6 +164,7 @@ jobs: df -h sudo apt-get -qq purge build-essential ghc* sudo apt-get clean + sudo apt-get update sudo apt-get install expect fzf coreutils -y docker system prune -af sudo rm -rf /usr/share/dotnet @@ -223,11 +223,14 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - name: Build and Test - run: bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}" + run: | + export PATH=$PATH:$GITHUB_WORKSPACE/scripts/cli + bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}" env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} + S3_PROVIDER_INTEGRATION_ID: ${{ secrets.S3_PROVIDER_INTEGRATION_ID}} + AWS_REGION: ${{ vars.AWS_REGION}} AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID}} AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY}} AWS_STS_ROLE_ARN: ${{ secrets.AWS_STS_ROLE_ARN}} @@ -236,22 +239,22 @@ jobs: AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}} CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}} CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}} - ENVIRONMENT: ${{ secrets.ENVIRONMENT}} - CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}} - CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}} - CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}} + ENVIRONMENT: ${{ vars.ENVIRONMENT}} + CLUSTER_NAME: ${{ vars.CLUSTER_NAME}} + CLUSTER_REGION: ${{ vars.CLUSTER_REGION}} + CLUSTER_CLOUD: ${{ vars.CLUSTER_CLOUD}} CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}} - AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}} - AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}} - AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}} + AWS_DATABRICKS_CLUSTER_NAME: ${{ vars.AWS_DATABRICKS_CLUSTER_NAME}} + AWS_DATABRICKS_CLUSTER_REGION: ${{ vars.AWS_DATABRICKS_CLUSTER_REGION}} + AWS_DATABRICKS_CLUSTER_CLOUD: ${{ vars.AWS_DATABRICKS_CLUSTER_CLOUD}} AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}} - GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}} - GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}} - GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}} + GCP_CLUSTER_NAME: ${{ vars.GCP_CLUSTER_NAME}} + GCP_CLUSTER_REGION: ${{ vars.GCP_CLUSTER_REGION}} + GCP_CLUSTER_CLOUD: ${{ vars.GCP_CLUSTER_CLOUD}} GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}} - AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}} - AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}} - AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}} + AZURE_CLUSTER_NAME: ${{ vars.AZURE_CLUSTER_NAME}} + AZURE_CLUSTER_REGION: ${{ vars.AZURE_CLUSTER_REGION}} + AZURE_CLUSTER_CLOUD: ${{ vars.AZURE_CLUSTER_CLOUD}} AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}} SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}} CONFLUENT_LICENSE: ${{ secrets.CONFLUENT_LICENSE}} @@ -267,7 +270,6 @@ jobs: SALESFORCE_CONSUMER_KEY_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_KEY_ACCOUNT2}} SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2}} SALESFORCE_CONSUMER_KEY_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_KEY_WITH_JWT}} - SALESFORCE_CONSUMER_PASSWORD_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_WITH_JWT}} DD_API_KEY: ${{ secrets.DD_API_KEY}} DD_APP_KEY: ${{ secrets.DD_APP_KEY}} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} @@ -318,7 +320,13 @@ jobs: MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST}} MONGODB_ATLAS_USER: ${{ secrets.MONGODB_ATLAS_USER}} MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD}} - + COUCHBASE_HOSTNAME: ${{ secrets.COUCHBASE_HOSTNAME}} + COUCHBASE_USERNAME: ${{ secrets.COUCHBASE_USERNAME}} + COUCHBASE_PASSWORD: ${{ secrets.COUCHBASE_PASSWORD}} + CLICKHOUSE_CLOUD_HOSTNAME: ${{ secrets.CLICKHOUSE_CLOUD_HOSTNAME}} + CLICKHOUSE_CLOUD_USERNAME: ${{ secrets.CLICKHOUSE_CLOUD_USERNAME}} + CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.CLICKHOUSE_CLOUD_PASSWORD}} + execute_one_test: if: ${{ github.event.inputs.test_name != '' }} runs-on: ubuntu-latest @@ -326,7 +334,7 @@ jobs: strategy: fail-fast: false matrix: - tag: ["7.7.0"] + tag: ["8.0.0"] test_list : [ "šŸš€ ${{ github.event.inputs.test_name }}" ] @@ -353,6 +361,7 @@ jobs: df -h sudo apt-get -qq purge build-essential ghc* sudo apt-get clean + sudo apt-get update sudo apt-get install expect fzf coreutils -y docker system prune -af sudo rm -rf /usr/share/dotnet @@ -411,11 +420,14 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD}} - name: Build and Test - run: bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}" + run: | + export PATH=$PATH:$GITHUB_WORKSPACE/scripts/cli + bash scripts/run-tests.sh "${{ matrix.test_list }}" "${{ matrix.tag }}" env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} + S3_PROVIDER_INTEGRATION_ID: ${{ secrets.S3_PROVIDER_INTEGRATION_ID}} + AWS_REGION: ${{ vars.AWS_REGION}} AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_ACCESS_KEY_ID}} AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_ACCOUNT_WITH_ASSUME_ROLE_AWS_SECRET_ACCESS_KEY}} AWS_STS_ROLE_ARN: ${{ secrets.AWS_STS_ROLE_ARN}} @@ -424,22 +436,22 @@ jobs: AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}} CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}} CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}} - ENVIRONMENT: ${{ secrets.ENVIRONMENT}} - CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}} - CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}} - CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}} + ENVIRONMENT: ${{ vars.ENVIRONMENT}} + CLUSTER_NAME: ${{ vars.CLUSTER_NAME}} + CLUSTER_REGION: ${{ vars.CLUSTER_REGION}} + CLUSTER_CLOUD: ${{ vars.CLUSTER_CLOUD}} CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}} - AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}} - AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}} - AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}} + AWS_DATABRICKS_CLUSTER_NAME: ${{ vars.AWS_DATABRICKS_CLUSTER_NAME}} + AWS_DATABRICKS_CLUSTER_REGION: ${{ vars.AWS_DATABRICKS_CLUSTER_REGION}} + AWS_DATABRICKS_CLUSTER_CLOUD: ${{ vars.AWS_DATABRICKS_CLUSTER_CLOUD}} AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}} - GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}} - GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}} - GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}} + GCP_CLUSTER_NAME: ${{ vars.GCP_CLUSTER_NAME}} + GCP_CLUSTER_REGION: ${{ vars.GCP_CLUSTER_REGION}} + GCP_CLUSTER_CLOUD: ${{ vars.GCP_CLUSTER_CLOUD}} GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}} - AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}} - AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}} - AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}} + AZURE_CLUSTER_NAME: ${{ vars.AZURE_CLUSTER_NAME}} + AZURE_CLUSTER_REGION: ${{ vars.AZURE_CLUSTER_REGION}} + AZURE_CLUSTER_CLOUD: ${{ vars.AZURE_CLUSTER_CLOUD}} AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}} SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}} CONFLUENT_LICENSE: ${{ secrets.CONFLUENT_LICENSE}} @@ -455,7 +467,6 @@ jobs: SALESFORCE_CONSUMER_KEY_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_KEY_ACCOUNT2}} SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2}} SALESFORCE_CONSUMER_KEY_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_KEY_WITH_JWT}} - SALESFORCE_CONSUMER_PASSWORD_WITH_JWT: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_WITH_JWT}} DD_API_KEY: ${{ secrets.DD_API_KEY}} DD_APP_KEY: ${{ secrets.DD_APP_KEY}} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME}} @@ -506,7 +517,13 @@ jobs: MONGODB_ATLAS_HOST: ${{ secrets.MONGODB_ATLAS_HOST}} MONGODB_ATLAS_USER: ${{ secrets.MONGODB_ATLAS_USER}} MONGODB_ATLAS_PASSWORD: ${{ secrets.MONGODB_ATLAS_PASSWORD}} - + COUCHBASE_HOSTNAME: ${{ secrets.COUCHBASE_HOSTNAME}} + COUCHBASE_USERNAME: ${{ secrets.COUCHBASE_USERNAME}} + COUCHBASE_PASSWORD: ${{ secrets.COUCHBASE_PASSWORD}} + CLICKHOUSE_CLOUD_HOSTNAME: ${{ secrets.CLICKHOUSE_CLOUD_HOSTNAME}} + CLICKHOUSE_CLOUD_USERNAME: ${{ secrets.CLICKHOUSE_CLOUD_USERNAME}} + CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.CLICKHOUSE_CLOUD_PASSWORD}} + post-build: name: Cleanup resources and Update README runs-on: ubuntu-latest @@ -552,22 +569,22 @@ jobs: AZURE_SUBSCRIPTION_NAME: ${{ secrets.AZURE_SUBSCRIPTION_NAME}} CONFLUENT_CLOUD_EMAIL: ${{ secrets.CONFLUENT_CLOUD_EMAIL}} CONFLUENT_CLOUD_PASSWORD: ${{ secrets.CONFLUENT_CLOUD_PASSWORD}} - ENVIRONMENT: ${{ secrets.ENVIRONMENT}} - CLUSTER_NAME: ${{ secrets.CLUSTER_NAME}} - CLUSTER_REGION: ${{ secrets.CLUSTER_REGION}} - CLUSTER_CLOUD: ${{ secrets.CLUSTER_CLOUD}} + ENVIRONMENT: ${{ vars.ENVIRONMENT}} + CLUSTER_NAME: ${{ vars.CLUSTER_NAME}} + CLUSTER_REGION: ${{ vars.CLUSTER_REGION}} + CLUSTER_CLOUD: ${{ vars.CLUSTER_CLOUD}} CLUSTER_CREDS: ${{ secrets.CLUSTER_CREDS}} - AWS_DATABRICKS_CLUSTER_NAME: ${{ secrets.AWS_DATABRICKS_CLUSTER_NAME}} - AWS_DATABRICKS_CLUSTER_REGION: ${{ secrets.AWS_DATABRICKS_CLUSTER_REGION}} - AWS_DATABRICKS_CLUSTER_CLOUD: ${{ secrets.AWS_DATABRICKS_CLUSTER_CLOUD}} + AWS_DATABRICKS_CLUSTER_NAME: ${{ vars.AWS_DATABRICKS_CLUSTER_NAME}} + AWS_DATABRICKS_CLUSTER_REGION: ${{ vars.AWS_DATABRICKS_CLUSTER_REGION}} + AWS_DATABRICKS_CLUSTER_CLOUD: ${{ vars.AWS_DATABRICKS_CLUSTER_CLOUD}} AWS_DATABRICKS_CLUSTER_CREDS: ${{ secrets.AWS_DATABRICKS_CLUSTER_CREDS}} - GCP_CLUSTER_NAME: ${{ secrets.GCP_CLUSTER_NAME}} - GCP_CLUSTER_REGION: ${{ secrets.GCP_CLUSTER_REGION}} - GCP_CLUSTER_CLOUD: ${{ secrets.GCP_CLUSTER_CLOUD}} + GCP_CLUSTER_NAME: ${{ vars.GCP_CLUSTER_NAME}} + GCP_CLUSTER_REGION: ${{ vars.GCP_CLUSTER_REGION}} + GCP_CLUSTER_CLOUD: ${{ vars.GCP_CLUSTER_CLOUD}} GCP_CLUSTER_CREDS: ${{ secrets.GCP_CLUSTER_CREDS}} - AZURE_CLUSTER_NAME: ${{ secrets.AZURE_CLUSTER_NAME}} - AZURE_CLUSTER_REGION: ${{ secrets.AZURE_CLUSTER_REGION}} - AZURE_CLUSTER_CLOUD: ${{ secrets.AZURE_CLUSTER_CLOUD}} + AZURE_CLUSTER_NAME: ${{ vars.AZURE_CLUSTER_NAME}} + AZURE_CLUSTER_REGION: ${{ vars.AZURE_CLUSTER_REGION}} + AZURE_CLUSTER_CLOUD: ${{ vars.AZURE_CLUSTER_CLOUD}} AZURE_CLUSTER_CREDS: ${{ secrets.AZURE_CLUSTER_CREDS}} SCHEMA_REGISTRY_CREDS: ${{ secrets.SCHEMA_REGISTRY_CREDS}} SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_ACCOUNT_NAME}} @@ -577,7 +594,7 @@ jobs: GCP_PROJECT: ${{ secrets.GCP_PROJECT}} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} + AWS_REGION: ${{ vars.AWS_REGION}} CLOUD_API_KEY: ${{ secrets.CLOUD_API_KEY}} CLOUD_API_SECRET: ${{ secrets.CLOUD_API_SECRET}} SALESFORCE_USERNAME: ${{ secrets.SALESFORCE_USERNAME}} @@ -592,36 +609,29 @@ jobs: SALESFORCE_CONSUMER_KEY_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_KEY_ACCOUNT2}} SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2: ${{ secrets.SALESFORCE_CONSUMER_PASSWORD_ACCOUNT2}} + - name: Update DOCS + run: | + sudo gem install bashly + cd ./scripts/cli + ./playground update-docs + - name: Update README run: | cd ./scripts/cli - ./playground update-readme --tags "7.7.0" + ./playground update-readme --tags "8.0.0" env: GH_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} - - - name: Pushes content.md - uses: dmnemec/copy_file_to_another_repo_action@main - env: - API_TOKEN_GITHUB: ${{ secrets.CI_GITHUB_TOKEN }} - with: - source_file: './docs/content.md' - destination_repo: 'vdesabou/kafka-docker-playground-docs' - destination_folder: 'docs' - user_email: 'vincent.desaboulin@gmail.com' - user_name: 'vdesabou' - commit_message: 'updating with latest versions' + AWS_REGION: ${{ vars.AWS_REGION}} - - name: Pushes introduction.md + - name: Pushes docs uses: dmnemec/copy_file_to_another_repo_action@main env: API_TOKEN_GITHUB: ${{ secrets.CI_GITHUB_TOKEN }} with: - source_file: './docs/introduction.md' + source_file: './docs' destination_repo: 'vdesabou/kafka-docker-playground-docs' - destination_folder: 'docs' user_email: 'vincent.desaboulin@gmail.com' user_name: 'vdesabou' commit_message: 'updating with latest versions' diff --git a/.github/workflows/update-changelogs.yml b/.github/workflows/update-changelogs.yml new file mode 100644 index 0000000000..823b537514 --- /dev/null +++ b/.github/workflows/update-changelogs.yml @@ -0,0 +1,69 @@ + +name: Update Change Logs + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + milestone: + types: [closed] + workflow_dispatch: + inputs: + milestoneId: + description: 'Milestone ID' + required: true + default: '1' + +jobs: + build: + name: Update Change Logs + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + repository: vdesabou/kafka-docker-playground + fetch-depth: 0 + # issues need to be closed to generate release notes + - name: Close issues + id: close_issues + uses: lee-dohm/close-matching-issues@v2 + with: + query: "milestone:${{ github.event.milestone.title }}" + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Create Release Notes + uses: Decathlon/release-notes-generator-action@v3.1.6 + id: Changelog + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OUTPUT_FOLDER: temp_release_notes + + - name: Read temp_release_notes/release_file.md + id: read_release_file + uses: juliangruber/read-file-action@v1 + with: + path: ./temp_release_notes/release_file.md + + - name: Update changelog.md + run: | + curl -s -o changelog_orig.md https://raw.githubusercontent.com/vdesabou/kafka-docker-playground-docs/refs/heads/main/docs/changelog.md + + tail -n +2 changelog_orig.md > changelog_tmp.md + + echo "# šŸ“œ Change Log" > ./docs/changelog.md + echo "" >> ./docs/changelog.md + current_month=$(date +"%B %Y") + echo "## ${current_month}" >> ./docs/changelog.md + echo "${{ steps.read_release_file.outputs.content }}" | sed 's/^#/##/g' | sed 's/###/#####/g' >> ./docs/changelog.md + cat changelog_tmp.md >> ./docs/changelog.md + + - name: Pushes docs + uses: dmnemec/copy_file_to_another_repo_action@main + env: + API_TOKEN_GITHUB: ${{ secrets.CI_GITHUB_TOKEN }} + with: + source_file: './docs' + destination_repo: 'vdesabou/kafka-docker-playground-docs' + user_email: 'vincent.desaboulin@gmail.com' + user_name: 'vdesabou' + commit_message: 'updating with milestone ${{ github.event.milestone.title }}' diff --git a/.github/workflows/update-readme.yml b/.github/workflows/update-readme.yml index 0fd4a7efb6..d83870b3f5 100644 --- a/.github/workflows/update-readme.yml +++ b/.github/workflows/update-readme.yml @@ -17,36 +17,29 @@ jobs: ssh-key: ${{ secrets.GH_SSH_KEY_FILE }} ssh-strict: 'false' + - name: Update DOCS + run: | + sudo gem install bashly + cd ./scripts/cli + ./playground update-docs + - name: Update README run: | cd ./scripts/cli - ./playground update-readme --tags "7.6.1" + ./playground update-readme --tags "8.0.0" env: GH_TOKEN: ${{ secrets.CI_GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - AWS_REGION: ${{ secrets.AWS_REGION}} + AWS_REGION: ${{ vars.AWS_REGION}} - - name: Pushes content.md + - name: Pushes docs uses: dmnemec/copy_file_to_another_repo_action@main env: API_TOKEN_GITHUB: ${{ secrets.CI_GITHUB_TOKEN }} with: - source_file: './docs/content.md' + source_file: './docs' destination_repo: 'vdesabou/kafka-docker-playground-docs' - destination_folder: 'docs' user_email: 'vincent.desaboulin@gmail.com' user_name: 'vdesabou' commit_message: 'updating with latest versions' - - - name: Pushes introduction.md - uses: dmnemec/copy_file_to_another_repo_action@main - env: - API_TOKEN_GITHUB: ${{ secrets.CI_GITHUB_TOKEN }} - with: - source_file: './docs/introduction.md' - destination_repo: 'vdesabou/kafka-docker-playground-docs' - destination_folder: 'docs' - user_email: 'vincent.desaboulin@gmail.com' - user_name: 'vdesabou' - commit_message: 'updating with latest versions' \ No newline at end of file diff --git a/.gitignore b/.gitignore index 86b9a44122..30ac39913c 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ myenv ._secrets.properties secrets.properties secrets.tar +variables.properties *.hprof *.pcap github_ssh_key_file @@ -75,4 +76,9 @@ playground_config.ini .ccloud .connector_config hsperfdata_appuser -*.lck \ No newline at end of file +*.lck +reproduction-models +**/bin/ +**/obj +zazkia +ccloud/connect-csfle-examples \ No newline at end of file diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile index 5721b5e447..271cc75a50 100644 --- a/.gitpod.Dockerfile +++ b/.gitpod.Dockerfile @@ -19,6 +19,8 @@ RUN curl -o /usr/local/bin/docker compose -fsSL https://github.com/docker/compos RUN echo "PATH="${PATH}"" | sudo tee /etc/environment RUN apt update && apt install fzf -y && apt install bat -y +RUN curl -L --http1.1 https://cnfl.io/cli | sh -s -- -b /usr/local/bin + USER gitpod RUN mkdir -p ~/.local/bin && ln -s /usr/bin/batcat ~/.local/bin/bat RUN echo "export PATH=~/.local/bin:/workspace/kafka-docker-playground/scripts/cli:$PATH" >> ~/.bashrc @@ -26,7 +28,6 @@ RUN echo "source /workspace/kafka-docker-playground/scripts/cli/completions.bash # Install Confluent Cloud CLI, with shell auto completion RUN mkdir -p ~/.local/share/bash-completion/ -RUN curl -L --http1.1 https://cnfl.io/cli | sudo sh -s -- -b /usr/local/bin && \ - touch ~/.local/share/bash-completion/confluent && \ +RUN touch ~/.local/share/bash-completion/confluent && \ confluent completion bash > ~/.local/share/bash-completion/confluent && \ echo "source ~/.local/share/bash-completion/confluent" >> ~/.bashrc \ No newline at end of file diff --git a/.gitpod.yml b/.gitpod.yml index c315a2343c..8cea322686 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -17,7 +17,7 @@ image: tasks: - name: prebuild init: | - cd environment/kerberos && source ../../scripts/utils.sh && docker compose -f ../../environment/plaintext/docker-compose.yml --profile control-center --profile ksqldb pull && cd - + cd environment/plaintext && source ../../scripts/utils.sh && docker compose -f ../../environment/plaintext/docker-compose.yml --profile control-center pull && cd - docker pull google/cloud-sdk:latest docker pull amazon/aws-cli docker pull mcr.microsoft.com/azure-cli diff --git a/.gitpod/automations.yaml b/.gitpod/automations.yaml new file mode 100644 index 0000000000..b0b0b99184 --- /dev/null +++ b/.gitpod/automations.yaml @@ -0,0 +1,19 @@ +tasks: + command-1: + command: "if [ -z \"$SECRETS_ENCRYPTION_PASSWORD\" ]; then echo \"\U0001F680 Enjoy the playground\"; else echo \"ā„¹ļø SECRETS_ENCRYPTION_PASSWORD environment variable is set\"; ./.github/scripts/decrypt_secret.sh;tar xvf secrets.tar;rm secrets.tar;mkdir -p $HOME/.aws;mv aws_credentials_with_assuming_iam_role $HOME/.aws/credentials-with-assuming-iam-role;chmod -R a+rw $HOME/.aws;mkdir -p $HOME/.confluent;echo \"$DOCKER_PASSWORD\" | docker login -u \"$DOCKER_USERNAME\" --password-stdin;fi" + dependsOn: + - init-001 + description: init task migrated from .gitpod.yml + name: 'playground: command' + triggeredBy: + - postDevcontainerStart + init-000: + command: "cd environment/kerberos && source ../../scripts/utils.sh && docker compose -f ../../environment/plaintext/docker-compose.yml --profile control-center --profile ksqldb pull && cd - \ndocker pull google/cloud-sdk:latest\ndocker pull amazon/aws-cli\ndocker pull mcr.microsoft.com/azure-cli\ndocker pull imega/jq\ndocker pull vdesabou/avro-tools\ngp sync-done prebuild\n" + description: init task migrated from .gitpod.yml + name: 'prebuild: init' + triggeredBy: + - postDevcontainerStart + init-001: + command: gp sync-await prebuild + description: init task migrated from .gitpod.yml + name: 'playground: init' diff --git a/README.md b/README.md index 586c1db615..6816f3362d 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,12 @@ Check out [kafka-docker-playground.io](https://kafka-docker-playground.io/) to l [![asciicast](https://asciinema.org/a/643687.svg)](https://asciinema.org/a/643687) +## šŸŽ™ļø Podcast + +[![podcast](./images/podcast.jpg)](https://youtu.be/zV56G1Gz1kI "Podcast") + +Made with love ā™„ļø by AI + ## šŸ Getting Started Check out the **[How to Use](https://kafka-docker-playground.io/#/how-to-use.md)** section, then select an example in the **[Content](https://kafka-docker-playground.io/#/content)** section and run it ! diff --git a/academy/connect-connect-aws-s3-sink/s3-sink-repro-000001-stackoverflowerror-with-s3-sink-connector.sh b/academy/connect-connect-aws-s3-sink/s3-sink-repro-000001-stackoverflowerror-with-s3-sink-connector.sh index 6a17a9e405..c5abee4e3b 100755 --- a/academy/connect-connect-aws-s3-sink/s3-sink-repro-000001-stackoverflowerror-with-s3-sink-connector.sh +++ b/academy/connect-connect-aws-s3-sink/s3-sink-repro-000001-stackoverflowerror-with-s3-sink-connector.sh @@ -6,41 +6,7 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" source ${DIR}/../../scripts/utils.sh -if [ ! -f $HOME/.aws/credentials ] && ( [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] ) -then - logerror "ERROR: either the file $HOME/.aws/credentials is not present or environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are not set!" - exit 1 -else - if [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ] - then - log "šŸ’­ Using environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY" - export AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY - else - if [ -f $HOME/.aws/credentials ] - then - logwarn "šŸ’­ AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are set based on $HOME/.aws/credentials" - export AWS_ACCESS_KEY_ID=$( grep "^aws_access_key_id" $HOME/.aws/credentials | head -1 | awk -F'=' '{print $2;}' ) - export AWS_SECRET_ACCESS_KEY=$( grep "^aws_secret_access_key" $HOME/.aws/credentials | head -1 | awk -F'=' '{print $2;}' ) - fi - fi - if [ -z "$AWS_REGION" ] - then - AWS_REGION=$(aws configure get region | tr '\r' '\n') - if [ "$AWS_REGION" == "" ] - then - logerror "ERROR: either the file $HOME/.aws/config is not present or environment variables AWS_REGION is not set!" - exit 1 - fi - fi -fi - -if [[ "$TAG" == *ubi8 ]] || version_gt $TAG_BASE "5.9.0" -then - export CONNECT_CONTAINER_HOME_DIR="/home/appuser" -else - export CONNECT_CONTAINER_HOME_DIR="/root" -fi +handle_aws_credentials PLAYGROUND_ENVIRONMENT=${PLAYGROUND_ENVIRONMENT:-"plaintext"} playground start-environment --environment "${PLAYGROUND_ENVIRONMENT}" --docker-compose-override-file "${PWD}/docker-compose.plaintext.repro-000001-stackoverflowerror-with-s3-sink-connector.yml" diff --git a/academy/connect-connect-debezium-sqlserver-source/debezium-sqlserver-source-repro-pipeline-example.sh b/academy/connect-connect-debezium-sqlserver-source/debezium-sqlserver-source-repro-pipeline-example.sh index c8eb79f362..17b487c190 100755 --- a/academy/connect-connect-debezium-sqlserver-source/debezium-sqlserver-source-repro-pipeline-example.sh +++ b/academy/connect-connect-debezium-sqlserver-source/debezium-sqlserver-source-repro-pipeline-example.sh @@ -6,7 +6,7 @@ source ${DIR}/../../scripts/utils.sh if ! version_gt $TAG_BASE "5.9.99" && version_gt $CONNECTOR_TAG "1.9.9" then - logwarn "WARN: connector version >= 2.0.0 do not support CP versions < 6.0.0" + logwarn "connector version >= 2.0.0 do not support CP versions < 6.0.0" exit 111 fi @@ -16,7 +16,7 @@ playground start-environment --environment "${PLAYGROUND_ENVIRONMENT}" --docker- log "Create table" -docker exec -i sqlserver /opt/mssql-tools/bin/sqlcmd -U sa -P Password! << EOF +docker exec -i sqlserver /opt/mssql-tools18/bin/sqlcmd -C -No -U sa -P Password! << EOF -- Create the test database CREATE DATABASE testDB; GO @@ -73,7 +73,7 @@ EOF sleep 5 -docker exec -i sqlserver /opt/mssql-tools/bin/sqlcmd -U sa -P Password! << EOF +docker exec -i sqlserver /opt/mssql-tools18/bin/sqlcmd -C -No -U sa -P Password! << EOF USE testDB; INSERT INTO customers(first_name,last_name,email) VALUES ('Pam','Thomas','pam@office.com'); GO diff --git a/academy/connect-connect-jdbc-mysql-source/mysql-repro-000005-insertfield-smt:-adding-topic-offset-and-partition-not-working.sh b/academy/connect-connect-jdbc-mysql-source/mysql-repro-000005-insertfield-smt:-adding-topic-offset-and-partition-not-working.sh index b255469d15..32e8eb6097 100755 --- a/academy/connect-connect-jdbc-mysql-source/mysql-repro-000005-insertfield-smt:-adding-topic-offset-and-partition-not-working.sh +++ b/academy/connect-connect-jdbc-mysql-source/mysql-repro-000005-insertfield-smt:-adding-topic-offset-and-partition-not-working.sh @@ -22,7 +22,7 @@ then docker run -i --rm -e KAFKA_CLIENT_TAG=$KAFKA_CLIENT_TAG -e TAG=$TAG_BASE -v "${PWD}/${component}":/usr/src/mymaven -v "$HOME/.m2":/root/.m2 -v "$PWD/../../scripts/settings.xml:/tmp/settings.xml" -v "${PWD}/${component}/target:/usr/src/mymaven/target" -w /usr/src/mymaven maven:3.6.1-jdk-11 mvn -s /tmp/settings.xml -Dkafka.tag=$TAG -Dkafka.client.tag=$KAFKA_CLIENT_TAG package > /tmp/result.log 2>&1 if [ $? != 0 ] then - logerror "ERROR: failed to build java component " + logerror "āŒ failed to build java component " tail -500 /tmp/result.log exit 1 fi diff --git a/ccloud/audit-log-connector/README.md b/ccloud/audit-log-connector/README.md index 3eefe7ba68..789cfbf42c 100644 --- a/ccloud/audit-log-connector/README.md +++ b/ccloud/audit-log-connector/README.md @@ -38,8 +38,8 @@ playground connector create-or-update --connector filestream-sink << EOF "consumer.override.sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/data_audit_cluster:sasl.username}\" password=\"\${file:/data_audit_cluster:sasl.password}\";", "consumer.override.client.dns.lookup": "use_all_dns_ips", "consumer.override.interceptor.classes": "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor", - "consumer.override.confluent.monitoring.interceptor.bootstrap.servers": "${file:/data:bootstrap.servers}", - "consumer.override.confluent.monitoring.interceptor.sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/data:sasl.username}\" password=\"\${file:/data:sasl.password}\";", + "consumer.override.confluent.monitoring.interceptor.bootstrap.servers": "${file:/datacloud:bootstrap.servers}", + "consumer.override.confluent.monitoring.interceptor.sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/datacloud:sasl.username}\" password=\"\${file:/datacloud:sasl.password}\";", "consumer.override.confluent.monitoring.interceptor.sasl.mechanism": "PLAIN", "consumer.override.confluent.monitoring.interceptor.security.protocol": "SASL_SSL" } @@ -59,8 +59,8 @@ Note that we also need to override monitoring interceptors to use the confluent ```json "consumer.override.interceptor.classes": "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor", - "consumer.override.confluent.monitoring.interceptor.bootstrap.servers": "${file:/data:bootstrap.servers}", - "consumer.override.confluent.monitoring.interceptor.sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/data:sasl.username}\" password=\"\${file:/data:sasl.password}\";", + "consumer.override.confluent.monitoring.interceptor.bootstrap.servers": "${file:/datacloud:bootstrap.servers}", + "consumer.override.confluent.monitoring.interceptor.sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/datacloud:sasl.username}\" password=\"\${file:/datacloud:sasl.password}\";", "consumer.override.confluent.monitoring.interceptor.sasl.mechanism": "PLAIN", "consumer.override.confluent.monitoring.interceptor.security.protocol": "SASL_SSL" ``` diff --git a/ccloud/audit-log-connector/start.sh b/ccloud/audit-log-connector/start.sh index fb9111fc24..1ec63a5991 100755 --- a/ccloud/audit-log-connector/start.sh +++ b/ccloud/audit-log-connector/start.sh @@ -34,6 +34,7 @@ sed -e "s|:AUDIT_LOG_CLUSTER_BOOTSTRAP_SERVERS:|$AUDIT_LOG_CLUSTER_BOOTSTRAP_SER -e "s|:AUDIT_LOG_CLUSTER_API_SECRET:|$AUDIT_LOG_CLUSTER_API_SECRET|g" \ ../../ccloud/audit-log-connector/data_audit_cluster.template > ../../ccloud/audit-log-connector/data_audit_cluster + playground start-environment --environment ccloud --docker-compose-override-file "${PWD}/docker-compose.yml" log "Creating FileStream Sink connector reading confluent-audit-log-events from the audit log cluster" @@ -50,12 +51,7 @@ playground connector create-or-update --connector filestream-sink << EOF "consumer.override.sasl.mechanism": "PLAIN", "consumer.override.security.protocol": "SASL_SSL", "consumer.override.sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/data_audit_cluster:sasl.username}\" password=\"\${file:/data_audit_cluster:sasl.password}\";", - "consumer.override.client.dns.lookup": "use_all_dns_ips", - "consumer.override.interceptor.classes": "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor", - "consumer.override.confluent.monitoring.interceptor.bootstrap.servers": "\${file:/data:bootstrap.servers}", - "consumer.override.confluent.monitoring.interceptor.sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/data:sasl.username}\" password=\"\${file:/data:sasl.password}\";", - "consumer.override.confluent.monitoring.interceptor.sasl.mechanism": "PLAIN", - "consumer.override.confluent.monitoring.interceptor.security.protocol": "SASL_SSL" + "consumer.override.client.dns.lookup": "use_all_dns_ips" } EOF diff --git a/ccloud/confluent-for-kubernetes/start.sh b/ccloud/confluent-for-kubernetes/start.sh index 4891c35865..ec05b9cae6 100755 --- a/ccloud/confluent-for-kubernetes/start.sh +++ b/ccloud/confluent-for-kubernetes/start.sh @@ -38,8 +38,8 @@ log "Install Confluent for Kubernetes" helm upgrade --install confluent-operator confluentinc/confluent-for-kubernetes log "Generate a CA pair" -docker run -u0 --rm -v $PWD:/tmp ${CP_CONNECT_IMAGE}:${CONNECT_TAG} bash -c "openssl genrsa -out /tmp/ca-key.pem 2048 && chown -R $(id -u $USER):$(id -g $USER) /tmp/" -docker run -u0 --rm -v $PWD:/tmp ${CP_CONNECT_IMAGE}:${CONNECT_TAG} bash -c "openssl req -new -key /tmp/ca-key.pem -x509 -days 1000 -out /tmp/ca.pem -subj '/C=US/ST=CA/L=MountainView/O=Confluent/OU=Operator/CN=TestCA' && chown -R $(id -u $USER):$(id -g $USER) /tmp/" +docker run -u0 --rm -v $PWD:/tmp vulhub/openssl:1.0.1c bash -c "openssl genrsa -out /tmp/ca-key.pem 2048 && chown -R $(id -u $USER):$(id -g $USER) /tmp/" +docker run -u0 --rm -v $PWD:/tmp vulhub/openssl:1.0.1c bash -c "openssl req -new -key /tmp/ca-key.pem -x509 -days 1000 -out /tmp/ca.pem -subj '/C=US/ST=CA/L=MountainView/O=Confluent/OU=Operator/CN=TestCA' && chown -R $(id -u $USER):$(id -g $USER) /tmp/" log "Create a Kuebernetes secret for inter-component TLS" kubectl create secret tls ca-pair-sslcerts \ diff --git a/ccloud/connect-centralized-license/docker-compose-connect-onprem-to-cloud.yml b/ccloud/connect-centralized-license/docker-compose-connect-onprem-to-cloud.yml index c8a7dd0d6f..656196ed87 100644 --- a/ccloud/connect-centralized-license/docker-compose-connect-onprem-to-cloud.yml +++ b/ccloud/connect-centralized-license/docker-compose-connect-onprem-to-cloud.yml @@ -2,7 +2,7 @@ services: zookeeper: - image: confluentinc/cp-zookeeper:${TAG} + image: ${CP_ZOOKEEPER_IMAGE}:${CP_ZOOKEEPER_TAG} hostname: zookeeper container_name: zookeeper environment: @@ -12,11 +12,9 @@ services: KAFKA_OPTS: -Dzookeeper.4lw.commands.whitelist=* broker: - image: ${CP_KAFKA_IMAGE}:${TAG} + image: ${CP_KAFKA_IMAGE}:${CP_KAFKA_TAG} hostname: broker container_name: broker - depends_on: - - zookeeper environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' @@ -24,7 +22,7 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' # Confluent Metrics Reporter for Control Center Cluster Monitoring - KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter + KAFKA_METRIC_REPORTERS: $KAFKA_METRIC_REPORTERS CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:9092 CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 CONFLUENT_METRICS_ENABLE: 'true' @@ -36,7 +34,7 @@ services: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 schema-registry: - image: confluentinc/cp-schema-registry:${TAG} + image: ${CP_SCHEMA_REGISTRY_IMAGE}:${CP_SCHEMA_REGISTRY_TAG} hostname: schema-registry container_name: schema-registry ports: @@ -47,7 +45,7 @@ services: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: broker:9092 connect: - image: ${CP_CONNECT_IMAGE}:${CONNECT_TAG} + image: ${CP_CONNECT_IMAGE}:${CP_CONNECT_TAG} environment: CONNECT_PLUGIN_PATH: /usr/share/confluent-hub-components/confluentinc-kafka-connect-replicator CONNECT_REST_EXTENSION_CLASSES: io.confluent.connect.replicator.monitoring.ReplicatorMonitoringExtension diff --git a/ccloud/connect-centralized-license/docker-compose.redshift.yml b/ccloud/connect-centralized-license/docker-compose.redshift.yml index 67dfca4265..88435eacf8 100644 --- a/ccloud/connect-centralized-license/docker-compose.redshift.yml +++ b/ccloud/connect-centralized-license/docker-compose.redshift.yml @@ -4,7 +4,7 @@ services: ports: - "5439:5439" volumes: - - $HOME/.aws/$AWS_CREDENTIALS_FILE_NAME:$CONNECT_CONTAINER_HOME_DIR/.aws/credentials + - ${AWS_CREDENTIALS_FILE_NAME:-/dev/null}:$CONNECT_CONTAINER_HOME_DIR/.aws/credentials - $HOME/.aws/config:$CONNECT_CONTAINER_HOME_DIR/.aws/config - ../../ccloud/connect-aws-redshift-sink/redshift-jdbc42-2.1.0.17/redshift-jdbc42-2.1.0.17.jar:/usr/share/confluent-hub-components/confluentinc-kafka-connect-aws-redshift/lib/redshift-jdbc42-2.1.0.17.jar environment: diff --git a/ccloud/connect-centralized-license/mqtt-source.sh b/ccloud/connect-centralized-license/mqtt-source.sh index d32c7cd34f..e6f012bbbb 100755 --- a/ccloud/connect-centralized-license/mqtt-source.sh +++ b/ccloud/connect-centralized-license/mqtt-source.sh @@ -5,13 +5,6 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" source ${DIR}/../../scripts/utils.sh -if [[ "$TAG" == *ubi8 ]] || version_gt $TAG_BASE "5.9.0" -then - export CONNECT_CONTAINER_HOME_DIR="/home/appuser" -else - export CONNECT_CONTAINER_HOME_DIR="/root" -fi - set +e playground topic delete --topic _confluent-command set -e diff --git a/ccloud/connect-centralized-license/redshift.sh b/ccloud/connect-centralized-license/redshift.sh index ce57989aeb..071cf5972e 100755 --- a/ccloud/connect-centralized-license/redshift.sh +++ b/ccloud/connect-centralized-license/redshift.sh @@ -13,41 +13,7 @@ then cd - fi -if [ ! -f $HOME/.aws/credentials ] && ( [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] ) -then - logerror "ERROR: either the file $HOME/.aws/credentials is not present or environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are not set!" - exit 1 -else - if [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ] - then - log "šŸ’­ Using environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY" - export AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY - else - if [ -f $HOME/.aws/credentials ] - then - logwarn "šŸ’­ AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are set based on $HOME/.aws/credentials" - export AWS_ACCESS_KEY_ID=$( grep "^aws_access_key_id" $HOME/.aws/credentials | head -1 | awk -F'=' '{print $2;}' ) - export AWS_SECRET_ACCESS_KEY=$( grep "^aws_secret_access_key" $HOME/.aws/credentials | head -1 | awk -F'=' '{print $2;}' ) - fi - fi - if [ -z "$AWS_REGION" ] - then - AWS_REGION=$(aws configure get region | tr '\r' '\n') - if [ "$AWS_REGION" == "" ] - then - logerror "ERROR: either the file $HOME/.aws/config is not present or environment variables AWS_REGION is not set!" - exit 1 - fi - fi -fi - -if [[ "$TAG" == *ubi8 ]] || version_gt $TAG_BASE "5.9.0" -then - export CONNECT_CONTAINER_HOME_DIR="/home/appuser" -else - export CONNECT_CONTAINER_HOME_DIR="/root" -fi +handle_aws_credentials playground start-environment --environment ccloud --docker-compose-override-file "${PWD}/docker-compose.redshift.yml" @@ -91,7 +57,7 @@ set -e log "Create AWS Redshift cluster" # https://docs.aws.amazon.com/redshift/latest/mgmt/getting-started-cli.html -aws redshift create-cluster --cluster-identifier $CLUSTER_NAME --master-username masteruser --master-user-password myPassword1 --node-type dc2.large --cluster-type single-node --publicly-accessible +aws redshift create-cluster --cluster-identifier $CLUSTER_NAME --master-username masteruser --master-user-password myPassword1 --node-type ra3.large --cluster-type single-node --publicly-accessible --tags Key=cflt_managed_by,Value=user Key=cflt_managed_id,Value="$USER" # Verify AWS Redshift cluster has started within MAX_WAIT seconds MAX_WAIT=480 @@ -103,7 +69,7 @@ while [[ ! $(cat /tmp/out.txt) =~ "available" ]]; do aws redshift describe-clusters --cluster-identifier $CLUSTER_NAME | jq .Clusters[0].ClusterStatus > /tmp/out.txt 2>&1 CUR_WAIT=$(( CUR_WAIT+10 )) if [[ "$CUR_WAIT" -gt "$MAX_WAIT" ]]; then - echo -e "\nERROR: The logs in ${CONTROL_CENTER_CONTAINER} container do not show 'available' after $MAX_WAIT seconds. Please troubleshoot with 'docker container ps' and 'docker container logs'.\n" + echo -e "\nERROR: The logs in ${CONTROL_CENTER_CONTAINER} container do not show 'available' after $MAX_WAIT seconds. Please troubleshoot with 'docker container ps' and 'playground container logs --open --container '.\n" exit 1 fi done diff --git a/ccloud/connect-centralized-license/replicator-onprem-to-cloud.sh b/ccloud/connect-centralized-license/replicator-onprem-to-cloud.sh index d1cc938332..e2c53c8548 100755 --- a/ccloud/connect-centralized-license/replicator-onprem-to-cloud.sh +++ b/ccloud/connect-centralized-license/replicator-onprem-to-cloud.sh @@ -35,9 +35,9 @@ playground connector create-or-update --connector replicate-onprem-to-cloud << "src.consumer.group.id": "replicate-onprem-to-cloud", "src.kafka.bootstrap.servers": "broker:9092", "dest.kafka.ssl.endpoint.identification.algorithm":"https", - "dest.kafka.bootstrap.servers": "\${file:/data:bootstrap.servers}", + "dest.kafka.bootstrap.servers": "\${file:/datacloud:bootstrap.servers}", "dest.kafka.security.protocol" : "SASL_SSL", - "dest.kafka.sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/data:sasl.username}\" password=\"\${file:/data:sasl.password}\";", + "dest.kafka.sasl.jaas.config": "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"\${file:/datacloud:sasl.username}\" password=\"\${file:/datacloud:sasl.password}\";", "dest.kafka.sasl.mechanism":"PLAIN", "dest.kafka.request.timeout.ms":"20000", "dest.kafka.retry.backoff.ms":"500", diff --git a/ccloud/connect-centralized-license/servicenow-source.sh b/ccloud/connect-centralized-license/servicenow-source.sh index 7008a96a07..16be5b774b 100755 --- a/ccloud/connect-centralized-license/servicenow-source.sh +++ b/ccloud/connect-centralized-license/servicenow-source.sh @@ -5,27 +5,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" source ${DIR}/../../scripts/utils.sh - -function wait_for_end_of_hibernation () { - MAX_WAIT=600 - CUR_WAIT=0 - set +e - log "āŒ› Waiting up to $MAX_WAIT seconds for end of hibernation to happen (it can take several minutes)" - curl -X POST "${SERVICENOW_URL}/api/now/table/incident" --user admin:"$SERVICENOW_PASSWORD" -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'cache-control: no-cache' -d '{"short_description": "This is test"}' > /tmp/out.txt 2>&1 - while [[ $(cat /tmp/out.txt) =~ "Sign in to the site to wake your instance" ]] - do - sleep 10 - curl -X POST "${SERVICENOW_URL}/api/now/table/incident" --user admin:"$SERVICENOW_PASSWORD" -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'cache-control: no-cache' -d '{"short_description": "This is test"}' > /tmp/out.txt 2>&1 - CUR_WAIT=$(( CUR_WAIT+10 )) - if [[ "$CUR_WAIT" -gt "$MAX_WAIT" ]]; then - echo -e "\nERROR: The logs still show 'Sign in to the site to wake your instance' after $MAX_WAIT seconds.\n" - exit 1 - fi - done - log "The instance is ready !" - set -e -} - SERVICENOW_URL=${SERVICENOW_URL:-$1} SERVICENOW_PASSWORD=${SERVICENOW_PASSWORD:-$2} diff --git a/ccloud/custom-connector-connect-aws-s3-sink/custom-s3-sink.sh b/ccloud/custom-connector-connect-aws-s3-sink/custom-s3-sink.sh index f82e0a9da2..fa7da2ce9a 100755 --- a/ccloud/custom-connector-connect-aws-s3-sink/custom-s3-sink.sh +++ b/ccloud/custom-connector-connect-aws-s3-sink/custom-s3-sink.sh @@ -74,34 +74,7 @@ then exit 1 fi -if [ ! -f $HOME/.aws/credentials ] && ( [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] ) -then - logerror "ERROR: either the file $HOME/.aws/credentials is not present or environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are not set!" - exit 1 -else - if [ ! -z "$AWS_ACCESS_KEY_ID" ] && [ ! -z "$AWS_SECRET_ACCESS_KEY" ] - then - log "šŸ’­ Using environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY" - export AWS_ACCESS_KEY_ID - export AWS_SECRET_ACCESS_KEY - else - if [ -f $HOME/.aws/credentials ] - then - logwarn "šŸ’­ AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are set based on $HOME/.aws/credentials" - export AWS_ACCESS_KEY_ID=$( grep "^aws_access_key_id" $HOME/.aws/credentials | head -1 | awk -F'=' '{print $2;}' ) - export AWS_SECRET_ACCESS_KEY=$( grep "^aws_secret_access_key" $HOME/.aws/credentials | head -1 | awk -F'=' '{print $2;}' ) - fi - fi - if [ -z "$AWS_REGION" ] - then - AWS_REGION=$(aws configure get region | tr '\r' '\n') - if [ "$AWS_REGION" == "" ] - then - logerror "ERROR: either the file $HOME/.aws/config is not present or environment variables AWS_REGION is not set!" - exit 1 - fi - fi -fi +handle_aws_credentials bootstrap_ccloud_environment @@ -160,6 +133,7 @@ playground connector create-or-update --connector $connector_name << EOF "confluent.connector.type": "CUSTOM", "confluent.custom.plugin.id": "$plugin_id", "confluent.custom.connection.endpoints": "s3.$AWS_REGION.amazonaws.com:443:TCP", + "connector.class": "io.confluent.connect.s3.S3SinkConnector", "kafka.api.key": "$CLOUD_KEY", "kafka.api.secret": "$CLOUD_SECRET", diff --git a/ccloud/custom-connector-connect-aws-s3-sink/stop.sh b/ccloud/custom-connector-connect-aws-s3-sink/stop.sh new file mode 100755 index 0000000000..7cd7cdca18 --- /dev/null +++ b/ccloud/custom-connector-connect-aws-s3-sink/stop.sh @@ -0,0 +1,10 @@ +#!/bin/bash + + + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +source ${DIR}/../../scripts/utils.sh + +docker compose down -v --remove-orphans + +maybe_delete_ccloud_environment \ No newline at end of file diff --git a/connect/connect-http-cdc-source/.gitignore b/ccloud/custom-connector-connect-iceberg-sink/.gitignore similarity index 100% rename from connect/connect-http-cdc-source/.gitignore rename to ccloud/custom-connector-connect-iceberg-sink/.gitignore diff --git a/ccloud/custom-connector-connect-iceberg-sink/README.md b/ccloud/custom-connector-connect-iceberg-sink/README.md new file mode 100644 index 0000000000..5a02ecf00d --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/README.md @@ -0,0 +1,22 @@ +# Custom Connector Apache Iceberg Sink Connector connector + + + +## Objective + +Quickly test [Apache Iceberg Sink Connector](https://github.com/tabular-io/iceberg-kafka-connect?tab=readme-ov-file) connector. + + + + +## How to run + +Simply run: + +``` +$ just use command +``` + +You can open the jupyter lab at http://localhost:8888/lab/tree/notebooks and use the sample notebook to query the table + +Refer to docker-spark-iceberg to check more details about it \ No newline at end of file diff --git a/ccloud/custom-connector-connect-iceberg-sink/custom-iceberg-sink.sh b/ccloud/custom-connector-connect-iceberg-sink/custom-iceberg-sink.sh new file mode 100755 index 0000000000..061f4e3f7f --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/custom-iceberg-sink.sh @@ -0,0 +1,199 @@ +#!/bin/bash +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +source ${DIR}/../../scripts/utils.sh + +if [ ! -f tabular-iceberg-kafka-connect-0.6.19.zip ] +then + log "Downloading tabular-iceberg-kafka-connect-0.6.19.zip from confluent hub" + wget -q https://d2p6pa21dvn84.cloudfront.net/api/plugins/tabular/iceberg-kafka-connect/versions/0.6.19/tabular-iceberg-kafka-connect-0.6.19.zip +fi + +plugin_name="pg_${USER}_tabular_iceberg_sink_0_6_19" + +set +e +for row in $(confluent connect custom-plugin list --output json | jq -r '.[] | @base64'); do + _jq() { + echo ${row} | base64 -d | jq -r ${1} + } + + id=$(echo $(_jq '.id')) + name=$(echo $(_jq '.name')) + + if [[ "$name" = "$plugin_name" ]] + then + log "deleting plugin $id ($name)" + confluent connect custom-plugin delete $id --force + fi +done +set -e + +log "Uploading custom plugin $plugin_name" +confluent connect custom-plugin create $plugin_name --plugin-file tabular-iceberg-kafka-connect-0.6.19.zip --connector-class io.tabular.iceberg.connect.IcebergSinkConnector --connector-type SINK --sensitive-properties "iceberg.kafka.sasl.jaas.config" +ret=$? + +function cleanup_resources { + log "Do you want to delete the custom plugin $plugin_name ($plugin_id) and custom connector $connector_name ?" + check_if_continue + + playground connector delete --connector $connector_name + confluent connect custom-plugin delete $plugin_id --force +} +trap cleanup_resources EXIT + +set -e +if [ $ret -eq 0 ] +then + found=0 + set +e + for row in $(confluent connect custom-plugin list --output json | jq -r '.[] | @base64'); do + _jq() { + echo ${row} | base64 -d | jq -r ${1} + } + + id=$(echo $(_jq '.id')) + name=$(echo $(_jq '.name')) + + if [[ "$name" = "$plugin_name" ]] + then + plugin_id="$id" + log "custom plugin $plugin_name ($plugin_id) was successfully uploaded!" + found=1 + break + fi + done +else + logerror "āŒ command failed with error code $ret!" + exit 1 +fi +set -e +if [ $found -eq 0 ] +then + logerror "āŒ plugin could not be uploaded !" + exit 1 +fi + +NGROK_AUTH_TOKEN=${NGROK_AUTH_TOKEN:-$1} + +display_ngrok_warning + +bootstrap_ccloud_environment + + + +set +e +playground topic delete --topic payments +playground topic delete --topic control-iceberg + +sleep 3 + +playground topic create --topic control-iceberg +set -e + +docker compose build +docker compose down -v --remove-orphans +docker compose up -d --quiet-pull + +sleep 30 + +log "Waiting for ngrok to start" +while true +do + container_id=$(docker ps -q -f name=ngrok) + if [ -n "$container_id" ] + then + status=$(docker inspect --format '{{.State.Status}}' $container_id) + if [ "$status" = "running" ] + then + log "Getting ngrok hostname and port" + NGROK_URL=$(curl --silent http://127.0.0.1:4040/api/tunnels | jq -r '.tunnels[0].public_url') + NGROK_HOSTNAME=$(echo $NGROK_URL | cut -d "/" -f3 | cut -d ":" -f 1) + NGROK_PORT=$(echo $NGROK_URL | cut -d "/" -f3 | cut -d ":" -f 2) + + NGROK_URL2=$(curl --silent http://127.0.0.1:4040/api/tunnels | jq -r '.tunnels[1].public_url') + NGROK_HOSTNAME2=$(echo $NGROK_URL2 | cut -d "/" -f3 | cut -d ":" -f 1) + NGROK_PORT2=$(echo $NGROK_URL2 | cut -d "/" -f3 | cut -d ":" -f 2) + + if ! [[ $NGROK_PORT =~ ^[0-9]+$ ]] + then + log "NGROK_PORT is not a valid number, keep retrying..." + continue + else + break + fi + fi + fi + log "Waiting for container ngrok to start..." + sleep 5 +done + + +log "Sending messages to topic payments" +playground topic produce -t payments --nb-messages $(wc -l <"../../ccloud/custom-connector-connect-iceberg-sink/data/transactions.json") --value ../../ccloud/custom-connector-connect-iceberg-sink/data/transactions.json + +connector_name="ICEBERG_SINK_CUSTOM_$USER" +set +e +log "Deleting confluent cloud custom connector $connector_name, it might fail..." +playground connector delete --connector $connector_name +set -e + + +log "Creating Iceberg sink connector" +playground connector create-or-update --connector $connector_name << EOF +{ + "confluent.connector.type": "CUSTOM", + "confluent.custom.plugin.id": "$plugin_id", + "confluent.custom.connection.endpoints": "$NGROK_HOSTNAME:$NGROK_PORT:TCP;$NGROK_HOSTNAME2:$NGROK_PORT2:TCP", + "connector.class": "io.tabular.iceberg.connect.IcebergSinkConnector", + + "kafka.api.key": "$CLOUD_KEY", + "kafka.api.secret": "$CLOUD_SECRET", + "name": "$connector_name", + "tasks.max": "1", + "topics": "payments", + "connector.class": "io.tabular.iceberg.connect.IcebergSinkConnector", + + "iceberg.catalog.s3.endpoint": "http://$NGROK_HOSTNAME2:$NGROK_PORT2", + "iceberg.catalog.s3.secret-access-key": "minioadmin", + "iceberg.catalog.s3.access-key-id": "minioadmin", + "iceberg.catalog.s3.path-style-access": "true", + "iceberg.catalog.uri": "http://$NGROK_HOSTNAME:$NGROK_PORT", + "iceberg.catalog.warehouse": "s3://warehouse/", + "iceberg.catalog.client.region": "eu-west-1", + "iceberg.catalog.type": "rest", + "iceberg.control.commit.interval-ms": "1000", + "iceberg.tables.auto-create-enabled": "true", + "iceberg.tables": "orders.payments", + "iceberg.kafka.sasl.jaas.config":"org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$CLOUD_KEY\" password=\"$CLOUD_SECRET\";", + "value.converter.schemas.enable": "false", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "schemas.enable": "false", + + "errors.tolerance": "all", + "errors.deadletterqueue.topic.name": "dlq", + "errors.deadletterqueue.topic.replication.factor": "3", + "errors.deadletterqueue.context.headers.enable": "true", + "errors.log.enable": "true", + "errors.log.include.messages": "true" +} +EOF + +sleep 30 + +playground connector show-lag --max-wait 300 + +if [ -z "$GITHUB_RUN_NUMBER" ] +then + # doesn't work on github actions + # not running with github actions + log "You can open the jupyter lab at http://localhost:8888/lab/tree/notebooks and use the sample notebook in notebooks/iceberg.ipynb to query the table" + + log "Verify data is in Iceberg" + docker exec -i spark-iceberg spark-sql << EOF +SELECT * +FROM orders.payments +LIMIT 10; +EOF +fi \ No newline at end of file diff --git a/ccloud/custom-connector-connect-iceberg-sink/data/transactions.json b/ccloud/custom-connector-connect-iceberg-sink/data/transactions.json new file mode 100644 index 0000000000..9508bdfa07 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/data/transactions.json @@ -0,0 +1,1000 @@ +{"id": "99e7a457-c86d-47de-824e-dcb90865a388", "type": "debit", "created_at": "2022-09-13T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 3814} +{"id": "90c2a935-26e7-4cc6-863c-c1c169348169", "type": "credit", "created_at": "2022-06-12T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 1917} +{"id": "7207dcf9-43b8-4d5b-854a-fed96d1e896f", "type": "bank_slip", "created_at": "2022-05-27T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 4265} +{"id": "22a6c526-b758-4035-91a9-c77a3f7dc9ed", "type": "debit", "created_at": "2022-06-05T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 3012} +{"id": "30c3fb33-8d78-4de9-9291-a3388c7b6014", "type": "credit", "created_at": "2022-11-06T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 1868} +{"id": "4ec089f8-d9b2-4bac-99ea-7efbf3ca67e4", "type": "bank_slip", "created_at": "2022-06-26T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 4596} +{"id": "4545f835-10ba-4033-b63a-118116d3d61b", "type": "bank_slip", "created_at": "2022-09-04T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 5024} +{"id": "99a7b931-7d02-4f00-91bb-59e875bcbc76", "type": "credit", "created_at": "2022-08-15T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 9018} +{"id": "5c95505b-35c5-4f35-a64b-d415708ab707", "type": "bank_slip", "created_at": "2022-03-27T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 8567} +{"id": "cc402b3f-189a-4251-85c7-f91c2bab7fa7", "type": "credit", "created_at": "2022-01-22T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 867} +{"id": "a1988618-e541-4ebd-b436-aa0e32a7983d", "type": "credit", "created_at": "2022-08-19T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 6258} +{"id": "5fe6df9c-4aa2-4f58-89e3-ee3a42013483", "type": "credit", "created_at": "2022-10-04T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 6312} +{"id": "54a272ac-1dc5-4f30-9ddd-27069b03b099", "type": "debit", "created_at": "2022-02-12T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 4057} +{"id": "a75766b2-d8aa-4f38-8533-1ee331b96a16", "type": "credit", "created_at": "2022-03-31T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 503} +{"id": "7ce9e050-0d47-4fb9-9eba-185f22200503", "type": "bank_slip", "created_at": "2022-03-02T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 7139} +{"id": "a6d8e76f-5c90-41c1-b98c-aa075bc41dbf", "type": "debit", "created_at": "2022-08-26T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 715} +{"id": "e83474f8-9573-4115-be5c-eb7feb3a4a46", "type": "credit", "created_at": "2022-07-07T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 7579} +{"id": "324f338b-0be4-450c-9a6d-547978a0ef00", "type": "bank_slip", "created_at": "2022-03-21T00:00:00", "document": "22", "payer": "Noemi Mcintyre", "amount": 9523} +{"id": "c551be48-7734-49cb-b3a1-5ab65ba1d7fc", "type": "bank_slip", "created_at": "2022-11-17T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 6968} +{"id": "79388493-61dd-4164-8e76-1cd662bae296", "type": "credit", "created_at": "2022-12-09T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 2732} +{"id": "c8936c42-2f02-415d-84b8-e478a2a061e3", "type": "bank_slip", "created_at": "2022-09-14T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 1055} +{"id": "52704039-8895-4da7-9ac2-88fb829444fc", "type": "credit", "created_at": "2022-07-05T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 1678} +{"id": "bbbeabcf-585d-41d5-9621-ca10ec1c624b", "type": "debit", "created_at": "2022-09-12T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 5950} +{"id": "36253bb1-245b-4472-8939-aea1316bc627", "type": "bank_slip", "created_at": "2022-12-20T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 292} +{"id": "24550eef-12d4-422e-b0ce-6c87e1309bbd", "type": "debit", "created_at": "2022-07-20T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 5827} +{"id": "0677ce3b-9164-4263-9e9d-0e3a9bf830f5", "type": "debit", "created_at": "2022-03-31T00:00:00", "document": "22", "payer": "Noemi Mcintyre", "amount": 5527} +{"id": "c4f6da0a-e304-4360-93c0-a6c7e0f1d80e", "type": "bank_slip", "created_at": "2022-09-10T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 5092} +{"id": "3d58b6f0-3560-47da-ad19-d4a2cfe4ee18", "type": "bank_slip", "created_at": "2022-12-11T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 3059} +{"id": "a9e9fd27-47f1-4688-b51d-e288f8cd80ce", "type": "bank_slip", "created_at": "2022-12-30T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 9794} +{"id": "b5c400f5-da44-499f-9983-1aefcb0ba109", "type": "credit", "created_at": "2022-03-16T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 4431} +{"id": "5c5ed237-b7a0-4b8f-afa7-d36ce088e6ec", "type": "credit", "created_at": "2022-01-03T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 5339} +{"id": "8e4910f3-f2bb-45fa-bf85-dc6811468038", "type": "bank_slip", "created_at": "2022-07-20T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 7020} +{"id": "a5a172ba-0ef4-4cba-a8d1-ea2580cde2d2", "type": "bank_slip", "created_at": "2022-04-10T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 5570} +{"id": "5b58f85a-e603-4d3c-be04-f3b7a16337fd", "type": "bank_slip", "created_at": "2022-11-16T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 6044} +{"id": "d82ad64e-98cc-4bac-a2c2-a230418003ee", "type": "debit", "created_at": "2022-12-20T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 7065} +{"id": "95d59330-318a-4b9c-9060-dd99cd003a81", "type": "credit", "created_at": "2022-07-09T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 993} +{"id": "dd2893ad-60c0-4f1d-ac9c-fbc69e706e42", "type": "bank_slip", "created_at": "2022-04-10T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 9482} +{"id": "0bb3d04b-fa41-4975-8d23-362bbd85613b", "type": "credit", "created_at": "2022-12-23T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 9160} +{"id": "db8cee5e-7847-4f28-be86-0b65f3c13ce7", "type": "credit", "created_at": "2022-09-12T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 7484} +{"id": "081294a6-b915-4762-8624-38a189f75f0a", "type": "debit", "created_at": "2022-09-15T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 2754} +{"id": "384f6f04-a518-46a8-bda9-e9d8148982d3", "type": "bank_slip", "created_at": "2022-11-14T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 1925} +{"id": "562c496d-4036-4e15-ab58-e42cc3b65623", "type": "bank_slip", "created_at": "2022-12-06T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 2427} +{"id": "dabebe12-4387-446d-b5ef-0e573e2d225c", "type": "credit", "created_at": "2022-03-16T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 8012} +{"id": "c261795c-2ab9-41c9-b46a-fe186d24a087", "type": "credit", "created_at": "2022-03-31T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 5235} +{"id": "b8be58a2-70a2-4473-948a-bea361970875", "type": "credit", "created_at": "2022-06-07T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 8557} +{"id": "95420e7a-2e6e-4c1b-a782-d39fd02de769", "type": "debit", "created_at": "2022-08-08T00:00:00", "document": "68", "payer": "Pauline Marcum", "amount": 3724} +{"id": "a4982461-12e5-4e31-93e7-0bc0c9a633e6", "type": "credit", "created_at": "2022-12-02T00:00:00", "document": "60", "payer": "Rachel York", "amount": 2036} +{"id": "592d21fa-c56f-44bb-bd58-0ed415f6ab05", "type": "credit", "created_at": "2022-12-20T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 7291} +{"id": "33fe6a56-1f47-4288-9775-0656f3991aed", "type": "debit", "created_at": "2022-10-30T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 8445} +{"id": "4e5fc495-dde8-4f13-9bb2-2390559ed963", "type": "bank_slip", "created_at": "2022-01-21T00:00:00", "document": "50", "payer": "Karen Lewis", "amount": 1505} +{"id": "61a606dd-9f4b-42a2-a95d-1f18f0ed753d", "type": "credit", "created_at": "2022-04-02T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 1933} +{"id": "5b78dfe7-ee5b-45f6-bb35-c1be27c47196", "type": "debit", "created_at": "2022-06-30T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 2966} +{"id": "e29cd169-b384-45df-8509-02c9020b4cc3", "type": "credit", "created_at": "2022-09-15T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 1620} +{"id": "f0eac1de-77cf-4262-88dd-13581de016e7", "type": "debit", "created_at": "2022-08-01T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 5748} +{"id": "d5af09db-96c1-46c6-aa24-ff1fb11c1e97", "type": "debit", "created_at": "2022-07-09T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 773} +{"id": "9fa70049-357d-4807-8b55-4c7ebc017cfc", "type": "bank_slip", "created_at": "2022-06-14T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 7525} +{"id": "f49eda9c-2e23-4d8e-85ba-770485597026", "type": "debit", "created_at": "2022-04-19T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 2235} +{"id": "925e1bee-431f-4444-b465-f02c5fd54ead", "type": "bank_slip", "created_at": "2022-07-02T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 3235} +{"id": "32d9daa8-70d0-4fe0-bf8a-1ce7dafa739e", "type": "bank_slip", "created_at": "2022-10-07T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 7260} +{"id": "3557642d-32b4-4ca1-81c7-401a8647c0af", "type": "debit", "created_at": "2022-09-19T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 6291} +{"id": "afd44e3d-6c7f-43d5-995b-7ce141aff9ec", "type": "credit", "created_at": "2022-03-28T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 1316} +{"id": "fd1709be-12fa-4357-b034-42c1b29921b0", "type": "credit", "created_at": "2022-11-17T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 996} +{"id": "127cc5c1-1e93-44ac-adbf-6ee7c0c98eb1", "type": "debit", "created_at": "2022-10-08T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 7110} +{"id": "7f2b1430-07db-4011-b2dc-212ac37fb092", "type": "bank_slip", "created_at": "2022-01-04T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 4836} +{"id": "fab4be9d-95bf-4f26-b051-2072cef7a8c3", "type": "debit", "created_at": "2023-01-01T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 3420} +{"id": "056ffd43-ddf9-4c62-a985-63ef90646bac", "type": "credit", "created_at": "2022-04-10T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 3488} +{"id": "3b024976-c89d-4f1f-a649-860be2a1a2a3", "type": "credit", "created_at": "2022-08-30T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 6451} +{"id": "0f4f0163-9d94-43f6-bf08-6231fc554d99", "type": "bank_slip", "created_at": "2022-01-06T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 9156} +{"id": "e51497ba-d05d-4f95-82a5-20ca9b1ed7ee", "type": "bank_slip", "created_at": "2022-07-01T00:00:00", "document": "22", "payer": "Noemi Mcintyre", "amount": 1852} +{"id": "304915df-5c70-4ea4-ba22-ca5207daf142", "type": "bank_slip", "created_at": "2022-09-28T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 7563} +{"id": "546072fd-4cfc-4e60-a511-80e6c46cb168", "type": "credit", "created_at": "2022-01-21T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 9313} +{"id": "f376ae8a-8448-4e40-ad61-e80308a7cd14", "type": "credit", "created_at": "2022-08-14T00:00:00", "document": "49", "payer": "Richard Isbell", "amount": 8899} +{"id": "284403bc-efe9-4517-83ab-81009c3ec163", "type": "credit", "created_at": "2022-08-25T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 9225} +{"id": "bc74fc02-9843-4163-aa57-d9b4e74f8c5d", "type": "debit", "created_at": "2022-04-20T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 2558} +{"id": "cdb2e945-26cb-492d-a148-a6cf25b6fe79", "type": "bank_slip", "created_at": "2022-08-16T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 9617} +{"id": "90145849-e9eb-4bdc-bb5a-4d6aa0af35af", "type": "bank_slip", "created_at": "2022-08-21T00:00:00", "document": "57", "payer": "David Landry", "amount": 7783} +{"id": "a8c2ccd2-38d5-4fde-8e01-478eee1dbf10", "type": "bank_slip", "created_at": "2022-02-25T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 847} +{"id": "7788a6bd-a6e9-4920-8700-c627f092dd29", "type": "debit", "created_at": "2022-11-16T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 1759} +{"id": "cd028c09-cbee-4216-92d3-098c8749f4dd", "type": "bank_slip", "created_at": "2022-06-29T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 1920} +{"id": "74397e97-e0bb-4fdd-896b-cbf5618942c6", "type": "debit", "created_at": "2022-11-20T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 7167} +{"id": "0aab19dc-f739-4f86-a917-81f8ee7cfabb", "type": "bank_slip", "created_at": "2022-02-03T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 6138} +{"id": "51355c97-e079-4286-9458-6bd0d4998e52", "type": "credit", "created_at": "2022-07-11T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 9095} +{"id": "f20508d6-de40-4489-955a-3200dd176c00", "type": "bank_slip", "created_at": "2022-03-24T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 3167} +{"id": "4376f178-dc1e-47e0-b5f3-c469cdb6ef7d", "type": "bank_slip", "created_at": "2022-09-25T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 3028} +{"id": "00ef44ac-c357-4016-b79a-d7e0b2576f8f", "type": "debit", "created_at": "2022-10-22T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 4051} +{"id": "cb116ac6-9b1a-45c0-89c9-67cb9bc74329", "type": "credit", "created_at": "2022-04-15T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 8215} +{"id": "10c3245a-bf10-4796-9989-947da6f058c4", "type": "credit", "created_at": "2022-03-29T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 3934} +{"id": "cc70cd4f-c32e-4b0d-8b20-b37b4c6df38c", "type": "credit", "created_at": "2022-09-21T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 1072} +{"id": "9cae835b-2d91-4e35-81fd-f90e90517399", "type": "bank_slip", "created_at": "2022-04-16T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 6318} +{"id": "a9608f38-fe7c-4d32-b836-60a8dca67f0a", "type": "debit", "created_at": "2022-11-09T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 1348} +{"id": "6655b159-6db6-4fe2-9666-5307657f38e5", "type": "bank_slip", "created_at": "2022-02-08T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 1861} +{"id": "d3b3bf58-d8bf-474f-83fa-41401ec28a1b", "type": "debit", "created_at": "2022-03-01T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 2600} +{"id": "466b9a06-4a32-4041-b81a-85d5d379bbee", "type": "bank_slip", "created_at": "2022-11-10T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 9164} +{"id": "75b74907-a83a-4e4c-b3b9-ec3fd0eb5132", "type": "debit", "created_at": "2022-12-25T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 7691} +{"id": "d1bd35d6-b6b6-4a24-b119-43d9f8724f25", "type": "debit", "created_at": "2022-12-01T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 3710} +{"id": "c80169ac-1a8c-4458-a786-1a6e71428da7", "type": "bank_slip", "created_at": "2022-08-12T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 8365} +{"id": "1ae27a72-b395-4cab-80c7-a93cc3a91e19", "type": "credit", "created_at": "2022-05-24T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 6474} +{"id": "d4dce8a1-7936-4072-a905-73017a3f2791", "type": "debit", "created_at": "2022-11-13T00:00:00", "document": "49", "payer": "Richard Isbell", "amount": 6402} +{"id": "f0441736-d940-47a5-b81d-2384e123a723", "type": "credit", "created_at": "2022-01-23T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 7239} +{"id": "24e3fcfa-f8a9-4083-a49c-a3fe1b6cc8c7", "type": "bank_slip", "created_at": "2022-10-30T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 2031} +{"id": "6c1649ce-1d1e-4f6e-8137-12ff3b1c957d", "type": "bank_slip", "created_at": "2022-11-09T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 9865} +{"id": "aad32abb-dd18-4b59-a47e-afeb0f7ed2ae", "type": "bank_slip", "created_at": "2022-10-06T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 3075} +{"id": "c0e81e7d-d0f9-4663-8fd7-756cf95052b3", "type": "credit", "created_at": "2022-07-23T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 660} +{"id": "bf2c0924-c49a-4cc5-a236-c5400a23270d", "type": "credit", "created_at": "2022-12-20T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 8966} +{"id": "c2809020-67b5-4006-930a-c992764a5740", "type": "credit", "created_at": "2022-12-18T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 1868} +{"id": "1bfa7e14-bbd2-41de-b8ba-b1a4bfbe88ef", "type": "debit", "created_at": "2022-05-24T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 333} +{"id": "3f193d33-f31b-4f52-95df-985335eb33bd", "type": "bank_slip", "created_at": "2022-12-25T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 9591} +{"id": "e80bab37-6be1-4b5b-84c1-f41634924492", "type": "credit", "created_at": "2022-05-08T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 14} +{"id": "c298ba67-cb71-4e09-982d-82d60f066521", "type": "credit", "created_at": "2022-10-04T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 5648} +{"id": "a5ef3b0c-00a3-43df-b5b5-f55e04ffb973", "type": "bank_slip", "created_at": "2022-01-03T00:00:00", "document": "50", "payer": "Karen Lewis", "amount": 9652} +{"id": "b49f7334-444d-427a-ae75-924cf9117d48", "type": "debit", "created_at": "2022-12-20T00:00:00", "document": "43", "payer": "Gary Jensen", "amount": 8169} +{"id": "da2afb6b-0501-40b1-b1ba-6f4488142f82", "type": "debit", "created_at": "2022-05-01T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 7492} +{"id": "e1190d99-d0cc-44b9-8501-a2cf4e0d0c28", "type": "bank_slip", "created_at": "2022-04-21T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 1114} +{"id": "29b670cc-81b2-4097-b49e-8947fc047075", "type": "debit", "created_at": "2022-03-01T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 4458} +{"id": "78d09ef7-d36f-4884-8d97-e2f4492ab002", "type": "credit", "created_at": "2022-09-08T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 482} +{"id": "0e366f1c-c071-4c68-bd30-9ed096be7d98", "type": "bank_slip", "created_at": "2022-01-05T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 5132} +{"id": "64c49c95-70bf-41b7-ad5c-bdc5a0bdf80f", "type": "bank_slip", "created_at": "2022-08-25T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 236} +{"id": "b31c5069-ded7-4de5-8c33-6abd72833f9d", "type": "credit", "created_at": "2022-11-17T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 469} +{"id": "6cea1aeb-342f-4083-bd8e-13b72a7c48e9", "type": "bank_slip", "created_at": "2022-08-01T00:00:00", "document": "78", "payer": "Paul debiten", "amount": 5615} +{"id": "2138b587-52de-46e0-9999-ee95de3e2418", "type": "bank_slip", "created_at": "2022-10-28T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 3256} +{"id": "480fd45d-2fc8-43f3-b3f6-4d23f4359403", "type": "bank_slip", "created_at": "2022-09-18T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 7032} +{"id": "257eb230-2593-4b91-87fb-aef944c4e219", "type": "credit", "created_at": "2022-07-17T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 3825} +{"id": "1e47d973-ef18-40f1-acff-8f123a779a36", "type": "debit", "created_at": "2022-06-06T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 3177} +{"id": "034f98ab-3935-4b13-936f-4d95a81bafb1", "type": "bank_slip", "created_at": "2022-04-06T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 9150} +{"id": "8c410f32-5e24-418b-99ef-4a5c07a18264", "type": "credit", "created_at": "2022-12-12T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 9551} +{"id": "5ad18ff0-187b-4199-9283-40d09873e015", "type": "bank_slip", "created_at": "2022-08-22T00:00:00", "document": "50", "payer": "Karen Lewis", "amount": 9184} +{"id": "8324dace-9740-4bac-aa89-5e87aefb22db", "type": "debit", "created_at": "2022-08-07T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 9378} +{"id": "e08858cd-a92d-4bd2-bf90-aefccb5f6d13", "type": "bank_slip", "created_at": "2022-05-24T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 1934} +{"id": "4ade39c7-06fc-4bde-a561-b22d5081cec1", "type": "credit", "created_at": "2022-03-13T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 2909} +{"id": "44484eca-18b1-416e-a31b-5722178f2c5b", "type": "debit", "created_at": "2022-11-17T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 3132} +{"id": "765ae679-6b07-4b0a-a42a-961ca076abc1", "type": "bank_slip", "created_at": "2022-11-20T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 9715} +{"id": "6a2afd30-ca61-4b89-a87e-5bd977d83ea3", "type": "credit", "created_at": "2022-04-11T00:00:00", "document": "2", "payer": "Ella Tatum", "amount": 1529} +{"id": "debfb244-b413-407f-bc1c-15d080342861", "type": "credit", "created_at": "2022-04-23T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 5081} +{"id": "ef12d207-50b6-46f3-8cfc-32ee79236cc9", "type": "credit", "created_at": "2022-03-23T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 9221} +{"id": "8f466d96-b38c-407f-ac2d-7e8441157c1f", "type": "bank_slip", "created_at": "2022-11-08T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 2346} +{"id": "dd4b0ab9-8e4d-4097-b62a-13f070b47395", "type": "bank_slip", "created_at": "2022-09-30T00:00:00", "document": "22", "payer": "Noemi Mcintyre", "amount": 9986} +{"id": "1a88f7c2-1091-4f12-a301-e295499d59b3", "type": "credit", "created_at": "2022-06-23T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 4343} +{"id": "28ffaf47-f630-449a-90aa-43ffbc2d1abd", "type": "debit", "created_at": "2022-10-29T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 4322} +{"id": "afb5df64-bac2-4e4b-b1e5-d7006ea77144", "type": "bank_slip", "created_at": "2022-02-11T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 3028} +{"id": "bf055716-c9ab-4928-a1f1-5e945f026ec1", "type": "bank_slip", "created_at": "2022-10-07T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 1573} +{"id": "48a4c3c8-5943-4211-9055-0b3e92696257", "type": "credit", "created_at": "2022-10-02T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 8256} +{"id": "b2005e5a-4a04-4dca-92b4-9677c2786e8d", "type": "debit", "created_at": "2022-01-27T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 5613} +{"id": "8ac40bfa-c471-40f8-b978-3d08b9bfd959", "type": "debit", "created_at": "2022-07-24T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 3855} +{"id": "a0317540-c57f-481f-af04-72a73bea84e3", "type": "debit", "created_at": "2022-01-27T00:00:00", "document": "23", "payer": "Donald Lee", "amount": 3845} +{"id": "3bca8a3e-424f-4cf8-80db-b077b7bc516f", "type": "debit", "created_at": "2022-04-22T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 8556} +{"id": "9fa8c316-7221-4e97-a0ad-8bd5f413ccfc", "type": "credit", "created_at": "2022-06-19T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 7791} +{"id": "fd101a02-413d-4be5-8130-a14bc8c6fa2d", "type": "bank_slip", "created_at": "2022-01-03T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 1959} +{"id": "42c97b43-ff2c-43f7-9548-e2ee2a229853", "type": "bank_slip", "created_at": "2022-09-06T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 2718} +{"id": "1f84193a-bf6d-4691-8724-41221de9c1ee", "type": "bank_slip", "created_at": "2022-06-04T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 2152} +{"id": "80643521-f11c-4e77-99fe-64a9f998b172", "type": "debit", "created_at": "2022-09-26T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 5011} +{"id": "3cba4844-aabe-4b4b-87ce-037bd7a0a011", "type": "debit", "created_at": "2022-09-12T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 3266} +{"id": "1a87edb7-d350-4e5b-9a55-07d033009e91", "type": "credit", "created_at": "2022-12-17T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 6307} +{"id": "e55579f0-9c6b-4f12-b04f-57b5295d2161", "type": "credit", "created_at": "2022-01-02T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 1696} +{"id": "6396c58b-79fa-4eb7-9c24-3f28f4e79ef5", "type": "bank_slip", "created_at": "2022-06-06T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 695} +{"id": "68160bd3-ad65-40f8-940b-ee4c39d04d30", "type": "credit", "created_at": "2022-11-30T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 5004} +{"id": "6cb336f3-f5e2-4cce-b5df-4d96a5a185c8", "type": "debit", "created_at": "2022-04-16T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 1897} +{"id": "a14683a5-5f91-4c58-a626-e4e0635abd5d", "type": "debit", "created_at": "2022-03-27T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 4050} +{"id": "77e42f86-135b-43ca-b621-056956274532", "type": "bank_slip", "created_at": "2022-11-25T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 8387} +{"id": "eabe69a7-dfcf-44b9-a16a-80665071ae62", "type": "bank_slip", "created_at": "2022-02-02T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 733} +{"id": "1a1b6a75-f1d3-433b-9277-2420a99b5848", "type": "credit", "created_at": "2022-06-26T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 1677} +{"id": "1edebad6-fccc-4b12-9459-ac688c60e28c", "type": "debit", "created_at": "2022-05-07T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 9173} +{"id": "eb74bb7e-fae5-428e-a483-cf256165d18e", "type": "debit", "created_at": "2022-07-28T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 1237} +{"id": "04af768f-a273-43c3-a01f-f22ec275cc6f", "type": "credit", "created_at": "2022-09-13T00:00:00", "document": "68", "payer": "Pauline Marcum", "amount": 5907} +{"id": "f954196f-baba-4a7d-a659-40d83d8e6ceb", "type": "debit", "created_at": "2022-06-01T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 1237} +{"id": "4d65b321-6ad9-495e-8b68-e6e6150fc990", "type": "debit", "created_at": "2022-04-11T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 4929} +{"id": "d9193514-bede-44da-92fe-c1fbb4c01cbc", "type": "bank_slip", "created_at": "2022-01-18T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 989} +{"id": "96da474b-627a-4a12-a1c9-a125e50db09c", "type": "debit", "created_at": "2022-09-29T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 5185} +{"id": "6fae629d-a9a8-4137-b7a9-cf87d0f057c0", "type": "debit", "created_at": "2022-04-13T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 4298} +{"id": "dc99be12-1ee8-4e28-8c0a-05aec86e402f", "type": "credit", "created_at": "2022-06-13T00:00:00", "document": "52", "payer": "Derek Corcoran", "amount": 1180} +{"id": "e0d9234b-16cb-465f-8693-ee11801719f8", "type": "bank_slip", "created_at": "2022-06-10T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 2592} +{"id": "a6333b5a-490b-4946-b8d3-c615f99886af", "type": "credit", "created_at": "2022-01-28T00:00:00", "document": "65", "payer": "William Boxer", "amount": 3432} +{"id": "c540a826-7ef7-4d83-a581-d911dd88ad7a", "type": "debit", "created_at": "2022-10-30T00:00:00", "document": "11", "payer": "Jenny Jensen", "amount": 1156} +{"id": "c8f45fea-10b7-4440-afaf-42ef6a757cd1", "type": "debit", "created_at": "2022-03-12T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 2844} +{"id": "5e22cc43-1e69-42dd-9871-a0ccfc6a1264", "type": "credit", "created_at": "2022-10-11T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 6638} +{"id": "f8867c83-b743-4909-9cfd-ca6414dc98aa", "type": "debit", "created_at": "2022-07-17T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 7644} +{"id": "bf1f117f-ef0f-4a40-808e-889742afdee8", "type": "bank_slip", "created_at": "2022-03-29T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 3355} +{"id": "facccf83-451f-441a-ad5f-7e6ed9deff19", "type": "bank_slip", "created_at": "2022-12-28T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 1247} +{"id": "8b2bdc58-322d-4674-93a7-c3e69ea15493", "type": "credit", "created_at": "2022-02-21T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 9930} +{"id": "b8440248-5c87-427e-9a39-33f9702a1765", "type": "credit", "created_at": "2022-02-05T00:00:00", "document": "46", "payer": "Shaunda Gallegos", "amount": 2014} +{"id": "b716ca18-5536-4931-88ab-bde265527cf9", "type": "bank_slip", "created_at": "2022-05-28T00:00:00", "document": "60", "payer": "Rachel York", "amount": 7575} +{"id": "d964b24a-f627-4818-985b-bbbcb9e98e11", "type": "debit", "created_at": "2022-03-10T00:00:00", "document": "57", "payer": "David Landry", "amount": 8591} +{"id": "a01a5fb6-ae21-427f-9c8f-297542857ece", "type": "bank_slip", "created_at": "2022-05-21T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 9774} +{"id": "68fd1b19-d461-4980-aba6-450949879472", "type": "credit", "created_at": "2022-06-21T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 7187} +{"id": "b97e9678-f813-4639-b86d-0c98dd8cb293", "type": "credit", "created_at": "2022-02-04T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 9117} +{"id": "743a0a20-511c-40fa-9ea7-3ff460d47d03", "type": "bank_slip", "created_at": "2022-01-14T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 4144} +{"id": "8678b5f4-0819-4641-998e-391f309c80c8", "type": "bank_slip", "created_at": "2022-05-11T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 5832} +{"id": "4be43919-cdab-40e1-9ecf-315d1a91129a", "type": "debit", "created_at": "2022-10-02T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 2384} +{"id": "e6adcfcd-ca19-4332-ab3f-7291cb328bf2", "type": "credit", "created_at": "2022-03-15T00:00:00", "document": "49", "payer": "Richard Isbell", "amount": 2357} +{"id": "cbeaa5ec-1434-4716-935f-1d216722765b", "type": "debit", "created_at": "2022-12-13T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 7642} +{"id": "af297dc2-86b4-4070-8fa7-7eb7c69d17fb", "type": "credit", "created_at": "2022-04-03T00:00:00", "document": "37", "payer": "Jane Olson", "amount": 7262} +{"id": "d57b168d-cf94-493c-9640-d3a915c8da2d", "type": "credit", "created_at": "2022-02-02T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 6188} +{"id": "60070a7a-3437-434f-b90c-25588c94a142", "type": "bank_slip", "created_at": "2022-10-25T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 6697} +{"id": "d2080e3e-c524-4002-96e6-c5ed0f9ec155", "type": "debit", "created_at": "2022-08-27T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 9847} +{"id": "e5d85b12-572d-4594-bc15-56d4792f7e1d", "type": "bank_slip", "created_at": "2022-08-07T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 7304} +{"id": "3f22e5bb-9cc0-4b9b-b742-b854f85f12c7", "type": "credit", "created_at": "2022-05-31T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 7402} +{"id": "1cb54ccd-ba8c-4ef7-b1c4-004c76f5a45c", "type": "debit", "created_at": "2022-02-28T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 4963} +{"id": "9ba95524-000e-4b32-b53f-106b75923a12", "type": "bank_slip", "created_at": "2022-08-01T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 9455} +{"id": "1c4b696b-6df1-4b92-9349-2710ec2b9940", "type": "bank_slip", "created_at": "2022-10-04T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 1724} +{"id": "fa384e35-b33f-4e4f-b309-f345a5aacac4", "type": "credit", "created_at": "2022-01-09T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 912} +{"id": "346b082c-312d-4f4e-ab7f-644c9d37896d", "type": "credit", "created_at": "2022-09-27T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 7928} +{"id": "346bac46-ba70-4033-ae84-ee33e8c89e38", "type": "credit", "created_at": "2022-06-03T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 2741} +{"id": "9e54a36b-c7ff-4654-b685-a24647d38c01", "type": "credit", "created_at": "2022-03-24T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 6390} +{"id": "c4108c2a-2295-4d1f-9836-be035cba2b90", "type": "bank_slip", "created_at": "2022-01-04T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 6372} +{"id": "33d8208e-de37-4887-8ccb-673831dc1f0b", "type": "bank_slip", "created_at": "2022-07-10T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 3778} +{"id": "4a0061d9-fe7f-4942-a94c-4da417b15024", "type": "debit", "created_at": "2022-12-06T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 3375} +{"id": "5e139790-138d-4ca3-aad4-dde983a82edb", "type": "debit", "created_at": "2022-12-17T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 4539} +{"id": "f783a2c1-0ba0-45c6-87c4-fd79ce859f76", "type": "bank_slip", "created_at": "2022-06-13T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 3259} +{"id": "ad8485ee-f9aa-4e9b-81ec-cc83b5f58c18", "type": "credit", "created_at": "2022-07-12T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 431} +{"id": "49b75b62-eabf-4865-8ee2-b8c0d36b09e5", "type": "credit", "created_at": "2022-10-19T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 8273} +{"id": "4ea31a13-bce4-494c-ac26-de8d137b20c7", "type": "bank_slip", "created_at": "2022-07-01T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 9545} +{"id": "419b823b-43c2-45cd-af36-aa038c9b3610", "type": "bank_slip", "created_at": "2022-01-30T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 4191} +{"id": "6a830415-159c-44e0-b5bf-32f0af7406cc", "type": "credit", "created_at": "2022-07-18T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 6320} +{"id": "7c057252-dc77-4cb7-8acc-e223a4f9aa21", "type": "bank_slip", "created_at": "2022-05-04T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 9456} +{"id": "ebe7a963-81d7-41e5-9cac-bdf60a389723", "type": "debit", "created_at": "2022-12-22T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 7526} +{"id": "e3a7cbe4-1061-4e2e-9403-cd97134e1603", "type": "credit", "created_at": "2022-06-09T00:00:00", "document": "61", "payer": "Ladonna Cooper", "amount": 79} +{"id": "330ce4b4-ce9a-4af3-a179-0a466765bbea", "type": "bank_slip", "created_at": "2022-11-24T00:00:00", "document": "68", "payer": "Pauline Marcum", "amount": 942} +{"id": "41fa6e2f-f3e8-4a61-b852-d6ad2a1a2d36", "type": "credit", "created_at": "2022-10-21T00:00:00", "document": "28", "payer": "Mark Williams", "amount": 6473} +{"id": "a9604c8d-47d3-4567-9351-76a2b4972560", "type": "credit", "created_at": "2022-01-25T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 696} +{"id": "384cae57-0080-460c-b774-d10da21830a5", "type": "debit", "created_at": "2022-10-29T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 9489} +{"id": "a9ec68f7-f7b3-447f-a13c-d76a61ddaec5", "type": "credit", "created_at": "2022-06-10T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 7701} +{"id": "8b976f9c-53bf-4d02-851a-8fc42df65c52", "type": "credit", "created_at": "2022-11-03T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 6907} +{"id": "58e29752-8088-4466-8b15-b182149a02c2", "type": "credit", "created_at": "2022-07-26T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 4753} +{"id": "12c79929-8623-4568-854f-36b185a7d6b1", "type": "credit", "created_at": "2022-03-06T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 5657} +{"id": "0178b124-8619-4fea-bff5-16e07537ec39", "type": "bank_slip", "created_at": "2022-12-30T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 3183} +{"id": "1803434e-4ff5-42b3-bae6-f17127fd87f9", "type": "debit", "created_at": "2022-04-14T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 7139} +{"id": "a71a2b75-ba54-4329-aff8-487ca0db5a07", "type": "credit", "created_at": "2022-12-06T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 5853} +{"id": "ed8f441f-a880-4ed3-bde6-2a69ff336af5", "type": "debit", "created_at": "2022-04-08T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 9484} +{"id": "00df54c3-881d-4170-8550-d29a995b22b7", "type": "bank_slip", "created_at": "2022-10-08T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 9854} +{"id": "38740701-365e-4e93-9dbf-d144d31e709e", "type": "bank_slip", "created_at": "2022-04-23T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 7680} +{"id": "60e26b84-f5eb-4bb4-98e8-7c655a0065cb", "type": "debit", "created_at": "2022-07-03T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 4252} +{"id": "95ccc77b-4c91-4713-b924-02fd82f22c30", "type": "credit", "created_at": "2022-01-31T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 9801} +{"id": "59d8e05f-ff37-47fb-8d4b-3d9665528c45", "type": "credit", "created_at": "2022-12-08T00:00:00", "document": "79", "payer": "Steven Rapp", "amount": 607} +{"id": "a8e7cb28-41ac-4b62-ba67-b0108609593e", "type": "debit", "created_at": "2022-05-31T00:00:00", "document": "33", "payer": "Charlotte Marczak", "amount": 8951} +{"id": "090b7835-39d1-4cfe-b60d-9f435ea00f3e", "type": "debit", "created_at": "2022-03-21T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 8798} +{"id": "b1f3ae84-88b7-4f2f-b1cb-8289fc5c79c6", "type": "debit", "created_at": "2022-07-15T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 8684} +{"id": "e4428cf9-e3c8-4a2a-9fee-836093c2588b", "type": "bank_slip", "created_at": "2022-07-06T00:00:00", "document": "33", "payer": "Charlotte Marczak", "amount": 3165} +{"id": "74b8dec0-3299-41d3-8262-25ea7e79550b", "type": "bank_slip", "created_at": "2022-01-24T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 2740} +{"id": "a607ab90-908d-4557-b9fb-8972008525b5", "type": "bank_slip", "created_at": "2022-07-22T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 7279} +{"id": "0508d6e6-a6cf-45e5-8e00-eee2487397e4", "type": "credit", "created_at": "2022-06-02T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 5046} +{"id": "1dc09c32-e7e4-4ad8-ae2a-b8bb05fb50a4", "type": "bank_slip", "created_at": "2022-06-11T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 9915} +{"id": "b158f765-5057-44cc-a4b2-94fbd7156451", "type": "bank_slip", "created_at": "2022-11-13T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 6326} +{"id": "e1e1688d-1b45-427e-bd8b-4f5778b1ed76", "type": "bank_slip", "created_at": "2022-04-16T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 6200} +{"id": "6000a90f-878c-451b-a762-acf5c7c77289", "type": "bank_slip", "created_at": "2022-03-24T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 6886} +{"id": "cae6cf8f-f871-4440-8405-8577c7f8d791", "type": "bank_slip", "created_at": "2022-11-08T00:00:00", "document": "46", "payer": "Shaunda Gallegos", "amount": 8513} +{"id": "945013f7-b0bd-4951-9f23-955f59cab2bc", "type": "credit", "created_at": "2022-12-07T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 3331} +{"id": "5ddcf2e4-c527-40b3-8899-2e3903fd79c4", "type": "credit", "created_at": "2022-03-02T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 9504} +{"id": "1972e75c-7207-4e2b-9e11-51eaceed0a38", "type": "debit", "created_at": "2022-09-06T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 5005} +{"id": "b3f87b17-9df1-437a-aaef-9265b98b2538", "type": "bank_slip", "created_at": "2022-11-19T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 6371} +{"id": "70b6acf2-c86f-4ffb-9420-4fcb1ab4f5cf", "type": "bank_slip", "created_at": "2022-07-29T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 8994} +{"id": "61fceb6b-dde9-4c66-b780-fbb42021bf2a", "type": "bank_slip", "created_at": "2022-09-11T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 8052} +{"id": "4e742551-ed93-402e-b7e2-0b6926e39c02", "type": "debit", "created_at": "2022-05-10T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 5976} +{"id": "174d38df-c0e0-4528-838f-62de8751f58f", "type": "debit", "created_at": "2022-08-05T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 825} +{"id": "281fb9fd-d293-4158-933e-e22a6ace2ad6", "type": "credit", "created_at": "2022-03-17T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 4068} +{"id": "874d6d43-878c-4b00-95bc-cbc4134247b8", "type": "debit", "created_at": "2022-08-05T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 299} +{"id": "bcf59d0a-2051-4efb-a654-0fc57b720790", "type": "bank_slip", "created_at": "2022-12-12T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 4436} +{"id": "96f4df1f-6f38-4a05-8f14-daf89eef4732", "type": "credit", "created_at": "2022-09-12T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 8003} +{"id": "cd30d475-751b-4ea3-8e8e-63703bc4b0f8", "type": "credit", "created_at": "2022-06-15T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 795} +{"id": "f82aec62-2bb1-49fa-baee-73085d95f2be", "type": "debit", "created_at": "2022-11-22T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 6967} +{"id": "b8eabddf-c2e3-4128-94b9-0627a9984c64", "type": "bank_slip", "created_at": "2022-04-25T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 1557} +{"id": "3d6f5b89-d3f5-4a31-acb7-412ac2f6ec2d", "type": "credit", "created_at": "2022-12-09T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 7450} +{"id": "de1a3f77-e4da-47aa-a1b2-d06f0b70b269", "type": "credit", "created_at": "2022-06-20T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 6650} +{"id": "92de1ed7-834d-4326-909f-cc04858a7d2a", "type": "debit", "created_at": "2022-09-07T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 6839} +{"id": "2b1700c0-4aee-4c1b-9adb-efd66c4c66be", "type": "debit", "created_at": "2022-06-02T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 6201} +{"id": "81e9896d-fbb2-4ccd-8ba3-82e93ddc9ba8", "type": "debit", "created_at": "2022-02-24T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 413} +{"id": "d69e01df-db4c-41a1-bd16-30b85b9afaac", "type": "bank_slip", "created_at": "2022-03-18T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 8230} +{"id": "66526dd9-14ff-4f6f-93e9-25bff28c4673", "type": "bank_slip", "created_at": "2022-08-26T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 5614} +{"id": "dd28ff56-db1b-40f4-9634-3b6249d4ead9", "type": "bank_slip", "created_at": "2022-04-16T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 7148} +{"id": "671a164b-72f5-4ad7-9d8d-c4231b1614a1", "type": "credit", "created_at": "2022-10-02T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 5898} +{"id": "56c50f35-5ce6-4e35-b086-10eb0e2c547a", "type": "bank_slip", "created_at": "2022-09-14T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 9324} +{"id": "d539ff03-875c-46e3-943e-23f4d597f95e", "type": "credit", "created_at": "2022-06-01T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 2084} +{"id": "a108ee67-56ca-4cfd-9545-10f4f0ba1ad2", "type": "debit", "created_at": "2022-05-20T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 1924} +{"id": "f94e04eb-f351-4de3-bc2b-a6f511b98673", "type": "bank_slip", "created_at": "2022-06-04T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 3248} +{"id": "e0554b28-8894-499d-843d-2588f815e22f", "type": "credit", "created_at": "2022-08-01T00:00:00", "document": "2", "payer": "Ella Tatum", "amount": 3240} +{"id": "b3583016-cd7f-4d56-b89e-edbd7d6c9a67", "type": "credit", "created_at": "2022-03-18T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 868} +{"id": "81191197-fda9-45c2-8f1b-89d997e9c541", "type": "debit", "created_at": "2022-02-13T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 9277} +{"id": "7e688a8a-4d7e-4e0a-9d85-28d748d460bf", "type": "credit", "created_at": "2022-11-24T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 1458} +{"id": "3c8c6c74-756c-407e-95bd-3f327b4fc2e5", "type": "bank_slip", "created_at": "2022-11-07T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 8079} +{"id": "7185488a-bd1e-4ca0-a9b3-ff54a7b696c1", "type": "debit", "created_at": "2022-06-21T00:00:00", "document": "65", "payer": "William Boxer", "amount": 7013} +{"id": "cae3f157-cfc8-43fd-98f1-a001d24e18f6", "type": "bank_slip", "created_at": "2022-12-04T00:00:00", "document": "38", "payer": "Sandra Gould", "amount": 9496} +{"id": "91e8146e-64a3-4a77-a665-edade9b83b8e", "type": "debit", "created_at": "2022-10-29T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 4049} +{"id": "bb1c1927-9023-42b1-851f-ee87db91fd69", "type": "credit", "created_at": "2022-03-03T00:00:00", "document": "57", "payer": "David Landry", "amount": 6480} +{"id": "35a4bd7d-c6af-4edd-8adc-b1415b964c5b", "type": "debit", "created_at": "2022-05-18T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 8914} +{"id": "11f5e727-aa77-49c4-bcb4-7fd914e11d53", "type": "bank_slip", "created_at": "2022-07-16T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 7814} +{"id": "3d2dc784-eefb-47f7-9eb0-9324909d1d31", "type": "credit", "created_at": "2022-06-12T00:00:00", "document": "93", "payer": "George Lesane", "amount": 6331} +{"id": "93846602-5234-4ed7-8275-762aa07aa507", "type": "debit", "created_at": "2022-09-01T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 8074} +{"id": "b32234bd-25f3-4d97-9c06-19c96126e4f8", "type": "bank_slip", "created_at": "2022-02-26T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 2045} +{"id": "1b2d7b36-ae8b-4c07-a08b-7b82975dcb7e", "type": "debit", "created_at": "2022-10-28T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 6388} +{"id": "9a25008e-92bf-43df-8b60-bf452c96f27c", "type": "bank_slip", "created_at": "2022-12-24T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 2249} +{"id": "589e81fe-1df0-4ed6-8d3a-a0bb4d21d966", "type": "bank_slip", "created_at": "2022-07-06T00:00:00", "document": "68", "payer": "Pauline Marcum", "amount": 8600} +{"id": "0d154442-0e4b-4853-8af3-d04500d683b0", "type": "debit", "created_at": "2022-02-27T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 7866} +{"id": "056cae01-c6e4-420a-9b9f-1b11976ca1ec", "type": "debit", "created_at": "2022-03-08T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 4245} +{"id": "686f011b-46bd-44ff-911f-2bf89ee9a9df", "type": "credit", "created_at": "2022-02-19T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 5758} +{"id": "50796dfb-e0ea-4b0d-aaf1-6b9447d509c0", "type": "debit", "created_at": "2022-10-26T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 270} +{"id": "0f839f6b-51ef-49ee-a51d-7403fc20227c", "type": "debit", "created_at": "2022-07-13T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 6472} +{"id": "0d58a904-b1e3-4d4b-85c3-0bb5cace2328", "type": "debit", "created_at": "2022-06-30T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 3911} +{"id": "5f10c1d3-44fd-4849-8351-9abd2f394931", "type": "credit", "created_at": "2022-05-01T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 7249} +{"id": "95eff0ee-dfef-43ef-a2df-4ca1a33c7d52", "type": "bank_slip", "created_at": "2022-12-11T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 6023} +{"id": "e86cbebd-2db2-41c9-8941-3a046f77bf78", "type": "bank_slip", "created_at": "2022-05-21T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 2102} +{"id": "27cbb55c-98cc-493d-bd7f-754e237f9a5d", "type": "debit", "created_at": "2022-12-01T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 3854} +{"id": "2198ec7b-7043-47ae-8603-2d24f6354dbe", "type": "credit", "created_at": "2022-04-17T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 2789} +{"id": "d456b85a-b5ec-42b9-886c-ae87ae90b07d", "type": "credit", "created_at": "2022-12-07T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 4779} +{"id": "bd6e862b-8a87-431f-945f-d465da0aa333", "type": "debit", "created_at": "2022-09-06T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 8759} +{"id": "4f769ff2-f90d-43a2-b5dd-3c457a6e7e22", "type": "credit", "created_at": "2022-01-17T00:00:00", "document": "33", "payer": "Charlotte Marczak", "amount": 1691} +{"id": "8d93986e-8e41-462b-8b79-c5b064709ec4", "type": "credit", "created_at": "2022-09-16T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 5124} +{"id": "dff818e6-3e79-44b5-ac04-faa8a3b250b9", "type": "bank_slip", "created_at": "2022-03-20T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 3167} +{"id": "b8657edb-d19b-4bb7-bc12-97ef17dfdf34", "type": "bank_slip", "created_at": "2022-12-23T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 6242} +{"id": "507b7db8-e752-4393-bdcc-6872e4f0c55e", "type": "credit", "created_at": "2022-04-08T00:00:00", "document": "24", "payer": "Susan Bostic", "amount": 1785} +{"id": "531060fb-0877-41cc-b5b3-c54ee2ec335b", "type": "debit", "created_at": "2022-08-10T00:00:00", "document": "37", "payer": "Jane Olson", "amount": 7241} +{"id": "d7d3bb6c-fc1c-47ab-a263-1d7cdf2b3f90", "type": "credit", "created_at": "2022-11-19T00:00:00", "document": "23", "payer": "Donald Lee", "amount": 1791} +{"id": "3c0218c1-ccb6-484a-bf4b-33f808006b6a", "type": "bank_slip", "created_at": "2022-08-01T00:00:00", "document": "2", "payer": "Ella Tatum", "amount": 6656} +{"id": "9e5c8c2a-f85f-4561-b1d2-417cf2ac626b", "type": "credit", "created_at": "2022-10-14T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 7351} +{"id": "a33a80ec-dda8-4dbd-ae48-acd59866e19a", "type": "bank_slip", "created_at": "2022-08-04T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 4291} +{"id": "fdd91e0f-8d3f-42e6-a05f-4711e1a4321f", "type": "debit", "created_at": "2022-03-13T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 8615} +{"id": "fcc99d46-0c90-44a2-9fb3-91b5fdadb409", "type": "bank_slip", "created_at": "2022-05-20T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 6819} +{"id": "44fe7e70-fe39-4387-aedd-a893e571a619", "type": "debit", "created_at": "2022-11-09T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 499} +{"id": "2033c783-0852-476d-9b09-100ec8e8e0e9", "type": "credit", "created_at": "2022-12-03T00:00:00", "document": "33", "payer": "Charlotte Marczak", "amount": 5115} +{"id": "abb0b84c-df6c-41ce-b3bd-cd8b3c8004c2", "type": "debit", "created_at": "2022-09-16T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 7134} +{"id": "371fb97f-ef8c-407f-95fa-1125111b06e2", "type": "bank_slip", "created_at": "2022-10-21T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 1229} +{"id": "7147e5d6-1fb3-427c-8553-2a1d67d048eb", "type": "bank_slip", "created_at": "2022-06-24T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 4844} +{"id": "3a8d43c1-9c50-462f-8041-878a85c02fa4", "type": "debit", "created_at": "2022-03-10T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 2794} +{"id": "0d4a985e-dd04-40aa-a7b0-3d308a3a4a5b", "type": "bank_slip", "created_at": "2022-06-09T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 272} +{"id": "d21ef233-3927-420d-8aed-cbbdaa0e0a71", "type": "bank_slip", "created_at": "2022-08-26T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 3472} +{"id": "100e2d0e-5129-42f4-baf0-5ed88a8aa3f4", "type": "credit", "created_at": "2022-01-08T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 7910} +{"id": "f73df99a-68ef-46a5-af2d-261657b318bb", "type": "credit", "created_at": "2022-11-09T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 7403} +{"id": "91f8f8c5-c46c-457d-9e9f-d251373fbc50", "type": "debit", "created_at": "2022-08-07T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 7117} +{"id": "de193bef-ba29-437a-9cc2-3d4941e4d5f3", "type": "debit", "created_at": "2022-10-04T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 216} +{"id": "d28196ea-02bf-42f9-826b-ff12edaffe06", "type": "bank_slip", "created_at": "2022-04-23T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 793} +{"id": "80a0e640-5f9d-4f30-9511-11d5d7d52719", "type": "debit", "created_at": "2022-05-11T00:00:00", "document": "78", "payer": "Paul debiten", "amount": 5580} +{"id": "d462c7a1-e780-4cb2-898d-f78e80775a54", "type": "bank_slip", "created_at": "2022-10-15T00:00:00", "document": "93", "payer": "George Lesane", "amount": 3000} +{"id": "35002dd9-6fb7-4913-894e-b5b5c8c4d352", "type": "debit", "created_at": "2022-09-10T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 7845} +{"id": "32390563-2173-449e-93f1-0657ba810ade", "type": "bank_slip", "created_at": "2022-08-16T00:00:00", "document": "61", "payer": "Ladonna Cooper", "amount": 8579} +{"id": "ba08e186-9dc7-4890-a38e-e8fba0fb0950", "type": "credit", "created_at": "2022-04-08T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 1813} +{"id": "dfedd01d-20ac-4ca3-a155-ed43ecb947d2", "type": "debit", "created_at": "2022-11-23T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 4845} +{"id": "bb9bc39f-6767-49c0-b6cc-49d88f70d300", "type": "credit", "created_at": "2022-10-15T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 8688} +{"id": "ade8abef-4292-4d78-9fc5-29518edbf3c0", "type": "credit", "created_at": "2022-08-05T00:00:00", "document": "65", "payer": "William Boxer", "amount": 8648} +{"id": "deb79c7b-b39d-4511-8852-03c3a2447cd5", "type": "debit", "created_at": "2022-05-23T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 5829} +{"id": "40e6de86-8803-49b5-8d70-53c476e869fb", "type": "credit", "created_at": "2022-11-07T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 9145} +{"id": "43553ccc-18cc-4f38-9bac-e20346c8a3b6", "type": "debit", "created_at": "2022-11-19T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 5335} +{"id": "6ecc9550-2a54-4cc7-9c2d-5054cd51dbf4", "type": "credit", "created_at": "2022-04-28T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 6657} +{"id": "a84fe461-732a-42de-83a8-498bc727176a", "type": "debit", "created_at": "2022-02-11T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 9496} +{"id": "6226c696-b995-4777-acfd-0f9cb55cc7a0", "type": "bank_slip", "created_at": "2022-12-01T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 5418} +{"id": "3c0e6bfc-7bc5-41a4-8ea6-5b09cd3ae919", "type": "debit", "created_at": "2022-07-28T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 4186} +{"id": "0aa46247-d07f-455b-8fa0-1e37009a5d17", "type": "credit", "created_at": "2022-07-02T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 2507} +{"id": "0005dde7-47cf-40df-a29a-8913f1ddede8", "type": "bank_slip", "created_at": "2022-10-11T00:00:00", "document": "24", "payer": "Susan Bostic", "amount": 1870} +{"id": "dcdf3b67-de56-4678-8258-b51ab9503175", "type": "credit", "created_at": "2022-04-24T00:00:00", "document": "61", "payer": "Ladonna Cooper", "amount": 2110} +{"id": "d1df5c89-9c94-42fd-bead-ef35b160fa25", "type": "debit", "created_at": "2022-05-23T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 7509} +{"id": "991f2326-06af-406d-a4c9-76ed6ca369b8", "type": "credit", "created_at": "2022-07-10T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 8614} +{"id": "30d2f2e7-4d6b-4176-86cd-3973adb72b4a", "type": "debit", "created_at": "2022-02-28T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 1436} +{"id": "df7bcbf4-a957-4eca-b09b-0501c9692378", "type": "credit", "created_at": "2022-07-06T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 7888} +{"id": "b3119075-f541-494d-a341-349f070c134e", "type": "bank_slip", "created_at": "2022-08-12T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 6377} +{"id": "a8f8640f-3569-48d9-bfd6-35caa52b564f", "type": "debit", "created_at": "2022-12-09T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 2914} +{"id": "288de595-555a-4462-a2d4-6ae185e7ea4a", "type": "bank_slip", "created_at": "2022-01-26T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 4945} +{"id": "5d325535-d498-4d19-af4e-e2f4739c6919", "type": "bank_slip", "created_at": "2022-09-10T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 1048} +{"id": "a960b3a2-3637-4a15-8a84-ae8002f641f6", "type": "bank_slip", "created_at": "2022-03-08T00:00:00", "document": "52", "payer": "Derek Corcoran", "amount": 79} +{"id": "267b99ae-a947-47b7-b5c4-ad3c9124c906", "type": "bank_slip", "created_at": "2022-03-25T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 2089} +{"id": "865f87de-f2ed-448b-a42b-97f742f3db53", "type": "debit", "created_at": "2022-09-21T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 8090} +{"id": "0b4da3a7-2ca3-4783-b1a6-acfe5954973f", "type": "bank_slip", "created_at": "2022-11-10T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 2647} +{"id": "a42b4caf-790a-4392-bf17-71cf499ba1b8", "type": "debit", "created_at": "2022-08-31T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 8100} +{"id": "f9092e68-be8c-4474-9378-005ea75e56e5", "type": "bank_slip", "created_at": "2022-05-20T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 3673} +{"id": "228c42d8-bf3a-414c-b3d5-e910b2c59eaf", "type": "bank_slip", "created_at": "2022-07-07T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 8590} +{"id": "2573250e-0171-43f9-acab-5c3a32bdfca7", "type": "debit", "created_at": "2022-05-07T00:00:00", "document": "79", "payer": "Steven Rapp", "amount": 6781} +{"id": "346dd59d-eb8f-4791-99a8-42df5bbe1af5", "type": "bank_slip", "created_at": "2022-03-09T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 316} +{"id": "4e87c320-cfa5-4d7b-8771-6b2fb9ca4035", "type": "debit", "created_at": "2022-02-17T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 1245} +{"id": "2a205cec-7c4c-42ee-bf15-7d7c24013485", "type": "bank_slip", "created_at": "2022-03-06T00:00:00", "document": "65", "payer": "William Boxer", "amount": 9582} +{"id": "ce659948-b47f-46f7-b00a-1bd8e9ca29cd", "type": "debit", "created_at": "2022-10-06T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 3537} +{"id": "c9c18fb7-e912-441f-8b35-a8d301c47f47", "type": "credit", "created_at": "2022-01-26T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 9514} +{"id": "84836abf-abbb-49b6-9989-8aa1699dc215", "type": "debit", "created_at": "2022-05-29T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 2790} +{"id": "0037af9a-24ce-4d11-9d72-fe409e919200", "type": "credit", "created_at": "2022-06-28T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 2014} +{"id": "3f560a93-9d04-45d6-b078-865700658c34", "type": "credit", "created_at": "2022-10-09T00:00:00", "document": "11", "payer": "Jenny Jensen", "amount": 1294} +{"id": "75377a17-6dbf-4732-9acc-1e3f49bb5c83", "type": "debit", "created_at": "2022-09-06T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 2709} +{"id": "11ff84ab-0d78-4e54-809a-cfa40598128d", "type": "credit", "created_at": "2022-06-08T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 2075} +{"id": "35e068ae-7561-4ed2-883a-cd81d342ae69", "type": "credit", "created_at": "2022-01-05T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 8302} +{"id": "622e9e7e-b231-4b77-9cb6-4a553cb69851", "type": "credit", "created_at": "2022-04-29T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 5416} +{"id": "5d0544bb-f9d9-42b7-bcd6-cc08bb4802da", "type": "bank_slip", "created_at": "2022-08-23T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 5993} +{"id": "3e3332e5-4bcc-4cee-95b9-d0d836b15c0f", "type": "credit", "created_at": "2022-01-28T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 6558} +{"id": "7ef8ebf3-228d-41ed-a8ec-4f34730bb000", "type": "debit", "created_at": "2022-03-04T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 5723} +{"id": "927a5ae9-4c82-48a7-8299-49066eee02a8", "type": "credit", "created_at": "2022-01-01T00:00:00", "document": "57", "payer": "David Landry", "amount": 9733} +{"id": "59f3c7e7-f47b-4eca-a529-5106073bab05", "type": "credit", "created_at": "2022-09-21T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 5590} +{"id": "c08612ce-1697-431c-9886-216616fad1e6", "type": "credit", "created_at": "2022-07-31T00:00:00", "document": "52", "payer": "Derek Corcoran", "amount": 9909} +{"id": "99efb935-f353-4327-803a-5eee67c23aa5", "type": "bank_slip", "created_at": "2022-12-19T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 8989} +{"id": "33ea5746-7a85-4397-a961-9e5466067fdf", "type": "credit", "created_at": "2022-05-19T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 2597} +{"id": "b44d6dc6-01ee-44f7-83b7-038e623c4c4c", "type": "credit", "created_at": "2022-01-03T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 6680} +{"id": "b8a8d195-79f2-4aeb-81bf-7fc6cba541b2", "type": "bank_slip", "created_at": "2022-12-29T00:00:00", "document": "46", "payer": "Shaunda Gallegos", "amount": 3360} +{"id": "4a00cb4f-7dbb-4b7d-9670-1dfad67c649b", "type": "debit", "created_at": "2022-05-26T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 9304} +{"id": "648cebc9-5c00-40d2-91dd-1b714b0e8800", "type": "debit", "created_at": "2022-02-13T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 7593} +{"id": "0b451864-adc8-4219-a4ae-5de3b432e025", "type": "debit", "created_at": "2022-01-01T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 3458} +{"id": "503f00fa-f849-407c-a91e-36658cb46182", "type": "debit", "created_at": "2022-03-20T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 9985} +{"id": "7cbf166c-11c3-42da-86bc-59341486e2f7", "type": "bank_slip", "created_at": "2022-01-25T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 1005} +{"id": "b5b80114-c8f5-48f1-b7fc-5c204ba9cdb8", "type": "debit", "created_at": "2022-05-31T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 7578} +{"id": "e207a460-7969-43e5-81aa-3e0a3a2677d9", "type": "debit", "created_at": "2022-11-11T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 7876} +{"id": "c526c7b4-f72a-4395-b47b-ef4596804a75", "type": "debit", "created_at": "2022-05-13T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 1495} +{"id": "eb7a22a9-eb3f-47a4-be76-b2786b020181", "type": "credit", "created_at": "2022-12-14T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 6318} +{"id": "73c89ff5-5dbb-4aaf-96c4-dd9353cdccf6", "type": "bank_slip", "created_at": "2022-08-09T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 4258} +{"id": "4837c455-ccfe-40c9-9151-89a64b39dde9", "type": "credit", "created_at": "2022-09-04T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 9160} +{"id": "da5f256d-0cf2-4e80-9c12-e3c029a12aeb", "type": "bank_slip", "created_at": "2022-10-25T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 6511} +{"id": "737ea891-a05b-49e6-8911-3a7f62f5317d", "type": "bank_slip", "created_at": "2022-01-26T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 509} +{"id": "db0f5de4-1a29-4d62-8c85-fa7c7e577da3", "type": "credit", "created_at": "2022-10-09T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 2057} +{"id": "5f36550a-c1f8-4998-a24f-ab38588e6c79", "type": "bank_slip", "created_at": "2022-06-23T00:00:00", "document": "68", "payer": "Pauline Marcum", "amount": 3069} +{"id": "9ab7914c-37a1-4013-8148-569b3eb5b135", "type": "debit", "created_at": "2022-05-18T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 2220} +{"id": "10186da8-e723-4739-a5a5-4de409038c3f", "type": "credit", "created_at": "2022-11-26T00:00:00", "document": "60", "payer": "Rachel York", "amount": 8861} +{"id": "7a6cacb5-fede-47b6-aa5d-6b8f35dd5dd7", "type": "debit", "created_at": "2022-06-03T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 7028} +{"id": "c285401b-dbbd-4e36-a093-61d1f0c1fab4", "type": "bank_slip", "created_at": "2022-01-08T00:00:00", "document": "93", "payer": "George Lesane", "amount": 4485} +{"id": "533b2039-82b3-4c00-adcc-22af9342d201", "type": "debit", "created_at": "2022-08-28T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 7084} +{"id": "dc62e67b-6242-45ef-a19d-e94fb1bf4382", "type": "credit", "created_at": "2022-02-02T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 4334} +{"id": "c3d6aaa8-95c9-4d85-bccb-71d16bd608ea", "type": "debit", "created_at": "2022-02-01T00:00:00", "document": "79", "payer": "Steven Rapp", "amount": 5340} +{"id": "2fe1fc6f-4755-4783-b60f-647353ae698a", "type": "debit", "created_at": "2022-09-07T00:00:00", "document": "60", "payer": "Rachel York", "amount": 5169} +{"id": "22d0b3f6-a5fd-49ea-bde1-49ee5df594c1", "type": "credit", "created_at": "2022-02-26T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 6836} +{"id": "fd109356-478f-47f3-9a4a-faa0d8bce7db", "type": "bank_slip", "created_at": "2022-05-10T00:00:00", "document": "33", "payer": "Charlotte Marczak", "amount": 1032} +{"id": "d3f68c72-e244-4c3e-9c59-ad6bb768b896", "type": "credit", "created_at": "2022-10-08T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 6481} +{"id": "d9ef7da1-3e22-42e0-a5f7-42f905c6d6f9", "type": "debit", "created_at": "2022-07-26T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 1810} +{"id": "b788ae42-4561-4f29-bc85-ce754805843f", "type": "bank_slip", "created_at": "2022-03-23T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 1311} +{"id": "7ad4808a-62c8-4aed-b3f6-e8dc2ba6cf4f", "type": "debit", "created_at": "2022-03-14T00:00:00", "document": "46", "payer": "Shaunda Gallegos", "amount": 7164} +{"id": "fbd50f3d-11b4-4b4d-86a3-8bf18441e8fc", "type": "bank_slip", "created_at": "2022-03-12T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 1128} +{"id": "cbaaca5f-25d6-4af0-9cc4-072da3ab8a51", "type": "debit", "created_at": "2022-05-11T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 1939} +{"id": "d3408fde-d404-4be6-9a54-97a9c31cc2a7", "type": "credit", "created_at": "2022-05-10T00:00:00", "document": "43", "payer": "Gary Jensen", "amount": 4497} +{"id": "c164899d-95ae-4851-b5e0-1c47953f0862", "type": "debit", "created_at": "2022-06-22T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 849} +{"id": "a1f76f23-7168-4875-9f09-8a8cb69e7b68", "type": "debit", "created_at": "2022-08-25T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 237} +{"id": "1f4a8f16-87bf-4f08-b942-444234b1bf79", "type": "debit", "created_at": "2022-05-15T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 6226} +{"id": "789aaea9-28ec-44dd-b8ae-37cb2ad46a97", "type": "credit", "created_at": "2023-01-01T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 5453} +{"id": "0163d1db-43c0-4f06-ad5e-b062ebda3c29", "type": "credit", "created_at": "2022-02-27T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 6678} +{"id": "6d9f8880-6a9d-4a87-813a-ca8fb19fd1c9", "type": "credit", "created_at": "2022-08-19T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 3265} +{"id": "fc5d2197-182f-4541-8084-94e86a5a4641", "type": "credit", "created_at": "2022-10-02T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 9599} +{"id": "f0e65c3b-b334-4e33-be89-8ac0e48804e6", "type": "debit", "created_at": "2022-12-09T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 3801} +{"id": "ab4b5d55-6cfa-41f8-bf0f-73d173a97918", "type": "bank_slip", "created_at": "2022-04-13T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 3963} +{"id": "310c0f54-ddc1-4b5b-848c-e22f0771c6d1", "type": "debit", "created_at": "2022-10-16T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 5987} +{"id": "a6b8d7d9-6d1c-4761-9d3c-3c3997076f7d", "type": "bank_slip", "created_at": "2022-01-30T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 8996} +{"id": "a4431727-2e71-4696-a618-1d38fcdeebd3", "type": "bank_slip", "created_at": "2022-11-21T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 2534} +{"id": "ca64b80b-c562-4dfe-85c9-53e3ed70c12b", "type": "debit", "created_at": "2022-11-10T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 8501} +{"id": "eb030a2d-1317-42c7-b6b8-056d562e699d", "type": "credit", "created_at": "2022-07-19T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 558} +{"id": "34f083ff-0eb3-4f41-93f6-18db7a52fa83", "type": "debit", "created_at": "2022-05-01T00:00:00", "document": "78", "payer": "Paul debiten", "amount": 3405} +{"id": "192544d3-ba5b-43b6-accf-e2e167d60942", "type": "credit", "created_at": "2022-08-23T00:00:00", "document": "61", "payer": "Ladonna Cooper", "amount": 9446} +{"id": "34cf3c42-8d2d-484f-9478-a3063783a536", "type": "credit", "created_at": "2022-12-10T00:00:00", "document": "38", "payer": "Sandra Gould", "amount": 6161} +{"id": "5a69e9a5-b6b1-4917-8a91-54aaa79b2ddd", "type": "bank_slip", "created_at": "2022-03-28T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 3896} +{"id": "59706a44-e8d3-4016-8c67-613d037a9daf", "type": "debit", "created_at": "2022-06-30T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 428} +{"id": "4f947780-8369-453d-b6f7-11818e202cb1", "type": "debit", "created_at": "2022-03-22T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 1433} +{"id": "d955296f-d669-4b58-ae97-0852004dcf71", "type": "debit", "created_at": "2022-06-28T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 6726} +{"id": "c302f1d4-add9-452b-ad02-6f1af6f79afe", "type": "credit", "created_at": "2022-06-25T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 9607} +{"id": "22c1f659-2116-43d7-ac63-b5d123859ea0", "type": "bank_slip", "created_at": "2022-11-11T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 6463} +{"id": "57cb714f-e967-473e-a25f-360e1ded0571", "type": "debit", "created_at": "2022-10-21T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 4374} +{"id": "58afc7b8-17bf-45d7-8f98-a2b6e5740db4", "type": "debit", "created_at": "2022-09-11T00:00:00", "document": "37", "payer": "Jane Olson", "amount": 268} +{"id": "6aa09d08-5076-490e-a461-f025ea5d29a8", "type": "debit", "created_at": "2022-08-03T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 5294} +{"id": "9236bf26-07a7-4ede-ad2f-0e90578c1002", "type": "credit", "created_at": "2022-06-22T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 7257} +{"id": "e22cf846-01a4-415c-b08b-8ee8798d9f86", "type": "bank_slip", "created_at": "2022-01-04T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 2249} +{"id": "d425ee69-7081-4b8d-8076-7ff8fe9e3ecd", "type": "debit", "created_at": "2022-08-27T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 5683} +{"id": "c1c8bacd-8733-4d8a-bd8d-f8816ccc851c", "type": "credit", "created_at": "2022-06-23T00:00:00", "document": "2", "payer": "Ella Tatum", "amount": 6650} +{"id": "eb61dd3e-dfa9-42c4-b9f2-6144f28080d2", "type": "debit", "created_at": "2022-12-21T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 903} +{"id": "762bd62f-5e31-4740-a0c5-ecd412e502db", "type": "debit", "created_at": "2022-05-17T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 7000} +{"id": "5406d961-d033-434f-a968-2e6f29a78184", "type": "bank_slip", "created_at": "2022-08-15T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 8355} +{"id": "79ce5093-8589-4886-b7ff-375dc85e2a32", "type": "debit", "created_at": "2022-08-20T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 7763} +{"id": "00e6c810-b055-4a3f-a9f2-12cbe7000eb2", "type": "bank_slip", "created_at": "2022-08-08T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 9171} +{"id": "1276c8ba-7b3a-46ca-b58e-b6a4c9b22930", "type": "credit", "created_at": "2022-06-06T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 798} +{"id": "a491718b-4903-46a4-91e7-467fcd98807b", "type": "debit", "created_at": "2022-02-23T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 1224} +{"id": "50390dff-40ff-4423-ba5d-edd72e97bd13", "type": "debit", "created_at": "2022-08-23T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 4977} +{"id": "f3f80e9b-5577-49eb-9260-3820129cab23", "type": "debit", "created_at": "2022-10-28T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 2164} +{"id": "dc108a63-d818-4f6f-b9bb-688dbf1ae90d", "type": "credit", "created_at": "2022-10-12T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 2873} +{"id": "66eef8d0-e53f-417e-a256-55aca06b2b68", "type": "debit", "created_at": "2022-03-30T00:00:00", "document": "60", "payer": "Rachel York", "amount": 8929} +{"id": "3d2a3252-9803-44ed-a476-24824b3f2e62", "type": "debit", "created_at": "2022-05-24T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 5696} +{"id": "946e64c7-227b-4d95-b1a9-7ed42932ec9c", "type": "bank_slip", "created_at": "2022-05-03T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 7103} +{"id": "a9a1f6ed-7003-4bf4-aeff-5b4ba425f59d", "type": "debit", "created_at": "2022-11-16T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 854} +{"id": "2a8474bf-bcb9-475d-968e-470c86a1cb47", "type": "debit", "created_at": "2022-04-05T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 4596} +{"id": "2a1d1577-0a20-4aa9-8a1b-2dca9230adbf", "type": "bank_slip", "created_at": "2022-12-17T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 538} +{"id": "656250f0-3df6-4755-a22e-bff8c494edd9", "type": "bank_slip", "created_at": "2022-12-24T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 9178} +{"id": "531901f8-9015-4a7f-a882-06dd525e563a", "type": "bank_slip", "created_at": "2022-05-13T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 9020} +{"id": "b5031d15-8aef-4c4e-908d-c20250b5219f", "type": "credit", "created_at": "2022-05-26T00:00:00", "document": "93", "payer": "George Lesane", "amount": 9809} +{"id": "0449d399-c41d-4c5d-bde9-f764b1a83447", "type": "credit", "created_at": "2022-07-24T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 3559} +{"id": "bf85aaa5-59c9-4610-9fcf-e963f025c774", "type": "credit", "created_at": "2022-09-23T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 6816} +{"id": "abca1f53-ff3c-470f-aac1-be4340a34ca2", "type": "debit", "created_at": "2022-06-05T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 4290} +{"id": "12bb90ec-31f0-47b7-b48c-3898a8c23eb3", "type": "credit", "created_at": "2022-10-08T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 3790} +{"id": "44f2077d-8562-4290-b1eb-3d1ad54b5926", "type": "bank_slip", "created_at": "2022-12-16T00:00:00", "document": "60", "payer": "Rachel York", "amount": 3964} +{"id": "38d82d54-2bf2-4928-a140-19cfd1bb5c31", "type": "credit", "created_at": "2022-01-30T00:00:00", "document": "24", "payer": "Susan Bostic", "amount": 2910} +{"id": "6fd2d0f2-8f73-41f1-944e-af779479d6b3", "type": "bank_slip", "created_at": "2022-08-05T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 8928} +{"id": "5d36bb22-b6d2-46cb-a4bd-62fbfa5af599", "type": "credit", "created_at": "2022-04-05T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 6257} +{"id": "afcd3409-fc14-4117-8ef4-6fc36e5e6f22", "type": "debit", "created_at": "2022-09-25T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 2461} +{"id": "2f74997e-ec59-4a65-bb5b-2bfb01d5fed4", "type": "bank_slip", "created_at": "2022-05-01T00:00:00", "document": "23", "payer": "Donald Lee", "amount": 1025} +{"id": "4c6468e7-452a-48d0-a1ac-2e9a01756c23", "type": "bank_slip", "created_at": "2022-06-13T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 7375} +{"id": "de2c714a-da40-4c34-a1c6-a938d305aef0", "type": "credit", "created_at": "2022-08-18T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 7779} +{"id": "24d34c2d-65d3-42f0-abbe-727290b56592", "type": "credit", "created_at": "2022-03-14T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 4503} +{"id": "c2702ab3-a1e7-4aeb-80d6-825a169bc1e9", "type": "debit", "created_at": "2022-06-06T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 6249} +{"id": "2d10c65c-9953-46a8-a164-f23b5d6134df", "type": "credit", "created_at": "2023-01-01T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 2646} +{"id": "63494b82-a4d7-42c9-aaba-e4ca7e5ccc80", "type": "debit", "created_at": "2022-10-27T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 6629} +{"id": "a6fe0e88-b5b9-4b31-8f9d-083d5c08f42c", "type": "credit", "created_at": "2022-11-20T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 9950} +{"id": "f8ab30d9-8427-4e13-a348-10dc761890bc", "type": "debit", "created_at": "2022-04-02T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 7211} +{"id": "117606af-db73-4266-b3be-f604ce2dcb0e", "type": "credit", "created_at": "2022-07-15T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 3835} +{"id": "2df45f39-90f0-4ce6-8e46-3cbef2fb5263", "type": "debit", "created_at": "2022-08-08T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 6504} +{"id": "40ac3296-355d-4c51-849c-5060ff087318", "type": "debit", "created_at": "2022-11-02T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 996} +{"id": "1f07e48c-0021-4e9e-bf17-36cca857de51", "type": "credit", "created_at": "2022-09-20T00:00:00", "document": "68", "payer": "Pauline Marcum", "amount": 4689} +{"id": "cc2f2e10-3423-4758-bbed-7c54e0218f94", "type": "debit", "created_at": "2022-11-11T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 1165} +{"id": "1777e2c7-f425-4842-b1fb-fb036f62d464", "type": "debit", "created_at": "2022-12-01T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 5129} +{"id": "9e759bad-5611-41c8-ba0a-0619f70f091b", "type": "debit", "created_at": "2022-06-28T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 1319} +{"id": "23c7eafc-2778-42bc-a296-1084a6a0374a", "type": "credit", "created_at": "2022-03-16T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 2647} +{"id": "04fb8785-a717-48fd-a8d5-4f55124c9255", "type": "bank_slip", "created_at": "2022-02-08T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 6687} +{"id": "aa5ee73f-67a2-4b45-a7e0-b885d96455dd", "type": "bank_slip", "created_at": "2022-04-11T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 7664} +{"id": "1d61824f-aff9-453a-99e0-af4f7c43541b", "type": "bank_slip", "created_at": "2022-12-28T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 6908} +{"id": "e6c99ea2-1581-42e8-8583-b9fda9ac7f60", "type": "credit", "created_at": "2022-10-25T00:00:00", "document": "37", "payer": "Jane Olson", "amount": 1718} +{"id": "9519d04f-5d16-424d-b167-0994187416d7", "type": "debit", "created_at": "2022-09-17T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 6916} +{"id": "e268fdb5-0a3a-49ab-b26a-c9c1cf300152", "type": "credit", "created_at": "2022-07-07T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 6558} +{"id": "f321ce19-4a88-4337-8188-cb7346fedb2f", "type": "credit", "created_at": "2022-04-19T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 7652} +{"id": "7d3b90ab-7cdb-4556-8df3-b35172cc06c8", "type": "debit", "created_at": "2022-07-20T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 7401} +{"id": "1e212636-5c17-48fd-9994-9e3ae34fecb0", "type": "credit", "created_at": "2022-08-04T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 8733} +{"id": "bb085244-0c29-441b-8784-222fc03adfd6", "type": "credit", "created_at": "2022-05-13T00:00:00", "document": "50", "payer": "Karen Lewis", "amount": 9688} +{"id": "5699fd40-7e2f-4907-9e98-d05f3b51ac55", "type": "bank_slip", "created_at": "2022-06-30T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 9716} +{"id": "7eddd545-fdec-4fbd-a47f-e4a1d3bc8f73", "type": "credit", "created_at": "2022-04-15T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 9609} +{"id": "f246362e-c8d4-4b33-9020-93d81da42818", "type": "credit", "created_at": "2022-09-01T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 6355} +{"id": "aaf9a491-b8bd-44b4-b719-d0c5d822633b", "type": "credit", "created_at": "2022-07-19T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 902} +{"id": "98db7397-808f-4f1c-b672-8a8881cbdd89", "type": "debit", "created_at": "2022-02-01T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 5368} +{"id": "3d3c536b-b7fc-4811-ad3b-1ee0d77102aa", "type": "debit", "created_at": "2022-02-24T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 5060} +{"id": "0d5a58b6-5769-4ffb-8bc1-a90e0c5ca414", "type": "debit", "created_at": "2022-02-20T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 3770} +{"id": "6e506e6e-e92f-490b-87be-1ea803509a7a", "type": "credit", "created_at": "2022-03-19T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 8861} +{"id": "41acdb82-e2c2-4409-a3f7-8d24c6ea2fdf", "type": "credit", "created_at": "2022-02-21T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 7142} +{"id": "3718e100-f711-4e1c-8332-2b13d85910ae", "type": "debit", "created_at": "2022-07-31T00:00:00", "document": "38", "payer": "Sandra Gould", "amount": 8402} +{"id": "17ce8de1-e96c-4ae9-b60a-a253cf722e60", "type": "debit", "created_at": "2022-12-10T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 4544} +{"id": "79e87a83-cff3-4c2d-8c88-21b406ac99e0", "type": "debit", "created_at": "2022-02-10T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 9989} +{"id": "d165c4a5-f0cb-4a42-a2a3-58f5d221d798", "type": "debit", "created_at": "2022-06-18T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 8410} +{"id": "c3724bb0-53d0-4b3f-9b96-2aac75f2861f", "type": "debit", "created_at": "2022-03-18T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 6472} +{"id": "557aa8a4-bc67-4642-adff-95a089d2f96b", "type": "credit", "created_at": "2022-02-13T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 1674} +{"id": "349286e0-bae2-4141-9c65-78ea5ae3b3b2", "type": "bank_slip", "created_at": "2022-08-20T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 7785} +{"id": "90f0bfcb-4aab-430d-8233-1e4199103824", "type": "credit", "created_at": "2022-10-11T00:00:00", "document": "78", "payer": "Paul debiten", "amount": 47} +{"id": "19d20a80-3be0-45b7-abae-df66e946e01b", "type": "debit", "created_at": "2022-02-16T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 5586} +{"id": "28e3e82a-6d57-4198-b17c-eb921ebb36bd", "type": "credit", "created_at": "2022-09-08T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 9965} +{"id": "29cbb909-15e3-4726-9cf4-f04582b076b6", "type": "bank_slip", "created_at": "2022-10-12T00:00:00", "document": "61", "payer": "Ladonna Cooper", "amount": 4119} +{"id": "522d1d42-8c25-4a4e-980b-e600ea74ec94", "type": "debit", "created_at": "2022-06-06T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 5654} +{"id": "19992f8f-4f17-4adf-8f80-c0c307a4c7c0", "type": "credit", "created_at": "2022-02-22T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 3872} +{"id": "00eb60ff-b689-47ea-90eb-2d50051bfe15", "type": "bank_slip", "created_at": "2022-11-11T00:00:00", "document": "28", "payer": "Mark Williams", "amount": 9296} +{"id": "d391c308-e9bd-4421-9a37-c14248b21d27", "type": "debit", "created_at": "2022-09-20T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 1528} +{"id": "00f15b85-e823-4d7b-a29d-b4f3111ecc27", "type": "credit", "created_at": "2022-02-27T00:00:00", "document": "49", "payer": "Richard Isbell", "amount": 3926} +{"id": "1562af89-2889-42ac-a35e-323f34e31640", "type": "credit", "created_at": "2022-05-14T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 3813} +{"id": "45826702-9787-4ac4-9b0a-566a9b6a2c8b", "type": "credit", "created_at": "2022-01-15T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 8450} +{"id": "d45a6fd6-4c9c-4dbc-8811-45d7894e7afa", "type": "credit", "created_at": "2022-07-26T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 7207} +{"id": "25f3d576-9bb7-49dc-9e96-cd88c0eb7c8c", "type": "credit", "created_at": "2022-05-25T00:00:00", "document": "23", "payer": "Donald Lee", "amount": 7086} +{"id": "1cf0a46d-21e7-4c45-8324-c396ce367b5c", "type": "debit", "created_at": "2022-03-18T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 6041} +{"id": "2385f28c-f4a9-43d3-9c00-6809696028b4", "type": "credit", "created_at": "2022-03-18T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 9156} +{"id": "62a94903-3788-4fa3-af82-25a2e1ac7404", "type": "credit", "created_at": "2022-06-27T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 8106} +{"id": "2dd57903-c970-4bea-9386-d5f46e90ae78", "type": "bank_slip", "created_at": "2022-08-15T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 7265} +{"id": "013ac330-a41b-4c86-a90c-0654695b3077", "type": "bank_slip", "created_at": "2022-08-04T00:00:00", "document": "93", "payer": "George Lesane", "amount": 5323} +{"id": "b51d4144-0005-4fa8-bf0c-c4c02d2fee1a", "type": "credit", "created_at": "2022-05-14T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 6230} +{"id": "e5b0cd99-d788-469b-b474-ffe18cf024b3", "type": "bank_slip", "created_at": "2022-08-22T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 965} +{"id": "abd8a33d-6f78-4d8a-aa6f-415344e863de", "type": "bank_slip", "created_at": "2022-05-24T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 4782} +{"id": "fb2ac759-3bb0-4133-9c4c-35c1a2b6cdf5", "type": "debit", "created_at": "2022-08-16T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 5046} +{"id": "d864c4f6-4417-47e9-977b-aa93e6f58a33", "type": "bank_slip", "created_at": "2022-09-11T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 1768} +{"id": "e7d15df8-67a9-43a3-bab1-cbb1a9ab1ea3", "type": "credit", "created_at": "2022-03-05T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 1450} +{"id": "b4bd4513-a269-40c6-bd61-b217b4745147", "type": "bank_slip", "created_at": "2022-12-20T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 3111} +{"id": "11595f7d-6e8f-4b9c-9890-c5fde6603015", "type": "credit", "created_at": "2022-07-10T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 7198} +{"id": "98222e26-74e6-4f51-9df3-8ee9a70606dd", "type": "debit", "created_at": "2022-02-22T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 8113} +{"id": "aa6463b9-8aae-439f-8cdb-fbb976d94804", "type": "debit", "created_at": "2022-11-11T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 7738} +{"id": "cf48e80b-7680-42af-b16e-1853cbecf8e8", "type": "bank_slip", "created_at": "2022-10-13T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 6488} +{"id": "64e199d3-57e6-4bbb-81e2-92374b967c05", "type": "debit", "created_at": "2022-11-12T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 6869} +{"id": "e99f4ad1-cc55-4e3f-b17b-f0b945c1c239", "type": "debit", "created_at": "2022-06-29T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 9402} +{"id": "fd3238b0-6034-4e5a-885a-6801533e2123", "type": "debit", "created_at": "2022-04-27T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 6024} +{"id": "d7e3a9e9-4b32-4c90-b751-33219a036030", "type": "debit", "created_at": "2022-03-10T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 3974} +{"id": "f5fd4419-5381-4ebf-abfa-81ec7065246e", "type": "bank_slip", "created_at": "2022-09-28T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 7297} +{"id": "c4947e8e-8bdb-44eb-b4f8-6f34e41330e4", "type": "debit", "created_at": "2022-12-31T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 8936} +{"id": "3b062e32-5618-4f0a-a46d-12e4d5dabcc4", "type": "debit", "created_at": "2022-12-31T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 3639} +{"id": "fcc7a6d6-766c-4fd4-9f54-a4a5352141bb", "type": "bank_slip", "created_at": "2022-01-03T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 7830} +{"id": "1590a0d0-62ff-49b1-8d0a-e7b75fbc2c89", "type": "bank_slip", "created_at": "2022-01-05T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 2606} +{"id": "ed5e55eb-42b2-4c50-9b6a-82025d4d9e35", "type": "debit", "created_at": "2022-09-08T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 8587} +{"id": "cc2f69d1-1d63-43b2-920b-9afb491045b1", "type": "credit", "created_at": "2022-04-16T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 5193} +{"id": "840a3b1b-0868-4c12-940c-0387b61cc02b", "type": "credit", "created_at": "2022-02-03T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 5952} +{"id": "3909d050-1960-4022-a7c8-b715a9cefe7a", "type": "debit", "created_at": "2022-06-20T00:00:00", "document": "60", "payer": "Rachel York", "amount": 9118} +{"id": "5fa538ee-d98c-49c2-9acb-dd42b7677d50", "type": "credit", "created_at": "2022-12-20T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 1162} +{"id": "23753b57-f76c-44ca-8a45-445e99377ad9", "type": "debit", "created_at": "2022-10-26T00:00:00", "document": "46", "payer": "Shaunda Gallegos", "amount": 2740} +{"id": "2ee8852c-2030-4631-9d01-a87fe70f4139", "type": "debit", "created_at": "2022-01-11T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 2489} +{"id": "c5a6bea0-7614-469a-bbdb-aef8eb78b657", "type": "credit", "created_at": "2022-04-28T00:00:00", "document": "93", "payer": "George Lesane", "amount": 7877} +{"id": "f47883f0-3464-47de-bebe-c2f4354ca3b0", "type": "credit", "created_at": "2022-02-27T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 8196} +{"id": "e25d96fa-ab8b-49c5-84cf-aac78c20985e", "type": "bank_slip", "created_at": "2022-08-29T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 7450} +{"id": "31345723-2f59-4631-9517-6e1221557f28", "type": "bank_slip", "created_at": "2022-11-12T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 4636} +{"id": "44d873e5-310f-4418-ab42-81ea7ca839e8", "type": "credit", "created_at": "2022-01-09T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 5998} +{"id": "dce99128-ead3-4053-aea3-ebe25fb41317", "type": "debit", "created_at": "2022-10-14T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 622} +{"id": "430fa3d7-781e-4898-8a8c-749adeb7aacc", "type": "credit", "created_at": "2022-11-24T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 1025} +{"id": "40033358-b3cb-4c7c-906b-007ea363de4a", "type": "bank_slip", "created_at": "2022-04-05T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 8828} +{"id": "55b1ed04-6869-42f5-920b-a78bc146111f", "type": "bank_slip", "created_at": "2022-01-07T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 859} +{"id": "16ece1db-6183-4f5e-a658-885171fff708", "type": "bank_slip", "created_at": "2022-11-04T00:00:00", "document": "50", "payer": "Karen Lewis", "amount": 2540} +{"id": "5475a970-cf88-42da-adf5-d3375eb42ce1", "type": "credit", "created_at": "2022-12-31T00:00:00", "document": "28", "payer": "Mark Williams", "amount": 3826} +{"id": "b27401f5-7d8d-45d9-b463-013b54d64c6b", "type": "bank_slip", "created_at": "2022-09-04T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 1180} +{"id": "241b425f-af21-412b-8cc1-11022e2626fc", "type": "bank_slip", "created_at": "2022-12-14T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 416} +{"id": "7f0e0089-889f-4550-bd10-700b31d6015f", "type": "bank_slip", "created_at": "2022-01-14T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 6309} +{"id": "a0452c91-fcc8-48ee-8599-cb06d3958417", "type": "debit", "created_at": "2022-11-09T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 4700} +{"id": "becf9308-6e64-4967-9e9f-67d1d6ac2d2b", "type": "bank_slip", "created_at": "2022-06-20T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 5194} +{"id": "26527782-5245-45d1-93a1-6ed47acd3bbd", "type": "bank_slip", "created_at": "2022-03-25T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 4629} +{"id": "4c213e1e-4f76-4c4d-a243-102d779fb895", "type": "bank_slip", "created_at": "2022-04-03T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 3538} +{"id": "0a497b85-6442-4898-acbc-6d33edf9440d", "type": "debit", "created_at": "2022-12-29T00:00:00", "document": "60", "payer": "Rachel York", "amount": 620} +{"id": "96ae15a4-da30-4244-93da-8372caee9abf", "type": "credit", "created_at": "2022-01-17T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 400} +{"id": "fedfadd1-533f-43e4-96b5-8ccb493699f8", "type": "credit", "created_at": "2022-10-08T00:00:00", "document": "24", "payer": "Susan Bostic", "amount": 5925} +{"id": "1d29fa59-b09a-4004-affa-9372ef58fbb6", "type": "bank_slip", "created_at": "2022-03-06T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 6904} +{"id": "723a6c3f-d4ef-4372-9546-df67eb6b839a", "type": "credit", "created_at": "2022-02-02T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 58} +{"id": "9eb8d5f1-560e-45df-93d0-0ff70b2fc306", "type": "credit", "created_at": "2022-07-30T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 1713} +{"id": "55d84a66-73fe-4f70-8c05-924dbf8e97c3", "type": "credit", "created_at": "2022-11-15T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 205} +{"id": "4abe59ef-9945-4324-8dec-98c4d086d083", "type": "credit", "created_at": "2022-12-23T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 4083} +{"id": "0142396d-b304-4959-a954-ab00a21ac21c", "type": "debit", "created_at": "2022-10-02T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 289} +{"id": "0a61dbaa-af99-463a-b1de-eb31ad0c00dc", "type": "bank_slip", "created_at": "2022-04-27T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 6023} +{"id": "6263f26e-2f0d-4d07-aa5e-05db60176e7f", "type": "credit", "created_at": "2022-06-09T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 377} +{"id": "2ed5b3e9-b1ab-4efb-a48b-3dc7795171e0", "type": "bank_slip", "created_at": "2022-04-16T00:00:00", "document": "42", "payer": "Carolyn Ponyah", "amount": 6562} +{"id": "c5eac5f5-f374-458b-b5c4-376d15c8fc66", "type": "debit", "created_at": "2022-01-23T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 413} +{"id": "383e9303-2cd3-424a-b7f5-b43cdf72c30a", "type": "credit", "created_at": "2022-05-22T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 8838} +{"id": "41894359-97c3-4852-b7bc-b55823bc9b06", "type": "debit", "created_at": "2022-11-20T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 4327} +{"id": "6407d992-3548-4818-9bb1-ba3196d61e7f", "type": "credit", "created_at": "2022-12-04T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 8916} +{"id": "72890e0d-4558-4b56-b8db-793b9c8bb3eb", "type": "credit", "created_at": "2022-02-11T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 2475} +{"id": "8197afca-940a-4cd5-a6f3-9ab380108f65", "type": "debit", "created_at": "2022-01-30T00:00:00", "document": "46", "payer": "Shaunda Gallegos", "amount": 5851} +{"id": "54db7fef-93c2-4b2a-b3d3-954999c3b5d8", "type": "bank_slip", "created_at": "2022-06-26T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 3249} +{"id": "e69ad003-1071-4b76-b400-c078173d0a70", "type": "bank_slip", "created_at": "2022-02-11T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 5876} +{"id": "59f025ef-967b-453f-b949-97bc7e15b7df", "type": "credit", "created_at": "2022-08-04T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 7786} +{"id": "a2c31c95-6454-4ff0-a350-0803bd11ff3e", "type": "credit", "created_at": "2022-05-11T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 7494} +{"id": "04e65d31-c96c-46fe-b307-5a39306f97fd", "type": "credit", "created_at": "2022-04-07T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 6376} +{"id": "e6d592fd-d201-4a16-9e74-1ff1e80a0c58", "type": "bank_slip", "created_at": "2022-03-10T00:00:00", "document": "82", "payer": "Juan Federico", "amount": 5609} +{"id": "230c88a9-c2f1-4c58-905a-4f1ee8a6c953", "type": "credit", "created_at": "2022-11-17T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 8453} +{"id": "e6a20a5d-b24c-42a8-a946-39dd51340c18", "type": "credit", "created_at": "2022-02-08T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 6165} +{"id": "3bf419f8-2baa-4afb-82bd-745cee7e0391", "type": "bank_slip", "created_at": "2022-01-06T00:00:00", "document": "93", "payer": "George Lesane", "amount": 2272} +{"id": "6c3340a9-06f7-4277-ad1b-961184606b58", "type": "credit", "created_at": "2022-08-07T00:00:00", "document": "57", "payer": "David Landry", "amount": 8608} +{"id": "015410ff-7ca4-4e99-a74a-919f4585b7fd", "type": "debit", "created_at": "2022-07-28T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 3082} +{"id": "27b1eece-c8d0-4ec3-896f-35d00d7903c0", "type": "bank_slip", "created_at": "2022-06-18T00:00:00", "document": "24", "payer": "Susan Bostic", "amount": 1759} +{"id": "3238a234-7c4c-40df-a5cb-51ac32bad998", "type": "debit", "created_at": "2022-03-26T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 6779} +{"id": "9fb649b6-4978-401b-ba1a-b2e2ef130b68", "type": "credit", "created_at": "2022-06-29T00:00:00", "document": "93", "payer": "George Lesane", "amount": 8483} +{"id": "08234bee-44b8-458f-884c-3a96ed6feef6", "type": "bank_slip", "created_at": "2022-02-06T00:00:00", "document": "49", "payer": "Richard Isbell", "amount": 2944} +{"id": "7d232a84-a803-4c94-9d91-baa7f0097d4d", "type": "debit", "created_at": "2022-06-25T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 4168} +{"id": "8845d914-9bee-47e4-8b0b-1749c783158f", "type": "debit", "created_at": "2022-09-22T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 1917} +{"id": "c186c318-e997-49d0-9a97-d81049108526", "type": "credit", "created_at": "2022-06-10T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 3265} +{"id": "7a8bcb45-add3-4575-9a27-c6185f02c6f5", "type": "credit", "created_at": "2022-03-31T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 4285} +{"id": "daa55b23-4d45-4b10-8c78-846aee814a0f", "type": "credit", "created_at": "2022-05-06T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 5109} +{"id": "a39cdd9d-98b4-476a-8167-a3c9c877bea8", "type": "bank_slip", "created_at": "2022-11-29T00:00:00", "document": "57", "payer": "David Landry", "amount": 7169} +{"id": "0e8d6ae6-3d8c-4682-b669-dbad419602d0", "type": "credit", "created_at": "2022-01-04T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 4621} +{"id": "d0825eff-2ee6-4977-ba97-6a2defc295cc", "type": "credit", "created_at": "2022-08-01T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 1190} +{"id": "ab53ceda-4c5c-4f36-aed0-4fb871e1aaad", "type": "bank_slip", "created_at": "2022-01-18T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 2489} +{"id": "dd81ef29-3b77-4ec4-9247-98f56cf9399b", "type": "bank_slip", "created_at": "2022-11-01T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 8933} +{"id": "5d177633-40fc-4c64-855b-61ed137abe0e", "type": "debit", "created_at": "2022-06-11T00:00:00", "document": "52", "payer": "Derek Corcoran", "amount": 5567} +{"id": "6bd71339-70ad-4811-8af2-68de79a8e0b9", "type": "debit", "created_at": "2022-02-05T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 1932} +{"id": "1a370c2c-7e16-439b-b18c-06feee34bbc2", "type": "credit", "created_at": "2022-04-28T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 3264} +{"id": "f7eb2dce-e3b7-43bf-8189-845b30c11e69", "type": "credit", "created_at": "2022-06-14T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 9098} +{"id": "d61fc3e1-3094-4a82-afda-54d9d6f535ca", "type": "credit", "created_at": "2022-02-12T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 5581} +{"id": "86bec851-2e9a-46d7-94d8-62bc4e1cb8fc", "type": "bank_slip", "created_at": "2022-02-12T00:00:00", "document": "57", "payer": "David Landry", "amount": 7346} +{"id": "9a577181-a1d2-417d-a79d-406dee830489", "type": "debit", "created_at": "2022-09-05T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 1666} +{"id": "ce29a78a-3ddd-448b-b939-1e2e65d495d3", "type": "bank_slip", "created_at": "2022-11-02T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 4397} +{"id": "f6c05b53-de54-446f-9c33-00cce430527c", "type": "credit", "created_at": "2022-08-23T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 2094} +{"id": "0c61ec29-3740-47ed-b2a1-1a0d34ab15bc", "type": "debit", "created_at": "2022-08-09T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 7673} +{"id": "d72a6477-5005-4903-b8e9-af0b762e69c0", "type": "debit", "created_at": "2022-12-10T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 2407} +{"id": "9976b0a5-7a98-41a8-9d6c-ae301a390869", "type": "credit", "created_at": "2022-06-16T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 5941} +{"id": "4ba11011-072f-44d0-9fbc-3e8ed396d41a", "type": "credit", "created_at": "2022-04-12T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 7996} +{"id": "60329630-362e-4144-b459-b188320dfa87", "type": "debit", "created_at": "2022-08-30T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 3020} +{"id": "d5c4ad11-67aa-475e-8d31-54fa39ad6258", "type": "credit", "created_at": "2022-04-27T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 9354} +{"id": "5eaccb32-38e2-4bef-bcc5-57874a375d23", "type": "credit", "created_at": "2022-01-13T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 1371} +{"id": "0a44ce10-9819-4a35-9ac1-7c805cea0935", "type": "bank_slip", "created_at": "2022-12-28T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 2158} +{"id": "e3e8361a-f06c-4205-b00b-35edc673fc04", "type": "debit", "created_at": "2022-04-24T00:00:00", "document": "78", "payer": "Paul debiten", "amount": 4700} +{"id": "196344b2-74f2-4325-a973-fa59589e5b41", "type": "credit", "created_at": "2022-10-25T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 2408} +{"id": "5da6c823-5e27-4847-96ee-890e328c205d", "type": "bank_slip", "created_at": "2022-01-23T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 3210} +{"id": "b0afc981-a193-46f6-9e7f-57bfaf02d36f", "type": "debit", "created_at": "2022-08-24T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 9658} +{"id": "d7c60606-7985-40bc-8a1f-27848c4e6807", "type": "debit", "created_at": "2022-09-29T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 1539} +{"id": "8de830ca-ef90-4b8e-8728-a053363c3fad", "type": "credit", "created_at": "2022-03-06T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 1297} +{"id": "5efaa8d7-4a7e-4081-8e56-688edea27a6a", "type": "debit", "created_at": "2022-08-07T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 4961} +{"id": "b5e18bef-de44-4a8a-8da4-3af37067a19c", "type": "bank_slip", "created_at": "2022-09-04T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 5698} +{"id": "c7c08d13-afc5-4f7c-8928-81110f3396a9", "type": "bank_slip", "created_at": "2022-03-13T00:00:00", "document": "79", "payer": "Steven Rapp", "amount": 9847} +{"id": "055b5913-e448-4a90-b57e-54c772681a1b", "type": "debit", "created_at": "2022-06-15T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 6557} +{"id": "60c548fa-e53d-49aa-b5c1-d22766f3d006", "type": "credit", "created_at": "2022-03-22T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 8664} +{"id": "2990eac1-d260-4f34-8aca-045e10353204", "type": "bank_slip", "created_at": "2022-07-17T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 8760} +{"id": "4e6a4b32-4406-47ee-9ff2-6f16ebd430a7", "type": "debit", "created_at": "2022-09-24T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 5327} +{"id": "6426ac86-b1d9-4018-99d3-8b59877e1d20", "type": "credit", "created_at": "2022-11-16T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 8984} +{"id": "3da26a80-1556-4285-bde1-33ef1d6ba2ae", "type": "credit", "created_at": "2022-08-27T00:00:00", "document": "65", "payer": "William Boxer", "amount": 9024} +{"id": "bac315b6-7839-4091-ade3-08d3dd3bf5ec", "type": "debit", "created_at": "2022-03-05T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 5703} +{"id": "7f81806a-22ed-4933-b0e8-8242484e8e20", "type": "debit", "created_at": "2022-10-13T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 2656} +{"id": "cda75900-6044-43dd-9013-b8c4ba6f92ba", "type": "credit", "created_at": "2022-12-23T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 2072} +{"id": "9efae137-0e99-4f9c-bd0f-be7d389863ff", "type": "bank_slip", "created_at": "2022-07-31T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 3805} +{"id": "47c4a36d-4df9-44ad-b887-d2d1fef8ee27", "type": "debit", "created_at": "2022-10-04T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 3008} +{"id": "419521c3-2878-4b1b-87e7-f3a65d5c8732", "type": "credit", "created_at": "2022-08-08T00:00:00", "document": "23", "payer": "Donald Lee", "amount": 8332} +{"id": "14ad48bf-fb02-4a21-9077-244e8c99577e", "type": "debit", "created_at": "2022-08-23T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 2788} +{"id": "c8a897bf-06a5-4a87-a58a-c4ea3b83ffa0", "type": "debit", "created_at": "2022-06-12T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 3671} +{"id": "886ced62-b0c9-43db-9955-c41579967460", "type": "credit", "created_at": "2022-06-15T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 6307} +{"id": "c86efe36-9c24-4120-8d28-e3e98e4f438f", "type": "bank_slip", "created_at": "2022-10-07T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 9583} +{"id": "2e4ef53c-085a-47b8-b0af-006f19a00994", "type": "bank_slip", "created_at": "2022-03-16T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 5883} +{"id": "91bfd964-ca18-4676-a00d-47712da89d5b", "type": "credit", "created_at": "2022-11-18T00:00:00", "document": "23", "payer": "Donald Lee", "amount": 79} +{"id": "1f5663d4-80b6-48bb-b007-83f83f251bd6", "type": "bank_slip", "created_at": "2022-08-16T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 3426} +{"id": "49a688a7-12e7-440f-b752-fb72b02fd096", "type": "bank_slip", "created_at": "2022-03-29T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 8594} +{"id": "6fdf7ff0-da08-4f23-bd64-010e39e0124e", "type": "bank_slip", "created_at": "2022-01-13T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 9107} +{"id": "f79c934b-7305-4459-a9c5-59b078822d18", "type": "debit", "created_at": "2022-03-13T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 4351} +{"id": "4851f348-a92f-4ba4-96e8-f7e1faf93ebd", "type": "credit", "created_at": "2022-10-14T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 8623} +{"id": "63130105-de1e-43f9-8e97-fc9ac8799674", "type": "credit", "created_at": "2022-12-09T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 7967} +{"id": "03e42ac6-819d-4aaf-a4ee-10726f37afcc", "type": "credit", "created_at": "2022-10-23T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 809} +{"id": "ac832803-5b7a-4410-bede-fcad470872dd", "type": "bank_slip", "created_at": "2022-11-09T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 6925} +{"id": "cfa87631-cf0c-42c2-bccd-185b4648116b", "type": "debit", "created_at": "2022-10-21T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 3922} +{"id": "45e1d1da-ed78-427c-9138-5ea916a9786e", "type": "debit", "created_at": "2022-12-08T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 7895} +{"id": "f32ca6b3-c041-43bc-b5e4-25f9e068453c", "type": "credit", "created_at": "2022-05-17T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 1455} +{"id": "14313bc2-dd44-4f9e-a735-6a569c084de6", "type": "debit", "created_at": "2022-02-08T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 5200} +{"id": "6849f463-9eeb-405d-8bd4-0e337dc3a7ea", "type": "credit", "created_at": "2022-06-27T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 3276} +{"id": "72330e22-f08f-4007-a2c6-982ae69e5423", "type": "credit", "created_at": "2022-04-25T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 2370} +{"id": "8ce6bc64-43b8-41d5-8a7a-2c42e9126f10", "type": "debit", "created_at": "2022-07-28T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 8736} +{"id": "594d20ce-c513-4a17-8683-b30b411416e8", "type": "credit", "created_at": "2022-09-10T00:00:00", "document": "76", "payer": "Megan Wallace", "amount": 7602} +{"id": "97927f5e-9df3-4149-a6a5-10698b906e86", "type": "credit", "created_at": "2022-10-04T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 9354} +{"id": "4375c507-84cd-4014-99eb-2f10251ba920", "type": "bank_slip", "created_at": "2022-12-16T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 6965} +{"id": "e92ae355-da2a-4fb8-ae50-bc2515c9f99a", "type": "bank_slip", "created_at": "2022-11-28T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 8968} +{"id": "66779765-8504-4ca6-91e8-5aa24d090d2f", "type": "debit", "created_at": "2022-04-23T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 222} +{"id": "bfb27110-e85e-460b-94ca-cc24cd243b02", "type": "credit", "created_at": "2022-04-01T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 9338} +{"id": "8acfb98a-a226-4011-9c63-6238af73c3c8", "type": "debit", "created_at": "2022-10-06T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 5439} +{"id": "f3b52c43-5a95-4595-8e18-1dd6347c020e", "type": "credit", "created_at": "2022-12-13T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 9093} +{"id": "17e7080a-188f-450b-a5b4-5880ea8467e2", "type": "bank_slip", "created_at": "2022-06-21T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 4850} +{"id": "b1e38fcd-a586-42cb-a862-326df03e06c1", "type": "bank_slip", "created_at": "2022-02-16T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 4205} +{"id": "dc765a5d-db73-4c08-86ba-09698e345f48", "type": "debit", "created_at": "2022-06-05T00:00:00", "document": "60", "payer": "Rachel York", "amount": 3020} +{"id": "45cc41a2-c3f1-4ffb-99ee-5bab70e5b22b", "type": "debit", "created_at": "2022-02-23T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 1805} +{"id": "73e13e3a-66fe-4795-b940-e2424db3aa02", "type": "debit", "created_at": "2022-07-25T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 462} +{"id": "baf4bc6d-770a-4bc7-90f1-fd02b1723752", "type": "bank_slip", "created_at": "2022-11-07T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 3930} +{"id": "bff12b0a-d51b-4d09-8460-776e665a4dab", "type": "bank_slip", "created_at": "2022-08-24T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 647} +{"id": "9820e6ce-0cb4-46fa-9c13-544b39db628c", "type": "bank_slip", "created_at": "2022-07-11T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 959} +{"id": "08bd67bb-6a6e-4ea8-86dc-ab75b620a342", "type": "bank_slip", "created_at": "2022-10-27T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 4995} +{"id": "cf89f963-2bc1-4ea3-93f7-daf4339573d8", "type": "debit", "created_at": "2022-01-14T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 3283} +{"id": "fac7326c-06ab-4ee5-88cb-0b3f59a46ac0", "type": "credit", "created_at": "2022-07-20T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 5154} +{"id": "6b1d5fde-777c-4f27-b9f9-1ac8a2cbc472", "type": "credit", "created_at": "2022-05-11T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 6818} +{"id": "9865286d-09fe-4f35-bb9a-34e739183947", "type": "credit", "created_at": "2022-07-05T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 6295} +{"id": "083de051-e82f-4ee5-992c-72f16cb483f7", "type": "debit", "created_at": "2022-06-08T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 7216} +{"id": "733196dd-154b-48d1-9890-38c7a35aa77d", "type": "bank_slip", "created_at": "2022-06-24T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 6531} +{"id": "98236706-11ce-4823-9c54-fea0241dd354", "type": "bank_slip", "created_at": "2022-09-08T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 878} +{"id": "771a03ea-c80d-44d0-94c2-654ea58a2d1b", "type": "credit", "created_at": "2022-07-13T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 6634} +{"id": "c441a45e-9346-4e98-8732-f1bcc3c4faf4", "type": "bank_slip", "created_at": "2022-12-31T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 6256} +{"id": "82402476-6938-4d01-80be-520a67cdda71", "type": "debit", "created_at": "2022-01-18T00:00:00", "document": "84", "payer": "Rick Reid", "amount": 6607} +{"id": "be37c578-641f-401d-9d7c-c3c816e860b0", "type": "bank_slip", "created_at": "2022-01-29T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 7802} +{"id": "19691dc1-0194-4df7-878e-f2de7495f114", "type": "bank_slip", "created_at": "2022-08-18T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 5179} +{"id": "e49819bc-8b0c-4b52-9371-4ded99107897", "type": "debit", "created_at": "2022-05-11T00:00:00", "document": "23", "payer": "Donald Lee", "amount": 62} +{"id": "fa494c73-f7df-4f23-90a7-9afadf23bda8", "type": "credit", "created_at": "2022-07-01T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 7268} +{"id": "75f73ebb-ad59-41fa-be2a-2ddd6e4b4f94", "type": "bank_slip", "created_at": "2022-03-14T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 5482} +{"id": "a3082785-e9d9-4edc-bd68-8489ee9c766f", "type": "bank_slip", "created_at": "2022-10-21T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 6031} +{"id": "3d1fbe06-5c12-49c0-8523-46554e57c528", "type": "bank_slip", "created_at": "2022-06-13T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 6919} +{"id": "3436c17e-3ed7-47da-83ed-26509ae8e6b5", "type": "bank_slip", "created_at": "2022-10-18T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 1582} +{"id": "bf124060-14f2-4b60-82ef-6e12a3fbb36e", "type": "credit", "created_at": "2022-12-28T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 9453} +{"id": "59c91201-8a3d-473f-a966-26ccd0806523", "type": "debit", "created_at": "2022-12-09T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 4836} +{"id": "a6a51b11-4c9f-4e2c-8960-db4c0c83ff8a", "type": "credit", "created_at": "2022-05-13T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 694} +{"id": "412b773c-9edf-4011-8751-89ec47758d64", "type": "bank_slip", "created_at": "2022-11-05T00:00:00", "document": "28", "payer": "Mark Williams", "amount": 5501} +{"id": "3c9eb77a-362a-4624-b141-5b5a8fcb3dea", "type": "credit", "created_at": "2022-01-16T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 7298} +{"id": "63db37a1-47ce-46eb-b406-d8e90cd07b7b", "type": "debit", "created_at": "2022-04-15T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 6250} +{"id": "92abe452-04fe-4a31-a8e9-2eb718c4a1a5", "type": "debit", "created_at": "2022-05-03T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 4884} +{"id": "d0711ce4-3d14-4650-954d-cad79e28dbd6", "type": "bank_slip", "created_at": "2022-01-23T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 4024} +{"id": "3334b048-274f-4671-bbfc-602cc844fde2", "type": "credit", "created_at": "2022-06-03T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 6850} +{"id": "74d530eb-351d-4c93-a4e7-9415fa03c8ca", "type": "bank_slip", "created_at": "2022-02-10T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 9887} +{"id": "d1fe81fd-bfe4-4201-8637-628d9382a85a", "type": "bank_slip", "created_at": "2022-07-13T00:00:00", "document": "49", "payer": "Richard Isbell", "amount": 9063} +{"id": "3f3869c0-7bf9-4360-bcf2-32beefdf5934", "type": "credit", "created_at": "2022-01-23T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 2655} +{"id": "5e9c1d4d-7f48-49d4-a82f-2de2d027135d", "type": "debit", "created_at": "2022-11-02T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 203} +{"id": "d376959e-8569-446b-9f17-66472465d45c", "type": "credit", "created_at": "2022-04-17T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 7486} +{"id": "3fcbdaa1-785a-4ec5-afd7-c6cfaa4729dc", "type": "debit", "created_at": "2022-05-28T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 8392} +{"id": "c3e6227e-39cc-4c96-83e2-480f4c9b4d2f", "type": "bank_slip", "created_at": "2022-07-19T00:00:00", "document": "60", "payer": "Rachel York", "amount": 5779} +{"id": "cb8a2a7d-42a8-42e0-9a8c-cfabac0cb106", "type": "debit", "created_at": "2022-09-12T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 9524} +{"id": "665fec6a-906f-4d7b-ae9f-61f967f8860d", "type": "debit", "created_at": "2022-12-02T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 6179} +{"id": "7e98fcc3-a125-4aea-8646-04bb8815e1c3", "type": "credit", "created_at": "2022-05-23T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 8979} +{"id": "4b63d7c7-aff1-4469-8eed-695dcc6ce6ea", "type": "credit", "created_at": "2022-06-30T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 9028} +{"id": "c5096fe2-5e01-4502-b14e-2c5136346d5f", "type": "debit", "created_at": "2022-03-13T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 572} +{"id": "2f29485c-51b3-4b37-92e4-78511fbcb9e6", "type": "bank_slip", "created_at": "2022-09-10T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 3807} +{"id": "f72cb722-0d95-4bac-9ba7-326cc50f15c9", "type": "bank_slip", "created_at": "2022-04-18T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 4080} +{"id": "de446df8-ec1a-400d-9b96-53216b004c41", "type": "credit", "created_at": "2022-07-07T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 997} +{"id": "42914b75-6857-4210-b2fd-3083345a090f", "type": "credit", "created_at": "2022-01-02T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 1477} +{"id": "0f22de01-2ae2-465f-8056-cbb83c65f8f7", "type": "credit", "created_at": "2022-02-02T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 4164} +{"id": "b9e770ea-5e13-4dba-b8d9-e264abdc50ae", "type": "bank_slip", "created_at": "2022-01-25T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 1792} +{"id": "a50035d3-a660-4989-b327-36eec1b6ef28", "type": "bank_slip", "created_at": "2022-08-08T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 7616} +{"id": "bdf6854c-8254-4872-b76d-cfb306d0f358", "type": "credit", "created_at": "2022-01-23T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 9736} +{"id": "68c32db9-373d-4ba8-a37a-dfe357306831", "type": "credit", "created_at": "2022-11-15T00:00:00", "document": "52", "payer": "Derek Corcoran", "amount": 3883} +{"id": "4554c1be-bd35-42ce-9827-87585a0b9210", "type": "bank_slip", "created_at": "2022-02-09T00:00:00", "document": "37", "payer": "Jane Olson", "amount": 2556} +{"id": "ea04dc87-2058-4048-8d0c-014ac7960455", "type": "bank_slip", "created_at": "2022-02-01T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 2701} +{"id": "df480b98-e2b0-4fb0-bc97-e2974a182dbe", "type": "credit", "created_at": "2022-09-18T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 7036} +{"id": "28dd086e-797f-4c2c-9c69-86bd391d93e7", "type": "credit", "created_at": "2022-01-22T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 5125} +{"id": "e45f2980-c2af-4137-9007-75d21f3f40f9", "type": "bank_slip", "created_at": "2022-12-24T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 8184} +{"id": "f213cee2-ae11-4ebb-8743-4493e615448c", "type": "credit", "created_at": "2022-07-11T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 7944} +{"id": "70cb0e3c-8e80-4151-b6f1-13656255e8ef", "type": "credit", "created_at": "2022-09-05T00:00:00", "document": "1", "payer": "Carl Teems", "amount": 6726} +{"id": "e9d9bc3e-02f9-4d51-b604-8ee37775833e", "type": "debit", "created_at": "2022-12-20T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 4963} +{"id": "934e183d-8fa8-4dc3-984e-ac9f1fadcb6c", "type": "debit", "created_at": "2022-10-10T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 1242} +{"id": "43f82a2c-4ca5-4401-b1ed-ddc09477de8f", "type": "bank_slip", "created_at": "2022-05-14T00:00:00", "document": "91", "payer": "Gertrude Leishman", "amount": 6829} +{"id": "ef37cc1e-e13e-4970-9a5d-f448a903e69e", "type": "credit", "created_at": "2022-08-18T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 4561} +{"id": "7fb2cc12-6ae7-484e-aef6-e0a45a47ca83", "type": "credit", "created_at": "2022-12-22T00:00:00", "document": "24", "payer": "Susan Bostic", "amount": 440} +{"id": "54424be3-90ff-4b60-a8fc-f31e6f586c14", "type": "credit", "created_at": "2022-06-19T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 9616} +{"id": "9b2e9e29-30a8-48ac-a45b-fef2f0f4356a", "type": "credit", "created_at": "2022-03-23T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 1750} +{"id": "35ee566f-b38b-44c4-8113-96af53ed1b0c", "type": "credit", "created_at": "2022-06-25T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 4950} +{"id": "b2cf74c6-6195-4a25-8f30-66e34db6adb0", "type": "credit", "created_at": "2022-04-22T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 6269} +{"id": "4babfc8a-36eb-46d0-9b37-aef39111bbc6", "type": "debit", "created_at": "2022-11-01T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 8719} +{"id": "bd1b44f9-428b-44d2-9b38-4d5c46a93a76", "type": "credit", "created_at": "2022-04-03T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 5112} +{"id": "4f8ce0d7-a62c-4ea3-9165-68c2640d8169", "type": "debit", "created_at": "2022-12-16T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 2276} +{"id": "b4e7ff8f-2dc8-45e7-9b3e-ddf63b4134e1", "type": "debit", "created_at": "2022-12-22T00:00:00", "document": "85", "payer": "Susan Wolcott", "amount": 1705} +{"id": "65a2fa21-826a-46fa-90e7-fa70a76b61e8", "type": "credit", "created_at": "2022-10-26T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 6906} +{"id": "b0d1bbec-a21b-45c4-9789-300ccb5feaf9", "type": "credit", "created_at": "2022-11-27T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 2064} +{"id": "3af2ebea-df2f-4d33-879a-1c9db68645e0", "type": "debit", "created_at": "2022-12-22T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 2217} +{"id": "d8791561-c081-403e-abba-dbfeaad0a189", "type": "bank_slip", "created_at": "2022-12-08T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 5159} +{"id": "57207b32-7e91-4ea3-b37e-4f614fa66cb4", "type": "credit", "created_at": "2022-12-03T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 5158} +{"id": "dfbc28f7-bcdf-425c-a96c-ba3bdf368165", "type": "debit", "created_at": "2022-09-22T00:00:00", "document": "70", "payer": "Patrick Davis", "amount": 8307} +{"id": "7a8ecf99-8f3e-4f6d-b1ce-9f7c26408e04", "type": "debit", "created_at": "2022-10-26T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 6268} +{"id": "476bd74f-039c-426f-8de7-635543ec124a", "type": "credit", "created_at": "2022-02-06T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 3854} +{"id": "6aa8f3e9-30da-4592-bc46-a1850ba20dec", "type": "bank_slip", "created_at": "2022-12-02T00:00:00", "document": "61", "payer": "Ladonna Cooper", "amount": 1349} +{"id": "efa65340-c6f2-4cbf-af44-f99f4d8b06e9", "type": "bank_slip", "created_at": "2022-02-25T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 4866} +{"id": "dbe2d7d0-47fd-4462-ab43-165b8683add3", "type": "debit", "created_at": "2022-05-11T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 295} +{"id": "3e449b5a-b57c-49d2-a377-b5ae8f5c51ec", "type": "bank_slip", "created_at": "2022-02-02T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 2678} +{"id": "c5c549b2-1164-4931-892f-b36322d89a4f", "type": "debit", "created_at": "2022-01-01T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 138} +{"id": "c91bb090-3a61-4dc1-bac8-c4ff0dd51796", "type": "debit", "created_at": "2022-09-08T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 4246} +{"id": "d03f52d9-dca7-4697-9931-85768e97a846", "type": "debit", "created_at": "2022-08-05T00:00:00", "document": "60", "payer": "Rachel York", "amount": 3695} +{"id": "55b0bd9c-625d-4584-8c32-a044c74b2073", "type": "debit", "created_at": "2022-09-10T00:00:00", "document": "22", "payer": "Noemi Mcintyre", "amount": 8593} +{"id": "28b82939-9e55-4b47-80fe-fbcf3cdcd89a", "type": "debit", "created_at": "2022-03-31T00:00:00", "document": "33", "payer": "Charlotte Marczak", "amount": 1573} +{"id": "e923c9bb-4ccf-4cad-b310-263c415bb34a", "type": "credit", "created_at": "2022-06-08T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 183} +{"id": "af9b2be2-86f7-411f-ae65-d05f1f4144ef", "type": "bank_slip", "created_at": "2022-01-13T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 1633} +{"id": "d8f291f5-658f-4668-b6e7-dc3a8a6f3e53", "type": "credit", "created_at": "2022-12-23T00:00:00", "document": "49", "payer": "Richard Isbell", "amount": 5380} +{"id": "a56f292f-8bb2-40c9-8ce2-a1ab0fbda5f1", "type": "bank_slip", "created_at": "2022-09-24T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 7181} +{"id": "c34c6dff-fd50-4c07-af03-b54bb43f0f35", "type": "bank_slip", "created_at": "2022-03-09T00:00:00", "document": "77", "payer": "Michael Fultz", "amount": 1356} +{"id": "f5814200-cfa1-4a4e-b9ab-207bcf277146", "type": "bank_slip", "created_at": "2022-03-19T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 9536} +{"id": "95b4547b-4128-480a-b2e7-331c4c5190fe", "type": "credit", "created_at": "2022-11-27T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 2921} +{"id": "ec77b16a-a67f-4bf6-b167-7a2895176b6e", "type": "bank_slip", "created_at": "2022-09-07T00:00:00", "document": "50", "payer": "Karen Lewis", "amount": 8080} +{"id": "e60d958a-a646-4a3a-9ba5-5394d38cc260", "type": "bank_slip", "created_at": "2022-09-15T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 758} +{"id": "25aa27c9-f66c-4161-b94a-3d621080867a", "type": "bank_slip", "created_at": "2022-04-26T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 4253} +{"id": "42af1641-b45a-4385-8dc6-7b14c26b7cf7", "type": "bank_slip", "created_at": "2022-09-11T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 3507} +{"id": "36e2bcc7-f96d-4205-8dac-17c669807373", "type": "credit", "created_at": "2022-03-20T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 7868} +{"id": "acb67761-c14e-4d04-b222-ff7a2681c475", "type": "debit", "created_at": "2022-06-08T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 9018} +{"id": "a7fd9162-0539-4b3a-9ec7-5fce8a701fd5", "type": "credit", "created_at": "2022-07-28T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 3909} +{"id": "5f0ebe16-fabd-4f2e-85a1-a849c0a9e985", "type": "debit", "created_at": "2022-06-19T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 4583} +{"id": "585c2b57-c4e2-42be-81d3-2b5866aa8643", "type": "credit", "created_at": "2022-03-13T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 2715} +{"id": "11b40cea-0be5-4be1-a6ee-e14dfe31a1a6", "type": "credit", "created_at": "2022-11-19T00:00:00", "document": "65", "payer": "William Boxer", "amount": 6516} +{"id": "a8aa09e0-2087-4b39-8515-ebe81a500f01", "type": "debit", "created_at": "2022-09-23T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 1732} +{"id": "6d0a0a8e-8417-4a36-ada2-4e69e084b349", "type": "credit", "created_at": "2022-01-31T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 2676} +{"id": "84c91a0d-8962-428d-ba0f-dc30838f07b2", "type": "bank_slip", "created_at": "2022-02-01T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 6965} +{"id": "7fb30b49-b2d9-4612-86c4-ebf6f3ddfe83", "type": "bank_slip", "created_at": "2022-08-25T00:00:00", "document": "28", "payer": "Mark Williams", "amount": 5646} +{"id": "9525530d-145b-4f61-a8cc-42c67eca402d", "type": "credit", "created_at": "2022-06-29T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 6797} +{"id": "e1a74e5a-03fb-4633-87eb-535a2868addd", "type": "credit", "created_at": "2022-12-10T00:00:00", "document": "89", "payer": "Antonio Cooley", "amount": 6366} +{"id": "caf51cb0-9701-475f-98bb-3f859d5d5c91", "type": "bank_slip", "created_at": "2022-12-08T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 4999} +{"id": "8ae97a5a-d2c8-4ce4-99ce-2f9553fcaee7", "type": "debit", "created_at": "2022-11-25T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 4408} +{"id": "002d6546-46fd-4e91-ba67-cba45c8d175d", "type": "bank_slip", "created_at": "2022-01-07T00:00:00", "document": "60", "payer": "Rachel York", "amount": 8978} +{"id": "73a77f94-4d97-4c8f-bb9f-72774a1f13aa", "type": "bank_slip", "created_at": "2022-05-23T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 1836} +{"id": "194741b3-64bd-489e-81f2-14e6da41a301", "type": "debit", "created_at": "2022-05-09T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 8286} +{"id": "e3f033dc-aa8c-4bc9-a000-97afe843c540", "type": "credit", "created_at": "2022-04-23T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 9289} +{"id": "4d3054b1-4922-4eef-bb67-863552f3b76a", "type": "debit", "created_at": "2022-08-22T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 2166} +{"id": "7b6c2166-d709-436e-a824-63e7835acfc1", "type": "bank_slip", "created_at": "2022-10-18T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 9836} +{"id": "85f77ea6-2d75-46e8-93d7-7c19a5d3498c", "type": "credit", "created_at": "2022-07-16T00:00:00", "document": "50", "payer": "Karen Lewis", "amount": 6112} +{"id": "ce1fb022-e6b5-4126-8219-cbbd6e2b905a", "type": "bank_slip", "created_at": "2022-01-03T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 1785} +{"id": "2a2a30db-3c27-47a7-b510-14fb7c54d417", "type": "bank_slip", "created_at": "2022-05-04T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 6575} +{"id": "d5508c30-2193-494d-9b3c-6bc560ef7d9c", "type": "credit", "created_at": "2022-10-10T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 1370} +{"id": "45a8f073-d40b-4c92-bfdc-a64fab60d08d", "type": "credit", "created_at": "2022-04-15T00:00:00", "document": "93", "payer": "George Lesane", "amount": 902} +{"id": "a56a619b-fc85-4f99-8643-83828557cea2", "type": "debit", "created_at": "2022-01-25T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 1345} +{"id": "049ea531-8385-4b3a-87c7-2f66830fd9aa", "type": "debit", "created_at": "2022-05-13T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 5622} +{"id": "0868437e-080a-4f34-9b2c-df37cc77acd0", "type": "bank_slip", "created_at": "2022-11-12T00:00:00", "document": "72", "payer": "Shirley Digiovanni", "amount": 2834} +{"id": "80622b63-23df-4606-aa83-30e3ed4223d7", "type": "debit", "created_at": "2022-12-05T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 9594} +{"id": "dfae87a1-42c8-49af-85d2-d10375b28139", "type": "credit", "created_at": "2022-08-02T00:00:00", "document": "98", "payer": "Elma Snapp", "amount": 9487} +{"id": "6cd5c547-5dae-4959-ac62-659e1756b3c1", "type": "credit", "created_at": "2022-02-06T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 4199} +{"id": "ae0075f8-2fc7-4d79-8c59-d3472551093b", "type": "credit", "created_at": "2022-03-11T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 8215} +{"id": "95816dfe-e28a-499f-a605-c117de7cb89b", "type": "bank_slip", "created_at": "2022-05-16T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 2405} +{"id": "fb5f0cbd-0842-4cf3-b6b1-f3e5f48c6a83", "type": "credit", "created_at": "2022-01-26T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 9178} +{"id": "6b0d3a74-7a1e-490e-aadd-3ac570f55524", "type": "debit", "created_at": "2022-09-10T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 2260} +{"id": "018f235b-f48e-4b88-8f06-6334a31d4996", "type": "bank_slip", "created_at": "2022-07-08T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 6218} +{"id": "1a3394bf-cc47-4987-b08b-2808a2c8430d", "type": "credit", "created_at": "2022-05-14T00:00:00", "document": "67", "payer": "Wiley Goetz", "amount": 3816} +{"id": "14ea3cb8-eb4c-4e14-a1a6-8d47bdc7c251", "type": "credit", "created_at": "2022-05-21T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 8543} +{"id": "e0509b13-678b-41e5-9d2c-0f2966e30b45", "type": "bank_slip", "created_at": "2022-01-04T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 8772} +{"id": "d9bc1e55-b4a4-4445-908b-bd939f4b2ccf", "type": "credit", "created_at": "2022-11-28T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 8767} +{"id": "d9c0028e-e252-406f-98e9-d4aa426687be", "type": "debit", "created_at": "2022-12-12T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 8990} +{"id": "34935509-387b-46c8-818d-6f511fadcda8", "type": "bank_slip", "created_at": "2022-03-31T00:00:00", "document": "36", "payer": "Elfreda Garza", "amount": 2026} +{"id": "7d7f280b-0307-4745-a917-af368c545128", "type": "bank_slip", "created_at": "2022-03-12T00:00:00", "document": "28", "payer": "Mark Williams", "amount": 1685} +{"id": "18394a1c-ba7b-4160-9156-ccb4e5326e24", "type": "bank_slip", "created_at": "2022-12-17T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 4015} +{"id": "5155a48a-b361-437d-a25e-098265022373", "type": "debit", "created_at": "2022-07-04T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 9492} +{"id": "b1b6b5b7-a26e-4579-bff9-ea97800598bf", "type": "credit", "created_at": "2022-10-06T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 4942} +{"id": "67d798a6-4d6a-4701-9fa4-007c2aa4ed4c", "type": "debit", "created_at": "2022-02-23T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 3145} +{"id": "89782482-abeb-40d5-a98d-40c827ab1fe7", "type": "credit", "created_at": "2022-10-02T00:00:00", "document": "60", "payer": "Rachel York", "amount": 4253} +{"id": "e2d1dde9-89ca-4597-901e-932e879c1a84", "type": "credit", "created_at": "2022-03-26T00:00:00", "document": "65", "payer": "William Boxer", "amount": 5407} +{"id": "7e8283d7-96f3-42eb-90b8-bb1a820d5a59", "type": "credit", "created_at": "2022-11-16T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 6274} +{"id": "d1fb3805-6a78-4ceb-bbc5-917ee6997131", "type": "bank_slip", "created_at": "2022-05-28T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 9223} +{"id": "3f4e46d4-3fbf-4d5f-9477-fb9bb5610481", "type": "bank_slip", "created_at": "2022-02-08T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 3204} +{"id": "164a911d-c24f-4659-9386-68270395a221", "type": "bank_slip", "created_at": "2022-07-14T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 2352} +{"id": "892d1cb5-7719-4be7-85f4-82c487e79972", "type": "bank_slip", "created_at": "2022-03-19T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 6525} +{"id": "0223bb00-5a41-4464-8c98-54adfeb441f7", "type": "debit", "created_at": "2022-08-13T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 5480} +{"id": "580dfaa4-546a-43b0-b4e8-8c61e28dd048", "type": "debit", "created_at": "2022-10-01T00:00:00", "document": "2", "payer": "Ella Tatum", "amount": 1976} +{"id": "925d0781-353a-43f5-bfd6-58dbf9ed0a94", "type": "debit", "created_at": "2022-07-01T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 5222} +{"id": "b2b3591f-8af6-44d0-8321-2e509e74e725", "type": "bank_slip", "created_at": "2022-01-17T00:00:00", "document": "6", "payer": "John Dickerson", "amount": 5929} +{"id": "91664d1e-a810-4f29-aabb-fd673f3c5cdf", "type": "debit", "created_at": "2022-06-12T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 6048} +{"id": "283ea219-1329-47f6-8331-277f0fef2f1d", "type": "credit", "created_at": "2022-06-12T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 8187} +{"id": "add68385-c8c0-4664-92b2-df245a1f5497", "type": "credit", "created_at": "2022-01-31T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 3645} +{"id": "d3ee74de-5592-470e-a400-a5f8e2d0ddfd", "type": "debit", "created_at": "2022-02-17T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 3906} +{"id": "8df655d0-a4a6-4560-a937-2b94089ed539", "type": "bank_slip", "created_at": "2022-03-15T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 8759} +{"id": "27577477-6619-45df-92f4-82d2a6dde973", "type": "credit", "created_at": "2022-03-22T00:00:00", "document": "15", "payer": "James Blomdahl", "amount": 8766} +{"id": "746b1aa8-8aeb-4412-ae9a-66a0436c308a", "type": "bank_slip", "created_at": "2022-02-02T00:00:00", "document": "10", "payer": "Emanuel Robinson", "amount": 1640} +{"id": "721904a9-dfbe-4738-b361-322dea29d473", "type": "credit", "created_at": "2022-05-24T00:00:00", "document": "63", "payer": "Pearl Lemaire", "amount": 9352} +{"id": "b04ae878-e67c-4cd2-985c-ec56fb728648", "type": "bank_slip", "created_at": "2022-07-29T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 4055} +{"id": "f782f1e2-3c4d-4c5b-b2ca-af3c1540dc09", "type": "debit", "created_at": "2022-12-25T00:00:00", "document": "46", "payer": "Shaunda Gallegos", "amount": 8207} +{"id": "3adabd10-6ea3-4548-bab2-0464cf689028", "type": "debit", "created_at": "2022-03-16T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 6310} +{"id": "2cba25d1-ab2a-4bdc-8597-5085114b23ec", "type": "debit", "created_at": "2022-12-21T00:00:00", "document": "25", "payer": "Calvin Reid", "amount": 1425} +{"id": "11e1d791-492f-413c-94bf-05550a98b142", "type": "debit", "created_at": "2022-01-29T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 5373} +{"id": "71ce76cf-940c-4ae4-8fa3-356cd75ee180", "type": "credit", "created_at": "2022-05-08T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 7118} +{"id": "4f79b3d9-66c8-4a4d-ac7a-94c2f2a35daf", "type": "debit", "created_at": "2022-07-29T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 4515} +{"id": "bdda3c70-c00c-4f32-bea9-8a65e28b2228", "type": "bank_slip", "created_at": "2022-09-19T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 2617} +{"id": "103dd83a-49e3-4f9d-a290-f32d9b80c9da", "type": "bank_slip", "created_at": "2022-03-05T00:00:00", "document": "94", "payer": "Rupert Peterman", "amount": 8398} +{"id": "22241bad-30a8-4d4c-b625-98bd17d9aec5", "type": "bank_slip", "created_at": "2022-05-30T00:00:00", "document": "40", "payer": "Charles Malik", "amount": 8884} +{"id": "ad07dc66-bbd2-4c6b-b66f-b230eea0009b", "type": "credit", "created_at": "2022-01-04T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 6773} +{"id": "a7eae518-ce66-41b8-87c4-07732ee37f04", "type": "credit", "created_at": "2022-04-19T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 9226} +{"id": "6671ef5d-a6b1-4a24-b4a0-007c2bf6a7a4", "type": "debit", "created_at": "2022-02-18T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 1566} +{"id": "29343b43-1469-4df4-b8d8-1cd405615e24", "type": "credit", "created_at": "2022-03-26T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 7622} +{"id": "7aa494ec-ccda-409f-ba1a-f359bb5cbd3a", "type": "bank_slip", "created_at": "2022-01-21T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 6001} +{"id": "a7d88c91-e76b-4d7f-a612-74c2ce7520f6", "type": "credit", "created_at": "2022-02-14T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 7148} +{"id": "e8a9018f-6ee1-48d8-947f-6e8fa34e906c", "type": "bank_slip", "created_at": "2022-04-13T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 2448} +{"id": "e52111a8-2a93-4ca1-9ee7-d40cac01e54a", "type": "bank_slip", "created_at": "2022-05-21T00:00:00", "document": "55", "payer": "Ashley Beauchesne", "amount": 133} +{"id": "5f9919db-2a4e-4088-9d29-a08b65dab3fb", "type": "credit", "created_at": "2022-10-06T00:00:00", "document": "43", "payer": "Gary Jensen", "amount": 7591} +{"id": "70725ddf-78da-472e-81c8-c1537b7c2f4b", "type": "credit", "created_at": "2022-08-02T00:00:00", "document": "61", "payer": "Ladonna Cooper", "amount": 9429} +{"id": "66146035-82ae-40c2-a12e-a85725da0661", "type": "credit", "created_at": "2022-10-20T00:00:00", "document": "11", "payer": "Jenny Jensen", "amount": 9908} +{"id": "ca897eb4-16ff-42fe-a417-203d912dcbe1", "type": "bank_slip", "created_at": "2022-04-13T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 9622} +{"id": "5b951992-0e05-4ea7-bbaf-6ab640d71cc4", "type": "bank_slip", "created_at": "2022-01-13T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 5952} +{"id": "ab86965d-f12f-482a-b1a0-a645a78fd02a", "type": "credit", "created_at": "2022-04-22T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 2112} +{"id": "e0a9d09d-2867-40b4-8891-84e1507b5905", "type": "credit", "created_at": "2022-12-08T00:00:00", "document": "57", "payer": "David Landry", "amount": 9061} +{"id": "038a1b7d-ae5d-43b8-a00a-357fb583970d", "type": "credit", "created_at": "2022-10-26T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 3849} +{"id": "0c511e8d-baf7-461c-b05d-5e095bd56411", "type": "debit", "created_at": "2022-09-05T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 9218} +{"id": "a608e21b-c779-4e6f-abc2-6fd5c5d1272c", "type": "debit", "created_at": "2022-03-26T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 1062} +{"id": "c113b25d-c6dd-4c8d-b842-05579a409cbb", "type": "bank_slip", "created_at": "2022-07-31T00:00:00", "document": "68", "payer": "Pauline Marcum", "amount": 3936} +{"id": "d741ad19-e007-42c9-9825-65e2411f006a", "type": "debit", "created_at": "2022-06-04T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 325} +{"id": "6d734a1b-58d2-48c4-8eea-5629aee365d3", "type": "credit", "created_at": "2022-08-12T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 7658} +{"id": "bd053a27-773f-45bd-a04f-d6c37bf8734b", "type": "bank_slip", "created_at": "2022-04-27T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 446} +{"id": "38b60a1c-14f3-4c7c-bec2-3bb8259c3d55", "type": "debit", "created_at": "2022-08-19T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 9295} +{"id": "34e1ef9e-8381-4fd3-872f-78932c3c2213", "type": "credit", "created_at": "2022-06-19T00:00:00", "document": "66", "payer": "Kimberly Scott", "amount": 3289} +{"id": "7ec1f52b-b9e0-4f7d-ac61-223515b7fbcb", "type": "debit", "created_at": "2022-09-15T00:00:00", "document": "57", "payer": "David Landry", "amount": 8492} +{"id": "a13bdec1-4d59-4625-b153-cc998ca426f9", "type": "bank_slip", "created_at": "2022-05-09T00:00:00", "document": "93", "payer": "George Lesane", "amount": 813} +{"id": "9a44edbd-a00f-4abc-ae8b-033b6ad29549", "type": "bank_slip", "created_at": "2022-07-05T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 230} +{"id": "5a32a6f7-7883-46e5-9434-902aff911077", "type": "credit", "created_at": "2022-09-16T00:00:00", "document": "5", "payer": "Paula Spencer", "amount": 877} +{"id": "dfab8323-9f20-423b-b8ba-c229a0c7f73c", "type": "credit", "created_at": "2022-12-18T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 5263} +{"id": "5fb02235-0205-405b-9da8-7ce38b995032", "type": "bank_slip", "created_at": "2022-02-05T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 7529} +{"id": "20ce820d-768b-4297-8c6a-fe44a2569062", "type": "credit", "created_at": "2022-05-21T00:00:00", "document": "57", "payer": "David Landry", "amount": 4038} +{"id": "0fb080db-e77b-4ec1-a1d1-ca8a9f81ddfa", "type": "credit", "created_at": "2022-05-05T00:00:00", "document": "11", "payer": "Jenny Jensen", "amount": 1478} +{"id": "3746be86-2d4c-431b-9c02-940efa02c22d", "type": "bank_slip", "created_at": "2022-07-21T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 1758} +{"id": "9a8eda79-16c8-4e5e-b781-c0ac342512a6", "type": "credit", "created_at": "2022-07-16T00:00:00", "document": "31", "payer": "Ridebito Little", "amount": 5558} +{"id": "b5248704-1ab9-441d-994c-b16a339c09bc", "type": "credit", "created_at": "2022-02-22T00:00:00", "document": "92", "payer": "Amber Hudson", "amount": 8641} +{"id": "5e16fdcd-d393-4a49-a511-dc8f277a0b75", "type": "bank_slip", "created_at": "2022-07-30T00:00:00", "document": "62", "payer": "Latasha Rubio", "amount": 9571} +{"id": "c2bfad88-7fb9-41f2-9139-96a898816c7d", "type": "credit", "created_at": "2022-02-26T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 2001} +{"id": "0e79b835-5b04-4f63-ab05-6d8415cb6e60", "type": "credit", "created_at": "2022-12-27T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 3864} +{"id": "d9c1dd36-74ba-45d8-a7ef-f7bc13a7f48b", "type": "credit", "created_at": "2022-04-19T00:00:00", "document": "11", "payer": "Jenny Jensen", "amount": 120} +{"id": "cf214a78-6cbd-4e0c-8d5b-f90942dfaaf4", "type": "bank_slip", "created_at": "2022-12-17T00:00:00", "document": "14", "payer": "Eleanore Hocutt", "amount": 5578} +{"id": "6bec32a2-4c33-4444-b2b4-de35b33cd16b", "type": "credit", "created_at": "2022-07-13T00:00:00", "document": "28", "payer": "Mark Williams", "amount": 5314} +{"id": "e2171948-f3dc-4d93-b2ec-91e315beb790", "type": "debit", "created_at": "2022-02-23T00:00:00", "document": "45", "payer": "Shanita Bergesen", "amount": 5599} +{"id": "3b7361c0-c275-433b-b6d3-bc2dbf12408c", "type": "debit", "created_at": "2022-03-14T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 6188} +{"id": "7773d0c9-4aaf-4a7e-b345-9d92fb8271ab", "type": "credit", "created_at": "2022-06-28T00:00:00", "document": "51", "payer": "Kenneth Stewart", "amount": 3573} +{"id": "0f3a61d0-477e-4a8e-8a95-93db63d4fdb7", "type": "bank_slip", "created_at": "2022-12-10T00:00:00", "document": "20", "payer": "Luis Moffett", "amount": 3326} +{"id": "51d081c9-7a4a-408b-b197-82a4f5ae8753", "type": "bank_slip", "created_at": "2022-06-24T00:00:00", "document": "48", "payer": "Lori Carter", "amount": 7334} +{"id": "2d72f135-a82d-425b-961b-0bd54330ad8c", "type": "bank_slip", "created_at": "2022-04-26T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 1778} +{"id": "01522cb7-c14e-4e37-b797-8d81488ffed5", "type": "debit", "created_at": "2022-11-28T00:00:00", "document": "37", "payer": "Jane Olson", "amount": 7805} +{"id": "439f70b9-fd9e-403f-8c1e-b3ac0cc6a745", "type": "credit", "created_at": "2022-03-13T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 5486} +{"id": "a8aedef6-4a6d-4021-94b6-f392de26c334", "type": "bank_slip", "created_at": "2022-12-08T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 8434} +{"id": "defe464a-246d-4c08-94e7-a15c11e91aad", "type": "debit", "created_at": "2022-11-17T00:00:00", "document": "65", "payer": "William Boxer", "amount": 2274} +{"id": "e260af4b-13c4-4e8f-bff9-765151865ba7", "type": "debit", "created_at": "2022-11-17T00:00:00", "document": "39", "payer": "Brendan Pease", "amount": 3105} +{"id": "19d17d8f-a6e7-4e08-9dc1-79d92476e11e", "type": "credit", "created_at": "2022-08-24T00:00:00", "document": "2", "payer": "Ella Tatum", "amount": 7972} +{"id": "c37827c5-5e67-4a28-ad75-40702ae8c6dd", "type": "debit", "created_at": "2022-11-16T00:00:00", "document": "90", "payer": "Michael Griffin", "amount": 8600} +{"id": "36391409-7999-41c9-ac53-07a9cb54a60f", "type": "bank_slip", "created_at": "2022-11-13T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 2684} +{"id": "e1cec55e-bd63-4d7d-9916-3b65d905cdf6", "type": "credit", "created_at": "2022-08-21T00:00:00", "document": "95", "payer": "Sarah Mcguire", "amount": 735} +{"id": "7e121e0e-5c17-4b85-af8d-0567ef56e029", "type": "bank_slip", "created_at": "2022-12-20T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 6789} +{"id": "34b68a91-6f3a-4925-aa21-c3ec41e95c26", "type": "debit", "created_at": "2022-07-05T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 2222} +{"id": "19121a0e-4377-4c7d-a5ee-9aae614c4839", "type": "debit", "created_at": "2022-08-06T00:00:00", "document": "96", "payer": "Earl Morgan", "amount": 4821} +{"id": "cb9f66a6-296b-4322-9714-a89138c62d63", "type": "debit", "created_at": "2022-05-15T00:00:00", "document": "30", "payer": "Ronald Shurtleff", "amount": 8215} +{"id": "6a9a914d-76af-483e-b9d0-66abfa6b9e95", "type": "debit", "created_at": "2022-02-10T00:00:00", "document": "16", "payer": "Gary Kozak", "amount": 7592} +{"id": "98741d86-984c-4c98-8bdc-93190550775b", "type": "credit", "created_at": "2022-06-05T00:00:00", "document": "58", "payer": "Erin Rhoads", "amount": 7028} +{"id": "b8a990ba-fd60-4a1c-9d47-2ad7c7e0512f", "type": "bank_slip", "created_at": "2022-11-01T00:00:00", "document": "59", "payer": "Elizabeth Robinson", "amount": 1647} +{"id": "2fa75936-3d67-41d6-a751-1ccb190a42da", "type": "debit", "created_at": "2022-06-20T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 9290} +{"id": "70ce7df9-0eca-47ba-9459-61358acc015f", "type": "credit", "created_at": "2022-07-18T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 6106} +{"id": "11c5f8c4-8017-456e-a26b-d13a83926737", "type": "credit", "created_at": "2022-02-22T00:00:00", "document": "44", "payer": "Kevin Parry", "amount": 4741} +{"id": "3d83d16e-9aca-44e6-a76e-d4280945ef3a", "type": "bank_slip", "created_at": "2022-05-09T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 4202} +{"id": "0ef18ef6-f824-49a7-b807-9f9129262347", "type": "debit", "created_at": "2022-03-31T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 6813} +{"id": "4fda02e5-c86a-4bc6-b40f-2c03dd6bfffd", "type": "bank_slip", "created_at": "2022-11-11T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 79} +{"id": "ada71770-7f42-42e5-93f3-d84e7a883a6a", "type": "bank_slip", "created_at": "2022-05-20T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 6661} +{"id": "7eb4f100-4135-4817-916c-3d3a95b964c8", "type": "credit", "created_at": "2022-08-08T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 8687} +{"id": "b93b07ca-1954-4347-970a-d39acc87228e", "type": "credit", "created_at": "2022-12-29T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 9100} +{"id": "8c0d9b63-af23-4984-97b2-a99d6b6f75c8", "type": "bank_slip", "created_at": "2022-06-22T00:00:00", "document": "34", "payer": "Sheila Ellard", "amount": 1941} +{"id": "63582bd7-6e81-41c9-86be-74269d674e13", "type": "bank_slip", "created_at": "2022-09-19T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 1275} +{"id": "bcb77f1f-decd-4ef6-9ab9-8aa148aad238", "type": "bank_slip", "created_at": "2022-09-09T00:00:00", "document": "3", "payer": "Conrad Hardin", "amount": 9389} +{"id": "9dcb9da3-06da-41c4-a140-aa40136aac5a", "type": "credit", "created_at": "2022-02-13T00:00:00", "document": "79", "payer": "Steven Rapp", "amount": 4451} +{"id": "f6dfa95f-7841-4f84-bf8c-db2d69ded334", "type": "bank_slip", "created_at": "2022-03-16T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 2842} +{"id": "5e0297be-194d-444e-8fb0-7e7e06a161ed", "type": "bank_slip", "created_at": "2022-02-03T00:00:00", "document": "54", "payer": "Carmela Ho", "amount": 4792} +{"id": "6252ab0e-ffbe-47d1-aa06-f1104c646434", "type": "bank_slip", "created_at": "2022-03-07T00:00:00", "document": "64", "payer": "Elizabeth Bobadilla", "amount": 9102} +{"id": "079859e6-6d46-45d0-b6c0-fa753aafb90d", "type": "credit", "created_at": "2022-07-10T00:00:00", "document": "97", "payer": "Marc Greer", "amount": 2163} +{"id": "c2a91737-0dfe-4b2d-ac70-780f40f2a189", "type": "bank_slip", "created_at": "2022-04-13T00:00:00", "document": "8", "payer": "Nicky Jones", "amount": 3824} +{"id": "613a7609-01ab-4c13-8e25-3eeb42873cbb", "type": "credit", "created_at": "2022-04-18T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 4522} +{"id": "457d0d3d-0ae8-4bde-9da0-b5cebfc3504b", "type": "debit", "created_at": "2022-07-21T00:00:00", "document": "75", "payer": "Edie Callam", "amount": 4538} +{"id": "72d6d64a-dfc0-4fb8-86ce-80da1bc60892", "type": "debit", "created_at": "2022-09-14T00:00:00", "document": "79", "payer": "Steven Rapp", "amount": 5419} +{"id": "80ddd00c-562a-44b5-8b5e-0e8bd61342c6", "type": "credit", "created_at": "2022-10-29T00:00:00", "document": "43", "payer": "Gary Jensen", "amount": 2311} +{"id": "9ea31658-2942-492b-9c58-2ecc12d65228", "type": "bank_slip", "created_at": "2022-12-08T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 816} +{"id": "388f1ef8-7d46-4d21-81f3-f6562dc327bf", "type": "bank_slip", "created_at": "2022-10-13T00:00:00", "document": "52", "payer": "Derek Corcoran", "amount": 6987} +{"id": "b01e19d9-522f-4e72-b5ce-27fa29fd3d20", "type": "credit", "created_at": "2022-07-08T00:00:00", "document": "41", "payer": "Tamera Boone", "amount": 2361} +{"id": "437f9247-65bd-4b0a-9d00-9646cf844e84", "type": "credit", "created_at": "2022-08-20T00:00:00", "document": "9", "payer": "Connie Randall", "amount": 750} +{"id": "d28c7489-cb4a-480b-9e46-e76dadd3e455", "type": "bank_slip", "created_at": "2022-01-27T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 4194} +{"id": "563d63ab-3303-444b-9498-42fc7651cc7f", "type": "debit", "created_at": "2022-03-13T00:00:00", "document": "12", "payer": "Susan Woodward", "amount": 4510} +{"id": "36a40ca1-b6dd-4626-8a2c-a1450a22ac2e", "type": "debit", "created_at": "2022-09-06T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 9728} +{"id": "02c523c1-aca2-4095-bbe0-112e5df3684f", "type": "bank_slip", "created_at": "2022-09-26T00:00:00", "document": "35", "payer": "Maggie Auten", "amount": 3404} +{"id": "c0f1e648-2028-48ad-a005-3cb2c100783b", "type": "credit", "created_at": "2022-09-24T00:00:00", "document": "74", "payer": "Norma Brumit", "amount": 7789} +{"id": "41626e98-2638-4e3e-a1c9-37be5dbb4c19", "type": "credit", "created_at": "2022-08-08T00:00:00", "document": "60", "payer": "Rachel York", "amount": 7230} +{"id": "5087fb2f-8d9b-465e-9c33-cc406a601075", "type": "bank_slip", "created_at": "2022-08-05T00:00:00", "document": "86", "payer": "Robert Kraft", "amount": 5116} +{"id": "cb10a857-9bb1-4ab6-bca3-dd5f734cf535", "type": "credit", "created_at": "2022-07-12T00:00:00", "document": "4", "payer": "Marcia Griffin", "amount": 8537} +{"id": "bc76c74f-8f6a-4103-ab94-ecbcd65c97ab", "type": "bank_slip", "created_at": "2022-04-06T00:00:00", "document": "0", "payer": "Ruth Sheth", "amount": 8291} +{"id": "070d0ed7-9cee-4e75-938e-404bb80a9dcc", "type": "bank_slip", "created_at": "2022-02-16T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 9812} +{"id": "90cd0a2f-2dc9-4872-8052-6f5e0fd1e876", "type": "bank_slip", "created_at": "2022-06-15T00:00:00", "document": "26", "payer": "Robert Krupp", "amount": 5053} +{"id": "222f8c46-15bb-43aa-800d-6dc2a05aa4a4", "type": "bank_slip", "created_at": "2022-10-24T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 8670} +{"id": "81d1a61a-4b80-4b79-878c-a4675480f52c", "type": "debit", "created_at": "2022-07-02T00:00:00", "document": "56", "payer": "Katherine Alexander", "amount": 781} +{"id": "9505fd83-2b83-412b-aa00-17bada1eb572", "type": "bank_slip", "created_at": "2022-11-13T00:00:00", "document": "13", "payer": "Janet Perry", "amount": 65} +{"id": "dd078cc4-4002-491a-8145-fdf2884348d0", "type": "bank_slip", "created_at": "2022-05-28T00:00:00", "document": "88", "payer": "Gabrielle Santoyo", "amount": 128} +{"id": "401d2f86-6cd4-4f0a-b775-9a9527cb9a51", "type": "debit", "created_at": "2022-12-18T00:00:00", "document": "83", "payer": "Filomena Suther", "amount": 691} +{"id": "3fd84e24-815b-42e7-98c1-607ddb61b375", "type": "bank_slip", "created_at": "2022-07-20T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 3101} +{"id": "72adbe0f-7109-431a-a863-a8e80187f760", "type": "credit", "created_at": "2022-03-05T00:00:00", "document": "33", "payer": "Charlotte Marczak", "amount": 3565} +{"id": "449068f7-e806-4bd7-8919-3d2c07ea76c1", "type": "debit", "created_at": "2022-11-03T00:00:00", "document": "65", "payer": "William Boxer", "amount": 9027} +{"id": "72063e36-ac6c-4fb3-a9bd-e2b7f281263f", "type": "debit", "created_at": "2022-09-24T00:00:00", "document": "32", "payer": "Barbara Harris", "amount": 6465} +{"id": "c3bbc3bb-d773-402b-954c-cebaddbbe8f0", "type": "debit", "created_at": "2022-10-07T00:00:00", "document": "17", "payer": "Janice Coughlin", "amount": 1572} +{"id": "79f4a37a-55e7-4887-a621-0f34bc559c52", "type": "bank_slip", "created_at": "2022-02-22T00:00:00", "document": "69", "payer": "Mark Evans", "amount": 1326} +{"id": "cfd9340a-d820-4acc-bffa-c981b1939550", "type": "credit", "created_at": "2022-01-18T00:00:00", "document": "53", "payer": "Louise Witherspoon", "amount": 1487} +{"id": "f03afd3c-4b44-4f03-852d-e2609b89f586", "type": "debit", "created_at": "2022-12-07T00:00:00", "document": "60", "payer": "Rachel York", "amount": 6945} +{"id": "430b590d-f93c-4c5b-972c-c06f5f78622d", "type": "debit", "created_at": "2022-05-26T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 8406} +{"id": "c172de72-e712-44d7-9595-59204d10ccbc", "type": "debit", "created_at": "2022-11-12T00:00:00", "document": "7", "payer": "Maria Bonney", "amount": 5313} +{"id": "02db4ec5-ab83-4ca7-a60a-dca48d782fa3", "type": "bank_slip", "created_at": "2022-01-30T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 8662} +{"id": "0feb1d52-0e1b-4fa6-9c42-99a58224473d", "type": "bank_slip", "created_at": "2022-02-18T00:00:00", "document": "27", "payer": "Devin Felt", "amount": 9956} +{"id": "f6f8ea73-442a-4551-b958-cb75fcd83ef5", "type": "bank_slip", "created_at": "2022-01-02T00:00:00", "document": "47", "payer": "Dustin Diller", "amount": 5694} +{"id": "3d7d28d0-b1d3-4a60-8042-808fe71133c9", "type": "credit", "created_at": "2022-03-20T00:00:00", "document": "73", "payer": "Fe Bartkowiak", "amount": 9905} +{"id": "776d87d3-fc3b-42ba-ac74-572798608535", "type": "debit", "created_at": "2022-08-16T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 9789} +{"id": "fcf569a2-205b-46f5-b8dc-49953d0eea6c", "type": "bank_slip", "created_at": "2022-04-01T00:00:00", "document": "87", "payer": "Emily Cockerill", "amount": 6698} +{"id": "daf72ab9-1d00-47b9-8d68-81dc0efa611c", "type": "bank_slip", "created_at": "2022-05-18T00:00:00", "document": "99", "payer": "Mark Seidel", "amount": 3616} +{"id": "03176959-14a8-43c6-b194-1048b7e6d394", "type": "credit", "created_at": "2022-02-10T00:00:00", "document": "2", "payer": "Ella Tatum", "amount": 8395} +{"id": "c030b899-67da-45f8-8f63-4815f5d93cd1", "type": "debit", "created_at": "2022-07-21T00:00:00", "document": "19", "payer": "Patrick Monson", "amount": 7327} +{"id": "0cc2cf12-43d0-4672-8c58-4d33ade4bed2", "type": "debit", "created_at": "2022-03-27T00:00:00", "document": "81", "payer": "Steven Delosantos", "amount": 1621} +{"id": "f19ad312-8f2f-41b7-b454-01503426094a", "type": "debit", "created_at": "2022-10-10T00:00:00", "document": "71", "payer": "Florence Henderson", "amount": 5475} +{"id": "12e4662e-0dfb-4347-a14c-9df153faddb6", "type": "debit", "created_at": "2022-09-05T00:00:00", "document": "21", "payer": "Richard Duckett", "amount": 7792} +{"id": "62b783ba-e6aa-45bd-8e85-8204364a1c2e", "type": "credit", "created_at": "2022-07-04T00:00:00", "document": "80", "payer": "Joyce Wimberly", "amount": 80} +{"id": "661a7644-ce14-49d4-b267-b2f72b0c9e8d", "type": "bank_slip", "created_at": "2022-09-21T00:00:00", "document": "29", "payer": "Timothy Short", "amount": 2140} +{"id": "745feb36-cad3-47a5-b38b-c62324dedbc2", "type": "credit", "created_at": "2022-10-28T00:00:00", "document": "18", "payer": "Armida Kostrzewa", "amount": 9459} \ No newline at end of file diff --git a/ccloud/custom-connector-connect-iceberg-sink/docker-compose.yml b/ccloud/custom-connector-connect-iceberg-sink/docker-compose.yml new file mode 100644 index 0000000000..a822d625d6 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/docker-compose.yml @@ -0,0 +1,106 @@ +--- +services: + # based on https://github.com/Wuerike/kafka-iceberg-streaming/tree/main + minio: + image: minio/minio + hostname: minio + container_name: minio + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + - MINIO_DOMAIN=minio + networks: + default: + aliases: + - warehouse.minio + ports: + - 9001:9001 + - 9000:9000 + command: ["server", "/data", "--console-address", ":9001"] + + aws: + image: amazon/aws-cli + container_name: aws-cli + command: | + -c "sleep 2 && \ + aws --endpoint-url http://minio:9000 s3 mb s3://warehouse --region eu-west-1 || exit 0" + entrypoint: [/bin/bash] + environment: + AWS_ACCESS_KEY_ID: "minioadmin" + AWS_SECRET_ACCESS_KEY: "minioadmin" + depends_on: + - minio + + spark-iceberg: + image: tabulario/spark-iceberg + hostname: spark-iceberg + container_name: spark-iceberg + build: ../../ccloud/custom-connector-connect-iceberg-sink/spark/ + depends_on: + - rest + - minio + environment: + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_REGION: eu-west-1 + SPARK_DEFAULTS: | + spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions + spark.sql.catalog.iceberg org.apache.iceberg.spark.SparkCatalog + spark.sql.catalog.iceberg.catalog-impl org.apache.iceberg.rest.RESTCatalog + spark.sql.catalog.iceberg.uri http://rest:8181 + spark.sql.catalog.iceberg.io-impl org.apache.iceberg.aws.s3.S3FileIO + spark.sql.catalog.iceberg.warehouse s3://warehouse/wh/ + spark.sql.catalog.iceberg.s3.endpoint http://minio:9000 + spark.sql.catalog.iceberg.s3.path-style-access true + spark.sql.defaultCatalog iceberg + spark.sql.catalogImplementation in-memory + spark.eventLog.enabled true + spark.eventLog.dir /home/iceberg/spark-events + spark.history.fs.logDirectory /home/iceberg/spark-events + spark.jars.packages org.apache.hadoop:hadoop-aws:3.2.0 + ports: + - 8888:8888 + # - 8080:8080 + # - 10000:10000 + # - 10001:10001 + volumes: + - ../../ccloud/custom-connector-connect-iceberg-sink/spark:/home/iceberg/scripts + - ../../ccloud/custom-connector-connect-iceberg-sink/notebooks:/home/iceberg/notebooks/notebooks + command: ["echo \"$$SPARK_DEFAULTS\" > /opt/spark/conf/spark-defaults.conf && spark-submit /home/iceberg/scripts/create_table.py && notebook"] + + rest: + image: tabulario/iceberg-rest + hostname: rest + container_name: rest + ports: + - 8181:8181 + environment: + - AWS_ACCESS_KEY_ID=minioadmin + - AWS_SECRET_ACCESS_KEY=minioadmin + - AWS_REGION=eu-west-1 + - CATALOG_WAREHOUSE=s3://warehouse/ + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio:9000 + - CATALOG_S3_PATH__STYLE__ACCESS=True + + # https://ngrok.com/docs/using-ngrok-with/docker/ + ngrok: + image: ngrok/ngrok:latest + hostname: ngrok + container_name: ngrok + ports: + - 4040:4040 + restart: unless-stopped + links: + - minio + - rest + command: + - "start" + - "--all" + - "--log=stdout" + - "--config" + - "/etc/ngrok.yml" + volumes: + - ../../ccloud/custom-connector-connect-iceberg-sink/ngrok.yml:/etc/ngrok.yml + environment: + NGROK_AUTHTOKEN: $NGROK_AUTH_TOKEN diff --git a/ccloud/custom-connector-connect-iceberg-sink/ngrok.yml b/ccloud/custom-connector-connect-iceberg-sink/ngrok.yml new file mode 100644 index 0000000000..0ed8fdded3 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/ngrok.yml @@ -0,0 +1,8 @@ +version: 2 +tunnels: + minio: + addr: minio:9000 + proto: tcp + rest: + addr: rest:8181 + proto: tcp \ No newline at end of file diff --git a/ccloud/custom-connector-connect-iceberg-sink/notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/ccloud/custom-connector-connect-iceberg-sink/notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 0000000000..363fcab7ed --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/notebooks/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/notebooks/.ipynb_checkpoints/iceberg-checkpoint.ipynb b/ccloud/custom-connector-connect-iceberg-sink/notebooks/.ipynb_checkpoints/iceberg-checkpoint.ipynb new file mode 100644 index 0000000000..15aa20543b --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/notebooks/.ipynb_checkpoints/iceberg-checkpoint.ipynb @@ -0,0 +1,103 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "29f8d24e-e4bf-484d-afd4-cb82ff6cd50d", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW DATABASES" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70349765-e5f1-43a5-a141-cc2d54c69a58", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW TABLES FROM orders" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fabaed9c-9049-4996-9d26-b20f66303911", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW TBLPROPERTIES orders.payments" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6317d9c6-140e-4a63-890e-2173fbb9503e", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT COUNT(*)\n", + "FROM orders.payments" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a1ff132-dc65-4943-a9be-416ba5a13c26", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM orders.payments\n", + "LIMIT 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2688a95-594c-45ad-9d49-70a1bcd59a1b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * \n", + "FROM orders.payments.partitions\n", + "ORDER BY record_count DESC\n", + "LIMIT 10" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/notebooks/Untitled.ipynb b/ccloud/custom-connector-connect-iceberg-sink/notebooks/Untitled.ipynb new file mode 100644 index 0000000000..d785557639 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/notebooks/Untitled.ipynb @@ -0,0 +1,33 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e5b4f2cb-d4fe-4376-a4e9-d6683887b1b7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/notebooks/iceberg.ipynb b/ccloud/custom-connector-connect-iceberg-sink/notebooks/iceberg.ipynb new file mode 100644 index 0000000000..152a1d29f8 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/notebooks/iceberg.ipynb @@ -0,0 +1,249 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "29f8d24e-e4bf-484d-afd4-cb82ff6cd50d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "24/10/16 10:11:21 WARN SparkSession: Using an existing Spark session; only runtime SQL configurations will take effect.\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namespace
orders
" + ], + "text/plain": [ + "+-----------+\n", + "| namespace |\n", + "+-----------+\n", + "| orders |\n", + "+-----------+" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "SHOW DATABASES" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70349765-e5f1-43a5-a141-cc2d54c69a58", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW TABLES FROM orders" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fabaed9c-9049-4996-9d26-b20f66303911", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW TBLPROPERTIES orders.payments" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6317d9c6-140e-4a63-890e-2173fbb9503e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
count(1)
0
" + ], + "text/plain": [ + "+----------+\n", + "| count(1) |\n", + "+----------+\n", + "| 0 |\n", + "+----------+" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "SELECT COUNT(*)\n", + "FROM orders.payments" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2a1ff132-dc65-4943-a9be-416ba5a13c26", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtypecreated_atdocumentpayeramount
" + ], + "text/plain": [ + "+----+------+------------+----------+-------+--------+\n", + "| id | type | created_at | document | payer | amount |\n", + "+----+------+------------+----------+-------+--------+\n", + "+----+------+------------+----------+-------+--------+" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM orders.payments\n", + "LIMIT 10" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a2688a95-594c-45ad-9d49-70a1bcd59a1b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
partitionspec_idrecord_countfile_counttotal_data_file_size_in_bytesposition_delete_record_countposition_delete_file_countequality_delete_record_countequality_delete_file_countlast_updated_atlast_updated_snapshot_id
" + ], + "text/plain": [ + "+-----------+---------+--------------+------------+-------------------------------+------------------------------+----------------------------+------------------------------+----------------------------+-----------------+--------------------------+\n", + "| partition | spec_id | record_count | file_count | total_data_file_size_in_bytes | position_delete_record_count | position_delete_file_count | equality_delete_record_count | equality_delete_file_count | last_updated_at | last_updated_snapshot_id |\n", + "+-----------+---------+--------------+------------+-------------------------------+------------------------------+----------------------------+------------------------------+----------------------------+-----------------+--------------------------+\n", + "+-----------+---------+--------------+------------+-------------------------------+------------------------------+----------------------------+------------------------------+----------------------------+-----------------+--------------------------+" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%sql\n", + "\n", + "SELECT * \n", + "FROM orders.payments.partitions\n", + "ORDER BY record_count DESC\n", + "LIMIT 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c576e9ab-b014-44f8-ba79-c5f16b4b719b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15bbf09b-c29e-4911-b82e-839e9f854756", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/.pyiceberg.yaml b/ccloud/custom-connector-connect-iceberg-sink/spark/.pyiceberg.yaml new file mode 100644 index 0000000000..7444ab0999 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/.pyiceberg.yaml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +catalog: + default: + uri: http://rest:8181 + s3.endpoint: http://minio:9000 + s3.access-key-id: admin + s3.secret-access-key: password diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/Dockerfile b/ccloud/custom-connector-connect-iceberg-sink/spark/Dockerfile new file mode 100644 index 0000000000..e2d163552b --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/Dockerfile @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# syntax=docker/dockerfile:1 +FROM python:3.9-bullseye + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + sudo \ + curl \ + vim \ + unzip \ + openjdk-11-jdk \ + build-essential \ + software-properties-common \ + ssh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install Jupyter and other python deps +COPY requirements.txt . +RUN pip3 install -r requirements.txt + +# Add scala kernel via spylon-kernel +RUN python3 -m spylon_kernel install + +# Download and install IJava jupyter kernel +RUN curl https://github.com/SpencerPark/IJava/releases/download/v1.3.0/ijava-1.3.0.zip -Lo ijava-1.3.0.zip \ + && unzip ijava-1.3.0.zip \ + && python3 install.py --sys-prefix \ + && rm ijava-1.3.0.zip + +# Optional env variables +ENV SPARK_HOME=${SPARK_HOME:-"/opt/spark"} +ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH + +WORKDIR ${SPARK_HOME} + +ENV SPARK_VERSION=3.5.6 +ENV SPARK_MAJOR_VERSION=3.5 +ENV ICEBERG_VERSION=1.5.0 + +# Download spark +RUN mkdir -p ${SPARK_HOME} \ + && curl https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ + && tar xvzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \ + && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz + +# Download iceberg spark runtime +RUN curl https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar + +# Download AWS bundle +RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar + +# Install AWS CLI +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ + && unzip awscliv2.zip \ + && sudo ./aws/install \ + && rm awscliv2.zip \ + && rm -rf aws/ + +# Add iceberg spark runtime jar to IJava classpath +ENV IJAVA_CLASSPATH=/opt/spark/jars/* + +RUN mkdir -p /home/iceberg/data \ + && curl https://data.cityofnewyork.us/resource/tg4x-b46p.json > /home/iceberg/data/nyc_film_permits.json \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-04.parquet -o /home/iceberg/data/yellow_tripdata_2022-04.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-03.parquet -o /home/iceberg/data/yellow_tripdata_2022-03.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-02.parquet -o /home/iceberg/data/yellow_tripdata_2022-02.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -o /home/iceberg/data/yellow_tripdata_2022-01.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-12.parquet -o /home/iceberg/data/yellow_tripdata_2021-12.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-11.parquet -o /home/iceberg/data/yellow_tripdata_2021-11.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-10.parquet -o /home/iceberg/data/yellow_tripdata_2021-10.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-09.parquet -o /home/iceberg/data/yellow_tripdata_2021-09.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-08.parquet -o /home/iceberg/data/yellow_tripdata_2021-08.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-07.parquet -o /home/iceberg/data/yellow_tripdata_2021-07.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-06.parquet -o /home/iceberg/data/yellow_tripdata_2021-06.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-05.parquet -o /home/iceberg/data/yellow_tripdata_2021-05.parquet \ + && curl https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-04.parquet -o /home/iceberg/data/yellow_tripdata_2021-04.parquet + +RUN mkdir -p /home/iceberg/localwarehouse /home/iceberg/notebooks /home/iceberg/warehouse /home/iceberg/spark-events /home/iceberg +COPY notebooks/ /home/iceberg/notebooks + +# Add a notebook command +RUN echo '#! /bin/sh' >> /bin/notebook \ + && echo 'export PYSPARK_DRIVER_PYTHON=jupyter-notebook' >> /bin/notebook \ + && echo "export PYSPARK_DRIVER_PYTHON_OPTS=\"--notebook-dir=/home/iceberg/notebooks --ip='*' --NotebookApp.token='' --NotebookApp.password='' --port=8888 --no-browser --allow-root\"" >> /bin/notebook \ + && echo "pyspark" >> /bin/notebook \ + && chmod u+x /bin/notebook + +# Add a pyspark-notebook command (alias for notebook command for backwards-compatibility) +RUN echo '#! /bin/sh' >> /bin/pyspark-notebook \ + && echo 'export PYSPARK_DRIVER_PYTHON=jupyter-notebook' >> /bin/pyspark-notebook \ + && echo "export PYSPARK_DRIVER_PYTHON_OPTS=\"--notebook-dir=/home/iceberg/notebooks --ip='*' --NotebookApp.token='' --NotebookApp.password='' --port=8888 --no-browser --allow-root\"" >> /bin/pyspark-notebook \ + && echo "pyspark" >> /bin/pyspark-notebook \ + && chmod u+x /bin/pyspark-notebook + +RUN mkdir -p /root/.ipython/profile_default/startup +COPY ipython/startup/00-prettytables.py /root/.ipython/profile_default/startup +COPY ipython/startup/README /root/.ipython/profile_default/startup + +COPY spark-defaults.conf /opt/spark/conf +ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}" + +RUN chmod u+x /opt/spark/sbin/* && \ + chmod u+x /opt/spark/bin/* + +COPY .pyiceberg.yaml /root/.pyiceberg.yaml + +COPY entrypoint.sh . + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["notebook"] diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/create_table.py b/ccloud/custom-connector-connect-iceberg-sink/spark/create_table.py new file mode 100644 index 0000000000..92525744d0 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/create_table.py @@ -0,0 +1,20 @@ +from pyspark.sql import SparkSession + +spark = SparkSession.builder.appName("").getOrCreate() + +print("creating database") +spark.sql('CREATE DATABASE IF NOT EXISTS orders') + +print("creating table") +spark.sql(''' + CREATE TABLE IF NOT EXISTS orders.payments ( + id STRING, + type STRING, + created_at TIMESTAMP, + document STRING, + payer STRING, + amount INT + ) + USING iceberg + PARTITIONED BY (document) +''') diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/entrypoint.sh b/ccloud/custom-connector-connect-iceberg-sink/spark/entrypoint.sh new file mode 100755 index 0000000000..2738b59bac --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +start-master.sh -p 7077 +start-worker.sh spark://spark-iceberg:7077 +start-history-server.sh +start-thriftserver.sh --driver-java-options "-Dderby.system.home=/tmp/derby" + +# Entrypoint, for example notebook, pyspark or spark-sql +if [[ $# -gt 0 ]] ; then + eval "$1" +fi diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/ipython/startup/00-prettytables.py b/ccloud/custom-connector-connect-iceberg-sink/spark/ipython/startup/00-prettytables.py new file mode 100644 index 0000000000..868f7e5ae0 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/ipython/startup/00-prettytables.py @@ -0,0 +1,81 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from prettytable import PrettyTable +from IPython.core.magic import register_line_cell_magic + +class DFTable(PrettyTable): + def __repr__(self): + return self.get_string() + + def _repr_html_(self): + return self.get_html_string() + +def _row_as_table(df): + cols = df.columns + + t = DFTable() + t.field_names = ["Column", "Value"] + t.align = "r" + row = df.limit(1).collect()[0].asDict() + for col in cols: + t.add_row([ col, row[col] ]) + + return t + +def _to_table(df, num_rows=100): + cols = df.columns + + t = DFTable() + t.field_names = cols + t.align = "r" + for row in df.limit(num_rows).collect(): + d = row.asDict() + t.add_row([ d[col] for col in cols ]) + + return t + +import re +import sys +from argparse import ArgumentParser +parser = ArgumentParser() +parser.add_argument("--limit", help="Number of lines to return", type=int, default=100) +parser.add_argument("--var", help="Variable name to hold the dataframe", type=str) + +@register_line_cell_magic +def sql(line, cell=None): + """Spark SQL magic + """ + from pyspark.sql import SparkSession + spark = SparkSession.builder.appName("Jupyter").getOrCreate() + if cell is None: + return _to_table(spark.sql(line)) + elif line: + df = spark.sql(cell) + + (args, others) = parser.parse_known_args([ arg for arg in re.split("\s+", line) if arg ]) + + if args.var: + setattr(sys.modules[__name__], args.var, df) + + if args.limit == 1: + return _row_as_table(df) + else: + return _to_table(df, num_rows=args.limit) + else: + return _to_table(spark.sql(cell)) diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/ipython/startup/README b/ccloud/custom-connector-connect-iceberg-sink/spark/ipython/startup/README new file mode 100644 index 0000000000..61d4700042 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/ipython/startup/README @@ -0,0 +1,11 @@ +This is the IPython startup directory + +.py and .ipy files in this directory will be run *prior* to any code or files specified +via the exec_lines or exec_files configurables whenever you load this profile. + +Files will be run in lexicographical order, so you can control the execution order of files +with a prefix, e.g.:: + + 00-first.py + 50-middle.py + 99-last.ipy diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API.ipynb new file mode 100644 index 0000000000..20af42b17b --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API.ipynb @@ -0,0 +1,469 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "16f6bb49", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "markdown", + "id": "c82657e9", + "metadata": {}, + "source": [ + "# An Introduction to the Iceberg Java API" + ] + }, + { + "cell_type": "markdown", + "id": "3ee90ad2", + "metadata": {}, + "source": [ + "## [Part 1 - Loading a Catalog and Creating a Table](https://tabular.io/blog/java-api-part-1/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72e68c62", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.catalog.Catalog;\n", + "import org.apache.hadoop.conf.Configuration;\n", + "import org.apache.iceberg.CatalogProperties;\n", + "import org.apache.iceberg.rest.RESTCatalog;\n", + "import org.apache.iceberg.aws.s3.S3FileIOProperties;\n", + "\n", + "Map properties = new HashMap<>();\n", + "\n", + "properties.put(CatalogProperties.CATALOG_IMPL, \"org.apache.iceberg.rest.RESTCatalog\");\n", + "properties.put(CatalogProperties.URI, \"http://rest:8181\");\n", + "properties.put(CatalogProperties.WAREHOUSE_LOCATION, \"s3a://warehouse/wh\");\n", + "properties.put(CatalogProperties.FILE_IO_IMPL, \"org.apache.iceberg.aws.s3.S3FileIO\");\n", + "properties.put(S3FileIOProperties.ENDPOINT, \"http://minio:9000\");\n", + "\n", + "RESTCatalog catalog = new RESTCatalog();\n", + "Configuration conf = new Configuration();\n", + "catalog.setConf(conf);\n", + "catalog.initialize(\"demo\", properties);\n", + "catalog.name();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4be615e7", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.Schema;\n", + "import org.apache.iceberg.types.Types;\n", + "\n", + "Schema schema = new Schema(\n", + " Types.NestedField.required(1, \"level\", Types.StringType.get()),\n", + " Types.NestedField.required(2, \"event_time\", Types.TimestampType.withZone()),\n", + " Types.NestedField.required(3, \"message\", Types.StringType.get()),\n", + " Types.NestedField.optional(4, \"call_stack\", Types.ListType.ofRequired(5, Types.StringType.get()))\n", + " );\n", + "schema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7299d16", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.PartitionSpec;\n", + "\n", + "PartitionSpec spec = PartitionSpec.builderFor(schema)\n", + " .hour(\"event_time\")\n", + " .identity(\"level\")\n", + " .build();\n", + "spec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d900c97", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.catalog.TableIdentifier;\n", + "import org.apache.iceberg.catalog.Namespace;\n", + "\n", + "Namespace nyc = Namespace.of(\"nyc\");\n", + "TableIdentifier name = TableIdentifier.of(nyc, \"logs\");\n", + "name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a4d8a6e", + "metadata": {}, + "outputs": [], + "source": [ + "catalog.createTable(name, schema, spec)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d8c46df", + "metadata": {}, + "outputs": [], + "source": [ + "catalog.dropTable(name)" + ] + }, + { + "cell_type": "markdown", + "id": "fe62e0a9", + "metadata": {}, + "source": [ + "## [Part 2 - Table Scans](https://tabular.io/blog/java-api-part-2/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1e7aa7a", + "metadata": {}, + "outputs": [], + "source": [ + "catalog.createTable(name, schema, spec)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78c95e06", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.spark.sql.SparkSession;\n", + "\n", + "SparkSession spark = SparkSession\n", + " .builder()\n", + " .master(\"local[*]\")\n", + " .appName(\"Java API Demo\")\n", + " .config(\"spark.sql.extensions\", \"org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions\")\n", + " .config(\"spark.sql.catalog.demo\", \"org.apache.iceberg.spark.SparkCatalog\")\n", + " .config(\"spark.sql.catalog.demo.catalog-impl\", \"org.apache.iceberg.rest.RESTCatalog\")\n", + " .config(\"spark.sql.catalog.demo.uri\", \"http://rest:8181\")\n", + " .config(\"spark.sql.catalog.demo.io-impl\", \"org.apache.iceberg.aws.s3.S3FileIO\")\n", + " .config(\"spark.sql.catalog.demo.s3.endpoint\", \"http://minio:9000\")\n", + " .config(\"spark.sql.defaultCatalog\", \"demo\")\n", + " .config(\"spark.eventLog.enabled\", \"true\")\n", + " .config(\"spark.eventLog.dir\", \"/home/iceberg/spark-events\")\n", + " .config(\"spark.history.fs.logDirectory\", \"/home/iceberg/spark-events\")\n", + " .getOrCreate();\n", + "\n", + "spark.sparkContext().setLogLevel(\"ERROR\");" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b17f820", + "metadata": {}, + "outputs": [], + "source": [ + "String query = \"INSERT INTO demo.nyc.logs \"\n", + " + \"VALUES \"\n", + " + \"('info', timestamp 'today', 'Just letting you know!', array('stack trace line 1', 'stack trace line 2', 'stack trace line 3')), \"\n", + " + \"('warning', timestamp 'today', 'You probably should not do this!', array('stack trace line 1', 'stack trace line 2', 'stack trace line 3')), \"\n", + " + \"('error', timestamp 'today', 'This was a fatal application error!', array('stack trace line 1', 'stack trace line 2', 'stack trace line 3'))\";\n", + "\n", + "spark.sql(query).show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15ca1822", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.catalog.Catalog;\n", + "import org.apache.hadoop.conf.Configuration;\n", + "import org.apache.iceberg.CatalogProperties;\n", + "import org.apache.iceberg.rest.RESTCatalog;\n", + "\n", + "Map properties = new HashMap<>();\n", + "\n", + "properties.put(CatalogProperties.CATALOG_IMPL, \"org.apache.iceberg.rest.RESTCatalog\");\n", + "properties.put(CatalogProperties.URI, \"http://rest:8181\");\n", + "properties.put(CatalogProperties.WAREHOUSE_LOCATION, \"s3a://warehouse/wh/\");\n", + "properties.put(CatalogProperties.FILE_IO_IMPL, \"org.apache.iceberg.aws.s3.S3FileIO\");\n", + "properties.put(S3FileIOProperties.ENDPOINT, \"http://minio:9000\");\n", + "\n", + "RESTCatalog catalog = new RESTCatalog();\n", + "Configuration conf = new Configuration();\n", + "catalog.setConf(conf);\n", + "catalog.initialize(\"demo\", properties);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a5cf423", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.Table;\n", + "import org.apache.iceberg.TableScan;\n", + "import org.apache.iceberg.catalog.Namespace;\n", + "import org.apache.iceberg.catalog.TableIdentifier;\n", + "\n", + "Namespace nyc = Namespace.of(\"nyc\");\n", + "TableIdentifier name = TableIdentifier.of(nyc, \"logs\");\n", + "Table table = catalog.loadTable(name);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e472d6a1", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.io.CloseableIterable;\n", + "import org.apache.iceberg.data.Record;\n", + "import org.apache.iceberg.data.IcebergGenerics;\n", + "\n", + "CloseableIterable result = IcebergGenerics.read(table).build();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d32f41c", + "metadata": {}, + "outputs": [], + "source": [ + "for (Record r: result) {\n", + " System.out.println(r);\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7dffc238", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.expressions.Expressions;\n", + "\n", + "CloseableIterable result = IcebergGenerics.read(table)\n", + " .where(Expressions.equal(\"level\", \"error\"))\n", + " .build();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec2b0431", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.CombinedScanTask;\n", + "import org.apache.iceberg.TableScan;\n", + "\n", + "TableScan scan = table.newScan();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09d13c6b", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.expressions.Expressions;\n", + "\n", + "TableScan filteredScan = scan.filter(Expressions.equal(\"level\", \"info\")).select(\"message\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1857c10f", + "metadata": {}, + "outputs": [], + "source": [ + "Iterable result = filteredScan.planTasks();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea206ec7", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.DataFile;\n", + "\n", + "CombinedScanTask task = result.iterator().next();\n", + "DataFile dataFile = task.files().iterator().next().file();\n", + "System.out.println(dataFile);" + ] + }, + { + "cell_type": "markdown", + "id": "41e9e10f", + "metadata": {}, + "source": [ + "## [Part 3 - Table Scans](https://tabular.io/blog/java-api-part-3/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81033412", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.Schema;\n", + "import org.apache.iceberg.types.Types;\n", + "import org.apache.iceberg.catalog.Namespace;\n", + "import org.apache.iceberg.catalog.TableIdentifier;\n", + "import org.apache.iceberg.PartitionSpec;\n", + "\n", + "Schema schema = new Schema(\n", + " Types.NestedField.optional(1, \"event_id\", Types.StringType.get()),\n", + " Types.NestedField.optional(2, \"username\", Types.StringType.get()),\n", + " Types.NestedField.optional(3, \"userid\", Types.IntegerType.get()),\n", + " Types.NestedField.optional(4, \"api_version\", Types.StringType.get()),\n", + " Types.NestedField.optional(5, \"command\", Types.StringType.get())\n", + " );\n", + "\n", + "Namespace webapp = Namespace.of(\"webapp\");\n", + "TableIdentifier name = TableIdentifier.of(webapp, \"user_events\");\n", + "catalog.createTable(name, schema, PartitionSpec.unpartitioned());" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12c45c6b", + "metadata": {}, + "outputs": [], + "source": [ + "import java.util.UUID;\n", + "import com.google.common.collect.ImmutableList;\n", + "import com.google.common.collect.ImmutableMap;\n", + "import org.apache.iceberg.data.GenericRecord;\n", + "\n", + "GenericRecord record = GenericRecord.create(schema);\n", + "ImmutableList.Builder builder = ImmutableList.builder();\n", + "builder.add(record.copy(ImmutableMap.of(\"event_id\", UUID.randomUUID().toString(), \"username\", \"Bruce\", \"userid\", 1, \"api_version\", \"1.0\", \"command\", \"grapple\")));\n", + "builder.add(record.copy(ImmutableMap.of(\"event_id\", UUID.randomUUID().toString(), \"username\", \"Wayne\", \"userid\", 1, \"api_version\", \"1.0\", \"command\", \"glide\")));\n", + "builder.add(record.copy(ImmutableMap.of(\"event_id\", UUID.randomUUID().toString(), \"username\", \"Clark\", \"userid\", 1, \"api_version\", \"2.0\", \"command\", \"fly\")));\n", + "builder.add(record.copy(ImmutableMap.of(\"event_id\", UUID.randomUUID().toString(), \"username\", \"Kent\", \"userid\", 1, \"api_version\", \"1.0\", \"command\", \"land\")));\n", + "ImmutableList records = builder.build();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "83bc5319", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.Files;\n", + "import org.apache.iceberg.io.DataWriter;\n", + "import org.apache.iceberg.io.OutputFile;\n", + "import org.apache.iceberg.parquet.Parquet;\n", + "import org.apache.iceberg.data.parquet.GenericParquetWriter;\n", + "\n", + "String filepath = table.location() + \"/\" + UUID.randomUUID().toString();\n", + "OutputFile file = table.io().newOutputFile(filepath);\n", + "DataWriter dataWriter =\n", + " Parquet.writeData(file)\n", + " .schema(schema)\n", + " .createWriterFunc(GenericParquetWriter::buildWriter)\n", + " .overwrite()\n", + " .withSpec(PartitionSpec.unpartitioned())\n", + " .build();\n", + "try {\n", + " for (GenericRecord record : builder.build()) {\n", + " dataWriter.write(record);\n", + " }\n", + "} finally {\n", + " dataWriter.close();\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "469e6af4", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.DataFile;\n", + "\n", + "DataFile dataFile = dataWriter.toDataFile();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "142b6ed1", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.catalog.Namespace;\n", + "import org.apache.iceberg.catalog.TableIdentifier;\n", + "import org.apache.iceberg.Table;\n", + "\n", + "Namespace webapp = Namespace.of(\"webapp\");\n", + "TableIdentifier name = TableIdentifier.of(webapp, \"user_events\");\n", + "Table tbl = catalog.loadTable(name);\n", + "tbl.newAppend().appendFile(dataFile).commit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c61e9e79", + "metadata": {}, + "outputs": [], + "source": [ + "import org.apache.iceberg.io.CloseableIterable;\n", + "import org.apache.iceberg.data.Record;\n", + "import org.apache.iceberg.data.IcebergGenerics;\n", + "\n", + "CloseableIterable result = IcebergGenerics.read(tbl).build();\n", + "for (Record r: result) {\n", + " System.out.println(r);\n", + "}" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Java", + "language": "java", + "name": "java" + }, + "language_info": { + "codemirror_mode": "java", + "file_extension": ".jshell", + "mimetype": "text/x-java-source", + "name": "Java", + "pygments_lexer": "java", + "version": "11.0.15+10-post-Debian-1deb11u1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Berlin Buzzwords 2023.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Berlin Buzzwords 2023.ipynb new file mode 100644 index 0000000000..66eaad26b1 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Berlin Buzzwords 2023.ipynb @@ -0,0 +1,361 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a5c8206", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "spark" + ] + }, + { + "cell_type": "markdown", + "id": "6f9a9f41", + "metadata": {}, + "source": [ + "## Load Two Months of NYC Taxi/Limousine Trip Data\n", + "\n", + "For this notebook, we will use the New York City Taxi and Limousine Commision Trip Record Data that's available on the AWS Open Data Registry. This contains data of trips taken by taxis and for-hire vehicles in New York City. We'll save this into an iceberg table called `taxis`." + ] + }, + { + "cell_type": "markdown", + "id": "747bee98", + "metadata": {}, + "source": [ + "To be able to rerun the notebook several times, let's drop the table if it exists to start fresh." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "930682ce", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE DATABASE IF NOT EXISTS nyc.taxis;" + ] + }, + { + "cell_type": "markdown", + "id": "5816de2e", + "metadata": {}, + "source": [ + "## First create the table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22ac5552", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DROP TABLE IF EXISTS nyc.taxis;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f918310a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE TABLE nyc.taxis (\n", + " VendorID bigint,\n", + " tpep_pickup_datetime timestamp,\n", + " tpep_dropoff_datetime timestamp,\n", + " passenger_count double,\n", + " trip_distance double,\n", + " RatecodeID double,\n", + " store_and_fwd_flag string,\n", + " PULocationID bigint,\n", + " DOLocationID bigint,\n", + " payment_type bigint,\n", + " fare_amount double,\n", + " extra double,\n", + " mta_tax double,\n", + " tip_amount double,\n", + " tolls_amount double,\n", + " improvement_surcharge double,\n", + " total_amount double,\n", + " congestion_surcharge double,\n", + " airport_fee double\n", + ")\n", + "USING iceberg\n", + "PARTITIONED BY (days(tpep_pickup_datetime))" + ] + }, + { + "cell_type": "markdown", + "id": "fcba103e", + "metadata": {}, + "source": [ + "# Write a month of data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c37ca92", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2022-01.parquet\")\n", + "df.writeTo(\"nyc.taxis\").append()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a69152aa", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis" + ] + }, + { + "cell_type": "markdown", + "id": "6fce6bb4", + "metadata": {}, + "source": [ + "## Metadata Tables\n", + "\n", + "Iceberg tables contain very rich metadata that can be easily queried. For example, you can retrieve the manifest list for any snapshot, simply by querying the table's `snapshots` table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fade1a3", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.snapshots" + ] + }, + { + "cell_type": "markdown", + "id": "4aa4a9cd", + "metadata": {}, + "source": [ + "# Write a month of data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed7a7b8f", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2022-02.parquet\")\n", + "df.writeTo(\"nyc.taxis\").append()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbfb160c", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.snapshots\n", + "ORDER BY committed_at DESC" + ] + }, + { + "cell_type": "markdown", + "id": "65deb074", + "metadata": {}, + "source": [ + "## Manifest lists\n", + "\n", + "Now we'll list all the manifests. This is the abovemention `manifest_list` of the current snapshot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bab64f90", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.manifests" + ] + }, + { + "cell_type": "markdown", + "id": "b11e64c9", + "metadata": {}, + "source": [ + "# Manifests\n", + "\n", + "The next layer is the manifests that has references to the Parquet files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c4a942c", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.files" + ] + }, + { + "cell_type": "markdown", + "id": "31567e4e", + "metadata": {}, + "source": [ + "# Flexibility of partitioning\n", + "\n", + "We can easily change the partitioning of the table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "156885c7", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.taxis.partitions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "184604d9", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis DROP PARTITION FIELD days(tpep_pickup_datetime)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c26dddb5", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis ADD PARTITION FIELD hours(tpep_pickup_datetime)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42ec7b70", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.taxis.partitions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d5dea98", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CALL system.rewrite_data_files('nyc.taxis')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5fdf3a22", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40447a02", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.snapshots\n", + "ORDER BY committed_at DESC" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Getting Started.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Getting Started.ipynb new file mode 100644 index 0000000000..fc817202e6 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Getting Started.ipynb @@ -0,0 +1,509 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "markdown", + "id": "247fb2ab", + "metadata": {}, + "source": [ + "### [Docker, Spark, and Iceberg: The Fastest Way to Try Iceberg!](https://tabular.io/blog/docker-spark-and-iceberg/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a5c8206", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "spark" + ] + }, + { + "cell_type": "markdown", + "id": "6f9a9f41", + "metadata": {}, + "source": [ + "## Load One Month of NYC Taxi/Limousine Trip Data\n", + "\n", + "For this notebook, we will use the New York City Taxi and Limousine Commision Trip Record Data that's available on the AWS Open Data Registry. This contains data of trips taken by taxis and for-hire vehicles in New York City. We'll save this into an iceberg table called `taxis`." + ] + }, + { + "cell_type": "markdown", + "id": "747bee98", + "metadata": {}, + "source": [ + "To be able to rerun the notebook several times, let's drop the table if it exists to start fresh." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "930682ce", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE DATABASE IF NOT EXISTS nyc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f918310a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DROP TABLE IF EXISTS nyc.taxis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c37ca92", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2021-04.parquet\")\n", + "df.write.saveAsTable(\"nyc.taxis\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9fddb808", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DESCRIBE EXTENDED nyc.taxis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bcf99fb3", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT COUNT(*) as cnt\n", + "FROM nyc.taxis" + ] + }, + { + "cell_type": "markdown", + "id": "cffd2c03", + "metadata": {}, + "source": [ + "## Schema Evolution\n", + "\n", + "Adding, dropping, renaming, or altering columns is easy and safe in Iceberg. In this example, we'll rename `fare_amount` to `fare` and `trip_distance` to `distance`. We'll also add a float column `fare_per_distance_unit` immediately after `distance`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efee8252", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis RENAME COLUMN fare_amount TO fare" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "794de3a0", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis RENAME COLUMN trip_distance TO distance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "adac7564", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis ALTER COLUMN distance COMMENT 'The elapsed trip distance in miles reported by the taximeter.'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32d7e6ef", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis ALTER COLUMN distance TYPE double;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6fb4b02a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis ALTER COLUMN distance AFTER fare;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81f7cc19", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis\n", + "ADD COLUMN fare_per_distance_unit float AFTER distance" + ] + }, + { + "cell_type": "markdown", + "id": "9416b498", + "metadata": {}, + "source": [ + "Let's update the new `fare_per_distance_unit` to equal `fare` divided by `distance`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18771ccb", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "UPDATE nyc.taxis\n", + "SET fare_per_distance_unit = fare/distance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09c72ca5", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT\n", + "VendorID\n", + ",tpep_pickup_datetime\n", + ",tpep_dropoff_datetime\n", + ",fare\n", + ",distance\n", + ",fare_per_distance_unit\n", + "FROM nyc.taxis" + ] + }, + { + "cell_type": "markdown", + "id": "37582e02", + "metadata": {}, + "source": [ + "## Expressive SQL for Row Level Changes\n", + "With Iceberg tables, `DELETE` queries can be used to perform row-level deletes. This is as simple as providing the table name and a `WHERE` predicate. If the filter matches an entire partition of the table, Iceberg will intelligently perform a metadata-only operation where it simply deletes the metadata for that partition.\n", + "\n", + "Let's perform a row-level delete for all rows that have a `fare_per_distance_unit` greater than 4 or a `distance` greater than 2. This should leave us with relatively short trips that have a relatively high fare per distance traveled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ded820f1", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DELETE FROM nyc.taxis\n", + "WHERE fare_per_distance_unit > 4.0 OR distance > 2.0" + ] + }, + { + "cell_type": "markdown", + "id": "faef3712", + "metadata": {}, + "source": [ + "There are some fares that have a `null` for `fare_per_distance_unit` due to the distance being `0`. Let's remove those as well." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18b69265", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DELETE FROM nyc.taxis\n", + "WHERE fare_per_distance_unit is null" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b92d7db", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT\n", + "VendorID\n", + ",tpep_pickup_datetime\n", + ",tpep_dropoff_datetime\n", + ",fare\n", + ",distance\n", + ",fare_per_distance_unit\n", + "FROM nyc.taxis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d5472b7", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT COUNT(*) as cnt\n", + "FROM nyc.taxis" + ] + }, + { + "cell_type": "markdown", + "id": "c4b157e5", + "metadata": {}, + "source": [ + "## Partitioning\n", + "\n", + "A tableā€™s partitioning can be updated in place and applied only to newly written data. Query plans are then split, using the old partition scheme for data written before the partition scheme was changed, and using the new partition scheme for data written after. People querying the table donā€™t even have to be aware of this split. Simple predicates in WHERE clauses are automatically converted to partition filters that prune out files with no matches. This is whatā€™s referred to in Iceberg as *Hidden Partitioning*." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30e3e3b7", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.taxis\n", + "ADD PARTITION FIELD VendorID" + ] + }, + { + "cell_type": "markdown", + "id": "6fce6bb4", + "metadata": {}, + "source": [ + "## Metadata Tables\n", + "\n", + "Iceberg tables contain very rich metadata that can be easily queried. For example, you can retrieve the manifest list for any snapshot, simply by querying the table's `snapshots` table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fade1a3", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT snapshot_id, manifest_list\n", + "FROM nyc.taxis.snapshots" + ] + }, + { + "cell_type": "markdown", + "id": "64887133", + "metadata": {}, + "source": [ + "The `files` table contains loads of information on data files, including column level statistics such as null counts, lower bounds, and upper bounds." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cb712f7", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT file_path, file_format, record_count, null_value_counts, lower_bounds, upper_bounds\n", + "FROM nyc.taxis.files" + ] + }, + { + "cell_type": "markdown", + "id": "65deb074", + "metadata": {}, + "source": [ + "## Time Travel\n", + "\n", + "The history table lists all snapshots and which parent snapshot they derive from. The `is_current_ancestor` flag let's you know if a snapshot is part of the linear history of the current snapshot of the table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bab64f90", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.history" + ] + }, + { + "cell_type": "markdown", + "id": "47129d69", + "metadata": {}, + "source": [ + "You can time-travel by altering the `current-snapshot-id` property of the table to reference any snapshot in the table's history. Let's revert the table to it's original state by traveling to the very first snapshot ID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c360238", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql --var df\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.history" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8df43d00", + "metadata": {}, + "outputs": [], + "source": [ + "original_snapshot = df.head().snapshot_id\n", + "spark.sql(f\"CALL system.rollback_to_snapshot('nyc.taxis', {original_snapshot})\")\n", + "original_snapshot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "955a4c52", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT\n", + "VendorID\n", + ",tpep_pickup_datetime\n", + ",tpep_dropoff_datetime\n", + ",fare\n", + ",distance\n", + ",fare_per_distance_unit\n", + "FROM nyc.taxis" + ] + }, + { + "cell_type": "markdown", + "id": "67b71c76", + "metadata": {}, + "source": [ + "Another look at the history table shows that the original state of the table has been added as a new entry\n", + "with the original snapshot ID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91b801d3", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.taxis.history" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85667efc", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT COUNT(*) as cnt\n", + "FROM nyc.taxis" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Integrated Audits Demo.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Integrated Audits Demo.ipynb new file mode 100644 index 0000000000..4b60b35e45 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Integrated Audits Demo.ipynb @@ -0,0 +1,626 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "markdown", + "id": "247fb2ab", + "metadata": {}, + "source": [ + "### [Integrated Audits: Streamlined Data Observability with Apache Iceberg](https://tabular.io/blog/integrated-audits/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd61c16f", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "spark" + ] + }, + { + "cell_type": "markdown", + "id": "747bee98", + "metadata": {}, + "source": [ + "To be able to rerun the notebook several times, let's drop the `permits` table if it exists to start fresh." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26245f7e", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE DATABASE IF NOT EXISTS nyc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08a13fcc", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DROP TABLE IF EXISTS nyc.permits" + ] + }, + { + "cell_type": "markdown", + "id": "eead44c0", + "metadata": {}, + "source": [ + "# Load NYC Film Permits Data" + ] + }, + { + "cell_type": "markdown", + "id": "6f9a9f41", + "metadata": {}, + "source": [ + "For this demo, we will use the [New York City Film Permits dataset](https://data.cityofnewyork.us/City-Government/Film-Permits/tg4x-b46p) available as part of the NYC Open Data initiative. We're using a locally saved copy of a 1000 record sample, but feel free to download the entire dataset to use in this notebook!\n", + "\n", + "We'll save the sample dataset into an iceberg table called `permits`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3cc669a", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.option(\"inferSchema\",\"true\").option(\"multiline\",\"true\").json(\"/home/iceberg/data/nyc_film_permits.json\")\n", + "df.write.saveAsTable(\"nyc.permits\")" + ] + }, + { + "cell_type": "markdown", + "id": "378cf187", + "metadata": {}, + "source": [ + "Taking a quick peek at the data, you can see that there are a number of permits for different boroughs in New York." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3170161", + "metadata": {}, + "outputs": [], + "source": [ + "spark.read \\\n", + " .format(\"iceberg\") \\\n", + " .load(\"nyc.permits\") \\\n", + " .groupBy(\"borough\") \\\n", + " .count() \\\n", + " .show()" + ] + }, + { + "cell_type": "markdown", + "id": "c85a71a2", + "metadata": {}, + "source": [ + "# Generate an ID for an Integrated Audit Session" + ] + }, + { + "cell_type": "markdown", + "id": "182510da", + "metadata": {}, + "source": [ + "An integrated audit session is a single cadence of:\n", + "1. Staging changes to a table\n", + "2. Auditing the staged changes\n", + "3. Committing the changes (optional)\n", + "\n", + "Each of these sessions must be represented with an ID. You can use any convention that makes sense in your environment but in this demo we'll simply use a UUID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e39d3d1", + "metadata": {}, + "outputs": [], + "source": [ + "import uuid\n", + "ia_session_id = uuid.uuid4().hex\n", + "ia_session_id" + ] + }, + { + "cell_type": "markdown", + "id": "fa31a9ea", + "metadata": {}, + "source": [ + "# The Setup" + ] + }, + { + "cell_type": "markdown", + "id": "d845953b", + "metadata": {}, + "source": [ + "Tables by default are not configured to allow integrated audits, therefore the first step is enabling this by setting the `write.wap.enabled` table metadata property to `true`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf29df0b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.permits\n", + "SET TBLPROPERTIES (\n", + " 'write.wap.enabled'='true'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1dc5ad69", + "metadata": {}, + "source": [ + "Next, the `spark.wap.id` property of your Spark session configuration must be set to the integrated audit session ID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65bc4280", + "metadata": {}, + "outputs": [], + "source": [ + "spark.conf.set('spark.wap.id', ia_session_id)" + ] + }, + { + "cell_type": "markdown", + "id": "3a34995b", + "metadata": {}, + "source": [ + "With a `spark.wap.id` value set, you can now safely write **directly to the permits table**--don't worry, these changes will only be staged, not committed!" + ] + }, + { + "cell_type": "markdown", + "id": "437088f6", + "metadata": {}, + "source": [ + "# Staging The Changes" + ] + }, + { + "cell_type": "markdown", + "id": "1c9fa6e9", + "metadata": {}, + "source": [ + "To stage the changes, you simply write directly to the `permits` table. This is awesome in situations where you're working with a large and complex data ingestion pipeline.\n", + "Instead of including hard-coded logic in your pipeline to switch between a sort of \"audit-mode\" as opposed to \"production-mode\", with integrated audits you simple run your\n", + "production code!\n", + "\n", + "For this demo, let's use a simple query that deletes all records for film permits in the manhattan borough." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14843243", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DELETE FROM nyc.permits\n", + "WHERE borough='Manhattan'" + ] + }, + { + "cell_type": "markdown", + "id": "56cc8478", + "metadata": {}, + "source": [ + "As described, even though the query was executed against the production table, these changes are only staged and not committed since we are within an integrated audit session. Let's confirm this by verifying that a count by borough still includes the Manhattan records." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95df15e9", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "3fe2c863", + "metadata": {}, + "source": [ + "# The Audit" + ] + }, + { + "cell_type": "markdown", + "id": "a7935b0d", + "metadata": {}, + "source": [ + "Once the changes for this session are staged, you can perform all of your audits to validate the data. The first step is to retrieve the snapshot ID generated by the changes and tagged with this integrated audit session ID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95d71430", + "metadata": {}, + "outputs": [], + "source": [ + "query = f\"\"\"\n", + "SELECT snapshot_id\n", + "FROM nyc.permits.snapshots\n", + "WHERE summary['wap.id'] = '{ia_session_id}'\n", + "\"\"\"\n", + "\n", + "ia_session_snapshot = spark.sql(query).head().snapshot_id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1035b246", + "metadata": {}, + "outputs": [], + "source": [ + "ia_session_snapshot" + ] + }, + { + "cell_type": "markdown", + "id": "4c602800", + "metadata": {}, + "source": [ + "This snapshot includes the staged (but not commited) changes to your production table. Once you have this snapshot ID, you can use Iceberg's Time Travel feature to query it!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c95130e9", + "metadata": {}, + "outputs": [], + "source": [ + "spark.read \\\n", + " .option(\"snapshot-id\", ia_session_snapshot) \\\n", + " .format(\"iceberg\") \\\n", + " .load(\"nyc.permits\") \\\n", + " .groupBy(\"borough\") \\\n", + " .count() \\\n", + " .show()" + ] + }, + { + "cell_type": "markdown", + "id": "0cab3813", + "metadata": {}, + "source": [ + "At this point, you can use any auditing tool or technique to validate your changes. For this demo, we'll do a simple audit that confirms that the only remaining boroughs are Queens, Brooklyn, Bronx, and Staten Island. If either borough is missing or any additional boroughs are found, we'll raise an exception." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82af5de3", + "metadata": {}, + "outputs": [], + "source": [ + "expected_boroughs = {\"Queens\", \"Brooklyn\", \"Bronx\", \"Staten Island\"}\n", + "distinct_boroughs = spark.read \\\n", + " .option(\"snapshot-id\", ia_session_snapshot) \\\n", + " .format(\"iceberg\") \\\n", + " .load(\"nyc.permits\") \\\n", + " .select(\"borough\") \\\n", + " .distinct() \\\n", + " .toLocalIterator()\n", + "boroughs = {row[0] for row in distinct_boroughs}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4fad7c2", + "metadata": {}, + "outputs": [], + "source": [ + "# Since `boroughs` and `required_boroughs` are both sets (array of distinct items),\n", + "# we can confirm that they match by checking that the lengths of the sets are equal\n", + "# to eachother as well as to the union of both sets.\n", + "if len(boroughs) != len(expected_boroughs) != len(set.union(boroughs, expected_boroughs)):\n", + " raise ValueError(f\"Audit failed, borough set does not match expected boroughs: {boroughs} != {expected_boroughs}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b1032255", + "metadata": {}, + "source": [ + "If the above check does not fail, we can go ahead and commit our staged data to publish our changes!" + ] + }, + { + "cell_type": "markdown", + "id": "2079435b", + "metadata": {}, + "source": [ + "# The Publish" + ] + }, + { + "cell_type": "markdown", + "id": "88d59f50", + "metadata": {}, + "source": [ + "After the audits are completed, publishing the data is as simple as running a `cherrypick_snapshot` stored procedure." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "056236ba", + "metadata": {}, + "outputs": [], + "source": [ + "publish_query = f\"CALL system.cherrypick_snapshot('nyc.permits', {ia_session_snapshot})\"\n", + "%sql $publish_query" + ] + }, + { + "cell_type": "markdown", + "id": "17b868e8", + "metadata": {}, + "source": [ + "That's it! Publishing the changes from this integrated audit session is a simple metadata-only operation that instantly makes the changes live for all downstream consumers querying the `permits` table! Query results will now include the commit that removed all Manhattan records." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "930682ce", + "metadata": {}, + "outputs": [], + "source": [ + "spark.read \\\n", + " .format(\"iceberg\") \\\n", + " .load(\"nyc.permits\") \\\n", + " .groupBy(\"borough\") \\\n", + " .count() \\\n", + " .show()" + ] + }, + { + "cell_type": "markdown", + "id": "d941b990", + "metadata": {}, + "source": [ + "# What Happens When The Audits Fail?" + ] + }, + { + "cell_type": "markdown", + "id": "f6b4084e", + "metadata": {}, + "source": [ + "What about when your audits fail? What happens to the snapshots generated? How about the data and metadata files?\n", + "\n", + "One of the best parts of Iceberg's integrated audits is that the cleanup of \"*staged-yet-not-committed-data*\" is part of the normal snapshot cleanup process of a typical Iceberg warehouse. To be more specific, let's say a daily snapshot expiration is performed on the data warehouse (using the [expire_snapshots](https://iceberg.apache.org/docs/latest/spark-procedures/#expire_snapshots) procedure) and all snapshots older than 7 days are expired. That means once your staged snapshot reaches 7 days in age, it will be expired.\n", + "\n", + "Additionally, since the changes were never committed, the underlying data files for the snapshot will be removed since they're not referenced by any other snapshots in the linear history of the table.\n", + "\n", + "Let's see this in action. First, start a new integrated audit session and stage a commit by inserting a single record." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25eff1ef", + "metadata": {}, + "outputs": [], + "source": [ + "ia_session_id = uuid.uuid4().hex\n", + "ia_session_id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4726b169", + "metadata": {}, + "outputs": [], + "source": [ + "spark.conf.set('spark.wap.id', ia_session_id)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31bf19f1", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "INSERT INTO nyc.permits\n", + "VALUES (\n", + " 'Hoboken',\n", + " 'Television',\n", + " '1',\n", + " 'United States of America',\n", + " '2021-11-24T23:00:00.000',\n", + " '2021-11-23T09:38:17.000',\n", + " 'Mayor\\'s Office of Film, Theatre & Broadcasting',\n", + " '613322',\n", + " 'Shooting Permit',\n", + " 'WASHINGTON STREET',\n", + " '100',\n", + " '2021-11-24T07:00:00.000',\n", + " 'Episodic series',\n", + " '07030'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "aa29184b", + "metadata": {}, + "source": [ + "Next, let's identify the snapshot that was tagged with the integrated audit session ID." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "682a5f52", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT snapshot_id\n", + "FROM nyc.permits.snapshots" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ef4dd148", + "metadata": {}, + "outputs": [], + "source": [ + "query = f\"\"\"\n", + "SELECT snapshot_id\n", + "FROM nyc.permits.snapshots\n", + "WHERE summary['wap.id'] = '{ia_session_id}'\n", + "\"\"\"\n", + "\n", + "ia_session_snapshot = spark.sql(query).head().snapshot_id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62f52c08", + "metadata": {}, + "outputs": [], + "source": [ + "ia_session_snapshot" + ] + }, + { + "cell_type": "markdown", + "id": "60561dff", + "metadata": {}, + "source": [ + "A quick check of the history table shows that this snapshot is not included as part of the current history of the table since it has not been published yet." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec96a9c0", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.permits.history" + ] + }, + { + "cell_type": "markdown", + "id": "6ec54f3a", + "metadata": {}, + "source": [ + "In a scenario where the audits fail and this change is not published, the `expire_snapshots` procedure will clean up the snapshot **and** the data files. Let's demonstrate this by calling the `expire_snapshots` procedure for all snapshots older than the current timestamp." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4727c61e", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "%sql CALL system.expire_snapshots('nyc.permits', {round(time.time() * 1000)}, 100)" + ] + }, + { + "cell_type": "markdown", + "id": "c8e47351", + "metadata": {}, + "source": [ + "The output from the `expire_snapshots` procedure shows that a data file, a manifest file, and a manifest list file were deleted. Furthermore, the snapshot no longer appears in the permit table's snapshots table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53f53072", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT *\n", + "FROM nyc.permits.snapshots" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Table Maintenance Spark Procedures.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Table Maintenance Spark Procedures.ipynb new file mode 100644 index 0000000000..8f3192b495 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Table Maintenance Spark Procedures.ipynb @@ -0,0 +1,220 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "markdown", + "id": "247fb2ab", + "metadata": {}, + "source": [ + "### [Table Maintenance: The Key To Keeping Your Iceberg Tables Healthy and Performant](https://tabular.io/blog/table-maintenance/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a5c8206", + "metadata": {}, + "outputs": [], + "source": [ + "spark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dab5ef0", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"DROP TABLE IF EXISTS demo.nyc.taxis_sample\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49a45d0b", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"\"\"\n", + "CREATE TABLE demo.nyc.taxis_sample (\n", + " `VendorID` BIGINT,\n", + " `tpep_pickup_datetime` TIMESTAMP,\n", + " `tpep_dropoff_datetime` TIMESTAMP,\n", + " `passenger_count` DOUBLE,\n", + " `trip_distance` DOUBLE,\n", + " `RatecodeID` DOUBLE,\n", + " `store_and_fwd_flag` STRING,\n", + " `PULocationID` BIGINT,\n", + " `DOLocationID` BIGINT,\n", + " `payment_type` BIGINT,\n", + " `fare_amount` DOUBLE,\n", + " `extra` DOUBLE,\n", + " `mta_tax` DOUBLE,\n", + " `tip_amount` DOUBLE,\n", + " `tolls_amount` DOUBLE,\n", + " `improvement_surcharge` DOUBLE,\n", + " `total_amount` DOUBLE,\n", + " `congestion_surcharge` DOUBLE,\n", + " `airport_fee` DOUBLE)\n", + "USING iceberg\n", + "TBLPROPERTIES(\n", + " 'write.target-file-size-bytes'='5242880'\n", + ")\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "997bb9df", + "metadata": {}, + "outputs": [], + "source": [ + "val df_202201 = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2022-01.parquet\")\n", + "val df_202202 = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2022-02.parquet\")\n", + "val df_202203 = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2022-03.parquet\")\n", + "val df_q1 = df_202201.union(df_202202).union(df_202203)\n", + "df_q1.write.insertInto(\"nyc.taxis_sample\")" + ] + }, + { + "cell_type": "markdown", + "id": "78cab088", + "metadata": {}, + "source": [ + "## Rewriting Data Files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ad64e6b", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"SELECT file_path, file_size_in_bytes FROM nyc.taxis_sample.files\").show(100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5d10355", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"ALTER TABLE nyc.taxis_sample UNSET TBLPROPERTIES ('write.target-file-size-bytes')\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f26228a5", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"CALL demo.system.rewrite_data_files(table => 'nyc.taxis_sample', options => map('target-file-size-bytes','52428800'))\").show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43a9ed67", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"SELECT file_path, file_size_in_bytes FROM nyc.taxis_sample.files\").show(100)" + ] + }, + { + "cell_type": "markdown", + "id": "523eb893", + "metadata": {}, + "source": [ + "## Expiring Snapshots" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98e8c5db", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"SELECT committed_at, snapshot_id, operation FROM nyc.taxis_sample.snapshots\").show(truncate=false)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b264c989", + "metadata": {}, + "outputs": [], + "source": [ + "val now = java.util.Calendar.getInstance().getTime()\n", + "val format = new java.text.SimpleDateFormat(\"yyyy-MM-dd HH:mm:ss.SSS\")\n", + "val now_str = format.format(now)\n", + "\n", + "spark.sql(s\"CALL demo.system.expire_snapshots(table => 'nyc.taxis_sample', older_than => TIMESTAMP '$now_str', retain_last => 1)\").show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "131e1f09", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"SELECT committed_at, snapshot_id, operation FROM nyc.taxis_sample.snapshots\").show(truncate=false)" + ] + }, + { + "cell_type": "markdown", + "id": "181212b6", + "metadata": {}, + "source": [ + "## Rewriting Manifest Files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49290e56", + "metadata": {}, + "outputs": [], + "source": [ + "spark.sql(\"CALL demo.system.rewrite_manifests('nyc.taxis_sample')\").show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spylon-kernel", + "language": "scala", + "name": "spylon-kernel" + }, + "language_info": { + "codemirror_mode": "text/x-scala", + "file_extension": ".scala", + "help_links": [ + { + "text": "MetaKernel Magics", + "url": "https://metakernel.readthedocs.io/en/latest/source/README.html" + } + ], + "mimetype": "text/x-scala", + "name": "scala", + "pygments_lexer": "scala", + "version": "0.4.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - View Support.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - View Support.ipynb new file mode 100644 index 0000000000..fa8dd7f6bb --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - View Support.ipynb @@ -0,0 +1,525 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a5c8206", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "spark" + ] + }, + { + "cell_type": "markdown", + "id": "6f9a9f41", + "metadata": {}, + "source": [ + "## Load Two Months of NYC Taxi/Limousine Trip Data\n", + "\n", + "This notebook uses the New York City Taxi and Limousine Commission Trip Record Data available on the AWS Open Data Registry. This contains data of trips taken by taxis and for-hire vehicles in New York City. This data is stored in an iceberg table called `taxis`." + ] + }, + { + "cell_type": "markdown", + "id": "747bee98", + "metadata": {}, + "source": [ + "To be able to rerun the notebook several times, let's drop the table and the views if they exist to start fresh." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "930682ce", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE DATABASE IF NOT EXISTS nyc.taxis;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22ac5552", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DROP TABLE IF EXISTS nyc.taxis\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cf5b4c0-89ac-4f79-8beb-fc55554bab22", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DROP VIEW IF EXISTS nyc.long_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a443155-85fe-4e7a-8216-9669e0765c93", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DROP VIEW IF EXISTS nyc.negative_amounts" + ] + }, + { + "cell_type": "markdown", + "id": "5816de2e", + "metadata": {}, + "source": [ + "## Create the table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f918310a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE TABLE nyc.taxis (\n", + " VendorID bigint,\n", + " tpep_pickup_datetime timestamp,\n", + " tpep_dropoff_datetime timestamp,\n", + " passenger_count double,\n", + " trip_distance double,\n", + " RatecodeID double,\n", + " store_and_fwd_flag string,\n", + " PULocationID bigint,\n", + " DOLocationID bigint,\n", + " payment_type bigint,\n", + " fare_amount double,\n", + " extra double,\n", + " mta_tax double,\n", + " tip_amount double,\n", + " tolls_amount double,\n", + " improvement_surcharge double,\n", + " total_amount double,\n", + " congestion_surcharge double,\n", + " airport_fee double\n", + ")\n", + "USING iceberg\n", + "PARTITIONED BY (days(tpep_pickup_datetime))" + ] + }, + { + "cell_type": "markdown", + "id": "fcba103e", + "metadata": {}, + "source": [ + "# Write a month of data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c37ca92", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2022-01.parquet\")\n", + "df.writeTo(\"nyc.taxis\").append()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a69152aa", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.taxis" + ] + }, + { + "cell_type": "markdown", + "id": "fd854d56-33d5-46a5-b552-869479b8e188", + "metadata": {}, + "source": [ + "# Create a view\n", + "\n", + "Let's create an Iceberg view to look at the longest distances travelled and the total amount of the trips." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fade1a3", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE VIEW nyc.long_distances (\n", + " vendor_id COMMENT 'Vendor ID',\n", + " pickup_date,\n", + " dropoff_date,\n", + " distance COMMENT 'Trip Distance',\n", + " total COMMENT 'Total amount')\n", + " AS SELECT VendorID, tpep_pickup_datetime, tpep_dropoff_datetime, trip_distance, total_amount FROM nyc.taxis ORDER BY trip_distance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfee5d8f-f862-4aa3-a096-8ff9ea66ba26", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.long_distances" + ] + }, + { + "cell_type": "markdown", + "id": "6fce6bb4", + "metadata": {}, + "source": [ + "## Update View to order results differently\n", + "\n", + "The output isn't as helpful as imagined, so let's update the view and change the order of columns and the ordering of the results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74c10267-d65b-4650-ab92-02a978f5872a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE OR REPLACE VIEW nyc.long_distances (\n", + " distance COMMENT 'Trip Distance',\n", + " total COMMENT 'Total amount',\n", + " vendor_id COMMENT 'Vendor ID',\n", + " pickup_date,\n", + " dropoff_date)\n", + " AS SELECT trip_distance, total_amount, VendorID, tpep_pickup_datetime, tpep_dropoff_datetime\n", + " FROM nyc.taxis\n", + " WHERE trip_distance > 35 ORDER BY total_amount, trip_distance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e764a28-297f-4c8d-87dc-45ae63380d6e", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.long_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "369b6e37-c7b3-4402-9087-2d9074b53dd7", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT count(*) FROM nyc.long_distances" + ] + }, + { + "cell_type": "markdown", + "id": "4aa4a9cd", + "metadata": {}, + "source": [ + "# Write a month of data\n", + "\n", + "Let's write another month of data and see how the results of the view change" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed7a7b8f", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(\"/home/iceberg/data/yellow_tripdata_2022-02.parquet\")\n", + "df.writeTo(\"nyc.taxis\").append()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbfb160c", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.long_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8085d47e-d629-408c-9753-95f58fac23c5", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT count(*) FROM nyc.long_distances" + ] + }, + { + "cell_type": "markdown", + "id": "35bf8f88-a493-42c0-b7a9-3941f8ebf4c8", + "metadata": {}, + "source": [ + "# Create another view\n", + "It appears that there are trips with negative total amounts. Let's display these results in a separate view" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa47bf43-2460-4990-88df-6040897c3386", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE OR REPLACE VIEW nyc.negative_amounts (\n", + " total COMMENT 'Total amount',\n", + " distance COMMENT 'Trip Distance',\n", + " vendor_id COMMENT 'Vendor ID',\n", + " pickup_date,\n", + " dropoff_date)\n", + " AS SELECT total_amount, trip_distance, VendorID, tpep_pickup_datetime, tpep_dropoff_datetime\n", + " FROM nyc.taxis\n", + " WHERE total_amount < 0 ORDER BY total_amount" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6c6a306-1752-4a6d-9213-d5b615110b1d", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.negative_amounts" + ] + }, + { + "cell_type": "markdown", + "id": "65deb074", + "metadata": {}, + "source": [ + "# Listing and describing views" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bab64f90", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW VIEWS in nyc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "be3b1930-e140-4795-81d9-9e5abe626fb7", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW VIEWS in nyc LIKE '*neg*'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfc2dcb3-4717-4730-94b6-18b9a239cf74", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DESCRIBE nyc.long_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7653ef78-f419-462b-915b-0cbd9f62d473", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DESCRIBE EXTENDED nyc.long_distances" + ] + }, + { + "cell_type": "markdown", + "id": "b11e64c9", + "metadata": {}, + "source": [ + "# Displaying the CREATE statement of a view" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c4a942c", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW CREATE TABLE nyc.long_distances" + ] + }, + { + "cell_type": "markdown", + "id": "42f6e042-00ce-4277-bf9a-16931f898d7b", + "metadata": {}, + "source": [ + "# Altering and displaying properties of a view\n", + "\n", + "This will add a new property and also update the comment of the view. \n", + "The comment will be shown when describing the view.\n", + "The end of this section will also remove a property from the view." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa823a4c-ede3-40d7-906e-27818070fa9b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW TBLPROPERTIES nyc.long_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c3b2fb4-4db9-408f-a36d-84970108dd5b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER VIEW nyc.long_distances SET TBLPROPERTIES ('key1' = 'val1', 'key2' = 'val2', 'comment' = 'This is a view comment')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a8dd804-d222-44e2-92b9-2069868e206a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW TBLPROPERTIES nyc.long_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1950bd5d-a5fc-4ee1-a4a9-1242261232f0", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DESCRIBE EXTENDED nyc.long_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88781989-0967-4349-b435-ad193c9697e7", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER VIEW nyc.long_distances UNSET TBLPROPERTIES ('key1')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6864599-b4fa-4525-8304-f1cb3ee7144a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SHOW TBLPROPERTIES nyc.long_distances" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Write-Audit-Publish (WAP) with Branches.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Write-Audit-Publish (WAP) with Branches.ipynb new file mode 100644 index 0000000000..b534c6e329 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/Iceberg - Write-Audit-Publish (WAP) with Branches.ipynb @@ -0,0 +1,788 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "markdown", + "id": "7b6633c1", + "metadata": {}, + "source": [ + "## Write-Audit-Publish with Branches in Apache Iceberg" + ] + }, + { + "cell_type": "markdown", + "id": "08d9d173", + "metadata": {}, + "source": [ + "This notebook runs using the Docker Compose at https://github.com/tabular-io/docker-spark-iceberg. \n", + "It's based on the [Iceberg - Integrated Audits Demo.ipynb](https://github.com/tabular-io/docker-spark-iceberg/blob/main/spark/notebooks/Iceberg%20-%20Integrated%20Audits%20Demo.ipynb) notebook. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd61c16f", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "spark" + ] + }, + { + "cell_type": "markdown", + "id": "747bee98", + "metadata": {}, + "source": [ + "To be able to rerun the notebook several times, let's drop the `permits` table if it exists to start fresh." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26245f7e", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE DATABASE IF NOT EXISTS nyc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f66e5810", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DROP TABLE IF EXISTS nyc.permits" + ] + }, + { + "cell_type": "markdown", + "id": "eead44c0", + "metadata": {}, + "source": [ + "# Load NYC Film Permits Data" + ] + }, + { + "cell_type": "markdown", + "id": "6f9a9f41", + "metadata": {}, + "source": [ + "For this demo, we will use the [New York City Film Permits dataset](https://data.cityofnewyork.us/City-Government/Film-Permits/tg4x-b46p) available as part of the NYC Open Data initiative. We're using a locally saved copy of a 1000 record sample, but feel free to download the entire dataset to use in this notebook!\n", + "\n", + "We'll save the sample dataset into an iceberg table called `permits`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3cc669a", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.option(\"inferSchema\",\"true\").option(\"multiline\",\"true\").json(\"/home/iceberg/data/nyc_film_permits.json\")\n", + "df.write.saveAsTable(\"nyc.permits\")" + ] + }, + { + "cell_type": "markdown", + "id": "378cf187", + "metadata": {}, + "source": [ + "Taking a quick peek at the data, you can see that there are a number of permits for different boroughs in New York." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3170161", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "fa31a9ea", + "metadata": {}, + "source": [ + "# The Setup" + ] + }, + { + "cell_type": "markdown", + "id": "d845953b", + "metadata": {}, + "source": [ + "Tables by default are not configured to allow integrated audits, therefore the first step is enabling this by setting the `write.wap.enabled` table metadata property to `true`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf29df0b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.permits\n", + "SET TBLPROPERTIES (\n", + " 'write.wap.enabled'='true'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "6405f8a7", + "metadata": {}, + "source": [ + "We create a branch for the work we want to do. This is a copy-on-write branch, so \"free\" until we start making changes (and \"cheap\" thereafter) since only data that's changed needs to be written. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14035a18", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.permits\n", + "CREATE BRANCH etl_job_42" + ] + }, + { + "cell_type": "markdown", + "id": "437088f6", + "metadata": {}, + "source": [ + "# Write" + ] + }, + { + "cell_type": "markdown", + "id": "a24b066e", + "metadata": {}, + "source": [ + "Before writing to the table we set `spark.wap.branch` so that writes (and reads) are against the specified branch of the table. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "842e361c", + "metadata": {}, + "outputs": [], + "source": [ + "spark.conf.set('spark.wap.branch', 'etl_job_42')" + ] + }, + { + "cell_type": "markdown", + "id": "6b749826", + "metadata": {}, + "source": [ + "Now make the change to the table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14843243", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DELETE FROM nyc.permits\n", + "WHERE borough='Manhattan'" + ] + }, + { + "cell_type": "markdown", + "id": "8eb688c8", + "metadata": {}, + "source": [ + "## Inspecting the staged/unpublished data" + ] + }, + { + "cell_type": "markdown", + "id": "ce1b1256", + "metadata": {}, + "source": [ + "### Staged/unpublished data" + ] + }, + { + "cell_type": "markdown", + "id": "1c4c04cd", + "metadata": {}, + "source": [ + "The changes are reflected in the table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfbd0d4b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "bf332168", + "metadata": {}, + "source": [ + "Note that because `spark.wap.branch` is set the above query is effectively the same as this one with `VERSION AS OF` for the branch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cd4b72b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits VERSION AS OF 'etl_job_42'\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "0ac96fd0", + "metadata": {}, + "source": [ + "Another syntax (albiet less clear IMHO) for `VERSION AS OF` is a `branch_` suffix to the table: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "169de151", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits.branch_etl_job_42\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "85332bf5", + "metadata": {}, + "source": [ + "### Published data" + ] + }, + { + "cell_type": "markdown", + "id": "5ad40cf1", + "metadata": {}, + "source": [ + "We can also inspect the unmodified `main` version of the table with `VERSION AS OF`: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95df15e9", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits VERSION AS OF 'main'\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "bf419df3", + "metadata": {}, + "source": [ + "The same `branch_` suffix words here too: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "270c09c6", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits.branch_main\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "c17c824f", + "metadata": {}, + "source": [ + "Any other user of the table will see the full set of data. We can reassure ourselves of this by unsetting `spark.wap.branch` for the session and querying the table without any `VERSION AS OF` modifier" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e9f7ea3", + "metadata": {}, + "outputs": [], + "source": [ + "spark.conf.unset('spark.wap.branch')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "935d46c8", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "a8ad4c4e", + "metadata": {}, + "source": [ + "# Audit" + ] + }, + { + "cell_type": "markdown", + "id": "ba43c910", + "metadata": {}, + "source": [ + "How you audit the data is up to you. The nice thing about the data being staged is that you can do it within the same ETL job, or have another tool do it. " + ] + }, + { + "cell_type": "markdown", + "id": "90485620", + "metadata": {}, + "source": [ + "Here's a very simple example of doing in Python. We're going to programatically check that only the four expected boroughs remain in the data. \n", + "\n", + "First, we define those that are expected: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d68a3b1", + "metadata": {}, + "outputs": [], + "source": [ + "expected_boroughs = {\"Queens\", \"Brooklyn\", \"Bronx\"}" + ] + }, + { + "cell_type": "markdown", + "id": "b17dfd72", + "metadata": {}, + "source": [ + "Then we get a set of the actual boroughs in the staged data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ce3d70a", + "metadata": {}, + "outputs": [], + "source": [ + "distinct_boroughs = spark.read \\\n", + " .option(\"branch\", \"etl_job_42\") \\\n", + " .format(\"iceberg\") \\\n", + " .load(\"nyc.permits\") \\\n", + " .select(\"borough\") \\\n", + " .distinct() \\\n", + " .toLocalIterator()\n", + "boroughs = {row[0] for row in distinct_boroughs}" + ] + }, + { + "cell_type": "markdown", + "id": "4a30827d", + "metadata": {}, + "source": [ + "Now we do two checks: \n", + "\n", + "1. Compare the length of the expected vs actual set\n", + "2. Check that the two sets when unioned are still the same length. This is necessary, since the first test isn't sufficient alone" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aeb78c2b", + "metadata": {}, + "outputs": [], + "source": [ + "if ( (len(boroughs) != len(expected_boroughs)) \\\n", + " or (len(boroughs) != len(set.union(boroughs, expected_boroughs))) \\\n", + " or (len(expected_boroughs) != len(set.union(boroughs, expected_boroughs)))):\n", + " raise ValueError(f\"Audit failed, borough set does not match expected boroughs: {boroughs} != {expected_boroughs}\")\n", + "else:\n", + " print(f\"Audit has passed šŸ™ŒšŸ»\")" + ] + }, + { + "cell_type": "markdown", + "id": "28a18925", + "metadata": {}, + "source": [ + "# Publish" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "7bc16e05", + "metadata": {}, + "source": [ + "Iceberg supports fast-forward merging of branches back to `main`, using the [`manageSnapshots().fastForwardBranch`](https://iceberg.apache.org/javadoc/latest/org/apache/iceberg/ManageSnapshots.html#fastForwardBranch-java.lang.String-java.lang.String-) API.\n", + "\n", + "This isn't yet exposed in Spark, so the existing [`cherrypick`](https://iceberg.apache.org/javadoc/latest/org/apache/iceberg/ManageSnapshots.html#cherrypick-long-) can be used as a slightly less elegant option.\n", + "\n", + "ā„¹ļø Note that `cherrypick` only works for one commit. " + ] + }, + { + "cell_type": "markdown", + "id": "4619fe57", + "metadata": {}, + "source": [ + "First, we need the snapshot ID of our branch, which we can get from the `.refs` table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cd5d318", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT * FROM nyc.permits.refs " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c5f09f1", + "metadata": {}, + "outputs": [], + "source": [ + "query = f\"\"\"\n", + "SELECT snapshot_id\n", + "FROM nyc.permits.refs\n", + "WHERE name = 'etl_job_42'\n", + "\"\"\"\n", + "\n", + "wap_snapshot_id = spark.sql(query).head().snapshot_id" + ] + }, + { + "cell_type": "markdown", + "id": "58520bc1", + "metadata": {}, + "source": [ + "Now we do the publish, using `cherrypick_snapshot` and the snapshot id:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b93d6f91", + "metadata": {}, + "outputs": [], + "source": [ + "publish_query = f\"CALL system.cherrypick_snapshot('nyc.permits', {wap_snapshot_id})\"\n", + "\n", + "%sql $publish_query" + ] + }, + { + "cell_type": "markdown", + "id": "d7546923", + "metadata": {}, + "source": [ + "Finally, we look at the table and revel in the glory that is our published changes šŸŽ‰" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d62ebc6", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits.branch_etl_job_42\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "0f60be58", + "metadata": {}, + "source": [ + "We can also inspect the unmodified `main` version of the table with `VERSION AS OF`: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97c7a98e", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits VERSION AS OF 'main'\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "b0072a24", + "metadata": {}, + "source": [ + "---\n", + "\n", + "# What if You Don't Want to Publish Changes?" + ] + }, + { + "cell_type": "markdown", + "id": "441d1e92", + "metadata": {}, + "source": [ + "If you don't want to merge the branch you can simply `DROP` it. " + ] + }, + { + "cell_type": "markdown", + "id": "49dd455a", + "metadata": {}, + "source": [ + "## Create a new branch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0dd25215", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.permits\n", + "CREATE BRANCH new_etl_job" + ] + }, + { + "cell_type": "markdown", + "id": "bc9bac1e", + "metadata": {}, + "source": [ + "## Set `spark.wap.branch`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b2e2284", + "metadata": {}, + "outputs": [], + "source": [ + "spark.conf.set('spark.wap.branch', 'new_etl_job')" + ] + }, + { + "cell_type": "markdown", + "id": "a5f65ce0", + "metadata": {}, + "source": [ + "## Write" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4dbd6ed3", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "DELETE FROM nyc.permits WHERE borough LIKE '%'" + ] + }, + { + "cell_type": "markdown", + "id": "0344e3d0", + "metadata": {}, + "source": [ + "## Audit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea44436a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits \n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "b5974d6e", + "metadata": {}, + "source": [ + "### Whoops šŸ¤­ \n", + "We deleted all the data" + ] + }, + { + "cell_type": "markdown", + "id": "74ebb57d", + "metadata": {}, + "source": [ + "### Reassure ourselves that the data is still there in `main`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "87d8ee7e", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "SELECT borough, count(*) permit_cnt\n", + "FROM nyc.permits VERSION AS OF 'main'\n", + "GROUP BY borough" + ] + }, + { + "cell_type": "markdown", + "id": "22d5d357", + "metadata": {}, + "source": [ + "## Abandon changes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0533a1f6", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "ALTER TABLE nyc.permits\n", + "DROP BRANCH new_etl_job" + ] + }, + { + "cell_type": "markdown", + "id": "328ed36c", + "metadata": {}, + "source": [ + "---\n", + "\n", + "# Where Next?" + ] + }, + { + "cell_type": "markdown", + "id": "cca251c2", + "metadata": {}, + "source": [ + "For more information about write-audit-publish see [this talk from Michelle Winters](https://www.youtube.com/watch?v=fXHdeBnpXrg&t=1001s) and [this talk from Sam Redai](https://www.dremio.com/wp-content/uploads/2022/05/Sam-Redai-The-Write-Audit-Publish-Pattern-via-Apache-Iceberg.pdf)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/PyIceberg - Getting Started.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/PyIceberg - Getting Started.ipynb new file mode 100644 index 0000000000..b26adc3b3f --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/PyIceberg - Getting Started.ipynb @@ -0,0 +1,382 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "markdown", + "id": "247fb2ab", + "metadata": {}, + "source": [ + "### [Docker, Spark, and Iceberg: The Fastest Way to Try Iceberg!](https://tabular.io/blog/docker-spark-and-iceberg/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a5c8206", + "metadata": {}, + "outputs": [], + "source": [ + "from pyiceberg import __version__\n", + "\n", + "__version__" + ] + }, + { + "cell_type": "markdown", + "id": "6f9a9f41", + "metadata": {}, + "source": [ + "## Load NYC Taxi/Limousine Trip Data\n", + "\n", + "For this notebook, we will use the New York City Taxi and Limousine Commision Trip Record Data that's available on the AWS Open Data Registry. This contains data of trips taken by taxis and for-hire vehicles in New York City. We'll save this into an iceberg table called `taxis`." + ] + }, + { + "cell_type": "markdown", + "id": "747bee98", + "metadata": {}, + "source": [ + "To be able to rerun the notebook several times, let's drop the table if it exists to start fresh." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84fd09a4", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "CREATE DATABASE IF NOT EXISTS nyc;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07bf9dc2", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "DROP TABLE IF EXISTS nyc.taxis;" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "363f815a", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "CREATE TABLE IF NOT EXISTS nyc.taxis (\n", + " VendorID bigint,\n", + " tpep_pickup_datetime timestamp,\n", + " tpep_dropoff_datetime timestamp,\n", + " passenger_count double,\n", + " trip_distance double,\n", + " RatecodeID double,\n", + " store_and_fwd_flag string,\n", + " PULocationID bigint,\n", + " DOLocationID bigint,\n", + " payment_type bigint,\n", + " fare_amount double,\n", + " extra double,\n", + " mta_tax double,\n", + " tip_amount double,\n", + " tolls_amount double,\n", + " improvement_surcharge double,\n", + " total_amount double,\n", + " congestion_surcharge double,\n", + " airport_fee double\n", + ")\n", + "USING iceberg\n", + "PARTITIONED BY (days(tpep_pickup_datetime))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47645b52", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql\n", + "\n", + "TRUNCATE TABLE nyc.taxis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9fddb808", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import SparkSession\n", + "spark = SparkSession.builder.appName(\"Jupyter\").getOrCreate()\n", + "\n", + "for filename in [\n", + " \"yellow_tripdata_2022-04.parquet\",\n", + " \"yellow_tripdata_2022-03.parquet\",\n", + " \"yellow_tripdata_2022-02.parquet\",\n", + " \"yellow_tripdata_2022-01.parquet\",\n", + " \"yellow_tripdata_2021-12.parquet\",\n", + "]:\n", + " df = spark.read.parquet(f\"/home/iceberg/data/{filename}\")\n", + " df.write.mode(\"append\").saveAsTable(\"nyc.taxis\")" + ] + }, + { + "cell_type": "markdown", + "id": "cffd2c03", + "metadata": {}, + "source": [ + "## Load data into a PyArrow Dataframe\n", + "\n", + "We'll fetch the table using the REST catalog that comes with the setup." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efee8252", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "from pyiceberg.catalog import load_catalog\n", + "from pyiceberg.expressions import GreaterThanOrEqual\n", + "\n", + "catalog = load_catalog('default')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "794de3a0", + "metadata": {}, + "outputs": [], + "source": [ + "tbl = catalog.load_table('nyc.taxis')\n", + "\n", + "sc = tbl.scan(row_filter=GreaterThanOrEqual(\"tpep_pickup_datetime\", \"2022-01-01T00:00:00.000000+00:00\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3ac7021", + "metadata": {}, + "outputs": [], + "source": [ + "df = sc.to_arrow().to_pandas()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e818e4a", + "metadata": {}, + "outputs": [], + "source": [ + "len(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7034fa26", + "metadata": {}, + "outputs": [], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32d7e6ef", + "metadata": {}, + "outputs": [], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6fb4b02a", + "metadata": {}, + "outputs": [], + "source": [ + "df.hist(column='fare_amount')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "81f7cc19", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from scipy import stats\n", + "\n", + "stats.zscore(df['fare_amount'])\n", + "\n", + "# Remove everything larger than 3 stddev\n", + "df = df[(np.abs(stats.zscore(df['fare_amount'])) < 3)]\n", + "# Remove everything below zero\n", + "df = df[df['fare_amount'] > 0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18771ccb", + "metadata": {}, + "outputs": [], + "source": [ + "df.hist(column='fare_amount')" + ] + }, + { + "cell_type": "markdown", + "id": "886c8408", + "metadata": {}, + "source": [ + "# DuckDB\n", + "\n", + "Use DuckDB to Query the PyArrow Dataframe directly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d5d6fb8", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext sql\n", + "%config SqlMagic.autopandas = True\n", + "%config SqlMagic.feedback = False\n", + "%config SqlMagic.displaycon = False\n", + "%sql duckdb:///:memory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b6f9522", + "metadata": {}, + "outputs": [], + "source": [ + "%sql SELECT * FROM df LIMIT 20" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5314f2b", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql --save tip-amount --no-execute\n", + "\n", + "SELECT tip_amount\n", + "FROM df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3dec260", + "metadata": {}, + "outputs": [], + "source": [ + "%sqlplot histogram --table df --column tip_amount --bins 22 --with tip-amount\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "989827d9", + "metadata": {}, + "outputs": [], + "source": [ + "%%sql --save tip-amount-filtered --no-execute\n", + "\n", + "WITH tip_amount_stddev AS (\n", + " SELECT STDDEV_POP(tip_amount) AS tip_amount_stddev\n", + " FROM df\n", + ")\n", + "\n", + "SELECT tip_amount\n", + "FROM df, tip_amount_stddev\n", + "WHERE tip_amount > 0\n", + " AND tip_amount < tip_amount_stddev * 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d1df179", + "metadata": {}, + "outputs": [], + "source": [ + "%sqlplot histogram --table tip-amount-filtered --column tip_amount --bins 50 --with tip-amount-filtered\n" + ] + }, + { + "cell_type": "markdown", + "id": "08d2c62d", + "metadata": {}, + "source": [ + "# Iceberg ā¤ļø PyArrow and DuckDB\n", + "\n", + "This notebook shows how you can load data into a PyArrow dataframe and query it using DuckDB easily. Iceberg allows you to take a slice out of the data that you need for your analysis, while reducing the time that you have to wait for the data and without polluting the memory with data that you're not going to use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72a9c64d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/PyIceberg - Write support.ipynb b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/PyIceberg - Write support.ipynb new file mode 100644 index 0000000000..e8b654fb40 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/notebooks/PyIceberg - Write support.ipynb @@ -0,0 +1,251 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1041ae6f", + "metadata": {}, + "source": [ + "![iceberg-logo](https://www.apache.org/logos/res/iceberg/iceberg.png)" + ] + }, + { + "cell_type": "markdown", + "id": "247fb2ab", + "metadata": {}, + "source": [ + "### [Docker, Spark, and Iceberg: The Fastest Way to Try Iceberg!](https://tabular.io/blog/docker-spark-and-iceberg/)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a5c8206", + "metadata": {}, + "outputs": [], + "source": [ + "from pyiceberg import __version__\n", + "\n", + "__version__" + ] + }, + { + "cell_type": "markdown", + "id": "6f9a9f41", + "metadata": {}, + "source": [ + "# Write support\n", + "\n", + "This notebook demonstrates writing to Iceberg tables using PyIceberg. First, connect to the [catalog](https://iceberg.apache.org/concepts/catalog/#iceberg-catalogs), the place where tables are being tracked." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47645b52", + "metadata": {}, + "outputs": [], + "source": [ + "from pyiceberg.catalog import load_catalog\n", + "\n", + "catalog = load_catalog('default')" + ] + }, + { + "cell_type": "markdown", + "id": "c531bd4b-9943-4516-9a6a-99fab016ed2b", + "metadata": {}, + "source": [ + "# Loading data using Arrow\n", + "\n", + "PyArrow is used to load a Parquet file into memory, and using PyIceberg this data can be written to an Iceberg table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9fddb808", + "metadata": {}, + "outputs": [], + "source": [ + "import pyarrow.parquet as pq\n", + "\n", + "df = pq.read_table(\"/home/iceberg/data/yellow_tripdata_2022-01.parquet\")\n", + "\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "bf1d58ad-5cc1-4e8c-9d7b-a54e67def783", + "metadata": {}, + "source": [ + "# Create an Iceberg table\n", + "\n", + "Next create the Iceberg table directly from the `pyarrow.Table`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47e5a21d-de87-4aaf-aa06-dc5048acba58", + "metadata": {}, + "outputs": [], + "source": [ + "table_name = \"default.taxi_dataset\"\n", + "\n", + "try:\n", + " # In case the table already exists\n", + " catalog.drop_table(table_name)\n", + "except:\n", + " pass\n", + "\n", + "table = catalog.create_table(table_name, schema=df.schema)\n", + "\n", + "table" + ] + }, + { + "cell_type": "markdown", + "id": "d612c035-4cf6-47a0-844b-165dfb463bbc", + "metadata": {}, + "source": [ + "# Write the data\n", + "\n", + "Let's append the data to the table. Appending or overwriting is equivalent since the table is empty. Next we can query the table and see that the data is there." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efee8252", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "table.append(df) # or table.overwrite(df)\n", + "\n", + "assert len(table.scan().to_arrow()) == len(df)\n", + "\n", + "table.scan().to_arrow()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9ce1cecc-8cb0-4622-b0eb-55880d091556", + "metadata": {}, + "outputs": [], + "source": [ + "str(table.current_snapshot())" + ] + }, + { + "cell_type": "markdown", + "id": "c029ea44-8ba6-4c08-a60d-5fffac6c3666", + "metadata": {}, + "source": [ + "# Append data\n", + "\n", + "Let's append another month of data to the table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "794de3a0", + "metadata": {}, + "outputs": [], + "source": [ + "df = pq.read_table(\"/home/iceberg/data/yellow_tripdata_2022-02.parquet\")\n", + "table.append(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3ac7021", + "metadata": {}, + "outputs": [], + "source": [ + "str(table.current_snapshot())" + ] + }, + { + "cell_type": "markdown", + "id": "85862bdc-7476-43f4-a604-5e4dfff065c9", + "metadata": {}, + "source": [ + "# Feature generation\n", + "\n", + "Consider that we want to train a model to determine which features contribute to the tip amount. `tip_per_mile` is a good target to train the model on. When we try to append the data, we need to evolve the schema first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72a9c64d", + "metadata": {}, + "outputs": [], + "source": [ + "import pyarrow.compute as pc\n", + "\n", + "df = table.scan().to_arrow()\n", + "df = df.append_column(\"tip_per_mile\", pc.divide(df[\"tip_amount\"], df[\"trip_distance\"]))\n", + "\n", + "try:\n", + " table.overwrite(df)\n", + "except ValueError as e:\n", + " print(f\"Error: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9aafd972-30d2-41ec-90e1-d5e17baeaf0b", + "metadata": {}, + "outputs": [], + "source": [ + "with table.update_schema() as upd:\n", + " upd.union_by_name(df.schema)\n", + "\n", + "print(str(table.schema()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea4ee286-1943-4a88-8d96-1e2a9e11faa1", + "metadata": {}, + "outputs": [], + "source": [ + "table.overwrite(df)\n", + "\n", + "table" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/requirements.txt b/ccloud/custom-connector-connect-iceberg-sink/spark/requirements.txt new file mode 100644 index 0000000000..3272073f6f --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/requirements.txt @@ -0,0 +1,7 @@ +jupyter==1.0.0 +spylon-kernel==0.4.1 +pyiceberg[pyarrow,duckdb,pandas]==0.6.0 +jupysql==0.10.5 +matplotlib==3.8.4 +scipy==1.13.0 +duckdb-engine==0.11.5 diff --git a/ccloud/custom-connector-connect-iceberg-sink/spark/spark-defaults.conf b/ccloud/custom-connector-connect-iceberg-sink/spark/spark-defaults.conf new file mode 100755 index 0000000000..42b6c73f93 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/spark/spark-defaults.conf @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default system properties included when running spark-submit. +# This is useful for setting default environmental settings. + +# Example: +spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions +spark.sql.catalog.demo org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog.demo.type rest +spark.sql.catalog.demo.uri http://rest:8181 +spark.sql.catalog.demo.io-impl org.apache.iceberg.aws.s3.S3FileIO +spark.sql.catalog.demo.warehouse s3://warehouse/wh/ +spark.sql.catalog.demo.s3.endpoint http://minio:9000 +spark.sql.defaultCatalog demo +spark.eventLog.enabled true +spark.eventLog.dir /home/iceberg/spark-events +spark.history.fs.logDirectory /home/iceberg/spark-events +spark.sql.catalogImplementation in-memory diff --git a/ccloud/custom-connector-connect-iceberg-sink/stop.sh b/ccloud/custom-connector-connect-iceberg-sink/stop.sh new file mode 100755 index 0000000000..7cd7cdca18 --- /dev/null +++ b/ccloud/custom-connector-connect-iceberg-sink/stop.sh @@ -0,0 +1,10 @@ +#!/bin/bash + + + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +source ${DIR}/../../scripts/utils.sh + +docker compose down -v --remove-orphans + +maybe_delete_ccloud_environment \ No newline at end of file diff --git a/ccloud/environment/docker-compose.yml b/ccloud/environment/docker-compose.yml index f969592593..7163e71713 100644 --- a/ccloud/environment/docker-compose.yml +++ b/ccloud/environment/docker-compose.yml @@ -1,14 +1,14 @@ --- services: connect: - image: ${CP_CONNECT_IMAGE}:${CONNECT_TAG} + image: ${CP_CONNECT_IMAGE}:${CP_CONNECT_TAG} hostname: connect container_name: connect ports: - "8083:8083" - "5005:5005" volumes: - - ../../ccloud/environment/data:/data + - ../../ccloud/environment/data:/datacloud - ../../confluent-hub:/usr/share/confluent-hub-components environment: CONNECT_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS @@ -45,7 +45,7 @@ services: CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_SSL CONNECT_PRODUCER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG CONNECT_PRODUCER_SASL_MECHANISM: PLAIN - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" + CONNECT_PRODUCER_INTERCEPTOR_CLASSES: $CONNECT_PRODUCER_INTERCEPTOR_CLASSES CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG CONNECT_PRODUCER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN @@ -53,7 +53,7 @@ services: CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_SSL CONNECT_CONSUMER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG CONNECT_CONSUMER_SASL_MECHANISM: PLAIN - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" + CONNECT_CONSUMER_INTERCEPTOR_CLASSES: $CONNECT_CONSUMER_INTERCEPTOR_CLASSES CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SECURITY_PROTOCOL: SASL_SSL CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG CONNECT_CONSUMER_CONFLUENT_MONITORING_INTERCEPTOR_SASL_MECHANISM: PLAIN @@ -69,7 +69,7 @@ services: CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n" control-center: - image: confluentinc/cp-enterprise-control-center:${TAG} + image: ${CP_CONTROL_CENTER_IMAGE}:${CP_CONTROL_CENTER_TAG} hostname: control-center container_name: control-center depends_on: @@ -78,6 +78,9 @@ services: - "${C3_PORT:-9021}:9021" profiles: - control-center + # needed for c3 next gen + command: "bash -c 'dub template /etc/confluent/docker/control-center.properties.template /etc/confluent-control-center/control-center.properties && \ + /etc/confluent/docker/run'" environment: CONTROL_CENTER_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS CONTROL_CENTER_KSQL_URL: "http://ksql-server:8089" @@ -140,9 +143,53 @@ services: profiles: - "conduktor" + prometheus: + image: prom/prometheus:v2.29.2 + hostname: prometheus + container_name: prometheus + profiles: + - "grafana" + ports: + - 9090:9090 + volumes: + - ./prometheus/:/etc/prometheus/ + + grafana: + image: grafana/grafana:11.1.0 + hostname: grafana + container_name: grafana + profiles: + - "grafana" + environment: + - "GF_SECURITY_ADMIN_USER=admin" + - "GF_SECURITY_ADMIN_PASSWORD=password" + - "GF_USERS_ALLOW_SIGN_UP=false" + ports: + - 3000:3000 + volumes: + - ./grafana/provisioning/:/etc/grafana/provisioning/ + - ./grafana/config/grafana.ini:/etc/grafana/grafana.ini + - ./grafana/confluent:/usr/share/grafana/public/img/icons/confluent + depends_on: + - prometheus + + confluent_cost_exporter: + profiles: + - "grafana" + image: docker.io/mcolomerc/confluent-costs-exporter:latest + #platform: linux/amd64 + container_name: confluent_cost_exporter + environment: + - CONFLUENT_CLOUD_API_KEY=$CLOUD_API_KEY + - CONFLUENT_CLOUD_API_SECRET=$CLOUD_API_SECRET + - CACHE_EXPIRATION=240m + - PORT=7979 + ports: + - 7979:7979 + # 75147 # schema-registry: - # image: confluentinc/cp-schema-registry:${TAG} + # image: ${CP_SCHEMA_REGISTRY_IMAGE}:${CP_SCHEMA_REGISTRY_TAG} # hostname: schema-registry # container_name: schema-registry # ports: diff --git a/ccloud/environment/grafana/config/grafana.ini b/ccloud/environment/grafana/config/grafana.ini new file mode 100644 index 0000000000..9fe327159a --- /dev/null +++ b/ccloud/environment/grafana/config/grafana.ini @@ -0,0 +1,894 @@ +##################### Grafana Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +;app_mode = production + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +;instance_name = ${HOSTNAME} + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +;data = /var/lib/grafana + +# Temporary files in `data` directory older than given duration will be removed +;temp_data_lifetime = 24h + +# Directory where grafana can store logs +;logs = /var/log/grafana + +# Directory where grafana will automatically scan and look for plugins +;plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +;provisioning = conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, h2, socket) +;protocol = http + +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +;http_port = 3000 + +# The public facing domain name used to access grafana from a browser +;domain = localhost + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +;serve_from_sub_path = false + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# Unix socket path +;socket = + +# CDN Url +;cdn_url = + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +;type = sqlite3 +;host = 127.0.0.1:3306 +;name = grafana +;user = root +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +;password = + +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# For "postgres" only, either "disable", "require" or "verify-full" +;ssl_mode = disable + +;ca_cert_path = +;client_key_path = +;client_cert_path = +;server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +;path = grafana.db + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +;log_queries = + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +################################### Data sources ######################### +[datasources] +# Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. +;datasource_limit = 5000 + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use Grafana primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +;connstr = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy waits before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +;timeout = 30 + +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# The maximum number of idle connections that Grafana will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. +;send_user_header = false + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs +;reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.net +# for new versions (grafana itself and plugins), check is used +# in some UI views to notify that grafana or plugin update exists +# This option does not cause any auto updates, nor send any information +# only a GET request to http://grafana.com to get latest versions +;check_for_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +#################################### Security #################################### +[security] +# disable creation of admin user on first start of grafana +;disable_initial_admin_creation = false + +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = admin + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,