diff --git a/.github/workflows/release_chart.yaml b/.github/workflows/release_chart.yaml index 4a5dd4d35..0cc3c4a4b 100644 --- a/.github/workflows/release_chart.yaml +++ b/.github/workflows/release_chart.yaml @@ -22,7 +22,7 @@ jobs: sudo install cr /usr/local/bin/ rm -f cr chart-releaser_1.4.1_linux_amd64.tar.gz - name: Package Chart - run: cr package deploy/helm + run: cr package deploy/helm/clickhouse-operator - name: Upload Release Artifacts run: | CHART_PATH=$(ls .cr-release-packages/altinity-clickhouse-operator-*.tgz) diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..a0c964a04 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,571 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.23.4](https://github.com/Altinity/clickhouse-operator/compare/release-0.23.3...release-0.23.4) + +### Changed +* Allow adding pod labels in Helm chart by @bruno-s-coelho in https://github.com/Altinity/clickhouse-operator/pull/1369. Closes https://github.com/Altinity/clickhouse-operator/issues/1356 +* The default service type has been changed from LoadBalancer to ClusterIP +* Cluster restart rules were fixed for some structured settings like 'logger/*'. Now changing of those would not cause a restart of pods. +* Operator does not fail if ClickHouserKeeperInstallation resource type is missing in k8s +* Added 'clickhouse_operator_chi_reconciles_aborted' operator metric + +> **NOTE:** There was a lot of internal refactoring related to Keeper code, but no functional changes yet. The Keeper configuration functionality will be improved in the next major release. + +## [0.23.3](https://github.com/Altinity/clickhouse-operator/compare/release-0.23.2...release-0.23.3) + +### Changed +* Enabled parameterized databases propagation to shards and replicas in ClickHouse 22.12+. That includes support for Replicated and MySQL database engines. Closes https://github.com/Altinity/clickhouse-operator/issues/1076 +* Removed object storage disks from DiskTotal/Free metrics since those do not make any sense +* Added ability to import packages with operator APIs by @dmvolod in https://github.com/Altinity/clickhouse-operator/pull/1229 +* Introduced number of unchanged hosts in CHI status + +### Fixed +* Fixed an issue with cluster stop could be taking long time. Closes https://github.com/Altinity/clickhouse-operator/issues/1346 +* Fixed a bug when inconsistent cluster definition might result in a crash. Closes https://github.com/Altinity/clickhouse-operator/issues/1319 +* Fixed a bug when hosts-completed could be incorrectly reported in status when reconcile is re-started in the middle + + +## [0.23.2](https://github.com/Altinity/clickhouse-operator/compare/release-0.23.1...release-0.23.2) + +### Changed +* Fix environment variables generation for secrets that might be off in some cases. Closes https://github.com/Altinity/clickhouse-operator/issues/1344 +* Golang is upgraded to 1.20. Closes CVEs in dependent libraries. + + +## [0.23.1](https://github.com/Altinity/clickhouse-operator/compare/release-0.23.0...release-0.23.1) + +### Changed +* Updated grants example by @lesandie in https://github.com/Altinity/clickhouse-operator/pull/1333 +* Upgrade ClickHouse version to 23.8-lts by @orginux in https://github.com/Altinity/clickhouse-operator/pull/1338 + +### Fixed +* Fixed generation of users that could be broken in some cases. Closes https://github.com/Altinity/clickhouse-operator/issues/1324 and https://github.com/Altinity/clickhouse-operator/issues/1332 +* Fixed metrics-exporter that might fail to export metrics in some cases. Closes https://github.com/Altinity/clickhouse-operator/issues/1336 +* Fixed Keeper examples +* Include installation of ClickhouseKeeperInstallations CRD in Helm chart readme by @echozio in https://github.com/Altinity/clickhouse-operator/pull/1330 + + +## [0.23.0](https://github.com/Altinity/clickhouse-operator/compare/release-0.22.2...release-0.23.0) + +### Added +* Kubernetes secrets are currently supported with the standard syntax for user passwords, configuration settings, and configuration files, for example: + ```yaml + users: + user1/password: + valueFrom: + secretKeyRef: + name: clickhouse_secret + key: pwduser1 + settings: + s3/my_bucket/access_key: + valueFrom: + secretKeyRef: + name: s3-credentials + key: AWS_ACCESS_KEY_ID + files: + server.key: + valueFrom: + secretKeyRef: + name: clickhouse-certs + key: server.key + ``` + > See updated [Security Hardening Guide](https://github.com/Altinity/clickhouse-operator/blob/0.23.0/docs/security_hardening.md#securing-clickhouse-server-settings) for more detail. +* Added **experimental** support for ClickHouse Keeper by @frankwg in https://github.com/Altinity/clickhouse-operator/pull/1218 + + ```yaml + kind: ClickHouseKeeperInstallation + ``` + See examples in there: https://github.com/Altinity/clickhouse-operator/tree/0.23.0/docs/chk-examples + The implementation is not final, following things yet needs to be done: + 1) dynamic reconfiguration, that is required in order to support adding and removing Keeper replicas + 2) integration with ClickHouseInstallation, so Keeper could be referenced by a reference, instead by a service +* CHI labels are now added to exported Prometheus metrics + +### Changed + +* Services are now re-created if ServiceType is changed in order to workaround [Kubernetes issue](https://github.com/kubernetes/kubernetes/issues/24040). Closes https://github.com/Altinity/clickhouse-operator/issues/1302 +* Operator now waits for ClickHouse service endpoints to respond when checking node is up. +* CHI templates are now automatically reloaded by operator. Before, templates were only reloaded during startup. In order to apply changes, CHI update needs to be triggered. +* Operator will now crash if operator configuration is broken or can not be parsed. That prevents the fallback to the defaults in case of errors. +* Improve helm, update values.yaml to properly generate helm/README.md by @Slach in https://github.com/Altinity/clickhouse-operator/pull/1278 +* Improve clickhouse-keeper manifests by @Slach in https://github.com/Altinity/clickhouse-operator/pull/1234 +* chore: remove refs to deprecated io/ioutil by @testwill in https://github.com/Altinity/clickhouse-operator/pull/1273 +* Update URL for accepted logging levels by @madrisan in https://github.com/Altinity/clickhouse-operator/pull/1270 +* Add a chi example for sync users by @ccsxs in https://github.com/Altinity/clickhouse-operator/pull/1304 +* Bump zookepper operator version to 0.2.15 by @GrahamCampbell in https://github.com/Altinity/clickhouse-operator/pull/1303 +* Optional values.rbac to deploy rbac resources by @Salec in https://github.com/Altinity/clickhouse-operator/pull/1316 +* update helm chart generator to treat config.yaml as yaml in values by @echozio in https://github.com/Altinity/clickhouse-operator/pull/1317 + +### Fixed + +* Fixed schema propagation on new replicas for ClickHouse 23.11 and above +* Fixed data recovery when PVC is deleted by a user. Closes https://github.com/Altinity/clickhouse-operator/issues/1310 + +## [0.22.2](https://github.com/Altinity/clickhouse-operator/compare/release-0.22.1...release-0.22.2) + +### Changed +* Fixed a bug when operator did not restart ClickHouse pods if 'files' section was changed without 'config.d' destination, e.g. ```files/settings.xml```. +* Fix ServiceMonitor endpoints [#1276](https://github.com/Altinity/clickhouse-operator/pull/1276) by @MiguelNdeCarvalho, and [#1290](https://github.com/Altinity/clickhouse-operator/pull/1290) by @muicoder. Closes [#1287](https://github.com/Altinity/clickhouse-operator/issues/1287) +* Disabled prefer_localhost_replica in default profile + +## [0.22.1](https://github.com/Altinity/clickhouse-operator/compare/release-0.22.0...release-0.22.1) + +### Added +* New 'Aborted' status for CHI is set when reconcile is aborted by an operator + +### Changed +* Allow shard weight to be zero ([#1192](https://github.com/Altinity/clickhouse-operator/pull/1192) by [maxistua](https://github.com/maxistua)) +* Removed excessive logging for pod update events +* Removed 30s delay after creating a service +* Allow empty values for CRD status and some other fields in order to facilitate migration from old operator versions that were upgraded without upgrading CRD first. Fixes #842, #890 and similar issues. + +## [0.22.0]( https://github.com/Altinity/clickhouse-operator/compare/release-0.21.3...release-0.22.0) + +### Added +* Support volume re-provisioning. If volume is broken and PVC detects it as lost, operator re-provisions the volume +* When new CHI is created, all hosts are created in parallel +* Allow to turn off waiting for running queries to complete. This can be done both in operator configuration or in CHI itself: + In operator configuration: + ```yaml + spec: + reconcile: + host: + wait: + queries: "false" + ``` + + In CHI: + ```yaml + spec: + reconciling: + policy: nowait + ``` +* When changes are applied to clusters with a lot of shards, the change is probed on a first node only. Is successul, it is applied on 50% of shards. This can be configured in operator configuration: + ```yaml + reconcile: + # Reconcile runtime settings + runtime: + # Max number of concurrent CHI reconciles in progress + reconcileCHIsThreadsNumber: 10 + + # The operator reconciles shards concurrently in each CHI with the following limitations: + # 1. Number of shards being reconciled (and thus having hosts down) in each CHI concurrently + # can not be greater than 'reconcileShardsThreadsNumber'. + # 2. Percentage of shards being reconciled (and thus having hosts down) in each CHI concurrently + # can not be greater than 'reconcileShardsMaxConcurrencyPercent'. + # 3. The first shard is always reconciled alone. Concurrency starts from the second shard and onward. + # Thus limiting number of shards being reconciled (and thus having hosts down) in each CHI by both number and percentage + + # Max number of concurrent shard reconciles within one CHI in progress + reconcileShardsThreadsNumber: 5 + # Max percentage of concurrent shard reconciles within one CHI in progress + reconcileShardsMaxConcurrencyPercent: 50 + ``` +* Operator-related metrics are exposed to Prometheus now: + ``` + clickhouse_operator_chi_reconciles_started + clickhouse_operator_chi_reconciles_completed + clickhouse_operator_chi_reconciles_timings + + clickhouse_operator_host_reconciles_started + clickhouse_operator_host_reconciles_completed + clickhouse_operator_host_reconciles_restarts + clickhouse_operator_host_reconciles_errors + clickhouse_operator_host_reconciles_timings + + clickhouse_operator_pod_add_events + clickhouse_operator_pod_update_events + clickhouse_operator_pod_delete_events + ``` + +### Changed +* fix typo in operator_installation_details.md by @seeekr in https://github.com/Altinity/clickhouse-operator/pull/1219 +* Set operator release date fot createdAt CSV field by @dmvolod in https://github.com/Altinity/clickhouse-operator/pull/1223 +* Fix type for exclude and include fields in 70-chop-config.yaml example by @dmvolod in https://github.com/Altinity/clickhouse-operator/pull/1222 +* change dashboard refresh rate 1m and add min_duration_ms, max_duration_ms dashboard variables, rename query_type to query_kind by @Slach in https://github.com/Altinity/clickhouse-operator/pull/1235 +* add securityContext to helm chart by @farodin91 in https://github.com/Altinity/clickhouse-operator/pull/1255 +* metrics-exporter collects all hosts and queries in parallel + +### Fixed +* Fixed a bug when operator could break multiple nodes if incorrect configuration has been deployed several times in a row +* Fixed a bug when schema could not be created on new nodes, if nodes took too long to start +* Fixed a bug when services were not reconciled in rare cases + +### New Contributors +* @seeekr made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1219 +* @dmvolod made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1223 +* @farodin91 made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1255 + +**Full Changelog**: + +## [0.21.3](https://github.com/Altinity/clickhouse-operator/compare/release-0.21.2...release-0.21.3) + +### Added +* Added selectors to CHITemplates. Now it is possible to define a template that is applied to certain CHI. Example here: https://github.com/Altinity/clickhouse-operator/blob/0.21.3/docs/chi-examples/50-CHIT-04-auto-template-volume-with-selector.yaml +* Added '.status.useTemplates' to reflect all templates used in CHI manually or auto + +### Changed +* CHITemplates are now re-loaded automatically without a need to restart operator. Changes in CHITemplates are not applied automatically to affected CHI. + +### Fixed +* Fix nil pointer deref in metrics exporter (#1187) by @zcross in https://github.com/Altinity/clickhouse-operator/pull/1188 +* Migrate piechart plugin on Grafana Dashboard by @MiguelNdeCarvalho in https://github.com/Altinity/clickhouse-operator/pull/1190 +* Permission error when deleting Pod sometimes + +### New Contributors +* @MiguelNdeCarvalho made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1190 + +## [0.21.2](https://github.com/Altinity/clickhouse-operator/compare/release-0.21.1...release-0.21.2) + +### Added +* Added support of clickhouse-log via podTemplates by @dmmarkov in https://github.com/Altinity/clickhouse-operator/pull/1012 +* Added SQL UDFs replication when adding shards/replicas. Closes https://github.com/Altinity/clickhouse-operator/issues/1174 + +### Changed +* Operator and metrics-exporter now automatically select 'http' or 'https' for connecting to cluster based on cluster configuration +* Changed statefulSet update behaviour to abort the update on failure. That protects the rest of the cluster from a breaking changes +* Removed unneeded operations on persistent volumes +* Run tests in parallel by @antip00 in https://github.com/Altinity/clickhouse-operator/pull/1171 +* fix build script to adapt to macos m1. by @xiedeyantu in https://github.com/Altinity/clickhouse-operator/pull/1169 +* Improvements to Grafana and Prometheus manifests + +### Fixed +* fix crash when reconcilePVC() failed by @jewelzqiu in https://github.com/Altinity/clickhouse-operator/pull/1168 +* fix reconcile threads number by @jewelzqiu in https://github.com/Altinity/clickhouse-operator/pull/1170 + +### New Contributors +* @jewelzqiu made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1168 +* @xiedeyantu made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1169 +* @dmmarkov made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1012 + + +## [0.21.1](https://github.com/Altinity/clickhouse-operator/compare/release-0.21.0...release-0.21.1) + +### Added +* Added configurable shard-level concurrent reconciliation by @zcross in https://github.com/Altinity/clickhouse-operator/pull/1124. It is controlled by the following operator configuration settings: + ```yaml + reconcile: + runtime: + # Max number of concurrent CHI reconciles in progress + reconcileCHIsThreadsNumber: 10 + # Max number of concurrent shard reconciles in progress + reconcileShardsThreadsNumber: 1 + # The maximum percentage of cluster shards that may be reconciled in parallel + reconcileShardsMaxConcurrencyPercent: 50 + ``` +* Added default configuration for ClickHouse system.trace_log table with 30 days TTL + +### Changed +* ZooKeeper manifests were rewritten to store configuration separately + +### Fixed +* Fixed a bug in metrics-exporter that might stop working on ClickHouse nodes with certain types of system.errors. Closes #1161 +* Fixed a bug with ClickHouse major version detection for Altinity.Stable builds + +## [0.21.0]( https://github.com/Altinity/clickhouse-operator/compare/release-0.20.3...release-0.21.0) + +### Changed +* Changed the way Operator applies ClickHouse settings. In the previous version, every change in settings required a restart via re-creating a StatefulSet. In this version it does not re-create StatefulSet anymore, but maintains a logic that decide if ClickHouse needs to be restarted in order to pickup a change or not. In case of restart it is performed using scaling stateful set down and up. The restart logic is controlled by configurationRestartPolicy configuration setting. Here is the default configuration: + ``` yaml + configurationRestartPolicy: + rules: + - version: "*" + rules: + - settings/*: "yes" + - settings/dictionaries_config: "no" + - settings/logger: "no" + - settings/macros/*: "no" + - settings/max_server_memory_*: "no" + - settings/max_*_to_drop: "no" + - settings/max_concurrent_queries: "no" + - settings/models_config: "no" + - settings/user_defined_executable_functions_config: "no" + + - zookeeper/*: "yes" + + - files/config.d/*.xml: "yes" + - files/config.d/*dict*.xml: "no" + + - profiles/default/background_*_pool_size: "yes" + - profiles/default/max_*_for_server: "yes" + - version: "21.*" + rules: + - settings/logger: "yes" + ``` +* Improved reliability of schema creation for new shards/replicas +* Added "secure" option for Zookeeper connections by @Tvion in https://github.com/Altinity/clickhouse-operator/pull/1114 +* Added "insecure" flag that closes insecure TCP/HTTP ClickHouse ports. See [security_hardening.md](https://github.com/Altinity/clickhouse-operator/blob/0.21.0/docs/security_hardening.md#disabling-insecure-connections) for more detail +* Added an option to disable metrics exporter by @roimor in https://github.com/Altinity/clickhouse-operator/pull/1131 +* Added system.errors scrapping into metrics-exporter +* Refactor Registry internals to make concurrent access safe by @zcross in https://github.com/Altinity/clickhouse-operator/pull/1115 +* Changed Grafana deployment to allow persisting custom dashboards +* Changed ZooKeeper version to 3.8.1 + +### Fixed +* Fixed a bug that might result in PDB being deleted. Closes https://github.com/Altinity/clickhouse-operator/issues/1139 +* Fixed propagation of podTemplate environment variables from ClickHouseInstallationTemplate to ClickHouseInstallation +* Fixed propagation of startup probe from ClickHouseInstallationTemplate to ClickHouseInstallation + +### New Contributors +* @roimor made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1131 +* @Tvion made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1114 +* @zcross made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1115 + +## [0.20.3](https://github.com/Altinity/clickhouse-operator/compare/release-0.20.2...release-0.20.3) + +### Changed +* Use alpine base image instead of UBI +* Fix error handling when table already exists in ZooKeeper for new ClickHouse versions + +## [0.20.2](https://github.com/Altinity/clickhouse-operator/compare/release-0.20.1...release-0.20.2) + +### Changed +* Added 'hostsCompleted' to the CHI status and events +* Changed some 'default' profile settings: + * Enabled 'do_not_merge_across_partitions_select_final' + * Set 'load_balancing' to 'nearest_hostname' + * Set niceness ('os_thread_priority') to 2. +* Improved stability of metric-exporter when some ClickHouse nodes are responding slowly +* Changed sequence of LB service creation to avoid a situation when service exists with no available endpoints +* Added 'secure' flag at cluster level for enabling distributed queries over TLS +* Addressed CVEs in dependent packages + +**Note**: datatype for 'secure' flag has been changed from boolean to String (accepting, 'true', 'yes', '1' etc.) + +## [0.20.1](https://github.com/Altinity/clickhouse-operator/compare/release-0.20.0...release-0.20.1) + +### Changed +* Improvements to Grafana dashboards by @Slach +* Generate operator helm chart by @slamdev in https://github.com/Altinity/clickhouse-operator/pull/1049 +* Added cluster to PodDesruptionBudget selector. Closes #996. **Note**: PDB requires Kubernetes version 1.21 or above. +* Added status.hostsCompleted to track reconcile progress +* Do not restart CHI with restart attribute when operator is restarted +* Address https://nvd.nist.gov/vuln/detail/CVE-2022-27664. Closes #1039 +* Fixed RBAC permissions for secrets. Closes #1051 + +### New Contributors +* @slamdev made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1049 + +## [0.20.0](https://github.com/Altinity/clickhouse-operator/compare/release-0.19.2...release-0.20.0) + +### Changed +* Add support for setting secure communication between clickhouse instances. (Inspried by @chancez in https://github.com/Altinity/clickhouse-operator/pull/938). See examples: + * [auto secret](https://github.com/Altinity/clickhouse-operator/blob/0.20.0/docs/chi-examples/21-secure-cluster-secret-01-auto.yaml) + * [plain text secret](https://github.com/Altinity/clickhouse-operator/blob/0.20.0/docs/chi-examples/21-secure-cluster-secret-02-plaintext.yaml) + * [k8s secret reference](https://github.com/Altinity/clickhouse-operator/blob/0.20.0/docs/chi-examples/21-secure-cluster-secret-03-secret-ref.yaml) +* Operator managed PV provisioning. (Inspired by @chhan-coupang in https://github.com/Altinity/clickhouse-operator/pull/947). Allows to re-scale volumes w/o a downtime. In order to enable, use: + ```yaml + defaults: + storageManagement: + provisioner: Operator + ``` +* Use secret for clickhouse_operator credentials as default +* Improve zookeeper scaleout by @Slach in https://github.com/Altinity/clickhouse-operator/pull/984 +* add loadaverage + network + disk panel to clickhouse dashboard by @Slach in https://github.com/Altinity/clickhouse-operator/pull/1015 +* comment out chopusername check by @SuzyWangIBMer in https://github.com/Altinity/clickhouse-operator/pull/1009 +* Reduced a size of status field +* Improved logging + +### Fixed +* Fixed a bug with cluster definitions being reset when CHI has been updated via k8s API POST method. +* Fixed a problem of metrics exporter not being able to scrape any managed cluster when one of clickhouse nodes is slow +* Fix zookeeper advanced setup with empty dir by @luluz66 in https://github.com/Altinity/clickhouse-operator/pull/993 +* Delete duplicate return by @Abirdcfly in https://github.com/Altinity/clickhouse-operator/pull/987 +* Fixes #991 - fix watch namespace regex by @mcgrawia in https://github.com/Altinity/clickhouse-operator/pull/992 +* Address CVE-2022-32149 by @bkuschel in https://github.com/Altinity/clickhouse-operator/pull/1035 + +### New Contributors +* @Abirdcfly made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/987 +* @mcgrawia made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/992 +* @chhan-coupang made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/947 +* @luluz66 made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/993 +* @chancez made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/938 +* @antip00 made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/1034 + +## [0.19.3]( https://github.com/Altinity/clickhouse-operator/compare/release-0.19.2...release-0.19.3) + +### Changed +* Fixed a possible loose of cluster state when operator has been restarted in the middle of reconcile. +* Backport of https://github.com/Altinity/clickhouse-operator/pull/1009 +* add loadaverage + network + disk panel to clickhouse dashboard by @Slach in https://github.com/Altinity/clickhouse-operator/pull/1015 + +> **NOTE:** There is a change in ClickHouseInstallation custom resource type, so make sure you deploy full installation manifest when upgrading. + +## [0.19.2](https://github.com/Altinity/clickhouse-operator/compare/release-0.19.1...release-0.19.2) + +### Changed +* Fixed a bug in 0.19.0 and 0.19.1 versions when operator could be rebuilding remote-servers.xml from scratch after restart. When it happened, distributed queries might behave unexpectedly. + +**Full Changelog**: + +## [0.19.1](https://github.com/Altinity/clickhouse-operator/compare/release-0.19.0...release-0.19.1) + +### Changed +* Fixed MVs in Atomic databases support for ClickHouse versions 22.6 and above. Closes https://github.com/Altinity/clickhouse-operator/issues/989 +* Fixed tables with '{uuid}' macro in Atomic databases for ClickHouse versions 22.6 and above. +* Fixed watch namespaces regexp by https://github.com/Altinity/clickhouse-operator/pull/992. Closes https://github.com/Altinity/clickhouse-operator/issues/991 +* Added "clickhouse.altinity.com/ready: yes|no" annotations for replica services for the use by external load balancer. +* Updated podDisruptionBudget API version + +## [0.19.0](https://github.com/Altinity/clickhouse-operator/compare/0.18.5...release-0.19.0) + +### Changed +* Now 'clickhouse-operator' user is restricted to an IP address of the pod where clickhouse-operator is running. It can not be used from anywhere else. +* In addition to hostRegexp, 'default' user is restricted to IP addresses of pods of the CHI. This fixes connectivity issues between shards in GKE. +* ClickHouse 'interserver_http_host' is set to the service name and matches 'remote_servers'. It can be changed to full names with 'replicasUseFQDN' setting if needed. This fixes connectivity issues between replicas in GKE. +* Changed the way schema is propagated to new shards. It is currently controlled by [two cluster level settings](https://github.com/Altinity/clickhouse-operator/blob/7f5ee13e9e2185e245c12e9821db188e542aa98e/deploy/operator/parts/crd.yaml#L668): + ```yaml + schemaPolicy: + replica: None | All (default) + shard: None | All (default) | DistributedTablesOnly + ``` +Previously, when adding shards only distributed tables and dependent objects were created (DistributedTablesOnly). +* Added support for adding custom CA for operator connection to ClickHouse by @SuzyWangIBMer in https://github.com/Altinity/clickhouse-operator/pull/930 +* Fixed support of '{uuid}' macro for Atomic databases. Closes https://github.com/Altinity/clickhouse-operator/issues/857 +* Fixed a bug when replica sometimes was not removed from ZooKeeper when scaling down the cluster +* Set container-log on ubi instead of busybox by @SuzyWangIBMer in https://github.com/Altinity/clickhouse-operator/pull/940 +* upgrade vertamedia-clickhouse-grafana to 2.5.0 in dashboard and grafana 7.5.16 by @Slach in https://github.com/Altinity/clickhouse-operator/pull/951 +* Improve grafana dashboards by @Slach in https://github.com/Altinity/clickhouse-operator/pull/952 +* Improve grafana-operator + prometheus-operator by @Slach in https://github.com/Altinity/clickhouse-operator/pull/953 + +### New Contributors +* @roshanths made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/936 +* @SuzyWangIBMer made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/940 +* @meob made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/949 + +## [0.18.5](https://github.com/Altinity/clickhouse-operator/compare/0.18.4...0.18.5) + +### Changed +* Dependencies were updated to address CVE-2022-21698 and CVE-2021-38561 +* generate_chart.sh by @ganievs in https://github.com/Altinity/clickhouse-operator/pull/925 + +### New Contributors +* @ganievs made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/925 + +## [0.18.4](https://github.com/Altinity/clickhouse-operator/compare/0.18.3...0.18.4) + +### Changed +* Base image has been updated to address https://access.redhat.com/errata/RHSA-2022:0896 (closes https://github.com/Altinity/clickhouse-operator/issues/913) +* Https support for health checks by @bkuschel in https://github.com/Altinity/clickhouse-operator/pull/912 +* Fixed number of SERVERS from 1 to 3 for a 3 node clickhouse-keeper deployment by @a-dot in https://github.com/Altinity/clickhouse-operator/pull/902 +* clickhouse-keeper and ZooKeeper manifests were updated by @Slach in https://github.com/Altinity/clickhouse-operator/pull/911 and https://github.com/Altinity/clickhouse-operator/pull/916 + +### New Contributors +* @a-dot made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/902 +* @bkuschel made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/912 + +## [0.18.3](https://github.com/Altinity/clickhouse-operator/compare/0.18.2...0.18.3) + +### Changed +* Fixed a bug that might result in broken CHI if operator has been restarted during the reconcile +* Added 'AGE' to CHI status +* Added ClickHouse Keeper examples + + +## [0.18.2](https://github.com/Altinity/clickhouse-operator/compare/0.18.1...0.18.2) + +### Changed +* Operator now correctly respects configuration settings to exclude annotations from managed objects. Fixes [#859](https://github.com/Altinity/clickhouse-operator/issues/859) +* Make CHI status messages more human friendly +: + +## [0.18.1](https://github.com/Altinity/clickhouse-operator/compare/0.18.0...0.18.1) + +### Changed +* fix non pointer mistake in metrics-exporter by @adu-bricks in https://github.com/Altinity/clickhouse-operator/pull/870 +* Helper files for operatorhub.io integration + +### New Contributors +* @adu-bricks made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/870 + +## [0.18.0](https://github.com/Altinity/clickhouse-operator/compare/0.17.0...0.18.0) + +### Added +* arm64 packages are published (closes https://github.com/Altinity/clickhouse-operator/issues/852) +* 'access_management' can be specified when defining users in CHI. +* k8s secrets can be referenced when defining user passwords. See [05-settings-01-overview.yaml](https://github.com/Altinity/clickhouse-operator/blob/0.18.0/docs/chi-examples/05-settings-01-overview.yaml). + +### Changed +* When CRD is deleted, operator keeps all dependent objects now (statefulsets, volumes). That prevents from incidental delete of a cluster. +* When operator restarts it does not run a reconcile cycle anymore if CHI has not been changed. That prevents unneeded pod restarts. (Closes https://github.com/Altinity/clickhouse-operator/issues/855) +* Operator configuration file format has been changed. See https://github.com/Altinity/clickhouse-operator/blob/0.18.0/config/config.yaml. Configuration in old format is supported for backwards compatibility. + +### Fixed +* Fixed a bug 'unable to decode watch event: no kind \"ClickHouseOperatorConfiguration\" is registered' that could appear in some k8s configurations. +* Removed INFORMATION_SCHEMA from schema propagation. (closes https://github.com/Altinity/clickhouse-operator/issues/854) +* Added a containerPort to metrics-exporter (https://github.com/Altinity/clickhouse-operator/pull/834) + +### New Contributors +* @johnhummelAltinity made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/838 +* @jiangxinqi1995 made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/846 +* @johnny made their first contribution in https://github.com/Altinity/clickhouse-operator/pull/834 + +## [0.17.0](https://github.com/Altinity/clickhouse-operator/compare/0.16.1...0.17.0) + +## Added +* Labels and annotations from auto templates are now supported +* Macros can be used in service annotations the same way as in generateName. Fixes https://github.com/Altinity/clickhouse-operator/issues/795 + +## Changed +* Status object has been cleaned up + +## Fixed +* Database engine is now respected during schema migration +* Fixed schema migration for ClickHouse 21.11+ +* Removed extra waits for single-node CHI changes +* Fixed a possible race conditions with labeling on operator startup + +## [0.16.1](https://github.com/Altinity/clickhouse-operator/compare/0.16.0...0.16.1) + +### Fixed +* CRD definition of Status has been modified. **Note:** CRD needs to be updated with this release +* Default terminationGracePeriod for ClickHouse pod templates was moved to operator configuration. Default is 30 seconds as in 0.15.0 and before. +* Improved installation templates +* Fixed a bug with replicas not being correctly added when CHI has been modified from a Python Kubernetes client. + +### Upgrade notes +* CRD needs to be updated with this release +* 0.16.0 had hardcoded 60 seconds for terminationGracePeriod that resulted in ClickHouse restarts when upgrading from 0.15.0 to 0.16.0. Upgrade from 0.15.0 to 0.16.1 does result in ClickHouse restarts. If you are upgrading from 0.16.0 to 0.16.1 set [terminationGracePeriod](https://github.com/Altinity/clickhouse-operator/blob/50134723c388eda208a8a02a3c333a8fca73133a/config/config.yaml#L163) to 60 in operator config file. Refer to [Operator Configuration](https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md) for more details. + + +## [0.16.0](https://github.com/Altinity/clickhouse-operator/compare/0.15.0...0.16.0) + +### Added +* PodDisruption budget is configured automatically in order to prevent multiple pods being shut down by k8s +* topologyKey is now configurable. It addresses https://github.com/Altinity/clickhouse-operator/issues/772 +* spec.restart: "RollingRestart" attribute has been added in order to initiate graceful restart of a cluster. Use it with a patch command. +* Added a support for Kubernetes secrets in settings section, e.g. for ClickHouse user/password. + +### Changed +* Pod maintenance logic during all operations that require pod restart has been improved: + 1) Wait condition is enabled for a pod to be removed from ClickHouse cluster. + 2) Wait condition is added for running queries to complete (up to 5 minutes). + 3) terminationGracePeriod has been increased to 60 seconds. +* Timeout for DDL operations has been increased to 60 seconds. That addresses issues with schema management on slow clusters. +* Base image is switched to RedHat UBI +* ZooKeeper image version has been rolled back to 3.6.1 since we have some problems with 3.6.3 +* CRD apiVersion has been upgraded from apiextensions.k8s.io/v1beta1 to apiextensions.k8s.io/v1. The v1beta1 manifests are still available under descriptive names. + +### Fixed +* Fixed a bug with non-working custom ports +* Fixed a bug with replicas being incorrectly deleted from ZooKeeper for retained volumes + +**Note**: Upgrade from the previous version will result in restart of ClickHouse clusters. + +## [0.15.0](https://github.com/Altinity/clickhouse-operator/compare/0.16.0...0.15.0) + +### Added +* Added 90s delay before restarting the stateful set, when ClickHouse server settings are modified. That makes sure ClickHouse server starts with an updated configuration. Before that there was a race condition between configmap updates and statefulset restart. The default timeout can be modified by 'spec.reconciling.configMapPropagationTimeout' property. See [example](https://github.com/Altinity/clickhouse-operator/blob/0926d00a22be23499d633ce455bc5473a9620d15/docs/chi-examples/99-clickhouseinstallation-max.yaml#L28) +* Added 'troubleshooting' mode that allows pod to start even if ClickHouse server is failing. In troubleshooting mode liveness check is removed, and extra 'sleep' is added to the startup command. Controlled by 'spc.troubleshoot' property. See [example](https://github.com/Altinity/clickhouse-operator/blob/master/tests/configs/test-027-troubleshooting-2.yaml) +* Added a cleanup reconcilation logic that will remove k8s object that are labeled by a particular CHI but do not exist in the CHI manifest. The default behaviour can be altered with 'spec.reconciling.cleanup.unknownObjects' . See [example](https://github.com/Altinity/clickhouse-operator/blob/0926d00a22be23499d633ce455bc5473a9620d15/docs/chi-examples/99-clickhouseinstallation-max.yaml#L30) +* ZooKeeper manifests have been modified to use 'standaloneEnabled=false' for single node setups as recommended in [ZooKeeper documentation](https://zookeeper.apache.org/doc/r3.6.3/zookeeperReconfig.html#sc_reconfig_standaloneEnabled). ZooKeeper version has been bumped to 3.6.3 + +### Fixed +* Fixed a bug when replica has not been deleted in ZooKeeper when scaling down (https://github.com/Altinity/clickhouse-operator/issues/735). The bug has been introduced in 0.14.0 +* Fixed a bug when PVCs modified outside of operator could be re-created during the reconcile (https://github.com/Altinity/clickhouse-operator/issues/730). +* Fixed a bug when schema was not created on newly added replica sometimes + diff --git a/deploy/devspace/dev-install.sh b/deploy/devspace/dev-install.sh index 461d76fa1..f0ea45db1 100755 --- a/deploy/devspace/dev-install.sh +++ b/deploy/devspace/dev-install.sh @@ -31,7 +31,7 @@ kubectl -n "${OPERATOR_NAMESPACE}" apply -f <( \ ) if [[ "${MINIKUBE}" == "yes" ]]; then - echo "Preparing for minikube" + echo "Building in minikube" case "${DEPLOY_OPERATOR}" in "dev") echo "Clean images in minikube" @@ -40,17 +40,17 @@ if [[ "${MINIKUBE}" == "yes" ]]; then echo "Remove errors will be ignored." minikube image rm "${OPERATOR_IMAGE}" > /dev/null 2>&1 minikube image rm "${METRICS_EXPORTER_IMAGE}" > /dev/null 2>&1 - - echo "Build images" - echo "VERBOSITY=${VERBOSITY} ${PROJECT_ROOT}/dev/image_build_all_dev.sh" - VERBOSITY="${VERBOSITY}" "${PROJECT_ROOT}/dev/image_build_all_dev.sh" - echo "Load images into minikube:" - echo " 1. ${OPERATOR_IMAGE}" - echo " 2. ${METRICS_EXPORTER_IMAGE}" - echo "Load images into minikube" - minikube image load "${OPERATOR_IMAGE}" - minikube image load "${METRICS_EXPORTER_IMAGE}" - echo "Images loaded" + echo "Build images in minikube" + echo "VERBOSITY=${VERBOSITY} MINIKUBE=yes ${PROJECT_ROOT}/dev/image_build_all_dev.sh" + if VERBOSITY="${VERBOSITY}" MINIKUBE=yes "${PROJECT_ROOT}/dev/image_build_all_dev.sh"; then + echo "OK. MINIKUBE build successful." + else + echo "########################" + echo "ERROR." + echo "MINIKUBE build has failed." + echo "Abort." + exit 1 + fi ;; esac fi diff --git a/deploy/helm/Chart.yaml b/deploy/helm/clickhouse-operator/Chart.yaml similarity index 62% rename from deploy/helm/Chart.yaml rename to deploy/helm/clickhouse-operator/Chart.yaml index a0e47af07..77db0ea65 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/clickhouse-operator/Chart.yaml @@ -7,13 +7,13 @@ description: |- For upgrade please install CRDs separately: ```bash - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml ``` type: application -version: 0.23.4 -appVersion: 0.23.4 +version: 0.23.5 +appVersion: 0.23.5 home: https://github.com/Altinity/clickhouse-operator icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg maintainers: diff --git a/deploy/helm/README.md b/deploy/helm/clickhouse-operator/README.md similarity index 89% rename from deploy/helm/README.md rename to deploy/helm/clickhouse-operator/README.md index d81cc7ca5..e3e4216cc 100644 --- a/deploy/helm/README.md +++ b/deploy/helm/clickhouse-operator/README.md @@ -1,6 +1,6 @@ # altinity-clickhouse-operator -![Version: 0.23.4](https://img.shields.io/badge/Version-0.23.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.23.4](https://img.shields.io/badge/AppVersion-0.23.4-informational?style=flat-square) +![Version: 0.23.5](https://img.shields.io/badge/Version-0.23.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.23.5](https://img.shields.io/badge/AppVersion-0.23.5-informational?style=flat-square) Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). @@ -8,9 +8,9 @@ The ClickHouse Operator creates, configures and manages ClickHouse clusters runn For upgrade please install CRDs separately: ```bash - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml - kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml ``` **Homepage:** diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml similarity index 99% rename from deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml rename to deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml index f3eff2751..70beee4cb 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml similarity index 99% rename from deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml rename to deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml index d8ef8ba5e..fa5b50c4c 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml similarity index 99% rename from deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml rename to deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml index 07fdeca31..6fa5be4dc 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml @@ -1,13 +1,13 @@ # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced diff --git a/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml similarity index 99% rename from deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml rename to deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index b53ef91d1..fe76e8728 100644 --- a/deploy/helm/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -7,7 +7,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced diff --git a/deploy/helm/files/Altinity_ClickHouse_Operator_dashboard.json b/deploy/helm/clickhouse-operator/files/Altinity_ClickHouse_Operator_dashboard.json similarity index 100% rename from deploy/helm/files/Altinity_ClickHouse_Operator_dashboard.json rename to deploy/helm/clickhouse-operator/files/Altinity_ClickHouse_Operator_dashboard.json diff --git a/deploy/helm/files/ClickHouseKeeper_dashboard.json b/deploy/helm/clickhouse-operator/files/ClickHouseKeeper_dashboard.json similarity index 100% rename from deploy/helm/files/ClickHouseKeeper_dashboard.json rename to deploy/helm/clickhouse-operator/files/ClickHouseKeeper_dashboard.json diff --git a/deploy/helm/files/ClickHouse_Queries_dashboard.json b/deploy/helm/clickhouse-operator/files/ClickHouse_Queries_dashboard.json similarity index 100% rename from deploy/helm/files/ClickHouse_Queries_dashboard.json rename to deploy/helm/clickhouse-operator/files/ClickHouse_Queries_dashboard.json diff --git a/deploy/helm/files/Kafka_dashboard.json b/deploy/helm/clickhouse-operator/files/Kafka_dashboard.json similarity index 100% rename from deploy/helm/files/Kafka_dashboard.json rename to deploy/helm/clickhouse-operator/files/Kafka_dashboard.json diff --git a/deploy/helm/files/Zookeeper_dashboard.json b/deploy/helm/clickhouse-operator/files/Zookeeper_dashboard.json similarity index 100% rename from deploy/helm/files/Zookeeper_dashboard.json rename to deploy/helm/clickhouse-operator/files/Zookeeper_dashboard.json diff --git a/deploy/helm/templates/_helpers.tpl b/deploy/helm/clickhouse-operator/templates/_helpers.tpl similarity index 100% rename from deploy/helm/templates/_helpers.tpl rename to deploy/helm/clickhouse-operator/templates/_helpers.tpl diff --git a/deploy/helm/templates/additional-resources.yaml b/deploy/helm/clickhouse-operator/templates/additional-resources.yaml similarity index 100% rename from deploy/helm/templates/additional-resources.yaml rename to deploy/helm/clickhouse-operator/templates/additional-resources.yaml diff --git a/deploy/helm/templates/dashboards-secret.yaml b/deploy/helm/clickhouse-operator/templates/dashboards-secret.yaml similarity index 100% rename from deploy/helm/templates/dashboards-secret.yaml rename to deploy/helm/clickhouse-operator/templates/dashboards-secret.yaml diff --git a/deploy/helm/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml b/deploy/helm/clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml similarity index 100% rename from deploy/helm/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ClusterRole-clickhouse-operator-kube-system.yaml diff --git a/deploy/helm/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml b/deploy/helm/clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml similarity index 100% rename from deploy/helm/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml diff --git a/deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml similarity index 100% rename from deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-confd-files.yaml diff --git a/deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml similarity index 100% rename from deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-configd-files.yaml diff --git a/deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml similarity index 100% rename from deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-files.yaml diff --git a/deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml similarity index 100% rename from deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-templatesd-files.yaml diff --git a/deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml b/deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml similarity index 100% rename from deploy/helm/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ConfigMap-etc-clickhouse-operator-usersd-files.yaml diff --git a/deploy/helm/templates/generated/Deployment-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml similarity index 98% rename from deploy/helm/templates/generated/Deployment-clickhouse-operator.yaml rename to deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml index 03b52fc56..64b10d959 100644 --- a/deploy/helm/templates/generated/Deployment-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml @@ -2,9 +2,9 @@ # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.4 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.5 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.4 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.5 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator diff --git a/deploy/helm/templates/generated/Secret-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml similarity index 95% rename from deploy/helm/templates/generated/Secret-clickhouse-operator.yaml rename to deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml index a5f68b9cc..9110a3ebc 100644 --- a/deploy/helm/templates/generated/Secret-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml @@ -3,7 +3,7 @@ # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # diff --git a/deploy/helm/templates/generated/Service-clickhouse-operator-metrics.yaml b/deploy/helm/clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml similarity index 100% rename from deploy/helm/templates/generated/Service-clickhouse-operator-metrics.yaml rename to deploy/helm/clickhouse-operator/templates/generated/Service-clickhouse-operator-metrics.yaml diff --git a/deploy/helm/templates/generated/ServiceAccount-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml similarity index 100% rename from deploy/helm/templates/generated/ServiceAccount-clickhouse-operator.yaml rename to deploy/helm/clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml diff --git a/deploy/helm/templates/servicemonitor.yaml b/deploy/helm/clickhouse-operator/templates/servicemonitor.yaml similarity index 100% rename from deploy/helm/templates/servicemonitor.yaml rename to deploy/helm/clickhouse-operator/templates/servicemonitor.yaml diff --git a/deploy/helm/values.yaml b/deploy/helm/clickhouse-operator/values.yaml similarity index 100% rename from deploy/helm/values.yaml rename to deploy/helm/clickhouse-operator/values.yaml diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index f84a2c198..b71ef8a6f 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1237,14 +1237,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2466,7 +2466,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2881,14 +2881,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3157,7 +3157,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 --- # Template Parameters: # @@ -3183,7 +3183,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 rules: # @@ -3402,7 +3402,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -3424,7 +3424,7 @@ metadata: name: etc-clickhouse-operator-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: config.yaml: | @@ -3796,7 +3796,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: --- @@ -3812,7 +3812,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3911,7 +3911,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 001-templates.json.example: | @@ -4011,7 +4011,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4066,7 +4066,7 @@ data: # Template parameters available: # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN={{ password }} # @@ -4076,7 +4076,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator type: Opaque stringData: @@ -4087,9 +4087,9 @@ stringData: # # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.4 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.5 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.4 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.5 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4100,7 +4100,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: replicas: 1 @@ -4136,7 +4136,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.4 + image: altinity/clickhouse-operator:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4204,7 +4204,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.4 + image: altinity/metrics-exporter:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4287,7 +4287,7 @@ metadata: name: clickhouse-operator-metrics namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index dd57a874a..71d68a098 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1220,14 +1220,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2437,7 +2437,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2842,14 +2842,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3117,7 +3117,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 # Template Parameters: # @@ -3142,7 +3142,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 rules: # # Core API group @@ -3351,7 +3351,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3373,7 +3373,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: config.yaml: | @@ -3744,7 +3744,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: --- @@ -3760,7 +3760,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3854,7 +3854,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 001-templates.json.example: | @@ -3952,7 +3952,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4006,7 +4006,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4016,7 +4016,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator type: Opaque stringData: @@ -4027,9 +4027,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.4 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.5 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.4 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.5 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4040,7 +4040,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: replicas: 1 @@ -4076,7 +4076,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.4 + image: altinity/clickhouse-operator:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4142,7 +4142,7 @@ spec: - containerPort: 9999 name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.4 + image: altinity/metrics-exporter:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4224,7 +4224,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index f07a1c75f..cfd712e22 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1230,14 +1230,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2459,7 +2459,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2874,14 +2874,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3150,7 +3150,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 --- # Template Parameters: # @@ -3176,7 +3176,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 rules: # @@ -3395,7 +3395,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3417,7 +3417,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: config.yaml: | @@ -3789,7 +3789,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: --- @@ -3805,7 +3805,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3904,7 +3904,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 001-templates.json.example: | @@ -4004,7 +4004,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4059,7 +4059,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4069,7 +4069,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator type: Opaque stringData: @@ -4080,9 +4080,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.4 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.5 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.4 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.5 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4093,7 +4093,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: replicas: 1 @@ -4129,7 +4129,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.4 + image: altinity/clickhouse-operator:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4197,7 +4197,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.4 + image: altinity/metrics-exporter:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4280,7 +4280,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index e07ac0848..99775a392 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1220,14 +1220,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2437,7 +2437,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2842,14 +2842,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3117,7 +3117,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 # Template Parameters: # @@ -3142,7 +3142,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 rules: # # Core API group @@ -3351,7 +3351,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3373,7 +3373,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: config.yaml: | @@ -3744,7 +3744,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: --- @@ -3760,7 +3760,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3854,7 +3854,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 001-templates.json.example: | @@ -3952,7 +3952,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4006,7 +4006,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4016,7 +4016,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator type: Opaque stringData: @@ -4040,7 +4040,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: replicas: 1 @@ -4224,7 +4224,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index e81406fc2..1a400e0c5 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1230,14 +1230,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2459,7 +2459,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2874,14 +2874,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3150,7 +3150,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 --- # Template Parameters: # @@ -3176,7 +3176,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 rules: # @@ -3395,7 +3395,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -3417,7 +3417,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: config.yaml: | @@ -3789,7 +3789,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: --- @@ -3805,7 +3805,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3904,7 +3904,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 001-templates.json.example: | @@ -4004,7 +4004,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4059,7 +4059,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4069,7 +4069,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator type: Opaque stringData: @@ -4093,7 +4093,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: replicas: 1 @@ -4280,7 +4280,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 9623fca78..1124255e3 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1237,14 +1237,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2466,7 +2466,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2881,14 +2881,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3157,7 +3157,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 --- # Template Parameters: # @@ -3183,7 +3183,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 rules: # @@ -3402,7 +3402,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -3424,7 +3424,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: config.yaml: | @@ -3796,7 +3796,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: --- @@ -3812,7 +3812,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -3911,7 +3911,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 001-templates.json.example: | @@ -4011,7 +4011,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4066,7 +4066,7 @@ data: # Template parameters available: # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=${password} # @@ -4076,7 +4076,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator type: Opaque stringData: @@ -4087,9 +4087,9 @@ stringData: # # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.4 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.23.5 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.4 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.23.5 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4100,7 +4100,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: replicas: 1 @@ -4136,7 +4136,7 @@ spec: name: etc-clickhouse-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.23.4 + image: altinity/clickhouse-operator:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4204,7 +4204,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.23.4 + image: altinity/metrics-exporter:0.23.5 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -4287,7 +4287,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 8887b99d1..030ae766f 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1230,14 +1230,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2459,7 +2459,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.23.4 + clickhouse.altinity.com/chop: 0.23.5 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2874,14 +2874,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.23.4 +# OPERATOR_VERSION=0.23.5 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.23.4 + clickhouse-keeper.altinity.com/chop: 0.23.5 spec: group: clickhouse-keeper.altinity.com scope: Namespaced diff --git a/deploy/operatorhub/0.23.5/clickhouse-operator.v0.23.5.clusterserviceversion.yaml b/deploy/operatorhub/0.23.5/clickhouse-operator.v0.23.5.clusterserviceversion.yaml new file mode 100644 index 000000000..cda38898c --- /dev/null +++ b/deploy/operatorhub/0.23.5/clickhouse-operator.v0.23.5.clusterserviceversion.yaml @@ -0,0 +1,1636 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: clickhouse-operator.v0.23.5 + namespace: placeholder + annotations: + capabilities: Full Lifecycle + categories: Database + containerImage: docker.io/altinity/clickhouse-operator:0.23.5 + createdAt: '2024-04-03T19:55:41Z' + support: Altinity Ltd. https://altinity.com + description: ClickHouse Operator manages full lifecycle of ClickHouse clusters. + repository: https://github.com/altinity/clickhouse-operator + certified: 'false' + alm-examples: | + [ + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallation", + "metadata": { + "name": "simple-01" + }, + "spec": { + "configuration": { + "users": { + "test_user/password_sha256_hex": "10a6e6cc8311a3e2bcc09bf6c199adecd5dd59408c343e926b129c4914f3cb01", + "test_user/password": "test_password", + "test_user/networks/ip": [ + "0.0.0.0/0" + ] + }, + "clusters": [ + { + "name": "simple" + } + ] + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallation", + "metadata": { + "name": "use-templates-all", + "labels": { + "target-chi-label-manual": "target-chi-label-manual-value", + "target-chi-label-auto": "target-chi-label-auto-value" + } + }, + "spec": { + "useTemplates": [ + { + "name": "chit-01" + }, + { + "name": "chit-02" + } + ], + "configuration": { + "clusters": [ + { + "name": "c1" + } + ] + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseOperatorConfiguration", + "metadata": { + "name": "chop-config-01" + }, + "spec": { + "watch": { + "namespaces": [] + }, + "clickhouse": { + "configuration": { + "file": { + "path": { + "common": "config.d", + "host": "conf.d", + "user": "users.d" + } + }, + "user": { + "default": { + "profile": "default", + "quota": "default", + "networksIP": [ + "::1", + "127.0.0.1" + ], + "password": "default" + } + }, + "network": { + "hostRegexpTemplate": "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" + } + }, + "access": { + "username": "clickhouse_operator", + "password": "clickhouse_operator_password", + "secret": { + "namespace": "", + "name": "" + }, + "port": 8123 + } + }, + "template": { + "chi": { + "path": "templates.d" + } + }, + "reconcile": { + "runtime": { + "reconcileCHIsThreadsNumber": 10, + "reconcileShardsThreadsNumber": 1, + "reconcileShardsMaxConcurrencyPercent": 50 + }, + "statefulSet": { + "create": { + "onFailure": "ignore" + }, + "update": { + "timeout": 300, + "pollInterval": 5, + "onFailure": "rollback" + } + }, + "host": { + "wait": { + "exclude": "true", + "include": "false" + } + } + }, + "annotation": { + "include": [], + "exclude": [] + }, + "label": { + "include": [], + "exclude": [], + "appendScope": "no" + }, + "statefulSet": { + "revisionHistoryLimit": 0 + }, + "pod": { + "terminationGracePeriod": 30 + }, + "logger": { + "logtostderr": "true", + "alsologtostderr": "false", + "v": "1", + "stderrthreshold": "", + "vmodule": "", + "log_backtrace_at": "" + } + } + } + ] +spec: + version: 0.23.5 + minKubeVersion: 1.12.6 + maturity: alpha + replaces: clickhouse-operator.v0.23.4 + maintainers: + - email: support@altinity.com + name: Altinity + provider: + name: Altinity + displayName: Altinity Operator for ClickHouse + keywords: + - "clickhouse" + - "database" + - "oltp" + - "timeseries" + - "time series" + - "altinity" + customresourcedefinitions: + owned: + - description: ClickHouse Installation - set of ClickHouse Clusters + displayName: ClickHouseInstallation + group: clickhouse.altinity.com + kind: ClickHouseInstallation + name: clickhouseinstallations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Installation Template - template for ClickHouse Installation + displayName: ClickHouseInstallationTemplate + group: clickhouse.altinity.com + kind: ClickHouseInstallationTemplate + name: clickhouseinstallationtemplates.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Operator Configuration - configuration of ClickHouse operator + displayName: ClickHouseOperatorConfiguration + group: clickhouse.altinity.com + kind: ClickHouseOperatorConfiguration + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Keeper Installation - ClickHouse Keeper cluster instance + displayName: ClickHouseKeeperInstallation + group: clickhouse-keeper.altinity.com + kind: ClickHouseKeeperInstallation + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + description: |- + ## ClickHouse + [ClickHouse](https://clickhouse.yandex) is an open source column-oriented database management system capable of real time generation of analytical data reports. + Check [ClickHouse documentation](https://clickhouse.yandex/docs/en) for more complete details. + ## The Altinity Operator for ClickHouse + The [Altinity Operator for ClickHouse](https://github.com/altinity/clickhouse-operator) automates the creation, alteration, or deletion of nodes in your ClickHouse cluster environment. + Check [operator documentation](https://github.com/Altinity/clickhouse-operator/tree/master/docs) for complete details and examples. + links: + - name: Altinity + url: https://altinity.com/ + - name: Operator homepage + url: https://www.altinity.com/kubernetes-operator + - name: Github + url: https://github.com/altinity/clickhouse-operator + - name: Documentation + url: https://github.com/Altinity/clickhouse-operator/tree/master/docs + icon: + - mediatype: image/png + base64data: |- + iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAYAAAB5fY51AAAAAXNSR0IArs4c6QAAQABJREFUeAHs + vQmgZ2lVH3j/r6p676abpSNLE2TrRlwSQBoVtVFHiUGFLKOEKCQTTUzGmTExhmhiSJBoTMZEs2KQ + BsWNREWdyCTOGOMEQZbI0i0NyCaIxAB203tXvfef33LO+c697/+qqqu3V1Xvvq773fOd3/md5Tvf + 9+57/erVajq4DiqwoQLnvf4PH3Pe4ekx29P0oNVqfTEgl6xWE8eLV6uVxgnjCjLHab0TI+ZW00PW + 6zWgq0+up/XNkG+edjiubt6Zppu31tOnd1brm7fwvL1e3by1Bf165+b1+vDN02r7Jhh+6MZnXfYh + jAfXQQVmFUC/HVxnbQV+aX3Bg875xJOOracrD21NV00705U4XK6aVtMTVqut81WX6BAcNpOOIIye + x4iJFMfDhmry8CIwRh5mZBHfppH61XT7znp672pavQeH3g07W1s3rLbX77npQQ+6YXra6rYNXg6m + zoIKRPedBZmerSmu16vzfvnGRx9a3XXlar2+CscEDqXVVTgSrtzaWj2SZRmHURRJh0uf9/EycNE2 + cVpxPg+jLLMOJczPRiiPd0j1Q634eMgtr/X693CIvWe9OnQD+G/goXbXzs4Nt3/FZR8BxwaDJcGB + fLpW4ODAOl1Xbq+4cUBd+Mv/4/Om9bFrcF5cs1qvvgSvM5fpMIDN8tCxzDcaHjoCiFkyn+psur/f + sOaHneLfdHgxRs7zcNxZfwrjryOPX5u2pl+78VmXvgsyvgo9uM6UChwcWKf7Sq7XWxf+8u997s72 + 1jWHpvU169X6S/Dl3GVKi4cQrjp8JO11aGnPGxGHlw+ztPeh5jOtTjHhfdj50AgX8zcrHib8Mg9K + 2W8a49DJw2c2Jmkf85Aib/LnGPxw9ik8/joyODjAeu1O4+eDA+t0WzwcUBf84keeslof4uF0DRbw + mTgJ8I1xHArIRYdHjD4c4piAntdm3JnxhjU75BaHF7PH98Q+hfHgAFMnnJ43d/HpGfvZEXUcUOtt + fHm3tb4Gp9Izp62tBzF5HU4+pVQLnkn90AIg5ufLvPnQIp/gfgDRHHf6vWExHR/abWxvcsgIB9hO + HGBbB19CxvLv5yFbdD/HeFbGdu7PffSJ07T9l7ZWqxegAI/YdJioMFLsPkzqsMkvxNrh1Q81486O + N6xNh5fyzy8rd3Y+tl5NP74znfPKm7/ikveelY23z5M+OLD20wL9Xx++7Nw7tl+wNW29EBvnadxM + vOrwydVq8/FKFbiDN6xT+l6Zqje/gectWINXr849/JM3ffGlfzjXHkgPVAVyCzxQ/g/8vnZ9zjmr + D/0JLMQ34bh5ztbW1jkqSjuU/EYUpeI8JvIw89dxB29YqkP7co/yyRxeszcs2vcLMip7Fwr+Szs7 + 61fffM5DXz89a3WsQw6e798KHBxY92+9y9uRf//Bq7d2dl6IDfH1+HmoB0uhw+g4h0+uVjvMDt6w + ol44XOrwQTF3ffmHOZaPh9iuw03FX9wC1w89PP8BiH9ie3341bc++7J3LCwOxPuhArkF7gdXBy6m + 137wM86Zdl6EHfNN+N7Uk+JVaVaY+ZuT36S0+XKlDt6wZvWSsOkQupfesDa+qcEfjsd3Y0lefezI + zqtvfdblH98d1MHMfVGB3Ab3BfcBZ1bgtR958Dnru/43NP//ii9UHqI3AehYfB9GsQwHb1h8Bbr7 + b0B5OOWYdd00ngp/8GBwfHrwbWe988nVtPXPp/U5//zTz34Qf+7r4LoPK3BwYN2HxZ1+9n2POnJs + 62+gyN+MQ+pCHk/jsIrjiodUuw7esHgmtC/v4hCqL+Narepx0yF0koeX1qP5K04+BG//slCrxvn6 + dBO4abp1Z1r/yLHtwz94+1c/5KM0P7ju/QrMd8u9z39WMp772g9cubOz/bfwd9z+PPr6SB1C0eTj + 0OIR5i/7VCgeXrl52nzhc7XikBOvCYZ5s9Mm77JQ9tf9buQHYMxrmy5kEYvRccggPGw+dMwytvpM + jsMhD4nZWKztoR8meTjlCJjy2zRu8tNo67HzB490nG+XDzP+0C4OWczvrNdHkeGPT+vVD9z8Jx72 + ngY9eLwXKsAaH1z3UgWO/NT78aMI67+Nln4uvkeF357SN7G3Zx0C+Rk6Dp8MQZufQjuUtPlypTgv + 2pgQrr25Le0Wfsr/DGd78na/iqnczH+SXrhZehmgrOa3xSGx642FbvFH7jkCrzjbaH9EbLgW/HnY + nZKfTh+8u3g4XxHjMeQ8tCjiz860Wv/89tbW99/2lQ97a6c9eD71Chyny06d9GyzPPxT7//y1Wr7 + b+OM+vI8o9zS8Zk3Dods8jo0UCjhUs8RnV762aFSZ0k9EDc/ZFKMZW32fU1Oih+BzXG749IhAmLH + IYNys+nQYVSuy4aRuzzy3zUWa3sI/L3ip9HWY+fHJOPWxfl2+TAbb1iUkQj+GBc0/w9+LOL7bvnq + z/jVZnrweAoViM4+Bcuz3eQl661DV33guVvrHRxU09O8yWLPoTb4chB3NG0cGtnEdQjs0rug2vx8 + bIeNtkCulDY11TGhcfdhspefmp/x296niXkH/4jLcTS/s/QyQONn99i19+jNR3kzgg3Xgv8e+en0 + wetDqR2ynM/1Iz7k/oZlmog34Ov1zlumaev7bn725b+ABTz4LRIu0t26H6fL7hbPWQU+8tPv+Tz8 + FpeX48u+q3OvqADaVD5r3KMHb1izNyAUqW91Nl/JWchN46buCtyMH/XfdbjA9oR+TsQfcQpGv+2y + vxO+YUVcjg8B/cb26tC33fbsh/23RnXweBIVODiwTqJIBcGPJxzZvvNl+Ez5Lfhhz63abQRsOKy0 + HTmvy9um3nByG5U+UCnHWPiiwQP2zHgDWvAu7RZ+Bp8JLR+8YakOi8Nozzc14Vx3rVrIJ37DQp3x + wUMOF355xPrlq3Mu/Lv4e4uf9Oof3E9UAXftiVBnux5f/h258n3fggZ7GX4h3oN5JujNAA/svTgj + Nh5aauIBQCXbl2+SFocPCDcfKgs/sCUuAtEKDTGWNfwKJ4RvJ8Ufh2LmOYs78+n8s0IAnXn0Ee7F + t2lM+01ji70eA3ev+CnS9tD5Mc24dXG+Xaf4hsUCgQXrtLPzyenQ9F03P/vhr8CCHnyZ2Gq76TE6 + e5PqYI4VOPJT770azfVyNBN+i6cPiTqEoudUqTgtYtBnUrV5bm42JwjqsAgZEzLPWx0uMV/4hIWD + Oa7xLu0WfgafCS3b3qfJmHdejmxpp7hVj4h8kUfmozE2f57u3uQ+BOgty1gj8PLXRvsjYsO14L9H + fjp98O6Kl/NZV+JDVl+kyHllFgMSrcMNeC1j8mDE7zb7b9s7h77t9uf8kd+Q6cFtYwXcnRtVZ/nk + j/3O5UcOb/9jvLd/I75XxXbjadWH2FSeVrWWejR1HW4G4N4OF0k+BId905MP1zgsJJbDDEtxCafw + hBey2YdlTDOu4Xcjv9LtuN1xDb+sS9QnHGlzwv9shE5+N41pv2kMztkQuBl/+tvEjzlWk3jF3ccZ + cQidn3aJ4Xy75D/XGfPib4fZcIP6EacJAXHLutFOpFCvf2xaHX7xrX/y4K/7qCKLm3fEYvKsFv/z + +vDWx977bfghqpegOJd4U2aTe5PXIcQmywrycBgwTFMREyqo5TocdulddR1CfGyHjdzs8hMTwu0+ + TPbyU/Mzftv7NDHviGPE5Tia31l6GaDxs/vYtcqLm5Zo8W0aqUd8wsVYh8yMOIQFv3Z/2m/ix5z8 + b/LT+YN3V7ycrwzowPI9ecMindyRblp/Gs9/79YLH/4vpoPfDtFXRFWfTZzNwuGffN+XTjvb/wab + Bf+qTG4a7rHYXhxjk6pFtSm0B122pR7lTZ4AYAhePAVr8HPCXavNKpEI+7c/ieVQcTVFuJ/zhX1Y + ajgpfuXJ+O1/Fjcd8YrRccjA87j3w0b+sAMrX+pp3kftVsxsGoHbdQXuXvGzixwTnZ9iYjjfLh9m + sc6YpzwyKxrXg/0gXgGNC7mm05Pd/Pb2tP7m25/zyIMvE1EtXtF5Fs7a+2vwd/5W2z+Ipvmz3sw+ + VGK3oizonjgNduujaqV3cx+8YbVu0m4ch5E3edZpwwh8HXKoqzd52Dfaelzww0DrdUp+ihQPe/Fw + vo7bwPEwwgc3lNQYnVkMCp9656N2SR75CXeCDx54orODLxNRBF6s79l9/eT1Tz60fegX0ECPc7ex + 16P5tFksq2/UZZTdRd5UllXEpZ7NySbmvAG4W+4tX3rZN33YOZ6FHzDJTkTmD/fDX7O3f98HX9ox + zgU/Jua43XEBIELHIYNyo8MC+tkIrfxsGrVpwbdpLNb2ELgZf/rbxI85Rku84u5jo63Hzk+7VHC+ + XfKf64x58WcjSU53qB9x4g0FcSHXdHoqN3yA3c76evxWiK+/5Wsffj0mztoL36o5e69DP/7ubzm0 + c+gt6PPHae+hN7xJvTnZO9qMflDXzjax9FE/EoSMQc3JCdsTo+0S/KlP/uBA1ya+j+KjOa/0o2YP + WX7kfre9cGTwNYsfU5bpF4JgmYcdaj7ycBwRSMVBO2gMtJPgpaA8FnmJB7rZGPYzfLNb8qe8F955 + wf/J+Mk4Mdal/LweGSd18idQ1scedBht1GNS5eEnhVjfkEPRB8SbvHJCRhGstlZPXk/bb7nglz72 + zak5G0dX52zL/LXXX3ToztWPo/me610TZXBvqCmzubwXSzFvLjT1bK+qydnUgqn5ksclNs+uzUQD + XjJsmyTmCx8w4QRPR1aU386XOPLHNfjSjpvJ7gUJojlud1zzOLQL04XeJGQfh47fRLIuG8Ys5Kax + WNtD4MSLeGcjYMpn03gq/MEj77Rvl/OKwwjzlPOQIWy4Q/3wIT3LnjgBAsdpRa4H3HiZz36sB8/r + brnoyDdOz7r8FmPOnrs79uzJd5p+4t1P3dpe/wx+Uv1x7oXctNE0bH58cLMNPZpmtom7PopX+mwx + dWU/BQBsmz4+c+amzyWQXwrk08B4SpzFEQAMjXdpt/AzP4RItylfz5tf98D1edcn3DkuQ3ffx64V + bmw+iED3LS4ZeMXVRtWDPJuuBX+eEqfkp/MH7y4exZGRwyBk9U2K4ol4I0Hz+NBi3SirAvJjGrMi + /108MghW6d+/s9r6+tue84i3afIsuZ09XxLiL/9t/fi7//rWsZ034ueqHpe94marnlMT9c+Eu5sq + mwnNNnqoui16D5vQzWh7dtOQsyk1q0Ci23h4hNzHWfOGA/GXgnax+Zf29FunCsNs8TMqybbXLhHe + 846P99hkgedMOmRWmw6THj/1XXb+ES/NRScm4xKfI31EXnzU1fNMXI4AVJ54nvnbZBd48eaNuOBL + e6oyDzwJaRn54aPnMfSAQeF4og4hh6IP4rEf0eNGP61+8kMZ33Pd3n7j+b/40W9P5NkwssZn/vUT + H77s0M5tP41V/srxhoG01QsuAXqTTYAejebTBDAcCQx5tz7KV/pssWYniOXyn9tI/MGBgXrDPRY+ + pnscA4fNAjuns7Bb+NmMs/28HhlHhjPnH3FYLzkedw2x+aPAGGJzA0gviruP1DOfNtofkRuuBf89 + 8tPpg3dXvJyvyPEY8ji0kiTiDXg/tLjMlMUjP6ZxPaI+YV4VEp4S9a4PPUF+3W1bF//F6Tln/j9H + Fl2ZBT4Dxx+7/plb6+knsQGuyB4bm1arzVPCibtbvFnYFNo0VJdCvaNmg8XQR91CUXo2VfB0B+Uf + k2pZ8YsQE+FXouMqfIQJx6JTXCYIeQNf4xEo5O53Iz8AY975z2URidJxyCBdaDMKn/lwhFZ+N43c + jCrchrFY20Pg+6FW/jbxY07+N/lptPXY+YNPOs63qw4hrjPm6xBSplrOSIv6OGQMBBoP8lMDrIK/ + 3PAhDm/y46N4IOeF37f1kdXhQ887079EPHO/JOSXgK+6/nu2ptWvoZGv8KbinuAhwhG3ehiy9WgK + bR7jBWyyzdhsYceuKX3QshnZXHIkQMmylz75qceF5k18H1uYww/tS0G7FOl38LK5tSk063mbDZwP + VQCoCN7hn3OOq9ulQ7HE5iYyr2Fv/10WD4CzMeokXNYhR5JWHcPDCfDkmfGnv012onc9gt3+wn/y + UJd5qE4loz74EK7izPoCBIXjifUNORR9EI/98M6LPGEniX6GTAQvfE/2iunYMX+JiN737Jl3PzMT + ++X3nbv1B3e+arW19Q3+DBZpeu2jedhEWFB3mVd2pndTeC+WYt5c3BRqvmgMEoYsWjVxby7z7NpM + 2eSyD7+gzM1ReJrzCrz0Lf5wX3YznC3DfL65NvIrj47bHdeMf1YIlCE3ex/h3XXZMKb9prHFXo+B + u1f8FGl76PyYZty6ON+u/fKG5Tqw7jzi1j9965ErXjR99erOFuoZ8Zhb4IxIRkn82DsuXK0P/RJ+ + XdWz+mmiTYnVnG1O7R6XIM6K0MchU3p2AXAauIm7PkpX+tyM5A07QSyX//jMmZs+WOTf8IwrD41A + lB/rbd/zWtgt/JR/8uCy3PMZ8wboHjjjPQ/cLD3bGb24x+bP9fAmzzptGHOd2qg8ybPpWvDfIz+d + P3h3xas4mG/EE7K/XMNshZl6QFkuKPLLOS1j8nCUXgNYiVvyBEBeBw/Ecc15/vOt5x35mumrPuPW + ATj9n86sLwnxGxZWO4d+FV8GPovN0ZvccjZNNEPritE7rRnUI2y6aJaQbTaaTG0wbxY1He3k1wDx + uGnhHx+8hp7qgc/5whvuwBkH7R0IDbVJKq7Gaxw1vgbf8O9NBL1g5h3+aee4On86FIt3nx3EfdiT + b56XeMTKPOx2I77ZyV/3oAWL+iUuR+Aqz+TPcZNd4DGMS+vZ8g5NxunIwz/rg49ZfZQZ9TCEwvFE + HUIORR/EQ1cRJp9EkH7tZ9STWF7Si7fq+azzbz/6q9PPffxyI86MO2t8ZlyvfNcVq0MrHlaP1xKj + ebXqHHHpTQJdMN4wMGlg6BMezSd7GRoY8u43EpnTgfjKzO3reXsQT/nfpTcP9Ya3uDGR08NPx/W8 + FnYLP+U/CHfnM+wdR4bDuriOngcOE7O4DN19j82f6+HNG+UHWrx9zHVqo/MmcsO14L9Hfjp98O6K + l/NZV+JDHodWkkS8ATePDxvWTYcMeeSnBtQjD7/OEzi6k373obWLx/H/znq99WW3P/eRH0m203k8 + M96wrn3356AB3oA3hcdXc3mx3AVc5GqK1gzoAS1yDtFE+druXqpuU4/O+cvcD6P31FQ0cFOyRcxT + 9pJpFk1NCJJIuY8tzOGH9qWgXYrmK3vhCPTFeZsN3MEb1mIdWCoV1OuherX6RSU1uM7A4aPXFQSh + xwCF6x7rG3Io+iAeGo62IE/YcV5+hkwsL8UhXtGFrInHT9P2G8553Uc+x8jT+46MTvPr1dc/fbXe + +Y/4ntWltZhISW8q0XTMUG8S3Kw8FCTjVg/Uu0k2v3EAKEDwsGlCVpfagfjGNEsbdtS7nYf/kMUr + vW/iFdxLM4s350UbS0eHEY/TWdgt/Ay+juv5jHlFZHHELX/QRKIpLvOQbd5yHWKsQ5M0+KO4+5jr + 1Eb7I3LDteDXbkdgp+Sn0+8VL+crcjyGPA6tJIl4A94PLZUveeTHNK5HHn6dBxrhWS/qdx9azrvx + AK/1lt36xqPrra86+rwr3pysp+N4er9hvfpdz8Fh9atYvEv1GSYWUYuuJmiLLJmL2ZrBQK2b1lTw + 1gwhqzl32aMLB311ScDUVAQ4Lrpw15Z/yY6nGic2mdAKaBGvFfZL+4qfflKkn8HL5u6HSfmf8dte + fMGbcRvmPOzO/OlQknef/OZt2Nt/l8UD4Gxs8TBe4XMkaeiTP+UZruE5P+NPf5v8iD7ySgfEBV/y + UJV54ElIy4gXH8JVnKkHDArHk3kljxShz3o0O3sQQfrd67CSftA5zlk9pksPTzu/et7PfvRZoj1N + b6fvgXXtdd+wWk8/j8PqQtbebyZoGi5SytF0pYdi6DFrINXsTcnWm4f65BVgZs/uCLskCBkDetSE + thdADsq/9MlPPS7Em/g+io90vBgo/dC+FLRL0cCyF45AX+VfCdu/NgHNBDNR2YtuUVdSVRy0Y0D2 + SxWvYU++eV4KG5jZ2OIpfLNb8qcsP4nLMfzP+NPfJj8tXjz6Un4t75rOPD06T+DwIX9Vh9TDEAri + XGfWI+ujh5CzHs1OPim3+snPkCMs13vQhTxwsR4XrraOvf78n/vgN6Td6TaengfWq971HVj+n0ST + HM6C12c6Ni0m/RmHzYFVLNnz1mOyHgImOO3BTruQ46Fkm6Ve9CYQPmjZlckjyJBlL73jCQbAwy8m + nI9H+wtUxqWuFxA32oXfni/VwpHBF3nNx3iS3/ZSBO/wT5Tj6nbpUCyxuYnMa9gzsHle4gFwNva4 + E58jSUOf/CnLT+JyFDzyxPMJ/QS+uPlAf8GX9p523VjZISM/fAhXcaYeKChc96hDyKHog3jMK3rc + yNPqJz9DLpTijbBp1eInxv75sDoXP1L9k+f93IdPy19Tc9odWFuvetc/xv8J/Mf4jMH+qMsimoZN + hlnJsWgEWfZoPSbrgfOWbW8e6mXnB+Gt73xkx0UC4YM2tontBcCtxSd98lOPC/Emvo8tzOFHuwA2 + EXi4320vHBl8zeLHlGX6hSBY5sGJoO9xCxgK5ktQqzNFXj3+ZV7KB5jZGLyyyzrkaELex3UCfOUJ + ixP6ASbjLQfkD/9pT93AZX04on74EC7r48pUXR1PrC+A5tGDApQ78nc7OpTc+gJyvalJ75v4Bp35 + W/3sn37xBz8Jj+FHzvvZD31XozgtHk+rA2vr2uu+H2vwHaosHvqlzyhcTC4SFPkZhk3Hy7JH6zFZ + D5y3bHvzUC87PwhvfecjOy4SCB+0sU1sLwBuLT7pk596XGijxPexhTn80L4UtEuRiQzegzesqAvL + 2+uigu2uN2unq+llV9OuLyocMI7gwUfnH3qpta7Hf3PPOJM3HMpPi1N+hlyoZf+1+IkZfQuBxxV5 + VtPLzvvZD34/9afLxchPj+va6/4P/FPL/9TF3h1yfQZT83BNkFosGtHSQ+6fadwLLsGAR/NpQoa4 + oYlCLj+lJzuu0guNKNwUmjcA9+Z/l14gxWd4xsV4TF/zCsd68zdexiF82C38jPw7zvbdEXEmigGy + 48h5jFEW42Le8Pk916FtIqLFt2nMdWqj60uLDdeCP9e9NilMNvpr/KoLefoVvLt4hEtGGIQ8Dq0k + Cb5I1DyoI/Bql+ThCLoYUBfok1ZURRAS9eZJT1YseORn4CqP9KsV4D9Bvf72O//0Z/6zGdc+FViX + /X+96rrno6qv4T8P78XO1V2E7y7QptcSh+xNjTTVBdz8ufkwZ6BrMNN7E6uJyEOg+GowT0yLYKln + U+GjNn8EUP7LfeQh+/ALXdoVPmCKI/Ut/nBfdjOcAvRt8JlwyAGqPLJOzn/gWrwVRy8EN1/Wr43A + KtxNowtNwyhwGyOs2RC4e8XPjDiEzo8pxq2L8+3yIRDrjHnKSIBPQgWN68F+0ISAxoVc0+mp3JjP + fsiKeiaPPPgmPdwWjx5iQu5oZ/14oAz+Nf796Wn9wrv+zGNf0yj35SNz2N/Xte+6Bl9z/wr+IvNh + r8bmkL2ZYjGREeWOlx6LMzYdQOoF8w14O2RKj4fZJjaP7aN8pQ9ad4XtBKGf5n+X3jyKm4/k05CH + hsRZHAHA0HiXdgs/I//O3/MZ8+a336XdqIf1GW9I86FvHsQ3Nh/SAVJl7mOuUxvtj8gN14I/1/2U + /HT64N3Fw/mKHI8hb34zot5w80T+SJyyeOTHNK4H1iNpMTaCkKg3jybytuSBrHWTH/sz7zx+x7E+ + Nq22vuqOP/WYX026/Tju7+9hXXvd52Oxf5GHlYoaza5CxiJkUXPxOdaicPO2xaJi6GFp4KArfTRD + yPOm6vxF7wfhs5XdFI6LLoYst5IdjwLgreWXdrN4iWE+9EN7ESVvipQHr3EE+hp8A+fmh14w8w7/ + tGuHTNa94qCaAZmPaF7Dnnxjc5V/YBR+jmEvu8TnaELex3UC/N3yA9aMtxwov5Z3KAZu1E/1wXoo + n6pD6mGo8lB/vL7KejQ7+aTc6ic/Q46wHH8sQ8XR6lf1EJ0QZke8lEB7eL2z87ojr/3A05NzP46K + dT8GNr3ynY9Fo78Zv874IdoMKH6NGwL2G0A0BfSUOz4/0xhHPUC1eJbdo2gGNoXsRWRgyLbveqh5 + lT5o1QZwIEcC4GY7t2AEUHpiCOe8HmJgPJ2GeS1xjTfsK89FHDU/w/V87H/EoTAUl+No8SkO60ee + IfchNn+uR20eYMgm3j5yEyE+4WIUP3k2XQv+e+Sn8wfvrngVR0YOg5BP5zcsZaM8pk+uDx2++s7n + Pfr9vRT75Xl/vmH96A2PQIF+BcE9hM3CZp2NrJ6LW3WUHu1fzRV2ibM9zbAZ0rwequdCn/6M17YS + X7dnXGGX8YQs2tiGjksA3Fp80ic/9bgyTzym3SxeYjIOnVohi1fhzOwEF44R+Rp83vyWGRf0gmVe + qaddi1tATFUcVEfihMbV41/m5fpEvMBLDl7ZZR1yJGf6Df6U98I7L5glf46b/Ije+Sa982t5hyLz + MjPDsodxWCWPx1ALt5/fsCqP9fSQ6ejR/zj9wu9yD+67Sy26r6L60Rsunrbuwj8UsfVkNV1uhj5u + CNhvDjyM0GTQ6w2B3UK7kJPPekzWg2GG0375xhE8GpZ60QdBDYpDDsI/NHLoOClZzviCxXFT6HEP + sRwov8LFG4lwLV/KCz/lf8a/zJduzCPz4HW5ch5jlIVhZLx6Xt5yHWLUOgAjvk0jcIqzjeKn/aYr + ePOQyfGU/HT+4N3FozhYh4gn5HFoJUnqIbNcwOWhxfJSDkUfwEoc9Z1HBpqw3odpIqyAQcBsP+pI + feWRfiP++TyBwl5357mXfOH0dQ+7Wdz75La/3rCu/eB509bRX8QPhvqwiiKzWbW4ObJ4YzVVylz8 + WfHdFUOvxWzN4FUNPYbSpz8vcu+COX8Lg/HIXoOajk+Oiy7YfZbltjULtbpafmk3/AUm/dC+4idv + ivSTcdMrFQT6GnwD500EvWAmGv5p1+LOulccVMMw5wnHNeztv8sKG5jZGPbCZR1yNCHv4zoBnjwz + flhK3mQHXcZXDogL/8lD3cCN+pF5HDKeB1JUcgcCx+M6MhDz6EGByR35u50ZcA+70OehJ3XcxDfo + zN/qZ//0SzoAy0/uBymsJ+dq+uxz7/z06ybuyX107Z8D6yVr/JjVLf8OAV3jmrIJWFuvwmxkAVX0 + UUm/EcRnlGaXONubb7ZmwZNrSJyajxMAJq94Qh5xtTAK71bINxvbK2DcWnzsWlxDT7X1fd7xbPCj + rhcBDTO84ktexaEuJav9Vfwl2575AqEEyl5htrhVKBkaRxPtNudDkdewJ988L/kHZjYGr+wSn6MJ + eR/XCfDkmfHDUvImO+gy3nJAXPhPHuoGzvlaRn746Pyqo/C4QeF4og4hh6IP4rEf3nnRT6uf/AxZ + EKIUL0fRhTxw9m+91ku8thtxE29G9+/0ZedcuP0zE/fmPrn2TSDTFe/8l1ur6TkqHoozPiNodSWz + mv7MJMCshJ63XkuMZvPqkdF8XKPBi0kDQ2/Z+vATeAHF1+3JT1nmfghZtG7fEW+Th958waB4M78+ + Cr/0o10ASwXAeDMdA8teODL4cn7Dr2Xbu4dNVPaiW9SVVFkPPmuX2C9FXsOefFHPmHf+ES/mnJ/t + ZZf4HE3I+7ii8HvhOX/SfsCa8ZYD5dfyDsXAtXjh6Ux7w9KbHgqI/7723Cd94IerLg/wA9f0gb+u + fddfwcn5r9kC1WTRrP7M4ab3Jtkccn0GU/OAB/YdLz2bmLz0Mxwp/wGP5tNEA4ZcfkovcxNmmDTL + TIjTZYflf5c+UImPsfBFgwf5iQnhel6eL7uFn5qf8dt+FCbqx5DKTdatJiIOx+2CxvNyiM2f67Hn + YRLupEd8fRQ/eTZdC/575KfzB++ueBUH6xDxhDwOrSRJPWTAzZN5WQ5FH8Aah22lywcRiNh686Qn + K4ALmKJDXFpvxZf+m9+Iv/IL3EgrA8BPlq5X33r06x/3b2b+HgCBeT2w1yuv/yz8RsS3IJALajMh + olgij4vmJS6bchZ8zBfPEqdVxJqSD4ZUjwdOWLY+F9t4AcVnO9uzaYIHpt5UNYCOfuIwoD4clH/M + JI/Vzqv0CpB8LV7RMFDP9/gjPOEDMHB68m3wDZ6qByGVZ/p1YZZ2wgnuuFMmRT9sZMfNw3niN41Z + yE0j8LuuwN0rfnaRY6LzU0wM59tVhxAQykv6zLBoXA8giA8gWIALuabTU7nhg+3MSj/Bs4xj0IWf + mADOcdod3crvbH63H9ErH/71nem21aH10+/6M0+4XvMP0O2B/ZLw5R+7YDXt/Ds09O7Dqjapi86m + Z5E1sliU2+V561X6wCfO9jTLTQhjA8UiOsjWh5+QBRRf6j3SXnZkKH3QuivEJwdNllvJwWPALD/n + Y73wmW76oX0pGG+KBpa9cAT6GvkPnA9V6AUzUdkLtqgrqSoO2sGQcruGPfmintCXfzwr/BzDXnaJ + z5G8C/6U98LfLT+in8cv/vCfcTqMxHmUf2TiT07Ojzg8+c5B5WHfRR1CDkUfxCPrdCOeVj/IxSMP + vrkOLpPi1fo0O8ieZzx6kmHVSfUlPvjolzD5x797uJouwM/C/8z0Sx+7wIgH5v7AHlhHPvGvUBW8 + YbGGKBZH/NGYMovLYvfRBrzXJT0siyfwXhzzk3joYWpH4iB86NNf2FnR9J0vQpA/zgdtZOK4iLGD + 8u9uUDzBAMPwS7QC8tjCLAds2hE/7VKkn2YvHBl8lf8Zv+3FF7zDP+0cl+MwfzqUxC4PvnAzi3+Z + l3jECrMcWzyFb/VY8qesOBOXIzgrz+TPcZOfwGMYF3HBl/FSmXVx5CmjPviY1UeZUQ8jlYd61zHl + UPRBPPbDOy8TpF/7CR4DjFK89ldxbKqH6EZGVacIVIO8Mn9S08C8+CHuJx+55Rbs2QfueuAOrGvf + +SLU44X+DIXasLioQ5ayZMxz1YXLkfXifLvMY5zXxHaJs33zMxyJRXRcI8WR/ixr0RRH6mNe+Aii + 9F5iNiefHBcxQ3aelJNfjyPPNu94iAtM+iGfE4WCflKc8yqO7MLgtdnA+TO2aIgQUcZtv4u6ApUO + xUL+CpDKaTrv0Gr6zsecO7356RdPz3vYYai7P3nZtd60Ey7XOUcreB9X49vUH1U3WLjeG+rT+DO+ + ckD+0Kc9dQM38iHzmf6GpXIgz62trRce+an3vqjqdD8/uOr3s9OJ37daH3sL/o7gBfoUtKE5skk0 + Qq/PBG0sux77kifk2kzMFoTVzCFv1scmDXwY9sE80kcQ8kd+waKJzWOEHZZ/hyMe6TfFD0Xhac6L + OA3pKOXwG/oZTha+Db60i08WFiuBOW6PQ6viGIX42ocdmf7BY8+drjhvfD78zZuOTX/zvbdP1926 + wyNxfohQxuGgeDeNDnt+D9yyL+qNAeiT9jNnttT5MYPoxnw+Y7Q/1o+HVuRRnilnWtS7jwMINCwE + qEE8oh8OhRt5NR4BfXMdGs9wbDrIjk90BuKxeImP+Fy3CEDzc15obju6Pvzk6fmf+SEY3a8XY7t/ + r5e/9ch05Jy34/Xys1zk2CyIwiWLEZsu9W3VvfobIvbmisWEnnLZhZx88uNVKb4Bj+bThAxxg0XI + 5af0UPMqfeYRDjivy3IdAm6f8h8gx00h7ApfNHhQODEhHPLG6Lw8X/LCT83P+G0fBI429DJXOHP+ + nm8YTE84f2v6oSvPm65+UP3maqnyhm/cTq/5+F3TSz9wx/Spo9ziSBN/do3cXMynjfZH5IaLm4rx + LkbZA76LH3Py2/jT34w9+HbxcL4ipwPL49BKFs7jCrh5Mi+alUJ0cic4+zhoaT8IQqLePJrIWxAU + Dx56XpUHAS3++TxUGbZwJPdEjx/Pbz566OgXT//zk+8i4v66xqfA+8vjkSP89cY6rNhkLAKvKhqe + q5livnAN7yahpS/zRDNgSnI2ccnNT61Z+gdIPd+aIeTeTRUn45be/hVPyBiwxHbguIgZ8tBHnEFR + eRK9rIvDpMJ+ySei5E3RwLIXjkBfs/gxZZl1gyCYectedIu6kqrimKZLjmxNP/D4c6c3fP5Fex5W + NME3bqdvevg509uuvnj61ivOnY5ATrfDH/1HX+SY/jjm1eqzCV95As+0juuH+uBLeufX8g7FwKkw + YQccPuSneFIPQ+aJ+TpkQg5FH8RDV0UTkadf+4n6REzGs262qzha/ew/eLkvxEu5x01ek6p/CSsc + 6Wd+n37k2JEfMPr+uyuk+83dK9/5dXizep2KFMXUZwAEkCWsselVRRaZ1VSxd0dsnmgK8i3w0nNx + yCs9bvVgWtNTb9zQh18NS33EIn/BQ/7MqOJ1ZuV/l948ipuPYVd4mvMqPzEhXM/L82W38FPzM/5l + vnST/Ok265bzq4mf7f7CI8+d/ja+V/Vgnj5383r/bdvTd+DLxP9y43ZG6THXqY3Omwu24fLCjf4I + uTYjTBidlruPjV91oV2/9uIRLhlJaOZxaCVJ8IVjx4M6Aq9lTB6OoIvB/Ze0oiqCkGCPD/LMriUP + 5J5X1UN2I/75PBiDdvB7wrjmFzzb0/pPbT//yp+fxXEfCoz6/rmuffdjpvVdv4VCX1rN51WLTRjF + RTRZyhqBU7Ha6NVdhB98s02pRQxcEA49nKkXNulzsRleGQovN4xT8Th8FTEUpWdT4cP2RJin/GPG + 7tM/xmwyokkUYw8Tiprv8Zff0M9wsvCt/DeeXfyYmOPa+sDu6Zccmv7ZVRdMV114qDGf2uOvfPLo + 9F2/c/v0/tvhNNerj5toQ7/sC2+qqCvslFcfO68KRsSGq/NDXSjOt6s2MRBcFcq5zoQNd9THZjfQ + OAECR3x6KjfmG3k1HjqIS3rwBp3jaPmVfYVnB2N+tx9RK58FLxRph1+vfOOx1Tl//P76fpY7P7O+ + r0Z+3+qcc9+A3xz6+doEKMJshF+tYR9RbC9CrkIbN8TpzRWLSZ5cLI4hJ5+WphymPhaFftl8spch + brAIufyUnuy4Si+0mrPsDMA98iY8M6Zdu+SXcszbX4nNT9gJ13iXdgs/gy/z3pQv3SS/g0u7R553 + aPrex583Pffyc1rU9/wR39KafuSjd04/8KE7p5uP4Rvz8F/rpV2IeGLz7PLW9Yw75NxU3opajayG + R+B2+enke/FwvpjwGPLZ+IbFcu3srN987H1P/ILpJasdyvfldf98D4vft8JhxUTYRNyMszHmuUWq + FYCTvAfeTUJGX+KDRTVp2CXO/uzfvLCrh+q5sM/4Il4CxdftmUfYMYTSZys7E8clgBxWfMrUfNTq + yrpASLvCk45X+tEhGrLyznQMLHvhmKivwTdw/swPvWCYV16pp91qOhdzL/7M86e3PuPie/2wogd+ + RfnX8H2tt+L7W9/4iHMjz1gHAph3v0JWnlm3HAWP/sEz06K1xk12gccwLtW59VNosq5mZFhmHodV + xukx1ML1N6y0q/UE3HE2O/mkPOpgP0POgF0Hl6ny3FQP0WVFHH/h5ceM9KOAVLnkHX6zDopna/X0 + w0947/dkLPflyFjv2+tH3/W0abXzptVq6xAXZ/kZLT9zZwlrjGJT70Vt44aIzcMmRVGhn9mFzCKX + v+FIbMMN7Y0LIui5ePRv3rle5k0vmOIoO3sQT/lXNwRvUHBQ3H7QbOEZL68WR8kRb+UtWNRh4Wfw + mdDyMt8exzR9Hd6mXvqEC2Y/piDf9+Htulu2p2+/4bbpv92Cfx6Buz4XaJPPrk8cRq038KrLpjH7 + oY0z+uDdxcP5rCsNQlZfpMh5eY6B7UM/+NAYsnjkxzRclc08MiBp6M2jibwteegv6iA7+XccPf7K + j/YCxpBy5NHjN854zuNp+9g0ffb05668wdb3zf1+eMNa/2sskg4rpsDk2HyzMea9WFFK4CTvgQfB + rCLiy2ZofhJnf/ZvXoDqgfOWicumoj55BQjZZswj7BiJ4qxBTUWA7QWQA/MD15qAWl1ZFwhpV/hM + N/3Q3oEATT8pGlj2whHoa/ANnPOFXjATEcfvT73+qRdPr/qci+7Xw4qRfvZFh6ZfedrF07990vnT + I85FmzLBfoWsPLNuOQJXeeJZZcpxk13gMYyLuOBLeyrlT6hRP3oYh0zGmXqAQeB4su+TR4rQaxCP + /cgJbuQZfWQ/Qy6U4iVv8LT4ial6iG5kNObTjxnVn4TJf/IOv1kH49hB06HD6+lf2Pq+uyuk+4z+ + R/HT7KvpWvHjgdXME382AsBAXLIYs1nCjs3j1dgccr0pcHHJt8Dv8lcOzTfgtHecQeTIBDDvXA81 + r9JnHuGA87osO07mu9QHKvExFr5o8IAElR9NhIu6SjSw7BZ+an7GP8/3ksOr6Xsef8H0Fx513oQf + WH/Ar9u319MPfeSu6Yc+fMd0J3+Ya3nF5qz+aJuV4c/6qssb+nFGvRcP57OuNAhZfZEi5+U5Bi4b + /eFDY8jikR/TOF7igpY0mYH8UBo8UudtyUN/WOc6XCiTTTz2RFPHlfOc4GyTY8K4wQfDBY5/SXr1 + F7Zf8MRXSXEf3Bj1fXPxVx1Pd34ABXvoLgfcLEx2MdZmgkFviSx6H8u+kwdf8XR+4mKNhj4dRRlm + +lxshlkKBSZa0uEh0yC986kB7qDHh+0FwC14CZdkHmpt3/Tya73qEWEKJzgmmqLHVXyJ04RvI38T + pnwILzJ/8VHnT9/9uAumy07hxxSai/vk8aN37Ezf8/47ptf9AX5WMQqvTaR1yPXCCO+z/ulyLlgf + N0Xb+cNeMM63qzYx1xXzlMfKVpiaz8MqgMbJT+Bor8j1gBsv89mP9cVjgFHg0foDXnHkBFmo14hb + PfT53X6CuIbiFU/WOfJV3vpHWT+xfdH02OnrrrpPfrXyffcl4erOl2IjjMOKxcOlzaviQs4x5iN1 + LxGb8Dj43LTk5CVeWOTmSz+Js2yceWFUD5y3bHvzUC87PwhvfcxLT++41Bw1oCdMaHsBcGvxSZ/8 + 1ONa1INTwx8lXOmH9hU/eVOk38GrOKKZct5mA8dN8oxLD09veMal0z+56sJ9eVgx9kfhr/q88skX + TK9/ykXTlfHjFKpv1i3HyF954nk2aqGjPgs8fdSlgvb1sma+nlln4PDR6wqNDOQOCq+j+Qg0jx4U + YK1ftzMD7mGHJ/sZsiCcD4Li0cPA2T9xpFOkMh3zUlhPPsZBWIunf/J1/IkTUHz4OcuHHr5leqmE + ++Dmqt7bxPxG+7Ttb7Rv4o5iqjosXitulnI2Qq/PEG1U0WnXryVPyG0VojmyuWAsR8HDQeFQ78V2 + eKUIfZoxLoevMOQv0sHE7uYyTzVJuU//GEFY+shvyPJSDtU0Lf5wL/uKh3EET1jP+B+BQ+D7rrxo + eu5n4P/MnUYXvzL88d+/c3rp+2+fPoXv9s76A3moLJvGXLA+bso79PVmkhjOt8v66BfMU+bKOwLK + Xq7CaUJA40KuaUVe5vHAPkNfSOJoGWJd0rt9jBNhTABV9hWe8xjzVpRccSQusiJv58tKt3mot4+t + VvfJN+DvozcsfKOd/1ewX23zcRW92WIErjYlnlU64HPchIdBZzcfLIqH+ly04Cfh0Jcj8Yiu9Bmf + 8YpIfN2e/JQjjNJH/NFeytMe5LD8S5/8wZF1gZh2hV/6ob0LFLwpGlj2whHoK/nOx9d/L378hdNv + ffGDT7vDipnwr/m8ED/+8LYveND0rY/CX/PhxKJ+Kg+wszEWTPVZ4F2huGs9Wz/VtOuLFdKM6wwc + PrwcSz1gULjux+urjDN5w6H8hB2mdn8SNM750E/wtPiJsH/rtS/E2+fpl36Cj3omVDiKLY4AMh7z + hZ3nDx3eWd8n34CP8OzsXrnnN9rbYbGLN4rp6mo1lXQVFQZa/ByB92eQMapIUbTiD97i6X4IYrbV + PLl4nI8yzPReHKdRirCP+GA3S1P+TOf42cRjkTOAis/hqJnwGIbhV6LjKnyEmfFy3oVKXIoBjLyE + kwPfnvvw86eXXXnhdMX5888pDXLaPfKv+fwN/jWfTx3NZd485oL1cVO2oR9vHAHifLusj3XGPOVc + Z8KGG6wrPqTn8iROgMBxWguqB9x4mW/E0XgMMAo8agfATc8HPMkPB9qFWA99frefIK6heMUTfBHf + bj/IZDXhG/BPepUI7qUbY7j3rpe/56HT4TvejQ0yvneV7FE8bT4V14snGRgVo4/Aq8hZ9D4mZxvN + E4tJngV+5ld63LRGLsGAR/NpogFDLj+lB4ZX6YM2M+K8Lmdoe+YbGZc+UCnHWPiiwYN6MSaEY7My + 7sg7Rrfg8HMVflTghz/7kukZl927P6XuyPfH/T/hr/m8+L23TR+6AxsmNynrgT+uRxuXfQh5dlFm + XZc8wiUjic2sT04piij4rA6e7GualUIBUjRrO1yKBxrhGX/0eciC8BYExYMH9UXgKg/JI/75PHnM + 6PhErAnjHH9M1Lwe4pZ8+Gs7n9i+c/XY6X+5974Bf+9+SXjkzu9FzD6sokiVSCtaNkGNAGWSsYRD + pl00TY0kXfC7uK25wi5x0mvNWzNozbw6oit9LErIvZsqTvG3MJos2tgejksB49bii64YeqpHM+T8 + 8EcOXOmH9hU/7VLMfGIEjr9N4Z981sXTG5/50DP6sGJ5vvIhR6Y3Xf2g6e8/7rzpYvxMBqvg9dhQ + nw31JocuFbSvV067rliIgNnDOKyWesCij/obltdXitBnnMlrf/bT+gJ+iychGMU36EJudshHdVC4 + epL16K/II8JnPjKoPCnO+UhgHPl8dT7MPnTrnJ179Rvww1N6PNXxJL/Rnif+bITPLGGN+ZkNo3dj + GzfE6DeMWEzydbuQVUzySsatHohPN9SDRxMNGHL5KT0wvEoftJkR53VxNK/c7tIHKvEx2p/pTcNA + Iz9OCNd4mx0f/9KjL5z+zhP5f/7u3c9NjnZ/3//grp3pe/G7t17zsTsVqOue64ORmxhF6uMsow2H + llaR87l+NAh5HFrJQhyugHszpz/7F4/8mMZdgrhoFuaNIOioN48m8rbkgZz5EWL/Hnv883kCTcj5 + eNLQ44+JmjfO984Hhu3tra177Rvw92IX7+gb7ZVkJRuphCx9NAk3W+IrScCzKXIsXMO31ZQD80Qz + YCb9JM5yX7RyFPaWHUfEhQCSVzwhj7iol7kfpM9WoGLkB0DJso+uMH9wtPxy3vFs8EN7BxK8KTqg + Z1x2RG9U/+eTLz4rDytW9PJztqYfxm+U+LXPv2S6Gj+2cR6+Md/rWn0FbM7TThcXNtbD65XTrq/X + M+3Aiw8vx1IPOyi8jsfrq1i/6gv7s58Wt/wMuVCKl35aHyz6abTLyGj0F+Mmrxn95sRnT5h3+M16 + GUc+X3M+zK2nQ4e2t++1b8APT+nxVMZXvPO5q63p512tPShRPOrzxJ+N8EkrlyzGbJawY/Mcj198 + uZjkW+B3+SuHfCA+6bEo5NFEKBhZyOWn9DJv+swjHBCny7LtAc+MSx+olGMsfNHgQeHEhHBRV1D8 + 0QsOTy97Ev6CMr6xfnDNK/CxO3emv48fg3gtfuupVmNDP84stEu9SYWHMu3iyXDioBmHVrJwHpfV + aF8easEHIm96PYhO7gQnjnoa8yqCkAaPJvIWBMWDB/VPEM0PE3kwH3F4cjx8MGHJMWF7xx+GMVSg + Jc/5WJlDz9v+xitfZ+ZTv9/zN6w1/l/A1vrvKjlsnkoyilShtaJx8y/xLkYtjfSV9AY8AEXNB/uN + ZkiZm3nm1zjzAlQPARO8NUPIvZsqTvJKT++4mizaaDLHJQBuLb7WBNTqyjwhpN3wF5j0Q3s5Yh1W + 0/l4e/h7V148ve2ahx0cVlGq5cC/k/jyz7oQP3h68XTlBWj9DfUuG9W5r5c1uS5YIU1YBg4fXg7P + Dz1g6pMT9VUsZ/FmJF7f9Gs/jisRHKWXn+Bp8ad+tIueZE47z6cfTSsfKVo8edgmn0bquc/imvMx + LiqgX2+/ODH3ZByeTpXl2nc8G78F5/UKOoq0kSoOjzzxZyMMVLQ+RjMR58OgjRsciA/FU1HJ0+1C + VjHJKxm3enDNHX40n+wbMOTyU3pgeJU+aDMjzutyhrZnvpb7YhOmuP1gK9i3MJsf8xL/9fjrNC99 + 0iXTw/G7qg6uk6sA/nri9GO/d8f0vR/E75fH97q0Lt5dgyD6uTYhNFo14WL9iA55HFpJwZXDpQUk + bBxaapfk4Qi6GADPQ8TmjSDoBk8irFjw0B/7R37Sv8fIJMzSX4sXmrSz/7QffAp4hhOd7EadBMBt + NeHne581feOTfs2oU7vf8zesnenF3HRKLkfGEkWqsFrRNuG9mC5NJpvjJvyS38V1HN0ucY4vix7h + GagQFR5kx5H5WO7dVHHSQPjIsMmijS51XMSwGVp8kpM/OFr90m74C0z6gf0fu/TI9Otf/NDpFX/8 + soPDKspzsgP/Ujf/cvfbnvGg6ZuvOA+/7jk2axKozn29rMh18Xrm+gGHD6077XR5lKg+aYdMyNVA + kiWJh+ZFo7iiHzkvP0OWK+Hhr/O0+ImZ9RH6jEzzecrk1bT8KKHCUTv8Zh0YD/dnXsNP8lMDO3wc + Wq9fkrhTHYenU2H4t++6Bv989X9Wlgw6irSRKvR54s9GGGQJawReyXfe4/CLD0U5G96wHoZvJr/s + yZdOf+6KC1S3jfU+mLxbFXjvrfz9W7dOv3Ej3gPyapu++hI6b9ac0QRu3pSajU2fhwL3NBfKmzn7 + uvHID+XcB3n4wU5XEYTUDr9AWGGC4sFD7jPq54fJiH8+T6DYIk9ZaqLHHxM1r4e47eZz/JzfXq3u + 0VvWPXvD2tp5MYM4mTcgrQYS2gtfSQLjRc9FOzl+8bJpuEjNz9xvX7RyhAfOW7a9ebLJotti6Pxh + lwRqyog/utRxCSAHFV90xdBTHX7xmPOFR3z86yd/4wkXT9d9xcOnFxwcVizqvXY9EX+Z+j889ZLp + 1fm7v1pfj6091gVP8u11wrrho/rOmtBjUF9Qf7y+6n3T+kp+Wl/Iz5DlBDfF0fuvxU9M7yO/EY34 + R9zkNaPfnGSpCdFl/MFnLQz4UhHX8JP8VICXeQB26B5+L2t4So8nO77yHU9BDG/FCe4s+5vQJo7Q + 54k/G5WSW0DFoyxaLHLnjUXYTO+inKlvWF/7iPOnf4i3qsdeuPnf/NtUk4O5U6sAf+fWD+N3b/3g + B2+f8APzY7ODTv2pXZ2dismQx6GVfr1puVdp6M3M7cK+thyKPmhz20/nkYEm7Mc8ibDCh4K2CSbs + Z+Dmh4k8hNnisM2wlRchnujxh2HZ6yFucz+YzPpgxP+iW++st54+fdNVb+02J/t86m9Y+NVESBm/ + qp1FiqLkSO+VbIQS8l74ShJwL1YrYvLmuIFfvLAsnogr47DfWMQMz44UoMKDbPvMx3Lvpjl/S1P+ + LIs2ulR+7QH3Fl9rAql5a/ml3RPwmf8/PvNh008//aEHh1UV6r59OBdvsn+Tv7/+Cy+dnodfD+31 + tM9cF6y0JixjXfHhdordXnrATqqvBBMPiWO78EkE6Xevw0p6+Qme6P+yg+z4SKcnPNBPzqcfTTsO + wloe9YYYdtbCTnyUlnyUOev6MAAcGPj1w9t/h7Oncimku23It6tp9VY6V7BRHEXXgp/xcp7Fwagi + 9RFABuKSxdj0xZt+ZsQWxAuGM+UN65JzDk0veRK+IfzYi/fFb/3cUPKzZupNNx6dvv3dt07vxve5 + 1KfahdmxKEPI49DK0mi3VmP7cMj+p1l0vPraNGbNQ6TzQCM86aLPQ06U90njgT73GzHLw4lMu+cJ + 1HTEJ4QmevwxUfN6iNvcDyYVp/NmAXiYnWQAAEAASURBVKHnW9ZV0wuf9N5udzLPp/aGtbP+Tnhl + bZ1UHC48vBisrhwtRdB74ytJ4LVoLDaexZe8OZJzwW+/0QzNLnHmMZ95AaqHoIPsOGbFNZD+Sm+c + Zah5lT4PX9ah1UNd0OJrzSJ73pAf/2/VX37cxdO7v/IR01/BuB9+RXHFd5Y+PAP/N5a/3PCf4pcb + 4gfm43KfV99h3dxO0f+1voBX3xyvr3rfRD/K07yP9jqsRn8Hj/px9J/7OnjRZ3gyO3AjbuI1DS0e + qCgcxTmftcQJKMPhJ/k5Dbvksz/8nPn2d8ngbt6Gp5M1vPa3nzBt3/Xbq62tw96koIji1LiJi0kx + WIxKqo/AMxCmWGPTF2/62cB/JrxhPfOh507/4ikPna68+MiGDA+m9kMFbsQ/oPh9H7htesVH7pi2 + 2Y+8NHpTqn9j2h1NPf6o/Xk4ZP/TrBShz/7PQ4TkvBpO0uCROm/aH+QNHjzkfiNkfpjkTlvOE2hC + xydLTdje8cdEzeshbnM/mIw8xRcFwgF2bGd96Ml39y3r7r9hbR/9bhThcCbjICKJOGQii55DBO3i + 8ESe2SmnWCQ8O6dcNFZ/N95FGC4cj3FpT7vE2Z/9R83SkUiipoormyqbTMDoAvIM/qL3Q7iTPprM + cdGFm67soysoPxp/nea1X3D59Ctf+vCDw2os6b58uhS/6/4f4XeJvRG/OPCL8Lrl9UXf4aP6QpF7 + 1598X0U7Vl9k+qNvOGM/sR8Swvnqz2zrtm9C7/ggcF+Unx43eaGSlvZ+0l3i8Ou8HY/5iHIcvQ7m + c32iQAGcDm9NO3/dwsnfFdJJw1/+2w+fDt31uwjwME9uZbdp3EQYuDzxZyPwSrKPwLMoG/1s4Bef + mobF4Zq0+EJOPusxWQ9eQ605/ZJH9jI0MOTyU3qoeZU+aDMjzutyhrZnvquJ/8PvxU+6bPq2J1wy + 8Ru9B9fpV4Ff+oM7p+96z63TR27Hv5/IvkEKuendCZzAHyjUf3hwH1oORR/24BGBCrTXoSXHAXMc + 7mP642X/za8CW84TKLjw8aShxx8TNW+c73M/mJN/590LtLPeuX197Mjj8fuyPtbtj/d8996wDh39 + W9iY+EegMukYsSlVlBzpcaya/beicXMv8ZUk0Mmf4yb8kl982Qxyj6rzsJj5dbzmLUeKL2qquO7r + NywW/QWPuWi67tlXTN9x5YMODiutwOl5+5rLz53e8kWXTd/9+POnC/j7t6rv85BAXmpDHmbZ97Fv + rNh1WLESRROnXfLudVhJH+3u/nb/lx0IR9/rSQWnvvCKT9M6NKWoQ41hjvyKl3rus7jmfJkH7IQD + KBIjbmvaOn9ra/s70/ZkxuHpROiXv/XIdPjIR2FwOZdi9gbDIPJwaMHPKEOvNwwWCbKS40g+/JmN + Ta8kw74Xp/P7zSWagnwL/C5/5dAlGHAX1/YicmQCmFdNE7IXNXBZBoqZEXG67PApl+H7VE996MTx + 4DqzKvAx/DNkf/e9t07//vfviMTY0biisb2Zs++5d0uhDaBtJHgeIrLuBJrY69DyPvGZoG5r+4yG + 9t/8KrDlfLgLPO2A8J18+HDcnB7zAsRt7idxYefAhBTPev3fd+6884rpLz/taOfY6/nk37AOHfqT + 8OXDSjG0YLEp5TxHeotkynFPLnE5Bp+3tNZOfFl0HlIn4pc+itntMg7b98WBUwMVosKDTFwtSsi9 + m6w3jvaVJh9CxoAlZn3G4n4G/nWaa59++fSGr3jkwWGlip95N/4LRD/6uRdP/+/Vl+JfzfbWOvm+ + inaswyHrM++jvQ6r0d/Bo34c/Tf6lm3pDqWHMZ9+7Ff9S1iLp/ZF2FnLvheQ4oKPMmcRB3nkVhOB + 48Tqj0znXPAniTqZ6+QPrGn1DXQlnxwjyHyTmY303JKgmPIMh2w6z5JfMnkSl2PjE7foHRn5ut3c + r+O2Hkb1EOFBtn3EFbKAiiP1HrUGdMur9EEblToP/zrN38L3qa7/E4+env9HL9KsDQ7uZ2oFnoYf + g3jjFz1k+iH8WuoH4+99uk+8ad3/0T/ZQOqz3jduJ9dn9DVlHRqwy32TNRy8wdP3De1qX0DQKeLG + HfPpx4z0w/DoUXeJw2/6N07AwLX9R2uZwy75PBHxkA//AtK0/Q0yPombozkR8F9ef9F0ztHfX22t + LmJoNJqNLEYcJjmqKBHcLnrOs2iLsYp3qvzBVzydn0FE4EOfjqIMM70Xx2GWQomLlnR4yDRI73xq + mJ73qIum7/+8B0+PufDgxxRUn7PwdhN+DOIfvv/W6Uc+fCv/GXftE/dh7CA1UGwH1Mdv5nqIahlX + b0I8rPBBuV/ed40nGzNwZR9u1bh0A726W7jYx5zXDhdAbkQXeE6UnXBg2OXH8ZXdLr6RBzA3r3cO + P/Jk/rGKk3vDOm/7z+w6rLhrcXHTMtjZaIX0dTsBvg4RGKimPAzwPONNPyQNPj7yEg4WxRNxJc48 + xpkXRvUQdJBtn/kMXvGUPuYly70JQr7qkiPTr1zziOmnv/CPHBxWUZ6zdXgQfwziqoum38Qb1xde + dlj9xb7Lfh19Fe2o48Pt5JrpOAk85nlYRZ/3mo7+zrZu+xJA93Xw8hQpP22fiTe90p7P9G+77tfx + U2s/AgnX+WxHouO9YWGrXjxtbf/p5DjeeHIH1rR+0ThRnQJlXhp5uFDO0Qrp63YC/CZ+lmrGexx+ + 4VCW4ol4MKEQzGM+82K6Hjhv2faZT/jPRdHasPgZV9jRAwgefO6h6Yfwg59v+6pHT19y+cGvKGZZ + Di5X4IkXHZ5ef/WDp9f88UumKy44pD5VA6pP3UfqK58S2bYwdmO6vyl586ec9R39nW0934/ua/vR + IVl+Wj9r/5hRb1gMqHDkjX3B2dxX1GNf5jX8MO7wRzvhakL25hMMf8dw56S+LByebLf7/mPvuHw6 + On0cwJVL5xR8okewKjoQOe5mcVLQ66RfjsAzkCW/kmcxkjfHDfz+DBKLSb5uF3LyyU855EOF5/gQ + ie1DoWIzDvPWZ5oMOAg+9DV/dHr4+fV3Nzh7Vl//7HfvnF7z8aPT33z0OdPXf8aZ+28h3t1F/oWP + 3zF909tvik2f/R1tDjIfSpbNHY3G/i/9ODyMoYINWoP4c7/JDnp2uw8bPcl0eciwz3kZp6eS+6El + fzOcYPbb5xX3eAnodo5Hvo6uj01XTN/8Of/dLJvvJ37DunPnz4FUv5UhU7STKJ6KlMG0IirI5jRk + FSHflHKM5Jb85SdxOZJ2wZ+LMCs+8ImzXy+CecFRDwETPA495WV874I5f9Hr4ZIjB7+imEvznz55 + bHrKb948vfSDd04fwL/I/Fffc8f0rLfePL395m2qz/prHCLRn2PQYcUCjfbmPhv7yofZkLOYo7+z + rXl4DdzoW9Kp8WU65tOPGeuNKE4vb4c5H5HGkc/XnC/zgB155JZ+OJ/7zHZQ4cemVs+3tPf9xAfW + 1upFSoXJgydT1RsI5TgUZiP9qSh8iCvkGY5Bt/klf/rl6s3sSLngN49x3S5xtne81oOjHoIOMnFq + CvKHHA8l2yz18/xCOiuHD+GnvZ/3jlun51932/Rh/iIpXLlu77x1PX3F226ZvvWG26eP41+vOduv + 7Ff1p/os2lE7LPpRRfKOMx7z+Kj+bEV0nW03+nPD/hKdEGZXv8c6gTm3Ff2w/+lRd4lzPmupEDBw + cU4EkQfYJV/NZx4yU+B4K3pRSHsOxz+wrr3+jyHoz2PIdXLiOWWycp5ZzkYreB9XJDXDpR1Qm/jL + T+JyJGsrkkUvQvFEXImz3+bH8OIRnWrv4na8MhZft2feZd4eGM3Zdd2Cf9HhJfgHS5/xllumX7/R + b1GqH8rgOro/WPKfwT+x9bQ33zL9U/yCvLP53HJ9ooHGoMOI3TPae/S15nmo4CPryzlerrPtQDer + e+o9Dwn7CAhOC1d48WracVBROLb78Jv+7603LATCsD5vuvadVzmCzffjH1jbR7+BJ7dKlmOkUCc+ + k5ezNtIX5/sVsj8TMLg5vvzAZuav49IPeRf8jse8ac+4Emd/FDOfckQ2wwSnPuMzXhEpjm7P+MOu + CER11txY55/GAfRUfPn3zz9y14T/g1+X1yPq19aN87fhgPtefLn49DffPP2H/3FSP+BcvGfKg+sT + DTQGHQrMcbQ391H0I+d5aDSZWF7i6zzq12YHmcsjXu4L8PCinefTj6YdBxWFYxRzPmu5DwSU4Zwv + /NGOPHbUcOSTmED8TNbquN983/vA8i/n+7M8SZVKjuBPma58siNpBs3kM/gcI55MaoZreM6Lt/GX + n8TlaMfJrNF+7b/bzf06XuvLUdhbdhyZT+SnYjO/bp+yzKmIh7NjePvNx6Yvw/el/iq+xPsEz5xF + /l6PqF9bt77OH8VfZfnG62+bvua3bpn4j0CcTVf2q+qmvop9pV3dy8m+GvtKh0aTs2bi6zxcj011 + F512gExrPbR+9GNG+mG/I5LAUWxxBNA4AQM3zgtZyxx2yZd2GM0ns0p4Z71+AYIYhKHOYe8D6xXX + fSkMH9tPTPrOVHWiU0ZRmOVsJDvn+xXyDJd2gjOpOb/kk+R3PI6j22Uc9ut4rYezeohwIRPH4na8 + gIoj9R5pH2m1h570mff8P/Bv+H3bDbdNX/62W6d33Ix/z48psgmrEM7Z65F1inpC5fpipBll/HnD + TdvTM996y/Sd77tj+sP+mgbdmXq5PlG3MaAe3jejnK5U1RP66s9WHOk7j/p1Q91Fl5Vv6yGHxJu0 + 3ohaPN1vj0frH7FwXuxB5AG85LFCSOOGv3S8tVo9fnrVO69uqc0e9z6wVseeS5J+Art00WxsUlw+ + 2Y1LfCg01O0E+PIDA9eUSZ48v+KARfHkJpr5NZ95y5FCFAwK22c+4V/FRr6lz7g4yrw9hHyGDTxH + /tXv3jE95Y2fnn7i9++KdUfazJNdWYVw4l4PTrNuUU+oan1oRjlG/Da36RW/d+f0tN/8tEb+Q6dn + 8uX6RN3GgHo48VFOV6jqCb3fTOYFcp29DKrrXnUXXVa+rYcccp1cdcVBWIun++3xaP1jsThf/mkt + PvCSxwohjRv+0nHkv+eXhcc5sFbXkKSfmMo1UqgTNppVcuAVEef7FfIM1/DlBzbywybH8174XiS6 + EQ4WxRNxJc48xpm3HNHcMChsbx7VWHFLocDm/GFXBKI6427/5VNHpy94003Td//O7dOt+T/4VF/k + z2zZlarTSF31hui6Rz1DVv1pRjnHsOc/C/id7719+qK33Dy9of8bgcCdSZfr0/sq6+F9M8rpClU9 + UTFu6pSzJq6zl0F11foMHPWehwXXyysX/R7rJF4z1htR4RjfnI9I48jna/jpecCOPBWA/ZmvDPUg + 3A7Onj2uzQfWtb91Kcg/l03YT0yXzqnWCRvNKjnw8qWiNK8hz3ANX35gIj/A55hx1EjaBb/jmcdL + fOLsl2LyliOyGSa4i9vxioj+Sm8eyzIvPyGdEcMHb9+env+OW6bn4ntM78fPU9V6MLusB59bnSny + 8npknbwuOS8eCLMx1tN1X+l7Wl/z9lunb7ru1unD+HGJM+1yfdRQvb10KLhOmbEPl6onDw18pFyo + XA+3qfWb9lcdGnzw+ox1Ja8Z9aZDhU4Z4hjm8Jv+jRNQhpwffLajZfGFA+OGv3QcuM+ZXnH9gx3J + /L75wNqersFJKbZ+YirXSIHzvDQyyGjanFcTCxG3E+Bpt+SX3HnTjx13dvvnYiZP2GUcjs/xmhfm + 9cB5y7bPfCI/AsXX7WEQ0wpEBLOQTluB/xfvH7z/9unq37hp+r8/cZcKM6srM8t68JlNuMhf9RaM + dYp6hqyy04xyjmHvdcr6r6f/8Ilj0zPe/Gn9X0XGdaZcrk/UbQyoByvSy+kKVT1j86ec9XDdalmw + HHvUXXRZeeJj36n+rLsZFYfK7QnRYcXSb42Ml+sf15wv84CdcACFA+OGv5o3cms6tP0lydnHzQfW + tHWNTlCQ9xNTudIn/kifY+DoNOfxAG27Qu68HV9+YJL8ORbuOPz2O49XRZj5ddzmLUcKUjAoHEfk + EbIiIqD0kb/kyDH8hHTaDq/9/Tunp/zGjdMP4h9a4PetnNairswu68FnrMtyvb0eWafRF7vWmeai + c7/ILtc5Rv681g9+6PbpqW+6WT/HtegsRnDaXa5P1G0MOBKyDpkS5VY/HhpNLlSuB+BVz6wjQFV3 + 0Qkh0zGffsxYb0Qtnu7X8YOXeq5/XHO+bAvELxxAsU+MY15lqIeGuyY0s2GPA2vnGp2gYOsnplNi + kIzRQXYcvec8HmaOUt4Lz/klf/lJ3hzJvOC333m8qsYsTsdtXnDUQ9BBdhyRR8gCkqf0xlmONBfx + xOxpM7wLP6bw5b954/Qt190yfRw/buB6Rl2QaK1P5pn1YIZYl83rkXUafVE8NMMfdonG4JXfXOcc + gSH/f8f/ofzWd982fRm+v8UfqzidL9c36jYGHUaRbqTnChnPennzp5w1cN1UplHPVj/qXWdYcL1U + eeJzPv2Ysd6ICsd1mq8jkcaRz9ecz/HQsvjaOpuvDPVQuGnz97F2H1j8/tVq63N1gjIZJg2qGvGc + Mj10HJtKshW8j0tF2hu/ib/8JG+OZA2+dGC/83i1iWZ+Wx65ZqUHExw6jsgj5FA0febRwljEk3Ht + 9/ETOAT+99++ZfriN944ve3TPARQmMo781vUlUkxX+L4jHXZvB6cJm70xa51prnoxDTHN7vO/45b + tqcvx6H113B4/QHiPx2v7FflFeVTHVzRli7r0uoXm9/2I3PXuZZlXkfAqu6isydaj/n0Y85602nx + 8DBJvzVSz/WPa87neBS/cADVfvMhVtsm5ws3bfw+1u4Da/vQNSDFQYkgQNJPTKeEafiVPsfAJV6x + tyS63Hk7vvwAnPw5Fi79kHDB73jm8dIucfbruM1bjshmmOBYFHx0vCJSPbo961P07UF0+/52DLH/ + 6w/fNj3lv/7h9KqP3o74ETILw4eogyTORz2odp31YJzUMuBTXYlzHaOe0FIWD55nY6znDH+c9UaH + Tj+Fn7J/Cr5M/CH8NZ/T7dxyfaJuY0D1VXD3o6rphTGeq9P6s6od69J51K8b6i66rHxbD9WfeJO2 + Nx1HgfnaF5jp8WifRSycF3sQeQAv87Ii+DKPMvR84PAvreJs2v19rN0H1rR9DaPWCRqjc6TT1mQ8 + DCjHoTAbrZC+bifA18kMg5m/k+SXf1gWT9hlMR2f4zV/OVKICg8K22f+kZ+KiHxLn3lzjAzrIeR9 + PPzXT901Xf1fPzm9+IZbp5uO8oc/mS8CZmHYVZGnJOW3qGsqiOMzu3KRv9eD0+SLegLq+tqO7miv + Mexn+Ga35E/51mM709/H32O8+jdvml6vH7kH4WlwuT5RtzGgHqpopodMXCHjKXnzp5ypum7Qs9y0 + 2qvuohNCpsQVHk+xDPIjhfwn73wdSaB4uf5xzflsx4iMo0Hml3mUoR4GDuLW+prQ1rD7wFqNn78i + eT8xXToGyR50kBoDl3ixtyS6vBe+/ACc/DkWb/oh4YLf8czjVXFmcTpu85YjspkOCsdhHgaSvAKU + PuYlyzwI4nmfDr+LH1P487910/TVb75x+p3b/KXUyBdBIx8kotF5Z1qLugoWOD6zCTeuB6eJi3qS + HTLdsHtm42ydsv7DbsmfcvLzt0O84J34EYy3nx5/zcf1jbqNAXVhZXo5XSnjWTdv/pQFFp51tp3q + ulfdRSeETMlTeDzFMjgOKlo8Oiz7OklLvwJu4Ms8wEseO2q44S8dDxxg6xMdWIufvyJJPzGVq4Ik + l4PUGLjEK6KWRJf3wpefxu/aws9J8DueebwqwixOx23ecjTCY02BV1PQLuR4KHnERbzM20PI+2i4 + HT8O8LL33To99dc/Of0ifnlcxY8YR74QqFDXe16S8lvUNRWsD5/ZxFUITnR71tH2OS//EGZj2Gsd + E5+jDXkf1x54/pDrF+HHIF78vtunG/l17z69sl9Vtyif6+GYRzkpt/rxUGlypue6eRnEQ4JWP69z + LBPm8STTMZ9+zFhvOoVjFC2OrD/14gs7zJd/epEb2AlXE/O+o+mMjzInt3Z9H2v+hrX4+SuS9BPY + KZkrT3iNgUs8XfUkurwXvvwALD8stmiQ/knwO555vCpCFNN+GVbylqMRHhxabx7VWPZSKLBhz7ha + muFHZPvo9rP4N/Ke8l8+Mf2j37kFv85lUR/EOfKFwIKz3aIOkth9mJjlnQripO6F4IR5a8z1i3m6 + od1snK1T1j9GE/E+ruPg+eNaP/JR/HjGG2+arsVf99mPP77Fekah+4C6qKKod6ZqnPGsmzd/yoWi + QSyD6ip51G+sH91m5b1Ohdc6m1FxUNHiub/fsFar9a7vY80PrGm6xic1isKkkHQ/gV06pyC9ch+4 + xCtlFUVPvoXceTu+/AAtP8DnWLiIZxO/45nHSzsvDgfG6dG85WjQlT7zDzsbLuyTT+blJ6QHfHj3 + zUen/+mNn5xe9Fs3Tr/XfvFU1bmthz9zImQWRl2feWdai7oKFvnzudWZIi+vR4xt3co/MFqHHFs8 + J7Pe83XN9Yox/N+IHyT76++5bXrmm2+a3rTP/pqP66OGi77KeqCuil8Dn/Cn5cVDpcmFqv4OHsnN + DrLqLbqsvNfH8+nHjPVGJP/EkXfOR6RxZPBV60sDXB5gR54KIP2Sz3b5MHCcl8E1gdCw68DSyQ2W + PpKzTmg8p0yGjqNTyVbwPi42Na698Jv4y0/y5mgi8eXNfu2/28GhIPbb8lAtoCo9nmHoODL/iNeK + po954SMC8dDzA3+95Q/xTfX/75PTm/7wqJosm4GRVZ0rbzfTCB+FiToYz/uirqkgTupeCE5k3bJO + oy/KPzCs1lgGMSm+6qPjrPdYN8ab67XZzw34C5DPx/e39sulTZ0F5xjlcz2yDhkt5ZYXKsZNrX5O + CEbJnUe8A1d1F11W3naSIh4N5OPKUKEVIo7inM9aKgSkqDgGn+0Uf/KFA8dDPpklcPjVPJj4PfV2 + jQOL37+aVrOfvyJbPzHJoWA4RpAaA5d48bckurwXvvwALD9swvRzEvyOZx6vqjGL03GbtxyN8KBw + HOZRjWUvhQKrODkf0zMCCQ/s7SZ878b14Hpp0SugWfyYHflCYGGE97wkLjgUSztMOH+peyE40e3t + v+KBHd2QdjaqzmF3Eust/+lnA77ibX7wuC8ubVblG3Ubg9cLUUY5+IQ/0Y+SWL8hY0qX6tt5SJB1 + AaLqIbqsfJ9PP8EXfWD/xDGK4bfWUzjyhR2AYo8EPMAu+Wo+8yhDPQwcRSU0+z7WOLCOHf5jRLCY + jK6PzpFOReGROBvswodCQ91OgNcikg9/Zv42xCPO4Et+xQvL4gk7Lhov5+PR/JisB85btn3mH3YE + iq/bwyCmyZ9+9LwPbq4HwvKiV0Sz+ihsN1PmTwvnlXWj6aKunMp6SN0LwQmq0558Uc+YV9nxPBs3 + 4ZvdrvqeAF95hh8M++pyfaJuY/B6IdJIj0/40+oHmZs665tJSe48JGj1q3qILivvdZIkh+Q14355 + w8K/h7o1rY89MfMcB9bWscdw0ic1ioJkGf04mZEM9fjjnPm0GR8KDXUjH67Om/w5v+QvPxFHx4NI + fHkTLxeTi9T8JM5+7d96gOqB85Ztn/lHvAQSoKHzhx2DWMTDqQfycj0QFlcsuxABzepTMvOFwHoI + b5wkzkOxtJMB6yF11IfPcZV/1S3qCV3x4Jnu0u1GfK47OZf1DVl2ictR8Hm/kmI/Xc436jYG1EMV + bem6QlUf6PubTubkOrhMqutedRddVr6th+oZfQDSetNp8XS/PZ7j9xcjBC95HJhCpr35JNb6DlzY + Ma6tVf2e93FgrTzpkxpkbHKSsgnoMkc8pyzKhks853sSXe68Hb+Jv/xEHB2/5BcvIiueiCtx9tvy + yDUjLsOFQ9tn/paVsfhSH/PCy5wT8bA/BtcDYbFLuOhxzeqDuZEvBMGIj/ykp+GirpzKekjdC8GJ + bm//FQ/s6IbVmo1RP+FOYr2z3nvhndfww5j20+V6RN3G4PVCoKOdXCnjmY83f8qZk+TOo/WJPhZf + 7mMI6gfy0k/Opx9NOw4uEJ50x8DDJP3WSP1x+4vWsBMOj22dzUc9rpwvHCfD32pnw4G1vdakT2ok + wSCYDJuHpjmSO2QMM1ziOQ+FhrqF3Hk7fhN/+Yk4On7JL15EVjz0xyLM/LY8GJ4dKETBBHdxHWfk + R6D4uj35PT0IKtsH/MH1QHz4UB0ioll9MGeZdYPAegjveUmch2JpV/WQuheCE93e/iseOKKbdFej + Agi7k1hvB7w3vuKFL6XFoPbR5XpE3cbg9UKcUQ4+4Y/rz/C1yZvMOV7i6zwkyDqGnnUQL/eFeG3n + +fQDlbS095PuElscuV7kER9RS77wx3iFE6DhyCcxgQ3H+fC3s+kNa2v1GEHoHCw+sT2Ss05iPKe8 + F57zPYkud97yIziTYoiDv/ws4ul8eqZdLELFGXlkHPbb8hiOBh0c2j7zH7ziKX3MS44I2qJlTA/k + 6HogTla0uiLzy7xSZr6IlgUXvus5N+8DzlQ9+Ez+Rf7lP9ahy3ST7moMe+FOYr3T3154znc/DHM/ + Xa5H1G0MXi8EOsrpChnPujGv6M+WkOtgO+W9V91Fl5UhPuokh9EH9I8PFZAjZYnDb4/n+P1Fa9gl + n/yk3+EvEx64sCN+tX4MJV7+kvC160PTzvQ4TvikRhLRhBwZco14TnkvPOd7El3uvAxSsuDhB8/J + n2PhGn7Jbx7zdbvE2W/LI9eMecq/HROnplD+UQ9G1OTB39IMHpHtg1vVlV0XTcKwnF/mlTLrBiUT + E77rObeoq2DkFdr8i/zLv+p2nHUWe/fneGV/nPWer2vwN3zlGfwMeT9drk/vK5dfbyIIdJTTC1P1 + jM2fcuYkedBpnWvfiC/3MQT1A3lddy57xpN+642I/SA945uvo+ZP2F9EwU64INKQ+4x6XOF44DhZ + /j5z4hmFywfWrW+/At/YukgQJgNjn9geVTI2gyig5qikY1zgyeOi6Mm3E+DrpAc6+XNcxiPC4EsP + jmcer4ow8+t4zVuOBh0UjiPzj/xUbNYl9TEvOSJYxJNxPVBjrQ8WPZuBsVSdW13YJBJZGOEjP+E5 + t6grp2jA/KXuheBEt7f/igd2dEO72djiOZn1dsDhZ0P/VZ7hB8O+ulyPqNsYUBdVNNNDzK5U1Q+y + 18u4TEr6zqP1iT4mS9ZddFn5Pp9+zFhvOi2e7rfHc/z+Ih/iII/cOm7HE31HSK5/4cJO86tLJp5R + uHxgbW9dlU59UsfJxiTZDADWiOeUSbAJz/nk03OT98Jv4i8/EQeTkn3jS37Pz+NVERC/4VzNlkeu + WemBKn3mH/lZ0fSZd0szeDKeB3rMOvkzFivpq+pcebuZJAo26kQLwxZ1TQXrxWc21SL/8s/5tm7l + n2b4M5ZBTF7fxOdIHwv+lOUncTkKHn2LZ6VFjn10uT5RtzGgHlmHDNYVqnpC3990CqU6u0yq6151 + F50QMq31UH25zmZ03/B5xNP99ni0/jbT+pV/WsscvOSxQkj7Hf7S8cARNvpuOjY9hjM+sNb4LjwW + m5dPahSFMrz5JPQ8fctn4nIMXOIxTQMNdTsOv+FMas4vucVxPH7FC4Yer4ow89vyGIkoRMFYUzyw + aOILWYumOFLvkQEHfXuojB/QB9cDYXHFsgtZX+XnkQGOfCGw4MJ3PecWdRWMvEKbvwpBZbe3/4on + /QNDd2MZ+BR290E/iXwf3VyPaKAxeL1UhwzWFar6AVH9mRDhWWfWL+qqh+jj0KveotOTrL3+uV7E + m7TeiLRCyTvnI/LE/UUU7MhTAdif86AeVzgeOE7aH83yRxt8YK2mx2RT6+SEcR+dI52GT46xCTqO + TnM++ehL1wnwtFvyp9/iPQ6//dp/t8s4HKfjth5R1QPnLTuOzD/zBJAADRFnyU4v/YT0gA+uB8Jm + l6BueTm/zMsjmyTzp0XmSRvNS9/yTgXrwWfyG0hJV/lXnUZflH+gGBXtNYa97HKdcyTjgj/lvfBL + P6TYT5fidqGj3lkPVTTTQ8iukPGUuA6jnpmT6+AyVT1b/aoeosvKE9/XNfpAXgEkTP6Td/jt8Ry/ + v8gBO/I4ME6E3+EvEx44ouzPYfinGOINC//bkE1HCEcm0UbnSKfhM3E5LvCYJpGGuh2H3/Dd/Ol3 + GY84F/z+DOS4u13G4Xycn/VgqYcIFzJxagryhxwPJdss9ZHhIp6YfcAG1wP5sEvaZnd+zpPBjXwh + MDHhu55zi7oKFvlLrULxqa7yrzpGP0Fb/vGsOuYY9ZPdfdBPFdg+eXB9om5j8HohxtFOqHPUn6Hr + sGoy53i5brZTXfequ+iy8rYrvHiDL/qAHs3PKObrqHnhyOCr1rfWk/OwS76az31WhnoYuLADXuwr + fNsKV7xh4fSKpvZJDRBlgtk8ANaI55RJ0HGJ5zwUGup2HH7Dww+E5M+xeCMecS74FQcsK07qWZyZ + 35YHw7ODQSe4i+u8Ij8Cxdftye/pQaCnfXFzPRAfl5t1iGtWH8xZZt0gCJZ5cSLzW9Q1Fcyfz63O + FHmV/1iHLtNNuqtxtk72V+tuQt7HdQJ85QmLkf0wf6CfXA8WEBUYg9cLwUV6fMKfqIek1p+Q8xJf + 5xFvs4PMOohX/UBeyjmffjTtOGgg/7bTYSkC21nL+AWU4ZzPdoqfPBWA7c0nswQOvwpv9N3Eb1vh + 2pr+5fX8v4OPSqebPsMpFQSlEeCUSbAJz/nk03OT98LXyQxs8ufIKs/sGl/ySw/L4slNFMW0veM1 + bzkShWBQ2D79RX6MSHypj/mYHgQZzQM/uh6Ik10STcaoZvUpmflCYGGEj/yk59yirpzKekjdC8GJ + bm//FQ/s6Cbd1agAwu4k1tsB742vPOFLaTGofXS5HlG3MXi9EGeUg0/4E/0oifUbMqZ0ia/zaH0G + ruohOlaED66fJDkkXtOOg4rCMYo5n7UwOG5/EQU78tgRJ+An85DICc8XjqL9OYzVo/ijDVvTkbse + D/ShdOqTGmQMgqRsHprmiOeURdlwied88um5yZ234zfxl5+Io+OX/OJFZMUTcSXOflseKt6Ik3DV + Snlm/saHouljHgHKruXHx/1wuR6Ij10SzcC4ZvUpmflCYMGFN04S55d1TQXzl7oXghPd3v4rHtXX + dnSXbru+1jnX3YS8jysKL7vE5QhU5YlnpTUs98WT8426jQH1UEVHX0WFqj6Q+5tOJuM6MG/nazn6 + GCDKqrcKrieZjnmvRJTVcRDW4ul+xS8tHQq4gc/xqH/IUwFkPNF3ckP/9JY4So7//2fvXYB2za6y + wPc/nU5CEjAoXoay8IIZCAm5QBDRGhFrlKmxykJriDrKCASJiKMU4zhT5eCoU17GKRwZLEdrnBKm + RDOiI1rlBS1jB+SWEGIIBJAAIUAg5Nb3TtLd55/nsp611/ue7z//6aS7z59071Pn23vt9axnrfXs + /b3f13+fPl1l3LG9/y0vwP+b4tqvoitJ/aT2k43dryczgkXhFoRTGOgK1/Pg41KjmjrFT3/nwVp5 + ePlqv3mTxwF87eF69vUybt/XyCPxEN51YS0486Z/48sx/LW/6JunC7rNiz4f3hLqUKN17r7TLwCC + EZ++09ZBV3Ixnjiuh840OTq/cKVn7TMN43bzqOdWznudG+vIeV2cB+mu1Mh9bR0tJ3SRomkPNVup + 1hN238/Rkfw6t9L1It1FF+V9TrKkP/UzaX8jGvXMvLOem98v8oGXPE6kBIw3n/Ml8cJVnHB1n+44 + /7X4Gdb1ZyukLvXuyUwwLwMAPWMdW5R1WXdxdvB1jZvwE3SKv/NUHWxKeRywuCuelTVP1YUN4Vzf + yJMzaz9g1BQ2RZv4cgy/ccZXGcVT1m2fVD+q4KXIZWBR7q/qb5v9wqDgwk8/9w66CkZeoc1/6L/z + c3+cW+cXq+P7nLGnuOAzJx/njMp3Ef6YJ2FXZVbduUC6d5LT54Uil5w+GOOp17ifoxnr4LjWc+jX + eohOCEWv/eQxqe8N19wPb70vZNc+/ciTsedzHDtrvnFufp9VZPbFxwTcP96768/GA+uaH1iV1E9q + iEIbJH4SsiYmJYW55Kc9cMFjmw5N/VL2RfhT/Mp3i/yuZ1+vTn2Xd/SxGlGJgkl7i+s6qz+JSD1m + fOzqsPJ0v7d5YT1QL09s3f51jkMXvQmih/DpmzMbOejKLTqoh9wShqsenV+4uk/wch9oxe1mJ5K/ + 71HdP5GWfySobdZR/JnJf8jTcVdkYX1KtzVBFykqeV0q7aEf/D4v49KO+CYP9Tqlh+ii/NBJ+jJP + sjKea2+IbtZRwMvvFznASx6lDV/6oB9jx0ebm+7bYdy4hgfW9XP8TaP0qTpMmBE8Z0L7yYl1bIfd + iOd++LQe9uTtPIKzKZa4+DvPoR7SHfnFC4aus/oIznlHHyvRokNCx6d/26pIfPHXvvAK50YtrsZk + PVAWFa3LwMp2+rTNfmFQcOGNk6W2DrrGwf65Jv+h/84v3UpPQDs/w2hnrnjF3cJ5J99F+GMepLlS + Q3VTAenTE/SQomkPNVsh42n5zR87TcledNL55PtLdFGeeet9J/3rHigrgISNevjQSd6e6b/p/SIH + 4oQjHQtI3pWv9xtHlPOpWsbhWXUNe35gVVI/qf1kI0l/UvESicIt+BOCtVql3axc6pYrj5vwE9B5 + sFae5LtFftezr1ci7PKOPFJBiZnemiOx60j/tlWR6oi/9oVXODdqcTUm64Gy8CuXgZW1zkMXvQmi + h/DVn/CMOujKregh9xSCGzPe+bsexOl8gdnNo57cu55NyNc1LsF3n4hgnqs2rEfptiafF4qt9rjC + 77qPsqjfsrGlIb7JQ4J6/xLQeoiOinAx95NH266DsMaxipXX9dPrPI468tGmB3HCkW7mJV9FZr9x + FYd9VUs/nlXrgTWefGTxE9szOftJzJxli5JxBzz3KdZu3ITfcDbF1hZ/57kFfj/x9/VKjV3e0cdK + pDIFQ0L3mf5tqyIC2l/7shXOjVpcjcl6oCwqysOu0ec4dPEnJwCCrT4ZYthB1zjYP9fkP/Tf+bmf + 8xPf4ZwZXvuYpH/jR9yRP7byBJe5eMSLNeerNqxP6bYmnxeKXXJS4bqP3Mcvn5eU77asg+PU90W6 + i04IxTKu8cpjSt8brp1HdLOOKvDy+0UO1E8eJ+KGzrnvnTe83ziah3unB9b29DcsSVNn6E8iHCIu + v7Tj7EXbPvP4GY0hnJdX4fXWPwF9mdKmbhXvVvXjyXp03+m37qDeXYf+Vzx1Kj0lE/Mpy37ufAM/ + 4m7Q9xI88888V+FMZg3WhwKue+R6YWNUe1zh99Cv3vzRl1gO2Yuu7BEXPUQXZRwnSwmJLz7mpUP5 + iaO557OXDgEV2LoXkSfEha/3x70TkRMvHDedb9THH7qfP/1vCSmNVOFscdcnVjt0drtPpHlW49Ao + 9e0eqh9FXP4JmH4B1p1Bv+qLfZcu2Nj1HQdxXOs2G0+To/NTF/inzTRE7+bST7jgM5uQr2tcgu96 + EcE8V21YDwpIfXryeaHYao8r/B76wX7KfsPazvDAOj97+t8S8krUu8efEH6y8674k0wL3Z3dJ0ht + I7wItLoSL66bV73eDVXVrn6VPT7p+N4QPn2nLetBd3glGPtniN50WtHSCE4z/NMWD1C7WQdQ/MFn + JmP5RT7sHf/Ac3/yd9wVWVgPCsjz6cnnhRpXu9R16AfE/KaTdqzD4BHviIseoosyxJdOSki8Gfub + jk+4ytzzEXn5/SIKceRRWidw3pUviReu4mZ95/wZVv6REIctCGeCxuwembRycr4Jvog09csl+OMn + Ytujjq6LpMUXftfjulOvVZ59uW77EdmLooPtvOnftoCqI/7aF74qONSTum7XbD1QJ29JbiGK2ena + NvuFQT2Er/7k595BV25FD7mnENyY8eM+1T7TJF3PpZ/qPtw/8rlArfxyCb77BFptjdCrsPT5lG5r + 8nmhwGqPK/yu+yjLb37HY6OGdXOc9NX5jDjY3keA7gN5ic9+8mjbdTBA+cO757MXcTe9X0Qhjjxd + QPKSj36MWiwcN52v+8Gzav3QvZL6SY0maLMZzGolM7lJdRM8U80mpj15w2945YER/syNq3omn9Z4 + cT37eiXCrk7jzNuJFh0c7jf9L1710/7al10VVJ6ybvvU58Nb0rci/aWv2OwXJVMY4aefewddBSOv + 0OY/9N/5uT/Ozfo6jumS9iR+xLlAJq5R+RQXXGZAjnkSdlVm91sXaE3Qg4qw/lRqhVof+J+y37B2 + P3THYXP4Se0nG1WbT+C+XMFlLlzw5Jlvkmmf4jccbwqG4bdmXj7arOsW+IXjYR7iUod5zGfeToRF + lQuH49N/5WclqiP+2q/tRaDVlXixHqhTTxV27LHTB1urXxiCUe/qT37GHXTlVvSQewrBjRk/zq/2 + mabPGWvZ5OO+eKN/zXbI3y+X4LtPBJD/qg31aaFL79JBylhe12yljKdufF8MXaox6+a41jPvG2Ba + D9HlBOZ+8iQrbAnH/fCuvLOe8XQdeVYcidY3p+ynD+fTfWIe/FJewZyv+9E3LPxzoULQHMepTyzG + Hj+x+onPOFyeXZyJ+LrGTfgJOsWfvLfC73pcx4wDsWpwfSOPVFDi8mNCoOtIP7bLMfy1L7zCuVGL + qzFZD5TF0683Nytzf1V/2+wXBoUTfvq5d9BVMPIKbf5D/52f+7kfZIfNNEnXc8UrLvjMycc54xL8 + MU/CrspsfeoCrcnnhSKrPa7wu+6jLL/5HY+NGtbNcdKXBEO/1kN0OQHi6zyUsO4BOPtnScof3lFH + FXj5/WKBiCOPC1PFzrvypeGFq7h9fU//W8K8mXG2PCWYFnd9YrVj+I0zXvpzoxZXY7r1T8D06/51 + q0oHduK2bviks6PuoC7bof/Oz31euvJzRtiND63hb/yIu0HfS/DHPOzlKg3rIaHrXmmCLr5H1R5K + 9v1r/WDzTR07PcledPYP/eiX7qLLCfAYs588ZuxvOqOemTf5jSNfxe34fE3YWfNVY87LPjpQi4Wj + 6T5dN4D4F4T4ofvT/5ZQ0tQZric/NqAR7Vq0raPlvvyMxhDOy6vw6rp55KyTFXu4v/TlmZckbTJi + 9U0/4+jnpVtxcrB/uacQ3Fg41cFLXPo0DzDiyzz8rPcYp3wkzrgEf8yTsKsyW4/SbU0+LxRZ7XGF + 30M/2D4vKd/tWC/HSVcSREfx5fxIF+V9To1XHlP2Nx2fsOqZeV0/q3OeFNK69/nQg/qFw7L300dF + Zr9xFYf9VZ//LaEj1AR7gZugMUsy2JqZE7/7CTtwHadcTDMGcRiTd+K5f+TvPId6ikhTXlyP655x + SFhw9uX89mO7F9y37TrSf/pU4CE+fFVB5Snrtk/WA/XzuOsysKjWeejCy5T+GRGdjOfrQdc4KIvc + pQ/XNTo/iXN+8HV+rCV/5lFP40ecCyxyTpfgj3lG5JVYWp95r6KHFE17bBS/6z7K8ps/+qYZ2YtO + OreOjINe0lt0UX7uJ48ZfW+4XvX4nsSuWfeFfB4rT3DcR/3CYTnOre8dIdlvHDdvvHf8Yw1305VL + 7Sc1muMlZ5O8NHLXjHVshy1c8NwPn9bDnrwT33mADX/mxlU94mR9Y4gXkc1T9acO52VZ6QfBTiAW + 0cG2P/3bFlB88de+8FWECEZBt3mpflGDP9nYqMfq3/qtfuEXjOeZvjkz7qArt6KH3FMIbsx48pWe + tc80pN3NpZ/qDj4zsFUIVx6X4LtPoJnnqg31aaFVYMnp80Kx1R5X+D30gz2/6aQv6+Y46SrCEQfb + +6SL8sRnP3nM6HvDNffDu+fTPv3iE+zA5zjVL1wRaUofjkvDnVdpna/7wbMKf6zh/AMKqaR+UoOM + NpvBrFYyMxl+y5+5cMFPPq35chN+uysPjPBnbt7kGXxccriefb0SYZfXOPMiqBdVHmz3m/4Xr+pv + f+3LVvoiqPUVmPp88qboMtd5cmv1C4N6CF/9yc+9g67coq7sX+4pBDdm/LpH2Wcaxu3m3TlF/5od + yNc1LsG7r5VnBV6Nlc+ndFsTdJGikteVWql5nv5mYly6kX/y6HyWfq2H6KK8z0mW9CQ+WbGgY9Qz + 88569D6rQlYeE5kPvORxIiGNW/mSeOEIc/1dH55V/BmWH1h4GAjCGVn8xPbM1P0kxjr2RXjuzyam + PXk7j+BsiiUu/s5zqGfyac041b+vVyLs+hp9rESLDgndZ/pfvOqn/bUvuyqoPKnnds/WA3VS0dxC + FOX+0lds9gsnBRd++rl30FUw8gpt/kP/nZ/7OT+yw2aapOu54hUXfObk45xxCf6YJ2FXZbY+dYHW + 5PNCkdUeV/hd91EW9Vs2tjSsm+OkLwmGfq2H6HICxNd5KGHdAzD2Nx3lD+/K6/qDI5/Hns9xqp88 + LkxA41a+NNx5WWf12f3gWbX+kRDNCcKZTYzZPYJcFCySWlyMLyJN/XIJvp/MCAh/5mM9p/hdj+ue + cTw0Dvfj2X5s9oL7tl1H+k+fABKgqXRom+wYlcfG7X+1HiiLt0SX0TW5v/Tl2Z+c8FMP4aefewdd + BSs95C59uK7R+aVT6Qlf58ea6Up27TNUcYf7x/0b9C29L8If84jjCr2obl8oCVHXyee1a9cKGU+9 + eP+WnmnJOlgm6XqR7qKL8ta78eI1Y3/T0QmFd+Wd9dz8fpEPceRxIiVgfN877uQ8G8dN51v18R8J + z5/+GZakqTNcT35sUOP65NGhlO0zj5/RGOOh4I3b++q6eeSskxV7uL/05VlvguqfEatv+hmHy8WH + CFbhlYN6yM2FVrQ0gtPMy1l+zuIBajcPP+s9xh35Y+9wiQP3MY+rujqvqpsKsO81+bxUf2qlrkM/ + IHxeJ/SePOIdcdFddFF+6CT9iXde3xuuvSG6WUcBL79f5AAveZQ2fOmDfowdH21uun6HKR4PrO3p + n2FJmjrD9eTHBjXSm12Ltqml9mub8fOhIPs2v7hulMVbkluoMnl50pdnXqa0yYjVN/1shP59nBzs + X+4pBDfM2zPydz3hgVN1ZHYi44LPbCK+rnEJvutFBPNctWE9Src1+bxQbLXHFX4P/WD7vKR8tyW+ + yUOCoV/rIToq4vi1L0fn7W9EjWMVo47oTz/yZOz50gfihGPamZd8FZn9xnHf+VSt/XxgPf0zLElT + Z+hPZj/ZfRbt0Bn3JzcPaZ7VOLQ6gts6+RPcl2zdCt4tXh7P7tuXSeXr8qQv9k0cX63HjJOD/cvN + hfE0OTo/93HZpi0eYHZzxQsXfGYT8nWNS/DdJyKY56oN61G6rUkPBda65KSuQz++iYedvqyb46Tr + RbqLTgiFtk5KyDxm7G9EPmHtz7yuH/no98PkBF/6AK9wgFQC5135er9xpHPf3Y9+6H5W/0hYSf2k + 9pONJPOJaelYJGskTc2FC74cmvrlEnznQUD4Mzdv8pB0iGTTh9A89FOcXV7Xa95OxHDDBLe46q9s + VSS+GU/+ph8L0d32lz4fHLp0qIp2+qjv9AuDwgjvPmVRVjiOcRKM/cs9heDGjHf+rgc6Mg3jdvPu + nPb3j3w5R62HLd7ci8yC7/N03BVZWI/SbU3QRYqOdq1U6wf//KaTdqyDZZKuuq+l49RDdFHe59R4 + nbMZ+xvRqGfmnfXc/H6RD3WQx4mUgPHmc740vHAVl/ui+zF/6M43N4af1CCrN7ufhN63dHXZboIv + Ik39cgm+8yDAmrLJ0/WIs/jCr3oR0TxVf8SUn5ph37ydaNG1P/1XfkaIb8ZDr9reEaSg2zi/9BPu + 3H7nr6i/5oy3RIftglb/67x5SSQnhRE+fbttNnqMaz0YQn4R0PCQ3lhq5qUrf/PAx3SsQvPwk+8Y + d+SPvcMlrvLu+LF3VYberBGcc8nnenMuqdYKtX5QzOdlXKOOPLJP6C66KOPzkVX1aAJpfyPSCRHH + Mvd8zG0cGTz6fIvIE+LI40QCGke+DvR+42g636pv/tCdl44QzmCZs3tk0soZXOYDHtsk0tQvN+E3 + /Eb+5D3WI84Dv+pFhboMAKT+1GHb++YFqBdVLmzHp//ikYjUJf7al61quFGEZd/G6Zc/69r2jz/n + E7dvfcUnbr/mOfynfjbqsdMHW6tfGIKtPhlhmQ+6xsH+uSb/yfMwf5+f+A7nzPDax6R6Gp97ZQdf + 16h8PtecV81AdZ9Yi39F3vaV3qy5L5xLPtcpRYectEdffBMPO81YBx9D6zn0az1EtxRZ+8ljxv6m + 4xNWPTOv8gFqHPk89nyuR/WTx4UJaBz76kDvN46m++5+8C8I1w/d61LrSQ6WObtHBIuCRYLqJnim + 4qXbjUvwOkSG4Xf4M7OrWY94D/yux7gZlzoc77rt70SLDg7XkXy2VRHztb/2ZSucG7W4OtN/8Sue + tb3xt37S9uf+0+duH3eH63N/6cuzPzlRN4XhCVSfshR20DUO4rjWm878NDl8HjXn/GqfaYjezZfc + jxv0vQTffVYeTFdqWB8JXXpHD+u4rpOVaj2h3Pymk6bkX3TW/5TuoovyPh9ZSshzNmN/I/IJa3/m + nfWsp86Rjzb5wEseJ1ICxve9404lXjhuHu6dHljn9UN3XjpC6vLN2T0yaeUMLjOTzzjszyZoxt7h + Eif3jfzJy2Z2cYOPSw75USHnGbfPa5z9COpFlQfb8cm3eMXT/tqXrfRFUOsrND3z2tn2tZ/6vO1N + v/WXbV/8yc/e64M6V78wqAdvVfUpS9fioGscxHHNy1b3hyaHz6PmnF/tMw3jdnPFKy74zMAe+WNf + hHdfKw8prtKwPqXbmqCLFE17KNlKGU+L97vu52jIOlgm6Xp4P7YeoovyxI/3i3hNqjoIG/XMvLOe + PGyE3vG5Hp50841zNp/zpeGF4/7h3t3BP+l+7ezp/5aQ0tQZric/NnC4/iTRom2fefwluAhqfQWn + T372Hdv//dJfsr32837p9mnPu6P6cn96E1T/vCSr79IFG9YlepRglIW91ptjtm3dCs9LXPo0D8Pw + O2mnn3yyM5P4qO/gO4U/5iHFVRrud96r6CFFR7tWqPXhm7jOY/ZjvSzTup8ndBddlPf5NF68Zu1v + Oj5h1TPzznrysGEk9xdfjg11kMcOJTCO9TlfFgvHfdfffOfXnv6T7hGb7zlpyjcJfq1PrHYMP6Rk + AM+Abo5e2Lyqr694/p3b9/6WX7Z9/Quft33infhzw90vKtblSV9uzG0dPunYHB11B32bI4Q7709g + 4UpPhVFfhOP3bnYi1UM+67vilM/Ufr0E775Wnhl6FdbWhwKWjp6gi3Ws9lCqlWo9Yff9HI1Yr6a7 + Qb/WQ3RRnvg6DyWk3iZVHYSNembeWU8eNkLv+FwPT7r5KoHzrnxJvHBk8/mrWsfhZ1jXr/8CXUnq + JzWa4KOPyccT09K5BfkVtnDBTz6t+VKP0sk78Z0HUGvKJhl2a/yuZ1+vRNjlNZ95OxEWVR4criP9 + V35Wojrir/3aXgRaXfkX/FPi9hWf8nHbm/6zT8T8HLTGflE2heFjpHSQxX1s9PmUnq2H3FMIbpQ+ + meseZZ9pkq7n3TlF/5odyNc1LsF3vYhQWyvySqxyX1tHyw5dJLjPQ5VaIeOpm9/8sdOM7DoG6Ut9 + TukuupyAz6nxOmczqg46Rj18mCRvz/T7YaJA7i8+8nMbccKRThviMZ/C1n7jKm7y4Vl1bXvO834S + hI8mqZ6cBLGImt0jkzI1tjlXkRMXPFOFT+thX4Tn/pG/8xzqEecQyfSurHmq/tThvK7bvIjqRZUL + 2/Hp37aA4ou/9oVXNdyoxUfPxG9YX//C527f/XmfuH3WJzzDevCESwd24rash+WqPqMHQTifY//S + W/HforHqAABAAElEQVTkKz3LFg/DaGcu/XxO0X/FHfljX4T3OS5+pLlSw/qUbmuCHtZ3XScrZDz7 + 8Zs/dpqSPXl0Pks/+qW36LRS6NpPHjOqDsJGPXpYznOSF3E8/xp7PkSTFpmbb8SbrwO1WLiK67qv + P7o9/My3Xdv+0Avuhetnk9RPajRXl3A+Md1SXYIqcuJYnWzlWk2okkvwnQdg5QE+c/PehN95nX/G + 7fuituHtRKs8BNqf/m2rItbf/tqXrXBu1OKjb3rh856xvfZzn7/9nRd//ParnnVH98lO3NZB1zjY + P9e8hIf+fR7cpm7rXrT+DMNvxmuu+B1+xB35Y1+EP+ZBmis1VLcvlASQTKhQ30Q493WyQsbbr4fW + Aqgv6+C41nPo13qILsoTn/dD8lim/kbkE1Y9M++sR+fvsANf+sD5k8eFdb3m60DvN47mundg+tnt + q190P/5Yg/bfnqR+UqOJuoTziemWEMoQ+jOz6YEvh6Z+uQTfeRAQ/sxU6zJ+12PcjENglYMZjs4j + 8eBqvxPbn3zpU4GH+PCJvnnK+qic/qv/5FnbD/yW529f++s/bnuGblfaOujK7nTemLjmm6d0pMnh + 86g551f7Oh+sd3OfA3WN/jWbkK9rXILvc0YE81y1YX1KtzVBTyk65KS9dNCbfNjpS3yTR+cz4mBT + B8nG8+o82U8eM6oOwhrHKvZ89vq8HOXzdp7ZB+LI0wUER76KrMXCcd/5XMb527njB9bZ2Y/q0hFS + l2/O5OwnMdaxsTyJ5374tB725GW1sgVnUwjD7/BnbtzAH/nNs57IyROc7dHHSsTqDENC4nQppINt + VTTsVVfFNYGoPqpfnoM/r/VnP/U52/f95udvX/hJzyz5DrqmX+rFNS8b9RnD51H6jXOzvo6TjgzH + 75P4EXfkj6244DIX3+QfpV2Jpfst3dYEPa3jktMXtfWBv+/n6MQ6+Bhaz1N6iC7KWPfGgzl5+xvR + qGfmnfWsp86Rz/XwhJuvEjDefNVE9pmvy1v3DoX9KJH1Dev8R5PUT2qQoVlWz9k91oyg2CSYuOC5 + Hz6th30RvvMAG/7MzVv1iJP1jSFeRDZP1Z86nNf1mhfBvahyYTs+/dsWUHzx177wVcShnlHaR+Xy + 1z/nju01L3ve9m0v//jt1+CPROx0ZUfRg2tetkP/0luwdY8IbR6G0c5c8YrLOWcG5sgf+yL8MQ8p + rtJQ3VQgOlImFMg3MceS0woZb//8piOw8CYoOsTTrntcfvGLLsozz3p/s4LkVR2EjXpm3lmPzp9Q + jD0fbe6ClzxdQHArXxIvXMVVffCPB9bZtae/YdUZric/NnTm7WjbZx4/hcXgm/ZjcHz+L71ze/3n + /ZLtz/+G5+hPy+eT1Q8p30FdtkP/wWnmpSs/Z+kHrXbz8JPvGHeDvpfgj3mu2tFYD12wulfRw/eo + 2kPZvn/G0/KbP3b6kr3obtCPfuktuijPa5v95DFjfyNSfuJY3/4ciTSOfB57PscxsvmqMePI14Fa + LBxN5xM7n1EY/oZ1x/Wnv2HVGfoTAofIjwYeEmcv2tbRtp8yYgjn5cfa6zNxS/7Er3n29sbf9Anb + F//KO91e+qel26xr1a1bt9IP/mlLP4bhd6m785PP+q+4G/QtvXe4xJEX/snfhV2RhfVAhdHR1w16 + WMdqrxUynnr5zR877chedDfoR7/0kOBRZuikhNTbjP1NZ9TDh0ny9kw/z7/GyjP7QJxwAFUC41a+ + 3m8cCZ1P7HxGYfiB9dyX/Qx8jwrC5GxuzO6RScclqyInruNMxNc1LsH3kxkRu3yjjpvxqw5ENk/F + RUzXSW2rj5xZ1+XE9qd/41WR+GY8daJdLfZitfyxtvpV+I+q//ZnPHf715/9vO1F+NPy6p9N8hIe + +pfecFn30rPs3T1ieHCZD/cP2zfwJ9+OP3HFM/OI4wq9WJ/SbU14G/pCLTl9UVtPvonxK3Zasg6W + qfU8pYfooozPp/HiNWN/0xn1zLzJbxwZKg6FL74cG+oljx0CMt58Hej9xtF0n2jj/o3PKAw/sF55 + hj+Htb1NEHh5GfQErdk9Mmnl5Exc5gO+HJr65RJ8P5kRsMt3op5T/K7HdSdel3qX13Xb34kWHRyu + I/2nTzlUWNepuuhX+FiU/TE8vQJ/Zut1r/j47a992sdtz78T2uD8lxBu3OdR+uV+wNX6YY0o3+Ha + xyT/8f5x/8gfW3nCn7l4Jr84rtCL9Snd1qQ3Mcvse1UKtZ6w+eaPnZasg+PUt+7nwtHvfUTwvMRL + fPb1NOm8/Y2ocTyvPR9zG0c+jz2f62Fk81VjxpGvA7VYOJqV7+z8JzY+ozD8wOJq80/h/aRGE2yK + zfAS0JsZ69iKGrjguW9RtPILcRiTd+JP8XeeqmPij/ziRWXNU3UF57yjj5xZ14XikNDx6b/qtWP4 + 0wdntTUWZX+MT/zT8l/6yc/c3vi5H7+9CvM1bozh84hOpSf8fT5Y63wz9zmA5xbOO8L7XHNeF+cZ + pV2JpfXRhat7FT2sY98rvGnpaT3rzR87zVgHX8Mb3jcA0e990kX5uZ88ZuxvRMof3lFHzot+8VVc + 55l9IE44FSKg6yGf47JYOO5XvvOztxdqPrD8U3g++Rg8Z7XCSyQKuDlXkRPXcWQfTdCMfRGe+0f+ + znOoZ/JpLXofQvMwP9XY1em6zYugXhRMcNaR/o0XUHwznvwVxyIqD5dPpfH8Z5xtf/UFz96+87Of + u/3GT8A/JtbQOWO9O++yJTvWu3l3TtG/ZnIe9b0E3/eg8pDiKg3rUxdoTXiL8h7Pdte91j7fxLmf + QvrFOjtOulKfvG/EV+8v0UV5n0/jxVt8rIOOUc/M6/rpdR5HHflo04NzFI502tC9MF9FZr9xFcf9 + a342cWd9w6qfwvtJjebqzb6ezEw6Lhn9tAeOxci2Q/5+uQTfeRCgPMBnbt6b8Duv8884HhqH6/Rs + fycqv23Xkf4rTiKCB4Fdp/qmrfCxKPspNn36c+/Y/uXLn7v9nRd+3PYrn4mruNN93YvWz3LrPdHn + hT3F5ZwzU8sWmgbGKf6BP+Zx0NV5tT66UHWv6t77KZH22Ch+D/3qze/41Y91syyt5yk9RCeEglsn + 6ck85uTDRG/AUQ8fOsnbs3DkqzgQdH5smQ9x4asEzrvyJfHCka/ynfvfEHJnPbCuP/p2bvhJjaRo + liSc3WPNwMS+CM99PmR2o+zJG37Db+TvPFXHxB/5xYvKZr0SYZfX/ZkXWXtR5cJ2fPq3LSB52l/7 + sqvLY7+75p86xu/5FXfiHxOft33tpzxrw8/opec8tz6fyJ95d07Rv2bKd9T3EvwxDymu0mB9+3tV + 19FPidGuccZDBr6J657PfuSv+4jpYt1FJ4TCW6eqJzL3N6JRz8w769H7rIrZ8+XYcI7kcWEjL8+3 + A73fOJp1/tdPfcN6xjP1rw315ATLnN0jk0ZUzONJyawTr8zl15ovl+AZf+RP3lvhdz2uY8bt87pu + +1FTL6o82K4j/dgWkPW3v/ZlV4fw11/qWRtP3enj8POsP/PrnrV962fyb4KgbqUnJLnhnLHnY+Bt + jq7Rf8XlHAUyUMsd/yV5OvY2L/Smpi6+UHPSw4jlya06jVOf3OebGL9iCyI8dXZc63lKD9EJYXYk + arx4zdjfdJCPQ8c48ia/cWTw4P7iSx+olzx2CGgc++hA7zeOZvX5DP+RBu6sb1hf9iL8NTPnb9aT + k0nRLNnmE1O9Ikgz/RgTF3w5NPXLJfjOg4DwZ27e1EXS4gu/6kBk81T9wblO12veTiQK0cHh+PRf + /UlE6hF/7cuuCkDwqCVJSU/Z+QEI8b/81Ae3L37Lg9Kzzw+K9PlgrXPIXOfpc4r+NVPJ8nOpcQn+ + VJ6E3u5Zb1bVXxdoTXiL5n2VKmkvHfSwGnajyDd5ZI842NJbdFoptHWqejTB09+IRj16WBZA59Q4 + 8nns+XJsqIM8XUDuAevrQC0Wjib919+8fdnL/ZeMYmc9sATf7tKTEyxzdo9MWjk516Nx4pg9+1iY + Ma+X4Bl35E/e5r0Jv/M6/4xLHa7TdduPwnrBfduuI/2nTwAJ0FR1tl0NiiDNPjVnyvkP3/Xw9jmv + v3/76+/44PbB66XfOLcbzhkxPgbfF59T9H/87tNVOxH16QtV96p00Lu67qOK9sU0Hvt8E+NX7PRl + 3fqa2n9Kd9FJcbPj3i79yWvG/kY06pl5k984MlTcji99gJc8TiQg483Xgd5vHE0EnJ3dJUe93PDA + 0pOTZGiW1c8npqXDNqnoz3zAl0NTv1yC7zwICH/m1NEzSYsv/K5nXy/xwckvk+LVdi+W7TrSP/el + sgG7eOpTcSfqSV1PlfmH7n90+50/8MD2VT/60PauD+Eqnjpv6V76Yy35M5/C515RxMN5x/a55rxq + FnyfhxRXadx4r6JH3lep1vev9cSbWA+tgx7Woa+p9R/60d/Xne8LPT2Iz37yOC8fJgpoHM29vkQa + Rz6PPZ/rYWTzVd3Gka8DtVg4muC9frMH1p3P/k6ArpPFT2zPaoXNm8Kzmgaa8wGvzOXXmi+X4P3E + VYnNn7y3wq86ENk8Vdc+r+s1L2rqRZUH2/Hp37aA4ou/9oWvDo/91vbH+vQePJz+5I89tP22Nz6w + /cB9+rN9atnnEZ1KT3j6fLCW/JkvuR85R5Hz5RL8MU/HXZGF9akLtCY9FEZ7bBS/h3715o++aUf2 + 5NF9HXGwpbfotFJo6yQ9iTdjfyNSfsvNh0ny9kx/P3XG+RaRJ8QJB+7e90Ms+Xq/cewaxHc8epcr + 8uv+G9YffuF78Uh7C4P5BMzsHpl0XLIqcuKCF/VoYtoX4f3E3fMnb/OmLhIe+MWLCpun6g/OeRlW + feTMikeTtLe4Ew8hnK/95sn27E/rp8DLw9Dvb/7MB7fP+r77tr/3Cw9L19m29MOGdaz7VLbOFevd + 3Oew7l2fO4nLz6XGJfg+Z4CZ56oN66MLNa8X36QqdbXri9p68qGBX7HTl+xFd7HuoovyPh9ZSkhe + M/Y3nVHPzJv8xpGh4kCw+HJs4CWPHQIy3nwd6P3GwTw/+8H58ysC9g8shVy7i1X7ie3ZPTLpuGTj + SXnEi6YfnbIQ6KYmb8eRlw+jA3/yNi51kfLA7yf+vl7GBee8I4/EWzyiE9ziTjyUM0/7zZNtlpM8 + Wn+Mv7zufQ9vn/f6+7b/6W0f2PgD9qlzWvd5RKe6T5LpcM7YA4POn7HWfX//uH+Dvo/xPonjCr1Y + n3mvSge9q2e7vqitJ/x888dOS9bNca3neL/Q731E8H3RebKfPGbsb0SNY30rb/IbR76K6zx5v3Mf + ceTpAlhn+uhALRYO5tn5XeXt6cQD6/pdvBx+YntWK2weYWm1n7D1UJh4sUuUzoNAN7XDJQ95T/An + 77GeU/yuZ18v4/Z5R57VyKIT3OK6TuN1uOpzxoNg0Xee0fHH3PInH3p0+wM/eP/2e978wPaTD+If + /9g/uxw6p2mfR+l3s3NmOH6fxI+4nGP4Y/uc9veVGO6LF2vOV224Xwq47pHrlaJpD2XTrv5kjfs5 + mrIOTaf++33DuOghuigz95PHpP2NyCdcZY46WDd56ef511h5yq8JccIxIPvpowO1WDiYh59fEXDj + A4s/x7p+ft1PbJDy0gDYM9axSTBxLEa2HXxdo5q6CH+Kv/OENzNZh0g2fQjNQz/F2eUdfeTM2g8W + wS2u66z+7Bj+2l/0nYe1fKwNfov68z/x0Pabvvfe7dvf+4jbk75om9bQOb1LPxjWcd2LPh+G0Z+5 + z8HndozLOQLucQn+mCdhV2VWf7t7FT2k6GjXChlPvcb9HM1YL+pdPFqc0F10Ud7nI0t6El/y8mTo + 0AmFd89nL3ECKrB1LyJPiAtf76cPhTmBsjnvOd2Hn18ReeMDiz/Hura9xU9sRPEhAWDPIrVNgolj + t7Lt4Osa1dRF+FP8ydu8N+F33n29Pj2L6byjD50Syuu6sJb2FnfiyzH86bvDx2K1/NG+okSv+fkP + bi//7nu3v/72h7ZHsGGd4aBuvluYJdyu3eCs47oXtHWuQO/mPgfyFj4zmcvfSS7BH/N03BVZWJ/S + bU14l1L12S7toR/8emgd9LDOfSw+p6Ff6yG6KE/8en87j9K7DsJGPTOv66fX5+WoIx9telC/cKTT + RuVlXxWZ/eCubzf8/IrIGx9Y3D0/v8tPbDTDprHVM91lY9I+s04897GhqV/K3uESJ3jlwTr8mW+F + X7yI7DqZjyLs8o4+cmbtd2LHp5/qjxWJb8aT39vqsXi0/hh4+Q/3PrJ9wffds33VWx/Y3v2hR/e6 + sr/owfXQmSaHz6Pmm50zsH3OiQs+swn5ukafm89Z+Qa+7wEiyH/VhvWZ96p00GN83CvZdR/RxFPl + Gxb+g+e7Tp3Z6QfWdsddfmLzCQixENmzRLNNwonjJZZtB1/XAA/HRfhT/MnbvDfhd959vX5Tzbyj + D247QdVl23VUH3ovFJD1l73qIp/Cx6Lsj9LpFz90fftqPKT4sHoz/piC+zvoyt6iB9d6eEUIbljn + nse53XDOALWeiQs+s4n4ukYJr3MPLjNQxzwr8GqsVLcvVN2r0qEer32v/DhXP6yc31TmN510Yx36 + WIw/pYeusxRXaOukhDxnM/Y3olHPzOv6XY/OvwrZ87kedtZ849zM14Guh/lY3qP7P39VqAu+Yd15 + 5+7PY7lHJo2omCEGx+6TDcVkfzZRwJviGXfkT16quMvjxOLLi/MaN+NSh+Ndr/2I7AX3bbuO5Euf + ABKgqepsuyoQQar56Jv5xxS+8e0Pbi/793dv3/JzHyhBMh10ZXvpn2tewkP/Pg9uU7fSU2GHc2Z4 + 7WPa40fckT/2jn/gfY7gK35yX6VhfUq3NelhxDqXnO7AePZD/Zae6ck6OK71PKWH6IRQaOukhOQ1 + I/PoYDhj6BhH3llPP+WEG++PiiNR81UC5135krgeYid/fsU6Tn/DOvx5LPfIpHW5OEMMDs0oYjfb + IX+/XILvJzMCdvkYdwv8rsd1JN4qzzpdr/2dSCWqPDhcR/pJn3KosK5TddFfHfai7I+i6XXv/dD2 + G7/rfdvX/fiD24OP8r+nSd/p76Are0v/XOs2RwhuzPhxfrUv/bHezaWfzvEWzjvCX4Tvc6o8mK7U + UN0WuvSOHrmvKZd23UesnhLfsLbTP7+iIqcfWPRs689jSTJeIuz2JeMlpV2XdTfbIX+/XIL3E3fP + n7y8nJfxy1+fADNOb6au0/Xaj81ecN+260i+6o9A9TnjEVDb6rH60/qj5OUn8EcTft+b7t1+9xvv + 2X7qITyoqIfeROnbbdPR55M+owdD9PDyfaDJ4fOoOedX+5Id691cvIoLPrMJ+brGJfiuFxFqa0Ve + iZX1QWXR0dcJ6lvHag+10q77KIvvw2WnGevWdNZ/6Nd6iC7K+3xkKSF5zdjfiEY9M6/rZ3UIQJ6M + lcdE5gOvcEBVAuNWvt4nbjv98yvmuPiBdX7920jSn1RsnlT4rbmKvOgTbjbBRLEvwneewd95qo7U + M/m0Fr0rax7WR3F2ddKsPlYji05wi+s6jVfH4pvx5G/6sUhFV3e+H/+67+t+7IHtc//9+7Z/9Ysf + RBs8Z9RLwXnCpYMs7ssf3bThfomTewrBDbqNs46+R9lnGnp38yl8zt2BfF3jEnyfMyKY56oN61O6 + rQm6RLdUbKVaT/h9XsY1inpMHtkndBcdFXF86yQ96x7Ia76FI/2ej7lVL99nNfZ88CsN4phPaWfe + la+Ahbv2beE7zhc/sL7iZa/bzq+/bT4xmSqt9hO2HgqyUV329bCY2aqpHW7guX/kl32L/M7r/DMu + dTgv6k+e1YiqVHkItL/6KFuHpjri96wzyFmJYDZ89dbU5e//3EPbS7/jvdv/gZ9X8edWq184CeAJ + d9/0c++gq2CFk1sBXPWQ3rA03+ycgWHak/gRV4U0f+wd/8BzX7zFvwKvxsr9lm5rgvq+UOs60V7v + Kz00hp1urIPPq/U8pYfooox1b7x4zag66Bj1PBnfsJDwp7YvffHrXMWNrxc/sM7OzrdrZ/94PjEt + nVvoJz5E4eWRnZl5uD9H2TvcwHcexFhTHBLWF+GP/MIhonmqruDMYz7zdiJVqfLgcHz6qfysRHzx + 135tLwKtruTLm+55ePv873rv9kd/8N7tPfg3gdYr/bBflE1h+DYvHWTpGA+6xkEc13x3iYCGx+Tv + +wGX9XUc0yXtSfy4H0f+2IoLLvOJPK7q6ry639JtTdBDiqa9Vqj1gX9+00lH1sHHIF11X+sekwW2 + 92HwvDpP9n0SOcb+RtQ4Xos9H3MbRz6PlWf2gTjydAGph3wdqAVw/w/qy24513TxA4uYR85f059U + vAzYSqv+hGDv2KEYc2asROGiRtk7XOIA4f6RX/bkHfgjv3jB0DwVF5zzjjyrERWo8pDQ8enHtg5X + fPHXvvD7/sq6MtO78JdS/dEfvGf7bd/9vo1/tmqnD6pc/cKg4Dzh0kEWdTrqGgdxck8huFH6ZB7n + 1vnhY7p1DGJSPU/EfUKqKzWoQwk9J+gRHVKuccZTL7/5Yzeq72fpKrvuMUCtu+ii/NxPHjOqDsJG + PXzoJG/P9ON8M1Ye8pGfr4gTrjeqHvLRj7EWr/HG6debP7C+8uX/gX/jHzn7yUnusknpJzuflCga + STXbwdc1qqkdbuC5L15EhD9z8w48Ei1urJzX+WdccM47+mC4geIRHWzXkX4Wr3jaX/uyq4xDPbV7 + 26YPXT/f/vpP3L+95K534x8DP6CHA4tpnave1S+c1IPI6lOWZD7oGgdxXPOyHfqX3nBpHufW+RlG + f+ZRz62cd/Lt+G+SB2mu1LA+pduaoIcUTXuo2QoZT8tv/thpyjpQ79JVi/V+pF96iy7KE5/95DGj + 6iBs1KOHJXm5m5l+nn+NPZ/rYUXNlzjlZX0dyDX+dtGX6K9qD99xvvkDi+iza9+kVngZaOJ3bLlZ + LJPP2Q6+rlFN7XCJA6qfzFiHP/Ot8IsXkc1T9URM5x15ViOqUeUhoePTj21VJL74a1/4ajEEZd6u + 6UH8d39/9x0Pbq/4jvfgB+v3b/jvlXU+qWenDzZXv8QRxfOs/uTn3kFXbkUPuacQ3Jjx437UPtO0 + /FjLln4Vl3uRGZicI5cal+C7T4DJfy/+RcOrfviB7R0fwL8Nvc1Db+rcl+ho2aELlZntWin2o334 + 9dAqW5vCm6DodK79vim/dYYBXX0C1nvpz3OGS17zLRx1rPcF/aMe8ylM+4uPOO4jjn11Acm78hF4 + fnb2TWa5+PXyB9b1a/+AGfvJCYM10ObQzGS0M9shf79cgj/F33nCm/kEv+txHTMOhakE1+d67cd2 + L7hv23Wkn/QJoPqb8ey34roeEt6ewfK/5Wce3F702ndtf+It+GMKD/g/UvYn26qrdR66+JMTBIKl + LzKmv4OucbB/rnEu0Zkmh8+j5nFunR8YpmO85lFP36MRd+SPrTzBZa78O37s/ZN34c+bfc8921/4 + yfprcbB3O4YepuvCSYC6XtBj6s7qrFDrCXt+00n91oF6Dz1P6SG6KOPzWfrznM3oe5P84a33Basq + 4OX3ixyIYx9OxA3F972jff06/lK1Z32LnDd5ufyB9Uc+812I/47jJ1Y/YSEKu5SdmQm5P0fZO9zA + n+K3trfG73pcx4xLHc7LsiheldeLZduffoxHgwZoSjzrqjj2eex39v4Er99094e23/qd795e/ea7 + 8QN1NpW6MfOW1OXKPhHWyzMvk8pXaPrCLBxfD7rGwf7l5sJ4mhyTv+9H7Ss/1ru54hWXe5HZhHxd + 4xK8z9H1zTyU53/Hf8j9Wd9z7/YP8B9203c7hvVBdvaxJp8XClpyUte6j9zHL5/XCb0nj3hHHGz2 + Kl7dB8e3TnZ0Xt8bBPiEq8w9n72sn8wee770gTjydAHcTx8JvPba7ctf8O7wXDRf/sBS5Nk38YnK + FpWTcxXpJzuS02YRKT5zMl+CZ9yRXzbjwpuZnAd+53X+GRec63Pd9oOjF0UH23WkH9sCqo74a1/4 + avBQT+0+odMvfODR7Svf9H48rN6zvenuh6sdnlDqxswTg24ZrXPVu/oFQjDiR7zoDrqSLHpwTf5D + /9JbMOefNtOQdjePem7lvJNPvLkXmSvvjj/5Ks8vfvDR7Y/9yIPbF7z+nu3776m/MgeYJ2tYj9Jt + TT4vFLHktFKtHxB6aC2ASrYOfSyIv0B30UUZ4ut9Jz6esxXob0Q6qfDW+0L1GXj5/SIf4sijtBWn + vDPf9Zv+sN1V3ewPjgah+Zn/GJfoPqZKq/6EgF2XdTczhvtzlL3Dseixf+SXfYv85jHfjEsdzut6 + 7UdxvahyYRNHcSdeQNURv2edQdqsPmbLT9SaP1D/az9+3/bSf/uu7e//7ENVb9pxQdYDdfLEcgtR + kPur+ttmvzCoh/DTz72DroKRV2jzH/rv/NKt9CQ7bMkuVsfLrnjF5V5kTj7OGZfgbzXPD95/ffsd + 33/v9pX4D77fyf/Nz5M0rA8FLB09+bxQQ7XHFX4P/WD3/Ry1Wremk848d+fhfukuupzA3E8ek/Y3 + IuUP756PyMvvF1GII48PmhtVD/m4ff7Adsf5t8pxycutfcN61affh7/U71+4JRaJJPUm2D3JKUrt + zzeJargE3096gMOfmV3t8pAweUSeeoybccE53jj7O5EYRAeH60i+xSue9te+7C6gFk/s9C9+Af84 + g59Tfd2P3LM9gAeX+2O9SzdWoH4585bwVtRwf8OPQF6m9M8IHkDHw+TGMU4BxMk9hXCiFe/802Y1 + jNvNKqDy3sJ5u+CL8V3vreQB5lvxj4evwM+3/upPPbR9gLo+wcN6lG5rgi7OXXKgCivV+sH2ee1r + lH/ykCA6kgW29BadVupw7SePG/e94dp5RFf3QLtV4OX3i2jcH/J0Aamn7u352T/a/puXPkDkZePW + HlhkObv2GrdUEtabwE92PimtVp7o802iIi7B9ycAwMpDsZl28lL04jnye9/+GRececxnfyda5cFB + HMWdeFWkOuJPXZwVPhZlP87T2+5/ePtd3/WL2xe//r36gfr+E4v1Lt2Y2npg5i2py5X91qdw7hcG + HcKPePV30FUwJ7RbwnG3R+dXYaUnvNZXWXy+2DvWw3qt/4pTg80uIlk7XOLkrvuDtfgz14Ht4gr/ + EP4N61/+yYe2z/7ue7b/Dz+gfyKH8rOyOrg+P72rve38VHjpoIfVsFOj+2m6G/Sj3zqTLooQn/3k + SVafLxDacH2jDm5gXH6/iEIcebqA5CUf/ibk7eyW/nGQTLf+wHr0Q/8cCd+VVvuJz+bZ9JxVI5Fj + SCRqdRrfT3qEUIq2L8Bb9MUvXkQe44KTn5qBz/ydSCQqr/3pp+qV2Kx7xseuGkSw6nm8Vvc+fH37 + H95y9/ZZ//YXtrve80EW4Dp4+qOR2lZ/zG09MAtHoMfqH/EYttkvDMGKv/rxdNDVgVUH4yQMd3t0 + fhVWesLb+bFW+Zk7H/NH/xXnApueRDKU5wT+MeUBU+olKf/R8FVvuW/7z99wz/aW+56Yn285X+m2 + Jp+X6mElHD6Y1KcPFygX25iqf/JcpLvoorzjZElP8prR9yb5ieN5rbzJf/n9IgfidA+xrASMJ9/1 + 6+fv3j700L8l6lbGrT+wXv2Kh0H4mrTqTwjeLau0m5mZ+3OUvcOx6LHvo0FPCmeTt85vHvPNuNTh + vOazH+S9qHJhE6dLwbrKrkXbDou/mqw+yvqIJ/5Tyd99+/3bi//Nz2/f+BP3bf1//EtdOOxVP/uO + iQVG6yocK/Zwf8OPQPcLv2DpKzyMO+jKra6DbgTSHqPzC3eTc2Y4fp/Ej/tx5I+tuOAyF594w5+5 + 6tzFjfxYegD3Rvww/vNff+/2x/HzLf7Fho/ncL+l25r0JmaeJSd1HfoB0fdzFOR++lis5yk9RBdl + rLssJfQ9Uv7cL84YdM+8rh/7wpHBg/uLz3GqP3zVmHHId+3a39/8bAnFTedbf2CR5vr538ST8RH3 + 7CL1pGWREIddaSaW9hxl73ADz33xIib8mZt34I/8zuv8My4452VZlUeqIlnX5cT2px/jVRFxIF7x + savJ4inrI5re8L4Pbr/pte/cvvpN79veg3+b5X5GHtbBa9EO1hsTCwz1y1k4Aj129WNr9QtDsPQV + HsYddOVW9JCbBRlPk6PzC1d61r7Kxno3V7zics6ZTcjXNS7Bd5+IuDQPMKm3E1Td1zF/yzs/sL3s + u+7evgF/HOLxem45X+m2Jp+X6kkl1HXoB8T8ptMo1dvH4n6Gfq2H6KKI+5YlPX2PyNn3i/eHNqaZ + N3pdfr8YDV7yOBE3ovcj16+d/a/auMWXx/bA+sqX/Udk/Rb3zOywIAq72c12yN8vxGHscImrffFi + Hf7Mt8IvXkRynnHYYNrK69n+TlR+245PPxUnsdnnjI+tcDpq8eFPP/fQI9uXvuE92+ff9QvbD+G/ + ++NY/RQv87AOnv5opLbdZ8UpXjgCPRaf6139wi9Y8bdujDvoyq2ug24WZD66OMjb883OGSC3cQI/ + 4o78sZUnuMyVX7zhz3yTugBZQ/2tvvnzrf/5bQ9un/s9d2//4hc/8p9vWZ/SbU0+V1Sx5KQudR+5 + j19880ffFGwdHNd6ntJDdFHG59R48Zqx7xfvD4bkGHmT3zgyeHB/8TlO9ZPHDgEr/lu2L/mMn0/s + rcyP7YFFxuvnfwmveMC6SM0sknZm4srPpcYleMb7aECDgLYn7034lR+Rx7jUIT81Sx6Jp0SrvPan + H+NVkeqY8eyXtsLHouzHMH2AP+z9kbu3l/zrd27/7zvur3pMvKuXntTB07dQ2GS9MVec4MIR6LH4 + Fk5vApqCmUh6Ycv9HXQlVdfBuCkEnXQv/nkvOj8wKj/zKfxNzrsKc57gMlf+Hf/N8hQe0xrqb/Rd + np968JHtv37zvdvvesPd23+s/5pgBd36yvqUbmvCaUW3cPlgWk/4fV7GNSrngW31PeonpnUXXZSZ + +8ljxv5GNOqZeWc9Ov8qZOVxfT5W6EgeFyYkvrniD7df+4sVdsvTY39g8VvW+fbP+glbl1U2qsv+ + bELVEIexww089y0ZRCxcZl7OXZyJ+NrDeY2bcanD8c5vP0J7wX3briP5ql4C1eeMR0Btq4jqrwu6 + xcW3/dwD20v/9c9tf+Gtd28PPoJ/X1I8cx5lrjp4+u1gvTHZSOrGLByBHu5v+BHoT2z4BUtf4WHc + QVduVUKheCurbro4Zv19frWvsrHezRWvuFs47+S7CN993kqeUS+WHupv9N3b1uW77n50+zx82/pT + P/rA9j78i5HHOqxP6bYmnxfIlpzMV/eR+/jl83IdyWsdHCddR/3EtB6ii/JzP3nM6HujSG2IbtZR + BV5+vxiO+vELkwvEdO3s7J9tX/YZP07vYxmP/YFF9vPtr/QTti6rbDSRfV7S3Sh7hxt47lsy9MQU + sRkXXGYSF19yOK/zJ16nXjjnnbyINFAUgsF23uqjbAFVR/yedQZp81BP6rpo/rF7P7T9jrt+fvv9 + 3/OL2zvwqa0x+nM/M18xpQ6eftfPemO6oI4XjkAP91f1Y2v1C0MwE3W86A66kqrrYJyE4m6PFU++ + 0hPezo+1ys9c+iku+MxkLT+XGpfgH1MeEKbeYq/+Rt/lWLjzDV+Mt/8L/6H5y77z/dvf+mn/fxs7 + /pKFeUq3NeG0JPhol/bQD349tA56iG/y0D/0o196i04rVbj2k8eF9zeiUc/M6/ohE/3Ik7Hny7Gh + fuGAqrofffTsryTmscwf3gPrj7z0e1Hk65jIT3aIwaIpSorPnGrK3uEGnvuWjCKYN3PzDjwShVmz + 8zr/jAvOeScvwgyseNuuI/1UfxKb/c342AqnoxY3n96Pn9p+zX947/bZ/+ad279/zweWXgwb/bmf + ma94mYd18Pp1/aw3puvoeOEI9HB/5uXO6heGYMVf/Xg66OrAqoNxLMh56eLo/Cqs9Kx9lY31bu58 + zB/9V9yRP7bynMB3n7eSZ9SLpceoO3XSkb6wEo72vfhm/D/ir5z+HPyPPPg/9LiVYZ7SbU0+V+UJ + C/MsHfTQGHajVK+PQfWO+olpPUS3Olr7yWPGvl/dJ6sYdZCfvPTz/Gvs+VyP6heOAYp7Hb5dfW9i + Hsv84T2wmOH6uZ6QetKiiN1M/2iCZuwdLnFyQwzO+K2Zl7D22eQuDvvh45JDfkRwnnHBOd44+xHU + i6KD7fjkW7ziaX/ty1b6Iqj1iYmfxn/rbfduL/qX79j+NuZHSh/XXQHpE2b2Xc+gZxzzUinM7o/1 + xsRixgtHoMfiWzi9CWgKZqKVn3EHXbnVddDNgsxHF8eKJ1/pWfsqG+vdXPGKCz6zCfm6xiX47hMR + l+YBJvV2AvU3+i7Hwrlf28Dh10/hf+rxu7//7u2VP3DP9rYH+g+iNOVcJK51tOziIW7JyTxDPyB8 + Xs4fTvHVMajfUb/58r4gXRRx341XHjP2/UI+x7OKUUf0p198FYf9xZc+rE858L8cvPbnjH7srx/+ + A+tVL/l2lPr9+qRgkSw6M+sYTaissne4ge8nM8CUqO3JO/BHfvHyMIFPvC8D5TOftY0fmwaW37bj + 00/FOVD4Pf9o89ivWP3yHe9+aHv5t//s9jVves/2fvxF6kqr0wt/gUd/7sf+UaYTYsOfbIhTXtab + dka/dDMPeDN29Svcl6loFBGdGOO2DrrGoTpgkN9AejRm/fNedH6gWBWr1VzxiosOmcl44I99Ef4x + 5RG9dWMqDear/KmT+8pnQMHcgd7M5f/2d39o+1x82/oz+PvI+PdwnRrmgU95evJ5iSdRxR99gOAN + WnUYJ3vR2T/0o199iE4rBa795Ck+3RuuuZ/6Vt7kv/x+MRpxi+/7ti954V3c/XDGh//A4t+7fL59 + vZ/sEAPiUHx/cqAU2nOUvcMNPPctmSVqe/IO/JHfeZ3fZ+J6gnNellV5cmZdF4pFoP3px3Y5hr/2 + ha8mi2e2/NP4t0i//7vftf1O/KzqP97Hf1QY9en6hL+iRn/uJ/VwHnmYl/FutHhjGtjxwhHosfpf + OL3ZaApm3o4XbNSdQjirDsZxIWBlcd00xHPoi2mI3s0Vv8OPuCN/7Ivw3N/xJ9+pPKkTcw/iKn94 + 6FM+gdyvbeiDX8IVP79B/w38jz74P/z4pp95CP9A0syOFg4RytOTeAgoGq7wu+6jLOZZNrY0rMPg + Ee/CtR6iWx2t/eQpPuYlTPnDu+ezlzgBFbjnc5zqL75Hz7YP62dXIsfLh//AIsOnvuQf4RX/1hAi + sujM9I0maMbe4Qa+n/SAWlOIo7DBO/DhIzWHn/iuY8YF57zG2Y+gXlR5sF1H+lm84ml/7ctW+iLw + mn/r51/4ofdvL8E//vHfAnrwUEd9vg1Vd0FGf+4n9Qx66Qyb8V0/eWMyj+M0C0egh/sbfgT6Ext+ + wUy08jNu1J1z7TronkIQv+ef96LzA6PyMxev8kaHzCbk6xqX4B9THrCm306g/kbf5Vi4qTNw+KV+ + qi4wKoL/BvFP/vB92+fhfwDyve9fP98yT+m2JvEwsGnE4zq0rzzLVhLhc26l66hfcbBdHyyeV9XX + OikheYmm13wLR96VNzoYRz6PPV/6QBx+nV/ffnz7Qy/8p8F+OPNH9sD6gjP8662zv8wu/YSvmZVI + lFFS2Ttc4gRnUyX2tBkXXGbSHvjFS1GwL56KC855GRY/OAwkm2E8I/nTj20BxRd/7QuvcG6QZfuH + +HNUn/HP37H9xbe+b+PfVqK8gviSdH5dn+kHaPSXuMbnTqQOxnf9rDemgR0vHIEei2/hdJloCmai + jhfsoCupug7GTSHoXH2J59CXygZmN0u/igs+swn5usYl+O4TEZfmASb9dgL1N/oux8It/Zjh+A0L + jIpQmSjgR+5/ZPvC77t7+5I33bP9NP6tsHngUJ6exMPAao8r/HYd2oft8zI/9zjEt+jKHnEglA6i + 06rjvJ882nYddCi/65l5owP71vk7THkXn+Nan7OzvwTsvvCKu9VJJd0q+CTu350/Y/vJN//02dm1 + T5bKvLynBvcpWl3C3Qy8mpzzwDWvDvc0v5/sdZjkqXwR84Z8ndB8C16XTxsiwksdiqajH24O4D/n + lz57+/73+X/4wMPtOANku072e/QL5Lq5ZH5NzNemF1VHATCVroXn/kV5en/H7/iZiDiNTLBdR28c + 2qt9R+1fc24187ITLb5TM/3MN2bpwfhTg/us9zB/WHkmf/HdwKM60gECyl4PrZBUvdWoedIXw9qB + +qt80uGX2CucOw1ov3mSSbPqHTywoyP93Ufyive4T6DYqj5FamPWXxu9r0W97PNgE/lQyc8/+qs/ + /VM2fcmZ6Me2/si+YTGXv2X9bxK/LplKkCijmLJ3uIHvJhHiw8qhQb3gMpP2wC9eRDYP/cAH57w0 + w9uJyGaY4PTXZSi7L0v7zcNCuwws3sCHFdOST6c+LxVvwaivboXrZgUYo7/s7+olphKIX4nCW3mr + oI5nHupQY/Exzn24XxiCEe99+/k66i7+VQfdCiCwR+dXvUuHzg8k07n6mc/1Kn7osYSuFLPP4DKT + F/4df/Kdiit8MXsadYeHjvTlymOjP/wSrviXH0FwuJ7SoexyzEk8zsNXDiuUvM6z9DSm6hBv6Trq + Fwts1weL5yXe1FXxQKT8vl+NI+/KO+u5+f1idsX91Y/0YUWmj/yBRZYHnvN/oqgfY7frk5mijCGR + qJVV3c2A6ZOAM37riHj5ar95b8LvvM4/4yKm8408KxGycB8vCHQd1UfZ5Rh+44xXuAmEr/qrE9el + DEpgfsDlL56i6D5hJ67xrI9D+lW8G8Um6628amTEM09uocKHrm07XoILP+KV96Ar4pLQ7mqc+zVm + /ce+VDZwu3nWnXPOTM7yF33byhNcZsGrT6wvzVP45uZCgo6+y5m+ANCObeDwS3m6zvgBg4M4PWzE + a7sccxIPiZtGeVyH9pVn2dzjsA6O6zpO6cGysA8kw6quikeFyav7SVjjWObKGx2ME5DgAx9t7p7/ + 2PVPuhPPiI98PD4PrD/xAvxt/udfw+ry5LUoo0CJRK3QXHCZAeO+tMTampZ9Af7I77zOn3iptcs7 + 8uTM2u/ErqP6AFF4la/sxU9/9ag6bcuPw2Unjidm2csf/uI46KEo8DpfYZKHfO1gnpjMs3j9SUmg + h/sbfvE7XnzFm7rJmz6czvxJaDc8BlaWPX+fN7ydX6ywM1e88kaHzGQ98Me+CP+Y8oi++mIuDuar + /Orbu6rfS+OVn+eMXzt91FmVLXnor/tQtk5MeSod0844JbJCzmN/81RNnKzD4Bn1x+/6YKEvMu33 + k0fbroOwxrHaqp+7dR6X3y9yXPua7b/EM+JxGI/PA4uFfMVL/9X59fN/mievRRkVSiRqpdPaz4Bx + 35JZorYvwB/5nReXIjwVF5z8SLD8SKqEOhXD2m8e+sMrQPvTB+fqUflsixaHywSOJ2bZy188RcE3 + SfBzFv6Yh3ztYFxMAzteOAI9uO+whdObjaZgJup4wVzXjEtCu+FpIVYersRz6Es8lS5pVz7mLx0y + m4iva1S+Hf/Ac/+W84A1+TsB+YsvPPQtnDovG/Xil3CtQ/wIgsP1pK/wyFF+TeJxHr5ykKfiZDHP + songUF2LruyFc37iAEZf5nWcLDvslxdAOpTfcTNvdGDf5iP2yMeN83/6yJd8+r+S83F4efweWCzm + 2rX/Fk/e+1SXRBkVlu1PAohOGyL1kxq2jwbbCON+5sYN/BSJWcxjvhkXnPNNXgQZyHDDYDtv6lu8 + ArS/9mUr3ARlY9KlYgLXpQx4GfX5Ngw/3QufONdjerJ0HYxXIl23bDdfxwtHoMfi020U3p+c8Atm + 3o4XbNRd55iEdiMw+yMPl+I59KWyKx3jZVf8Dj/ijvyxL8Jz/5bzpE7MPVhP5Q8PfconkDovG/rg + 1+wDSKOqQddT5wugebSQAEpH/hnXeca9UJ5lC8K4ImieUT8xzs8ZBvpiJo6170Lllxc2YY2jufK6 + fnqJE5DgHR989z+63fnH5XicXh7fB9aXf+bPoK4/q9pGE9P2JwEOl36o009q2JbMEnFf9sQN/BSJ + /OYx34wLzvmMsx9BvSjNYTtv6lu84ml/7ctWdyYoG5MOlwlcFzG+JOaPn2HrsI96KKp1oIUhPSpe + icKbdswX3ss+AV0P6wQ3+Vin+ggP9w66ClY4uRXAVY/Or3qXDs6nLEqXtCfxNzlvF0w5WEfxZ0YV + jylP4bt4LgavZC5n6nQHlZ/64JdwElIEipBZeuqhJt4V5zyVjmmpP2dP2on+2leepSf3OKzD4FGe + hWs9JPjqaO3L0Xl9b8Rc/Kxiz2cv4qB7xuTD7tdtf+gFPxvf4zE/vg8sVvTxL/lG1K8fwO8KrKb8 + SVBPaojaT2peNgREyt2TP7jMJB4i2XTkMS4452VY5VmJGG6YtKc/9RnPayRA+2u/tptgwXS4jFNe + A/BqG7Dyh18A3bbg5yw86+Vg38xDpdpB3pgGdrxwBHpw32EL537hF8xEHS/YqJuJOLoOrOvNof16 + WfHkWzp0fobhN9mO9TR+xClfcWuqOpQnuMzkTZ/hz3wqrvCY1lB/o+/ypC9X7jzsQG9mYG70YxMN + up7SoexyzEk8TFVlciWC8DrP0pNYDvnFK7qyF875ixc6mddxspSQeNG5DjqU33H9vuBuAX0PBVTg + ynP+xuvPfOE3avNxfHn8H1ivPMN/9Xntq/pRnWIlEsTkjGZ3MzD9ZMZaR8TLV/un8Ed+8SGieSpP + cM438uiUlAAv3McLEjo+9dkux/DXvvAKN0HZqrvejq5LGZSg65M//MURXWAmrvG5E+oLfr0LBCze + pRvZOl44VuSx+Exom/3CLxgW6iN+xh105VbXQbcCuNuj8wtXesLb+bFmuqQ9iR96uMCmd/7iO3U/ + HlOe8Ax69zf6Ll/qdOXuR/qgE/UjIQke+pWeethEN+FKtzUhasQppxVKXj006jyqJKOat3SVfUJ3 + 0anSjpNV9WiCp+/XqKfrp7+AxpHBo3U/21696VkQz+MzP/4PLNb1qhf/O/wA/pt3JfJSY/iToJ78 + aLqf1Lyc9OO3NS273gy7OBPxtYd5zJf4XDqCHO/Zfmz2gvu2idOlUN6KI3DYi7/inEB8BUMfJnRd + AiiB+REnf/jpxzjowa3GWz4nZDmM7/rZd0wDk9c4Aj0W38K5X/gFM1HHC3bQlVTRg2teXtpjrHjy + 3eScGY7fJ/Ej7sgfW3HBZS4+8YY/c9W5ixv5sfRQf6Pv3k6fnl03cPg1++AJcSid5LnsXpUOM84M + eB36Kc+yBVEe6ux8XccpPVgW9ld9s27ymrHv16in3xfJJxbmJZ8H9Tg7P//mh//gC9+YvcdzfmIe + WKzw0Wf/KRT/ni62mtKTmU3RzgxQP5mxtqa8BN5v3MBPkZhDfDxM8M644Jxv5MmZdV1O7PjUt3jF + o7OZ/PQzO4b66Ql3xwlclwB4GfXJH376MUZ/iXM9J/LodiJGBZB36UaqjheOingsPhdu2/ESrng7 + XrBRdxpOv6RF3a6jkmBa8SzM8fR2fqx1TpmLV3HBZ3YgX9e4BP+Y8oA19XYC9Tf6LsfCLf3Yid7M + O574sdn3pnQouxxzEg9TVXtciSB5nWfpSSyH/HUM0nXUH7/3YfG8xOu4xrMPl+066Ggcy1x5Zz3m + I1bjvQ8/uv13MR7v+Yl7YL360/CwuvZ1XbBEolZQAarsZoD0ZOaM3zoiXtbaP4U/iGQ+RDRP5QnO + +UaelQhZuI8XJHR86rNdjuGvfeEVboKyVXd1oryCOEHXJ3/4iyO6wExc4xnOob4wMV6JwhvTwI4X + jkCPxbdw/uSEXzDzdrxgB11J1XUwDoESkA6PFU++0hOuzo+1ys9c8YoLPjMpD/yxL8I/pjyitx5M + paH+Rt+9HZxn5UcnT3/DKtm28z+1/eEXvrfketyndZMfd2oQ/sPzO7b73vK9uHevyKXmk1mXac6A + 7i4v7eHX5YTd84lahce12X3S1aUjfPLxqpGO72UvPBlel0/54gew7M7TfmA42l+06Yj7Gk7oeMBv + 8Bcq+Job3zRYqJzaEM6ffG7H+x13yNP7O/765NzpEf5V1+Sf/bq9whu+f805nHgIMEq8cwZOdY7Z + +Yg8MQ78uScXPrRAobyDP/l27MV7Aw/3oysDyl4PrbBUvQU3T+4/w9ohOqUjHX65vsmDHeHjN08Q + mougebCYfXUf4onyriN6mMesro9r9zHrN27tl/3Gh//gp79C6yfo5Yn7hsWC9UO3s6/CATzaYkvN + EhtvmojSYiIs4mXmZRUuM7klOhce5jFuxgXn+Hk4iDNQBDlD15F8xs/bZH/tI77LUF+2RVtvQ9fF + FL6cHT8ugQrgy+gvcY333egEvNSrftYb08COF44VeSy+hXv6GxZltR7RyYKO+zT0KyU1OQ44/PJ1 + Ck/0BUz3hP6b3as6P54XxiqHdsVxX3mWTSyH6lCecQ8O98n1AYx9MnHs7wN5ta08rHvhyLvyRi/f + Q+zjh0Db9oxXO/qJe31iH1is+1Wf+f14/RsWiVpZ1d0MgD4JOOO3johi1z5VPOLDB4iG/IhonsoT + nONHnpWo4jEhoeOTz3Y5hr/2hXd+5SlbdVcnrosYJ+z6ZIe/ONIn0bpUnsXHcA71hYnx7WC9MQ3s + eOEI9Oj8O37Hi694O150B11J1XVgzVtefM5Cc9Rx6EtlM4w0mU/hR9yRP7byBJeZvODb8d8sT+Ex + rcF6ii88dKYvVx4b+uCXcNXH8iMIDtdjHWOXY07icR6+clih5HWe4jHAKNXLPKJznaf0EN3qqHVS + 3eQ1ad8v5Q/vyjvr8VPu/G88/Ad/wxPyg/bR5uP0Hz9PxlPr5z3jT+N/QvZ6uvxJgMOFmFSnn9QU + l378tqZlT9zAWyQyepjHfIn36ZExeT3bj81ecN82eXQplLfiCBz24q84JxBfwdCHCdMfkEpgflqj + LsZzjP4S13jDVx2M7/rZd8w9bz4BnYDhQ1ds2na8+Ip35VdhN8QlobLxlrOAMVY8C7vJOSOm9cRa + ccFnJu+BP/ZFeO6LN/yZi2cXJ/p9/eKv/OEBzPVxsTs/9Idfs4/lBxQO11M6lF2OOYlH7F0OF0M/ + 5Vk2sRzuh7Poyl641kN0q6O1nzzFhzxqiLP4ae75tM+Kz7Y3PHLfp/33Aj7BL0/8Nyw28MoXfWi7 + 9szfh9bu1pOZlwmXger2k5qXA9BIyX3ZEzfwfBPMYR7zzbjgnA/8zYtoA0UjOtj2pz7bAqqO+Gtf + +Kqi/UVbnbguYtxZ55cd/uIY/SWu8Wk3eRjf9bPemAZ2vHAEeiy+hdObjaZgJup4wQ66kqrrYNwU + gs7Vl3gOfalsYHYz+RIXfGY75O+XS/DdJwIuzQNM+t3xV/7E07dwo15k8DefU34EgcD13Oxepc7w + phLaFcf8+OXzMq5ROQ9sq17ZIw6290m3OmqdpCfxZmQeBXDGEB020n9mPK3ufmS7/srt1WcPO/KJ + fX1yHljs4cte+Pbt/NqX+5MA4lE0iijxqCHEACxStj1xA2/RlzjmMZ94Ki44+XkGybMSiYRwnZH8 + qc/4cgx/7YuvalA+7guGPkzouohZtvukHX4tEbjXI373U5jkIV87GBdzz6s6eNs6vHRmAIb1cLzv + polSt2Guy+kcl4SyyF98lUa84T/2JR44d/Oop/FDjyN/bNUZXObuK6qP+VSewrPeHsQVX+qkL7qY + MTb0wS/hWgfrJBOOpTN5V5z6kB09RpyKoW39lV95li0IXqwD5+IZ9SsOtuuDhb6A5HbVVfHKo231 + o4DGkXfljQ7Xr13/0u0P4L39JI0n74HFhl71mf8ET+ZvoKp6QmeGnLO7BAAAKHpJREFUi7aPxlK2 + TXGDy0wuic6Fh/jAcIwLTn4eZvLkzIpHU/tTn/GqiID2177sLmDCdLjccF3EOGHnlx3+4hj9Ja7x + DOdIHYxHfvfHPDEN7HjhCPRYfAvnT2z4BTNRxwt20JVUXQfjphB0rr7Ec+hLZQOzm8mXuOAz2yF/ + v1yC7z4RcGkeYNLvjr/yJ56+hRv1IsNT8RsW5PmGR3//p39Ef0d7632Liyf3gYWirj/32p8+Pzt7 + vT8R9k9sXoFcDn8i8b2AHVzO3czmuD+G/Lg2x7jgHG8+5VmJxCI6OByffJU/b/r2177sKkJ1ch/l + Yuvpb1g+H+t+k3OGVtKrznOHz7lT4vJzqXEJ3ufIc8h53CQPMMprZr/qIMd9Kt/Cjf547/Br9uHM + VbbuST3UxJt8cqiw2q57U3HK6Q6S13mWnilZ/kXnfoZ+9Ls+ROhhv+r3fvKYUfeXDinoeuY3rOvn + 19/w8H0veFJ+buWK/KqS5saTsv67P/Jrz7aH3wQtnn+zT0KJTHHr8vR8okjzHD7pEge8/Dw0HqJs + vPTCZ2h4XT7lHcCyO0/7geFof9H6etTlEAAvI/8NfmJcZy3aHmWOPHV0zDt5ZZvHV9B+X9Kxv8O5 + rr0e4VcZQ7fsY0aCohkL43evOYea+80DUFW3n3NOY7a+7OjEOPDnnnxYeSb9RfVyvyvGsuz10ApJ + 1Vtw1+OHDXWjLR7lMY314D0uWlENHPd53vjleAH8cuSBnXtPQOuRvDrw4z6BoavFDneG/2XZ+d0P + nz/j5dsf+HVvN/LJe33Sv2GpNfw86/xR/wfSEb3FBMCHlUODaDhd4TKTRKKLTS/mqcsgt+OCc/w8 + HIDGrcgZuo7kM35/qWZdowwSgK8mHDEPe16qZSvtuARqgC+jP/dzqJeY5GF81888MZnHcZqFI9DD + /Q0/An354RfMRCs/4w66cqvroBuBtMdY8eRbOnR+huE3ozRXvOKCz0zeA3/si/CPKY/o9/W7v9E3 + a8BQPq+GDRx+zT7cWZUteaIz9QiPFmVHD9ex2rVCyXvRw8o6OF/XMfRrPUQnRNffeBSWvLq/dOiE + wms98D+Dws+tnvyHFau5PQ8sZv6KF78GIn4zPwE4/M0lhzZs+qGicJkdoLi8mMc4n4nj9GYqfmrf + eZi2D4/7tu1PPuMFVB0znvwVxyLaX7S+vspHNwD4PeqTbT65+TL6cz/2jzJXHsa3g7wxmWfx8mHU + t7D2HbZwehPQpKN4V37ujbolFGHkFdr82SccY8U7/7SVH5jdXPHCRYfMJuTrGpfgybPjR6TsU3Hw + pb5OQFzlDw99C0fBYkMf/Jr88JQfExyuxzrGLsecxGNeheOFPBUni3mW3SjVyzyic51Dv9ZDdKuj + tZ88ZvS94Zr74VVm/NzqBU/qz61UQL3cvgcWC3j4k/4Y/ln4h7nsTwCsrSkvgfd5Cv4EqdkBfO0h + PyKaB4fl0+PhhGfkyZkRJz9ekNDxyVdxdgz/5FM4N8qvSZeKK9elDErQ9QGh3covI33CSFzjDV95 + GG+BgGaemHtef1IS6LH4Fu7pb1hL7+hkQcd9GvqVkpp8TsDhV99Xe8qP6ZbuVZ1f3wuF44XntO6R + 8yy7Ucf7J3vh1rmTTpUqdO0njxl9b7jOPdH+Wx++51Of9J9buSK/3t4H1qs/+cHt2jN+H0R7sJ/0 + JVHb+sioTxQcAvc1MtuqffslfcX5cHhGvDWe7e9Ei679yVdxDjzEh68LKL+PWJ9QiOt6fZ1lK7/s + 8BfH6C9xnLtewtKH3gVlK0/lpV/bNQtHBo/Ft3B6E9AUDAvMKz/jDrpyq+ugWwHc7bHiybd06PxA + Ml3SnsSPOOVrdgYysuoMLnPt7/ixJ/tUXHgw91B/o+9ypE5XXvmpD35N/uVHIBzuu3QouxxzEg9T + VZlciSB5nWfpSSyH/OIdfZ7SQ3SqtONkKSF5te066FB+TQ9i+cVP1p+3chU3vt7eBxbr+dIX/TCE + +Op+0mPLmvISQHyIThV3M+O4P4b8iGieigvO8ebbnVnxaILD8clX+VmJ+OKv/dpWGe2v+n19xecy + dS2KH/HyF0/6SJ+w3Y/nrpe45NG7oGz1vXQzzPr4k5IMHjt9sLX6hSEY9Z75GXfQlVtdB90K4G6P + WX+fH7ydH2umsyozH/NH/5rJynxzlK08J/CPKY/oT/AXb+pk+vTlymOjTvwSrus0n0zJQ3/6WnGt + I9smvxShn9k4uFg6OM+yBSGKAcpTPLIXrvUQnTMxdu0njxl9b4TQBrxf/aFXvuCt9t6+19v/wGLv + X/aSb8InxDdYsjoiXha4/MnBJ79PQzNjaI/hfeNmXHCON5/9CO5F0cEmTpdC+WwLOOzFX3Gso/1F + W9ev6x224mWHvxrBJQt+zqPMlUe3s/Ki4kp/Y7xwZPBwfyuvbcdLj+Jd+Rl30JVb6Vdu8NMeY8Xv + z63zA2sdaq54xUWHzOQ98Me+CP+Y8oh+X7/7G32zBgzl82rYwOGXzyk8nlW25LnsXkWPEdd5XAfN + 2/ENCz9k/4aHX/kbvknl3OaXq/HAggjXf/rFX3t2vv29vsS8rNj3JwcOGzYvkWaKRnsM79s/44Jz + vPnsR3Avig42cTf/JJx1jTJUn23R1tux6x328jtftzH6S5zrOZGHfF0/+45pXTpeOAI9Ft/CuV/4 + BTNRxwt20JVU6Zdrvitpj7HiybfOrfMzDL8ZpbniFRd8ZvIe+GNfhH9MeUS/r9/9jb5ZA4byeTVs + 4PBr9uHOqmzJc9m9ih6uY7VrhZL3yf6GdX5+/e89/NZP/Vo1ewVeqPHVGf/u/Blnb3/Lt+Pofrs+ + IXFq+QTtNwXfHCeGP1HrUsC/iytbl5hvAtl46QXxvlzNo40BLPtGPzAc7S/avB25r8HZ/fgK2lZc + ITipbi+063ymrw0l2OMGb+XruEMdvb/DRWcV0Hm90KvqslzVz+i3Agw89cp3n/CeL3yYIFaq1Ln3 + ec34W+A/9bAptaPGxXkmf+W9oV7uNxOWZa+HVkiIwyi4eeohiIJoi0d5TOM6eY+LlvGLoCz6zaON + vBx5YOu8lcf5zLuvv/srnNIx6/n5az/4y3/9Fz4e/4v5lPiRzlfmG5Ya+YKzR86vP/OLsNafhG9x + cdl1uJkJjrgK9GHw8Hfi500iOA4JhMuPzXErRNf+5Fu8ytf+2pfdBcy7hzP3pVDdgixbaetWLD/r + WZcw+7t6ycNCmZfxIgpvTNqpOzgCPRbfwj39DWvpFZ2s87hPQ79SUpPPCTj88nFYVyrPcev3qs5v + xpkBr+NeKM+yBVEe5OO98ISZi4Vb5046VarQtc96idc2/3Do6z/4rE/4oqv0sGJl6ya7zqvx+s0/ + 8svOzh/9Pqj3qfomUeL7NE6X7G8O9cnDxvKw0uHY1uHwEOXHSy/or8OW359Myw+gAMXDS1N2K9j+ + opWj4qSqL4nrBM8NfoHMyyX5NI16vaEEyh+76nE7h7hDns6/45/9rHjSK5wT8JNf9e3ac5xiji+H + 8+s3SdGLF+uegVe+MTsfESfGgT/35MPKM+mL9wYe7kdX4steD62QVL0FNw8fCuyPYe0QHU2qeJpH + ASK23zzJZIcJmkd5Fq77SF4p7jqUV/uwr5//xAef8azP3X7vr37C/qrjXd2Pwbha37BSOP5OaBzK + F+JE36lD5ZtFp7DEx0bQmnP4u0PxrVj+uiQ+HGz3godm2/HJ58Oct2nPX3HMoPp60qVjnOsSQAk6 + flwWejXSJ4zENT7tJg/ju37miWlgxwtHoMfiW7inv2EtvaOTBfX5SeahXympyToDh18+DusKxvJj + gsO6130ouxxzEg8DdR/NIIJ5nqf+cVB+8Y57cLhPro90WpkdibzPelHf9e2d2513fuFVfFixYNZ6 + dcffeeuLz+64/t2Q8eP7E1Zi31iyvwFQfF4engla07vYLe4+seXHi4EiW3DG8xOQ8QNYdudpv8J9 + CYBf28xbG84g2/EU/ug3j/JySSJN6UdmJ9jjXK/LPcQd8nT+Hf+xX6Yxj8J3dWQf86692q8yd1PO + oWa/eUteAFX3nPkmQn7harawRJ4YB/6c+4eVZ9JfVC/3oyvxZa+HVkiq3oK7nvTFsHaITulIh19U + U25RDVz7zSN3XoqgebCIjoS0HiJWBkVmH3+I+77za3f+5g/93k/5oVBetflqfsOKSl/xGT90fn72 + RfgbDR/R4eby0i/RA/Rh8NQjfvDB2Z6HhthxK3KGjq/LAL/iCCSgbIfFrhraX7T1NnQ8Mb505qdF + O/xaInBdwsQ13vBVB+NdCIIZF3PPqzzgzVh8C/f0N6zDOVAsCTruUwmYcwFAO7aBw6++F/aUHxMc + 1r3Ot+xyzEk8DGR6Dy4qDivnWXaBxO88oit74da5k06VKjT716+ff+Ds2h1fdJUfViz4aj+wWOGX + v+i1+Jr6ZVD5Ok9xffJT9DW8X58o/397VxtraVWdzzn3zgxQgQEKSP1ATSWhlaaJ/dNUrYxNTEYL + CRoxVRtpy4/GtJo2TfnRxFT7o/5ppLRNjKlaCtoMatKG4h/LxJg2/eE/EezwVQlWKylcYGBm7syc + t896nrX2Xvs97713mLngPTPvvtyz9seznrX22muv+56Zwx1Mc+xJZyiNJZkCcWZ+meMMDceksAkA + gxedMpZ+jN2Hsk4YckcGpE8P8JL843rwO0faX+jJH8P17DA7MccF43W7Diz6xJnHapVPhBpL3/YL + wrJPjghLfocjblDLDJQM+GuxT5z0xWfxpZVWZr8jDiGl6MwutsCXfQLe2BvSAyb8LUaS36FvaxVX + 48dzRdyIc37t0PBQwoL88Tj42BeyAEvwhiciCLv84QLCGBcU/ZW94keKn+yHP0RQ1ebn3WSOSnDz + 0ZvecH/w7VS58wuWRe53rrPPZ/2xnb5+omEOh5Gb5rXOI7Z1HqJwXMdC/EShuoCkibHW3Y7jQSR7 + jb7xJzdor8CQUmY3+ZvGUPN1w8s/OpH2F/ONvwYKO8ZHorATQ/EVfeIMqFb5Km58wuqdg4WKcU75 + lOLnkaRQnIHDl45DcQWBr0NgQXHfLK/8/LKeGEiQz1PnFXYIUh7RTsqDXj7JP6NjT+zMv+kfHnvf + m/5FTDv7dTkKlsXwlrd8Fn8i+Jnyk4VBr8HVvH7y2FFy7ElnKI0ltY7J0qlnaLjxCcvjxbhZ9Hpx + tSmLP+JnqR+X27rRdB7Og3PIY4YdwEb6eeqcHJ/0aC/ITW6B1znKvy3tkI47qRa4v7RvX4l9+M59 + X8Dhi3bcr7oORSzIn9iXxr6QBXnMVKFhhF3P5mmnjg1rTXGTXvEjxa/Eg7UqIkLVPz/2/jf8LXtL + 8NI7pSXw+AsP/NlsNvn00CXhKfsh5UMrp8/D0uFqHfstHfSbdSUF6JQMyro2uZBVWve4MclT0iwk + lwyU5Cnm/Rio73bNHc/agndY7IfryX83X/QanLtoovKJsI4dVPaRL+GiXw1/EwiEKc4hS9DT3SEZ + +kPS3WqE47bFTkPsg8yPKfObzeZTK09OQPB0uc5egetYbd3yBfoCYh0dH5fpsFTMWEd6Yk08fT8q + ndvxCeDkp8yFfTB98uhNV38q0ez4rsVg+doXH7gVSfA5OF/816Xzw8SOdJktOQThOrKiuZzMhVj3 + 3AE+P2FJ33ko+usePmalzJFWWVHsowNgsr+wLh76bd3sdx0WAy0u8fb1enbq/mPf/f3UeXqkYYpb + mbDthJupQ632hbcRei7L5bF94VvxSjLOKUkaMv2h5rzBH/K07GR+513goR/hORR8XN8WBon76xsU + jxcfhqMsMBA0Z3SWJ0FLqoQr6+Lhcrw4QeFBh+dtE2ixD3woFBOz3zt60+u/EKrLIi0uy9m++MAH + 4fyXcD/3lA2waPQuL0/Pt8kssLtll9TvWOnYBL4x1noctsa+kIUng/OYE7RfBOhSMbB1N1DsY0bm + wz/ISDJDG5/L7CbtxHpacPNFr8GRSS/F/mb8JQ7hRy+uRpX0o0iEGV4OrDcSi3R3SMY5DckgzdJx + DX/YG+LHHI/3dPidj+ZNPzXaj3PGvI3dElHVHOJnOE4QKJyPyzQjZOtU947HkaPEExCbB4EdR+Fh + xydiHRLF6hj8+MiRm66+J6kvTVc3Ymnc7Tn6Dw9cP+26r+MA/HfD+2ECxsseh+ZjHaoVEb9rpaOx + 4LZuh2+HnYA+1mXP68BYK+uupuvhhgjAi/RIu7BuGIP7kbiUvUwjv1pc4u3r9exUPtnROO+nzssh + vtIvhSP5h4lwt3aEb17jHFzyHABQHAYkcPQrSfKb/lBzXt1Wi48cOy07mX8jHvphcXB/fDz8ZOQw + uoV94asWF48A7bjbhBtOYwjr4dv3xVHlsdXS+jwYRxwNg3/M+NlusnLj0Zte962is2Qdi8tyty8+ + dN10euKbSIQrIqmbS8lD9G2aYC4PFy1liYpGFC2pF0XX9xTCbc30sq9LrBRLxY9RFk/xT+4wqbSM + 9UgyTBjOWsFrKAOxTkOBq365ootQ5DDxhV6Kh0HMbhMnw3nyc9n5kn9tIGwb9bLQfxuTBfRDMgI5 + JIFfaI7bFjsL5JjI/DYMjM2npqLo54x5G9cdFhrFAzvnOgPgONpxnOmHpWJGONnRehS95Ib4wet0 + bscnjHc+f+rk6vT69Ruu5m/4zbrL1F+evyXcKKq3XPvdbmX2a7hkj9hx1ssdp2fZYXfQxpJMAZsu + HZvXWPriCbwr9vSDz9jRCr/T+rWkXQHwmvzjuvvFdVt2u+iGnvxx/wwXdky/+G96MbSNJH3iDKhW + +SpOxRnrhImo2je95DcDhanihy1DMeYNjlb1ja/dF90GppGuT73AhxShvda2Bb7sExpb2gEm/C0G + uL+0b1+ouBo/s8Ai0vDEOiYZnvTDy8e+kAV5zFQNp/Gk+AERP0wNF01xkx73m/3vJo/Op7NfXfZi + ZXtVVGPXyyzvfOSK6fwYnrQ6PHHZoXoS5suks2dyah2g0jEFjaXvyYp1JSk7bXKZHZ+GpgHb9YXk + koHin8w5f+inS2J8nI79cCg7Pp/9d/MtX+Bc1USxvxm/7xvC+Rb9wgJZGZ8mEAiDF5tGAk2+IRn6 + Q5JWei+Oa/jhT30Sgd+naqdHzWHmd54yn/CyZ+eD+BjO9IplG1v8PB6WD5wgUDgfl2lGyNbxzSa+ + ui+z4zwBgeQ67bh1N4z57xzZvWv/ZP9VTyX40naX/wkrQv/bP/+TrrvgHUiOgzxiZYmyBRhdKkmt + Y7J0HIaxLrMup61TTx3ite7zXHcHaE88pPWklb5hdH2KPsfB7xxIssBnST5TtxZ2TL8smF4MBSz6 + xBlQrdg3BTSNpU8+5y36hMkvmZNeGNQyVpxPVuq+yNPbF3kAbGTyx255X6/PH+MGF3plX/DjVOw4 + HqI2BjTt21ciLmKOfQKHryY+tGzrUGR4vKiRt+oRwPWIh+Lr4YCyCMKu7Hh8qregAS7zYIzfuvCt + Iyurv362FCvbrqKTNr703fse3jP7ybEv4QQ/GMmsrPHdluSxpMEcxsPrnqyOz9nguQE1+0lX1cnj + Y9IyiXNy1eTTepj3YyCx26Vbmqed5Gb4G0laxwN8iQddtsq3CT/3kS/hol/VLniaQNgw4pckrDf7 + zuPQH5Jyu3113LbYaZk1yvyYMb/ZbD412o9zxryNkRHWI8ppFA/DcYJA4XxcpsNSMSM+2TFWxDN4 + aEEvXI9jwBT+R+Z7jux+/Ucm+6fHEmzpu8rYpd9GbwP4P6Zn//jQ5/A3iLfmS6VDjUsIHeaCQuC1 + AnBb1yWr6wASYKK/7rbLutNG0to8m0m/vOhZ0hFZ1gkiP3s+X4uL1rMfFZd4+3o9O5Uv9t3fT50X + v+z29aofyS/vLoh6axnHevksDhGvJIGnvSRlr9zi1kSPH1Xh9O1k5o14bL54bhvQuL4tDBL31+Ha + N+INvB2TSfKY5DhYsR60pEo4m7c8wpf0CdCL8+CjC589csPr/ghGTPGsahaXs7at3Png7+JQ70Dy + n89NMguQFMgWpoCPmT0GaNbj0gjP68QsszFH4rFcMz1r/XVLKkuuCgDIeQ3Okak5AfXTus83/kIn + DFKvbKSYb/kIDwdNWfakpvlBfu4r4iRPKy7563y8tLEPzOlS6lJRzy6pzRt+SPKyYWVIAr/QHLct + dhbIMZH5bRgYm0+N9uOcMW/jusNCo3gYjrwECufjMh2WihnxyY75YXF0HjBEk93JEfyDER8/euPr + Ph/zZ5v0zDvbtpX2c9dD181Odl+bzaZv5qHjUjEFyq1RCLxW8LKXIpOBBMRlV1ERkdsq60ruxSco + GSyXHklHpOmlZutsLgs+YMVOxm1S5Hp2Kp/0Nc77qfPyI9yJuCW7iE+4WzvCN6+8jdBzWS9fvdrG + qqtpMN9PkuQ3/aHmvMEf8rTsZH7nXeChH+ExFHysYuJD8ri/vjHxqNjwGIOHdqQnVuw/aAsPZoiH + wFe/aIH70RPT1RvXb7hqqT+2wO1u8nL2/KH7Rpv88LXfnZ+YvRWfQ/mqZUFJvno7lBIDSRV4XquS + VJFMUKg5VLLNYUwqA5g9NRko9nk95U9xHVkc+Cyb5A0/TL8smF4MZa/oE2dAtWLf/dJY+uRz3qJP + Ovklc+IPg1rGStlntWM98vT2RR6sNTL5Y8Wpr9fnj3GDCz232/CHvSE74SdkaYZzvuCxtYgLeoRq + DH/xRVyJQ6wDhgXDlSLjY1/IgjyyQ3q8GI/Hg6PEE5DJ5K4Xj+355bO9WNl2LcbnTLO3iEjCO7Dh + 85GLngsKgY2Vo558nAAmgD5efCLx8JX1SDE3QH3DaFyecHwsfucwVOBdFrzclD+8Sz5BHJIYkqnd + 1+vZGcZJ3wnoTPVDvvX16Df9cN/dro9a4ZffA+yXN+I0IO1y236SlD3b4UDr8Z+RnUzvvCo2yU+b + j7ga3se1aAWJ++vwXLQsXDYmD+2IRlliRclpSZVwNo8vK374g/Wj+F1Wt+It4F2EnQMvnvXnwE5j + i3iLuNp1X8ORv5l3jLngYVC2+OWPSxNFBEBmWRF+qTQmfX8dScXkoiFDyEC5/JiR+bAPieQt665X + x7RSDNq8E3ChDhOfWS32pV/5hKvjxM/t5iK46BfjEfy8dG4Xc7nYkN/2ZfOGH5KhPySBX2iO2xY7 + C+SYyPw2DIzNp1aKEBDcF9djh4VG8QCCRUpAsKBDO0WAxfmLGet48UavFKv5/Jx4C5hCze7Z/5aw + v2O8RTyxPnsrcuBLzI1SFJgNXix0OW1dl50dyxZf93mfpgmvFqVoWDJCQfqGqGOoYWRj52EPL0je + wGdJvOCmID9MvyyYXgwFLPrEGVDN5qVWcSqqWCdMREWfMPmV9cKglrFiDqRW9Y2v3Rd53JxpZV7q + BT6k8fb4Y7wR3uZP2Q7pW/+1v7Rv8wEt9oVeGgOHr7yPug4YFuSPx8HHvpAFeWSH9HhRhMKuv628 + 68WLzo23gBGFkL1TiulzQ67c/dBH8cth/w7JMPC3iJ6smyWXXQque7zsUvkYwpPYk5SQmnxaJ7xe + Auq7XdO3sUvi47TSvN8Sxw3wuT4B/qLLswU/95Ev4aJfcNDtQjaBsGHEL0mgm33ncegPSfe7EY7b + FjsNsQ8yP6bMbzabT+2VfMKCF0dPTrtbj7733HkLmELNblyB/vy5M7a3iJPJ13CJ8beIusTlbRxv + l4WICy7sEusSMosjgiw2pq/k1hOU65VQp0vvPznj0kfAo0jFfC0ujih23DCLRuLlOPYB2bNT+aSv + cd5PnS9uo9PXi426OW08NtGXfvmjqOmSR5wGJPC0l6Ts2YEMtB7/GdnJ9M674K/NR1wN72O9XfMh + edxfh5fiBjyPMXhMWt5IKL+C1nnw0cLvTbvpzYfP8r8F5HY3efGs3wRxLiz9/fcvXNkz/Wt8zu6j + VoTyJWYSeVFQluluMgcxr3UPErMwrSMLS3EjxMKdigtHsqdlZW2/ONQxUTKArs2bv/TLRfhFJP1O + /JzUOOMG+QGo8z2/nbfald8xNjO8nMA10ubxLbaejEAOSSPsN8c1/GEP2Jdkp89t48zvfITZfGql + CNm5Yt7GdWeFRnGwfCAvgcL5uEzTc1vHNxsz6K4XfuaqW/GvMB+N2XNVWozH5hFYvfP7+6az6R3d + tPuFthgge+ySUqQiFLlp+mVdudZ/sokkLkUgrq3ppUa7Nvb5gg9YseMTxKUi2Nfr2al80tdY+rwk + SZ9uFTO2b3OrTHg8iCr++qgVvI3Qc6lLHnEakMDRryQVX/NgoPX4z8hOpt/IX5uPuBrexy/DE9aD + k9nsDw7vf/WO/9dscthezr5n38tpYsm4D3aru/7nvz6Gvy7+FD5sepFy1sKEJLXLWoVfqnRX++v2 + ExVf5ZJ7kqtI1JQv69TfpPjEaXnRoB7vjhbcfLWXcekYiv20nmjSPnORWvSL8QCv/PDAuB0WJfA3 + UlEsV92jqrEXhyg2jUy+l67jG/6w91LtFNLUyfzOx1WbT03F188Z8zauJ6taZmEuOPISKJyPy7Ql + 2GTyHMafxFPV3+Cp6oRNjE0RiCswxqMfgQOPv3rX+vrtk9n0A7lK6bJ7EYrcNF1mZRG4hH4dbZ5N + 41IsFtYdFXiXBV9o0IFdFglTIW6xmBS9np0y3/Dn/chQ5a9+cbvZP/qhdfnh/b7gbTS/paDLy22E + d60Ejn4mSX7TH2rOG/whT8tO5nfeBR76YXFyf3y8HU9YID2A3wr68Rfec8WPsytjXxFQdo7R2DAC + q1/G28RudscEbxN1aVJRwuWNu0ICu8x+iS2VrWiNT1ip+CAmisuAjEAOyaHTcRyLCc/hDOxsxe9+ + E2Z2U1Mx25YnrAeRX+PbvxTboe5YsIai0p+zt4n/+/DHupP+NhHXjk8gcfsMP1CsSvUin34ilyec + eKYwvdTqk43mCz5gxY5PUN8vK3hCv+j17JR5t6tx3k+1S7eKGbuUlT/vV7hwkKP2pVeEFp5YjBff + Ec6hIiR7bbEoRnr88VPktOwUUnMI9hCnBR4WrfDYcfHDKYbkcX99Y+JxPqjbGAaem8/n49s/xmvr + l02ybGvlcw5hbxNPrN+OS/4BphqTmTmtUFgRwIILpLD/5LUJNpOpuHBkeF+nYlr3+VpkxEID6FJP + jnCB6jFvM0mfAH+pfLJbxwXg+8hFatGvhj+KhlPwcsJ+I7FGd4dk6A9J52yE4xr+sDfEjzlG/3T4 + nY/2TT812o9zxryN3RJR1RziZzhOENfBnzvn3cpt49u/FNAtun5TtkCNy00EVr/8yL7ZrLsDtw9v + E33Jq4ULJqff+gBApktPRSS3KaTGImRjn9+smFCNuMTb1+vZqXyyq7H0VU3qvPj5Cndy8XL/svtu + V+jea7213JcuOaMT3rUSeNpLkvEwnqHW40dVOH07mX8jHvphcXJ/fHyqf4aF/wfwwenK7Nbn333l + f2RzY3/rCLS3ZWv8iIgIHOhWdp98+Gbk6p/ioxC/xGm7tH6JLZXHJ6zxCctrnp6suul38fuqPvPC + 4Sv+afKB6clIpVGeegTGgnXqsdoQufKVR9+DJ67b8Psd3+Y1i8WqVC9q6idyeVKJZ4rek4mtCy5Z + 8HFSpShm3PiEVZ7AGJ+d9YTVzSf/PlmZ/eXz77783g2TaFw4pQjENTgl8AjaPAKrBx59G35Z4G3T + 2eQ94xPW4ts6SzbFZUDWRxG9Hc7jobD7Ot9eokg18qXa2Yrf+Qgzu6np7a29XcZ+MW9je7a2HaJ/ + 34lu8ukj+1/9n0ll7J5BBMaCdQbB20h194FHr8P/VH0bPgpxM56QVoRTEpcnpvEJa9PixTjh8ocs + T1BDQffihQrRFDsVk4HiCI4oLsEfsqF3vgUem4/zMwUfW9HCf3ir1x2Yz1f/4vD+yx5s+MbBGUdg + LFhnHMJNCA789xt3zY//yXQ6uwUPAedFkvNyQE0lDNLexljztzNl3efrWDDiCIce7470qR7zBk36 + NoxW+UIv/kDdEU7U4hbfdjb8USycgpccPI3EWr7qZr2MQ39IOmcjHNfwh73Mi/6WdhpiH2R+5+OK + zaemYmbxm78AS59fX9312aPvuuQHCTJ2tzECythtJBypBiLw5ceu3L3afQIrvz+bTi/WJfVr5EUl + tKxIsLmsRcMRNg+CFrdYTIpeXNcFPtkRTvqqHnVefoQ7UdSSf/Qj+eXdBdErQrrkqVhBwVgVF0jg + 6VeSLI69YlHs9PjjSeu07BRSc0gbXOChH/IY3afxj2rdMb1gz+3Pvn3vM1l97G9/BDz7tp94ZByI + wIFHL95tRWsy/QQ+hHNluaS5SOEGlGKzUGScM823RcYvfeaDivHldkr8XhRZRKi/6BeLSPD75Q47 + Q0Wn7BegUpxM38ahPySDNEvHbYudzBv9zI8585fN5q11kyfR+6vn1n/2c5PfnL6oyfH15Y5Am8kv + t7WRv0Rg99efeMvsxPEP4dL/Fr5fHwuluKSixMsdJ2XzmGhxi8WkFKV4dlngE6Fw0lcVqfP0ye0W + PudhsaIf7nnMx0ay7BWhhScWYM0M92kSeNpLUvYMMdB6/Bs+GUF1SzuZ3nmTv0+A4CsnT87uOvzu + Sx/I0LH/ykTA0/GVMTZaGYgAfjPbeV99Yt98Mv8Q7vz7cCAX9YtDHbt+Kj5tkfFLH8Uj45Lpyqfj + r+PE70WRRYQ8i0WRRQQqph9FIswMFR2zVoqS6eXxBkWHvEGaZS4msN/Yy7zob2kn80Y/+OfdM/jL + k3tOzlbvPrxv77ex2Q2qZiiO8uWMgJ3l2HZKBA4+ft7up2fvXem6D+Et434Ugt2bFRO6vUExKXpx + XYlTcalFKMbjExbjZUUKDWIdJfAb+Fly97OvufSfJ784XefC+PJTj8BYsH7qR7CBA/f+4JLz1ifv + xyPJh/FJ+rcDZXdKLRWf8Qlr8e1j80RlgcN3ebLzJ6fyRBhFChDgvg22uyd7ZveMf4DuubbDRFyB + HebW6E4Tgft+dPn568fxa266fd1sdj3elOCfKPOjo1x8uzY+YfnbRASyFCv0c/HCb0l4ZDad3Y8P + Th2c7971b4ffceFTTdzHwY6LwFiwdtyRbO3Q+fc9+dru+ORd+Dfa9uE22q91fq1pWe3Kb/c4wfn2 + mEsxS09q0nPbTtTiFotiwx9PLk7R/JkS+NIfXDdFoxST0B+SztkIx700O92T3by7fzJbOXhsNv/m + kesve7LhHAc7PgJtJu94d0cHhyKw51+fvGblJJ68Jp0Vr+uBubwUmygPC8VJRy+cipFXO5qweXVc + WNFBt85jHRMBqx3hm9deETqV4kW/oBeS/MYz1Hr88Xavm89/BP2D80l3cNKt3P/sb+x9bEh9nFue + CHhWLo/Do6dbRAB/Uvyqb/z4WvwPItdPZngCm3TvxCftL81aLAKYMGmtjjlUFWIxykVq5z9hwfGn + 8Ynzb+Fzbvcfn3YHD++77Hu+o1GcJREYC9ZZcpAbbgMF7Px7/+/nVqbHr5lMZ9fgf8y+Bk8g7OPh + 60345P0q/8ddK156hCJVFDM+oGGmFjVPGce/0k9YcPEE/Hwcny44hBJ6CNs7BPcOrc92HzryzvN/ + CEc3eAzbMELjwhJFYCxYS3RY2+4qfvXzRcefeuNkPkUBQyHDNxLiGpSna3DrX4Nixtuv4vXKPWF1 + MDvt5j/sJrNDkIcwftgK1ImTq4eeX7nwsfFfktn2TFgawrFgLc1RvcKOfqfbdeHzz188Pd5dMpkd + 3zudrO6dTE/unUxWICeXwJu9+EN/jPGNMYqa+jEnd9cg1vBEt4aiszaxD2HOpmvzebcG3TX8Mru1 + ldnkGfwt3RrK4Rp41ubT2drhC161NvmV6XFRjK9jBGoE/h//xb5CiJqhJQAAAABJRU5ErkJggg== + installModes: + - supported: true + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: false + type: AllNamespaces + install: + strategy: deployment + spec: + deployments: + - name: clickhouse-operator + spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse-operator + template: + metadata: + labels: + app: clickhouse-operator + spec: + containers: + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.memory + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: docker.io/altinity/clickhouse-operator:0.23.5 + imagePullPolicy: Always + name: clickhouse-operator + - image: docker.io/altinity/metrics-exporter:0.23.5 + imagePullPolicy: Always + name: metrics-exporter + serviceAccountName: clickhouse-operator + permissions: + - serviceAccountName: clickhouse-operator + rules: + # + # Core API group + # + - apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + # + # apps.* resources + # + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete + # The operator deployment personally, identified by name + - apiGroups: + - apps + resources: + - deployments + resourceNames: + - clickhouse-operator + verbs: + - get + - patch + - update + - delete + # + # policy.* resources + # + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + # + # apiextensions + # + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + # clickhouse - related resources + - apiGroups: + - clickhouse.altinity.com + # + # The operator's specific Custom Resources + # + + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete + # clickhouse-keeper - related resources + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete diff --git a/deploy/operatorhub/0.23.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..70beee4cb --- /dev/null +++ b/deploy/operatorhub/0.23.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1219 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallation +# SINGULAR=clickhouseinstallation +# PLURAL=clickhouseinstallations +# SHORT=chi +# OPERATOR_VERSION=0.23.5 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.23.5 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallation + singular: clickhouseinstallation + plural: clickhouseinstallations + shortNames: + - chi + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + jsonPath: .status.status + - name: hosts-unchanged + type: integer + description: Unchanged hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUnchanged + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHI completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/deploy/operatorhub/0.23.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..fa5b50c4c --- /dev/null +++ b/deploy/operatorhub/0.23.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1219 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallationTemplate +# SINGULAR=clickhouseinstallationtemplate +# PLURAL=clickhouseinstallationtemplates +# SHORT=chit +# OPERATOR_VERSION=0.23.5 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallationtemplates.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.23.5 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallationTemplate + singular: clickhouseinstallationtemplate + plural: clickhouseinstallationtemplates + shortNames: + - chit + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: CHI status + jsonPath: .status.status + - name: hosts-unchanged + type: integer + description: Unchanged hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUnchanged + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more ClickHouse clusters" + type: object + required: + - spec + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + status: + type: object + description: "Current ClickHouseInstallation manifest status, contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other" + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized CHI requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHI completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "DEPRECATED! VolumeClaimTemplate is deprecated in favor of DataVolumeClaimTemplate and LogVolumeClaimTemplate" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouse clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouse servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + type: + type: string + description: "DEPRECATED - to be removed soon" + shardsCount: + type: integer + description: "how much shards for current ClickHouse cluster will run in Kubernetes, each shard contains shared-nothing part of data and contains set of replicas, cluster contains 1 shard by default" + replicasCount: + type: integer + description: "how much replicas in each shards for current ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, every shard contains 1 replica by default" + shards: + type: array + description: "optional, allows override top-level `chi.spec.configuration`, cluster-level `chi.spec.configuration.clusters` settings for each shard separately, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: "use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, More info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: "list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `Chi` manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/deploy/operatorhub/0.23.5/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml b/deploy/operatorhub/0.23.5/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml new file mode 100644 index 000000000..6fa5be4dc --- /dev/null +++ b/deploy/operatorhub/0.23.5/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml @@ -0,0 +1,263 @@ +# Template Parameters: +# +# OPERATOR_VERSION=0.23.5 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + labels: + clickhouse-keeper.altinity.com/chop: 0.23.5 +spec: + group: clickhouse-keeper.altinity.com + scope: Namespaced + names: + kind: ClickHouseKeeperInstallation + singular: clickhousekeeperinstallation + plural: clickhousekeeperinstallations + shortNames: + - chk + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: status + type: string + description: CHK status + jsonPath: .status.status + - name: replicas + type: integer + description: Replica count + priority: 1 # show in wide view + jsonPath: .status.replicas + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: + - spec + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one ClickHouse Keeper cluster" + properties: + apiVersion: + type: string + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + kind: + type: string + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + metadata: + type: object + status: + type: object + description: | + Current ClickHouseKeeperInstallation status, contains many fields like overall status, desired replicas and ready replica list with their endpoints + properties: + chop-version: + type: string + description: "ClickHouse operator version" + chop-commit: + type: string + description: "ClickHouse operator git commit SHA" + chop-date: + type: string + description: "ClickHouse operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this CHI" + status: + type: string + description: "Status" + replicas: + type: integer + format: int32 + description: Replicas is the number of number of desired replicas in the cluster + readyReplicas: + type: array + description: ReadyReplicas is the array of endpoints of those ready replicas in the cluster + items: + type: object + properties: + host: + type: string + description: dns name or ip address for Keeper node + port: + type: integer + minimum: 0 + maximum: 65535 + description: TCP port which used to connect to Keeper node + secure: + type: string + description: if a secure connection to Keeper is required + normalized: + type: object + description: "Normalized CHK requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized CHK completed" + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: KeeperSpec defines the desired state of a Keeper cluster + properties: + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + replicas: + type: integer + format: int32 + description: | + Replicas is the expected size of the keeper cluster. + The valid range of size is from 1 to 7. + minimum: 1 + maximum: 7 + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + settings: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-keeper` instance" + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes ClickHouseKeeper clusters layout and allows change settings on cluster-level and replica-level + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of ClickHouseKeeper servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + layout: + type: object + description: | + describe current cluster layout, how many replicas + # nullable: true + properties: + replicasCount: + type: integer + description: "how many replicas in ClickHouseKeeper cluster" + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: "allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else" + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/operatorhub/0.23.5/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.5/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..fe76e8728 --- /dev/null +++ b/deploy/operatorhub/0.23.5/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,415 @@ +# Template Parameters: +# +# NONE +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.23.5 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseOperatorConfiguration + singular: clickhouseoperatorconfiguration + plural: clickhouseoperatorconfigurations + shortNames: + - chopconf + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: namespaces + type: string + description: Watch namespaces + jsonPath: .status + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" + x-kubernetes-preserve-unknown-fields: true + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + Allows to define settings of the clickhouse-operator. + More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml + Check into etc-clickhouse-operator* ConfigMaps if you need more control + x-kubernetes-preserve-unknown-fields: true + properties: + watch: + type: object + description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" + properties: + namespaces: + type: array + description: "List of namespaces where clickhouse-operator watches for events." + items: + type: string + clickhouse: + type: object + description: "Clickhouse related parameters used by clickhouse-operator" + properties: + configuration: + type: object + properties: + file: + type: object + properties: + path: + type: object + description: | + Each 'path' can be either absolute or relative. + In case path is absolute - it is used as is. + In case path is relative - it is relative to the folder where configuration file you are reading right now is located. + properties: + common: + type: string + description: | + Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. + Default value - config.d + host: + type: string + description: | + Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. + Default value - conf.d + user: + type: string + description: | + Path to the folder where ClickHouse configuration files with users settings are located. + Files are common for all instances within a CHI. + Default value - users.d + user: + type: object + description: "Default parameters for any user which will create" + properties: + default: + type: object + properties: + profile: + type: string + description: "ClickHouse server configuration `...` for any " + quota: + type: string + description: "ClickHouse server configuration `...` for any " + networksIP: + type: array + description: "ClickHouse server configuration `...` for any " + items: + type: string + password: + type: string + description: "ClickHouse server configuration `...` for any " + network: + type: object + description: "Default network parameters for any user which will create" + properties: + hostRegexpTemplate: + type: string + description: "ClickHouse server configuration `...` for any " + configurationRestartPolicy: + type: object + description: "Configuration restart policy describes what configuration changes require ClickHouse restart" + properties: + rules: + type: array + description: "Array of set of rules per specified ClickHouse versions" + items: + type: object + properties: + version: + type: string + description: "ClickHouse version expression" + rules: + type: array + description: "Set of configuration rules for specified ClickHouse version" + items: + type: object + description: "setting: value pairs for configuration restart policy" + access: + type: object + description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" + properties: + scheme: + type: string + description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto" + username: + type: string + description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + password: + type: string + description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + rootCA: + type: string + description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + secret: + type: object + properties: + namespace: + type: string + description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + name: + type: string + description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + port: + type: integer + minimum: 1 + maximum: 65535 + description: "Port to be used by operator to connect to ClickHouse instances" + timeouts: + type: object + description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" + properties: + connect: + type: integer + minimum: 1 + maximum: 10 + description: "Timout to setup connection from the operator to ClickHouse instances. In seconds." + query: + type: integer + minimum: 1 + maximum: 600 + description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." + metrics: + type: object + description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" + properties: + timeouts: + type: object + description: | + Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances + Specified in seconds. + properties: + collect: + type: integer + minimum: 1 + maximum: 600 + description: | + Timeout used to limit metrics collection request. In seconds. + Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. + All collected metrics are returned. + template: + type: object + description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" + properties: + chi: + type: object + properties: + policy: + type: string + description: | + CHI template updates handling policy + Possible policy values: + - ReadOnStart. Accept CHIT updates on the operators start only. + - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI + enum: + - "" + - "ReadOnStart" + - "ApplyOnNextReconcile" + path: + type: string + description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileCHIsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + statefulSet: + type: object + description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" + properties: + create: + type: object + description: "Behavior during create StatefulSet" + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. delete - delete newly created problematic StatefulSet. + 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. + update: + type: object + description: "Behavior during update StatefulSet" + properties: + timeout: + type: integer + description: "How many seconds to wait for created/updated StatefulSet to be Ready" + pollInterval: + type: integer + description: "How many seconds to wait between checks for created/updated StatefulSet status" + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. + 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. + host: + type: object + description: | + Whether the operator during reconcile procedure should wait for a ClickHouse host: + - to be excluded from a ClickHouse cluster + - to complete all running queries + - to be included into a ClickHouse cluster + respectfully before moving forward + properties: + wait: + type: object + properties: + exclude: &TypeStringBool + type: string + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster" + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + queries: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" + include: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + annotation: + type: object + description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + include annotations with names from the following list + items: + type: string + exclude: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + exclude annotations with names from the following list + items: + type: string + label: + type: object + description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + include labels from the following list + items: + type: string + exclude: + type: array + items: + type: string + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + exclude labels from the following list + appendScope: + !!merge <<: *TypeStringBool + description: | + Whether to append *Scope* labels to StatefulSet and Pod + - "LabelShardScopeIndex" + - "LabelReplicaScopeIndex" + - "LabelCHIScopeIndex" + - "LabelCHIScopeCycleSize" + - "LabelCHIScopeCycleIndex" + - "LabelCHIScopeCycleOffset" + - "LabelClusterScopeIndex" + - "LabelClusterScopeCycleSize" + - "LabelClusterScopeCycleIndex" + - "LabelClusterScopeCycleOffset" + statefulSet: + type: object + description: "define StatefulSet-specific parameters" + properties: + revisionHistoryLimit: + type: integer + description: "revisionHistoryLimit is the maximum number of revisions that will be\nmaintained in the StatefulSet's revision history. \nLook details in `statefulset.spec.revisionHistoryLimit`\n" + pod: + type: object + description: "define pod specific parameters" + properties: + terminationGracePeriod: + type: integer + description: "Optional duration in seconds the pod needs to terminate gracefully. \nLook details in `pod.spec.terminationGracePeriodSeconds`\n" + logger: + type: object + description: "allow setup clickhouse-operator logger behavior" + properties: + logtostderr: + type: string + description: "boolean, allows logs to stderr" + alsologtostderr: + type: string + description: "boolean allows logs to stderr and files both" + v: + type: string + description: "verbosity level of clickhouse-operator log, default - 1 max - 9" + stderrthreshold: + type: string + vmodule: + type: string + description: | + Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. + Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. + log_backtrace_at: + type: string + description: | + It can be set to a file and line number with a logging line. + Ex.: file.go:123 + Each time when this line is being executed, a stack trace will be written to the Info log. diff --git a/dev/generate_helm_chart.sh b/dev/generate_helm_chart.sh index cba8278df..3d4f76356 100755 --- a/dev/generate_helm_chart.sh +++ b/dev/generate_helm_chart.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash function usage() { -cat << EOT + cat << EOT Script splits clickhouse-operator-install-bundle.yaml to separate files and adjusts them to conform the helm standards NOTE script requires some pre-installed tools: - yq ( https://mikefarah.gitbook.io/yq/ ) > v4.14.x. Do not use brew install yq in MacOS，Version is lower than it. @@ -13,13 +13,14 @@ cat << EOT EOT } -for cmd in yq jq helm-docs perl; do - if ! command -v ${cmd} &> /dev/null; then - usage - exit 1 - fi -done - +function check_required_tools() { + for cmd in yq jq helm-docs perl; do + if ! command -v "${cmd}" &> /dev/null; then + usage + exit 1 + fi + done +} set -o errexit set -o nounset @@ -29,7 +30,7 @@ readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pw readonly manifest_file_path="${script_dir}/../deploy/operator/clickhouse-operator-install-bundle.yaml" readonly release_file_path="${script_dir}/../release" readonly dashboards_path="${script_dir}/../grafana-dashboard" -readonly chart_path="${script_dir}/../deploy/helm" +readonly chart_path="${script_dir}/../deploy/helm/clickhouse-operator" readonly values_yaml="${chart_path}/values.yaml" function main() { @@ -45,7 +46,7 @@ function main() { done # update version in Chart.yaml - readonly detected_version=$(cat ${release_file_path}) + readonly detected_version=$(cat "${release_file_path}") readonly chart_yaml="${chart_path}/Chart.yaml" yq e '.version = "'"${detected_version}"'"' -i "${chart_yaml}" yq e '.appVersion = "'"${detected_version}"'"' -i "${chart_yaml}" @@ -73,10 +74,8 @@ function main() { function process() { local file="${1}" - local kind - kind=$(yq e '.kind' "${file}") - local name - name=$(yq e '.metadata.name' "${file}") + local kind=$(yq e '.kind' "${file}") + local name=$(yq e '.metadata.name' "${file}") local processed_file="${kind}-${name}.yaml" local copied_file="${processed_file}" @@ -127,7 +126,8 @@ function process() { CustomResourceDefinition) ;; *) - echo "do not know how to process ${kind}" + echo "Do not know how to process resource kind: ${kind}" + echo "Abort." exit 1 ;; esac @@ -354,7 +354,9 @@ function to_camel_case() { } function get_abs_filename() { - echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" + readonly filename="${1}" + echo "$(cd "$(dirname "${filename}")" && pwd)/$(basename "${filename}")" } +check_required_tools main diff --git a/dev/image_build_universal.sh b/dev/image_build_universal.sh index 607de62bf..ad982e15f 100644 --- a/dev/image_build_universal.sh +++ b/dev/image_build_universal.sh @@ -17,20 +17,28 @@ source "${CUR_DIR}/go_build_config.sh" source "${CUR_DIR}/build_manifests.sh" -# Build image with Docker +cat << EOF +######################################## +Build vars: +DOCKERHUB_LOGIN=${DOCKERHUB_LOGIN} +DOCKERHUB_PUBLISH=${DOCKERHUB_PUBLISH} +MINIKUBE=${MINIKUBE} +EOF + if [[ "${MINIKUBE}" == "yes" ]]; then - # We'd like to build for minikube + echo "Going to build on minikube, not on the build host itself." + echo "Minikube is expected to be run on build host, though" eval "$(minikube docker-env)" fi +# In case architecture of the host we are building on is arm，such as MacOS M1/M2, no need to install qemu +# We may need to install qemu otherwise ARCHITECTURE=$(uname -m) -# Do nothing if architecture is arm，such as MacOS M1/M2 - -# We may need to install qemu if [[ "${ARCHITECTURE}" =~ "arm" ]]; then - echo "arm host does not need qemu to be installed" + echo "Build host is arm and does not need qemu to be installed" else - echo "need qemu to be installed" + echo "Need qemu to be installed on build host" + echo "Check whether qemu is available" if docker run --rm --privileged multiarch/qemu-user-static --reset -p yes; then echo "qemu is in place, continue." else @@ -63,10 +71,10 @@ DOCKER_CMD="docker buildx build --progress plain" # Append arch if [[ "${DOCKER_IMAGE}" =~ ":dev" || "${MINIKUBE}" == "yes" ]]; then - echo "Building dev images, skip arm arch." + echo "Building dev images for amd64 only, skip arm arch." DOCKER_CMD="${DOCKER_CMD} --platform=linux/amd64 --output type=image,name=${DOCKER_IMAGE}" else - echo "Going to build for amd64 and arm64." + echo "Going to build for both amd64 and arm64." DOCKER_CMD="${DOCKER_CMD} --platform=linux/amd64,linux/arm64" fi @@ -93,10 +101,17 @@ if [[ "${DOCKERHUB_PUBLISH}" == "yes" ]]; then fi fi +echo "Docker build command ready:" +echo "${DOCKER_CMD}" +echo "Starting docker image build." +echo "Please, wait..." + if ${DOCKER_CMD}; then echo "OK. Build successful." else - echo "ERROR. Build has failed." + echo "########################" + echo "ERROR." + echo "Docker image build has failed." echo "Abort" exit 1 fi diff --git a/dev/run_operator.sh b/dev/run_operator.sh index 10029f367..b4bc609ef 100755 --- a/dev/run_operator.sh +++ b/dev/run_operator.sh @@ -10,20 +10,24 @@ source "${CUR_DIR}/go_build_config.sh" # Setup paths LOG_DIR="${CUR_DIR}/log" -echo "Building ${OPERATOR_BIN}, please wait..." -if [[ $1 == "nobuild" ]]; then +echo "Check whether we need to build the operator" +if [[ "${1}" == "nobuild" ]]; then echo "Build step skipped, starting old binary" else + echo "Building executable: ${OPERATOR_BIN}" + echo "Please wait..." if "${CUR_DIR}/go_build_operator.sh"; then echo "Successfully built ${OPERATOR_BIN}." else - echo "Unable to build ${OPERATOR_BIN}. Abort." + echo "Unable to build ${OPERATOR_BIN}." + echo "Abort." exit 1 fi fi if [[ ! -x "${OPERATOR_BIN}" ]]; then - echo "Unable to start ${OPERATOR_BIN} Is not executable or not found. Abort" + echo "Unable to start ${OPERATOR_BIN} Is not executable or not found." + echo "Abort." exit 2 fi @@ -37,7 +41,7 @@ rm -f "${LOG_DIR}"/clickhouse-operator.*.log.* -config="${SRC_ROOT}/config/config-dev.yaml" \ -alsologtostderr=true \ -log_dir=log \ - -v=${VERBOSITY} 2>&1 | tee operator_output + -v="${VERBOSITY}" 2>&1 | tee operator_output # -logtostderr=true \ # -stderrthreshold=FATAL \ @@ -46,7 +50,7 @@ rm -f "${LOG_DIR}"/clickhouse-operator.*.log.* # -logtostderr=true Logs are written to standard error instead of to files # -stderrthreshold=FATAL Log events at or above this severity are logged to standard error as well as to files -if [[ $2 == "noclean" ]]; then +if [[ "${2}" == "noclean" ]]; then echo "Clean step skipped" else # And clean binary after run. It'll be rebuilt next time diff --git a/docs/operator_installation_details.md b/docs/operator_installation_details.md index 3aa36f7e4..1c63275de 100644 --- a/docs/operator_installation_details.md +++ b/docs/operator_installation_details.md @@ -68,7 +68,7 @@ helm repo upgrade clickhouse-operator helm upgrade clickhouse-operator clickhouse-operator/altinity-clickhouse-operator ``` -Look https://github.com/Altinity/clickhouse-operator/tree/master/deploy/helm/ for details +Look https://github.com/Altinity/clickhouse-operator/tree/master/deploy/helm/clickhouse-operator/ for details ## Resources Description diff --git a/go.mod b/go.mod index 2a94758ea..9ff2047b5 100644 --- a/go.mod +++ b/go.mod @@ -29,22 +29,22 @@ require ( require ( github.com/MakeNowJust/heredoc v1.0.0 github.com/Masterminds/semver/v3 v3.2.0 - github.com/go-logr/logr v1.2.4 + github.com/go-logr/logr v1.4.1 github.com/golang/glog v1.0.0 github.com/google/uuid v1.4.0 github.com/imdario/mergo v0.3.15 github.com/juliangruber/go-intersect v1.0.0 github.com/kubernetes-sigs/yaml v1.1.0 github.com/mailru/go-clickhouse/v2 v2.1.0 - github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_golang v1.18.0 github.com/sanity-io/litter v1.3.0 github.com/securego/gosec/v2 v2.8.1 github.com/stretchr/testify v1.8.4 - go.opentelemetry.io/otel v1.19.0 - go.opentelemetry.io/otel/exporters/prometheus v0.42.0 - go.opentelemetry.io/otel/metric v1.19.0 - go.opentelemetry.io/otel/sdk v1.19.0 - go.opentelemetry.io/otel/sdk/metric v1.19.0 + go.opentelemetry.io/otel v1.24.0 + go.opentelemetry.io/otel/exporters/prometheus v0.46.0 + go.opentelemetry.io/otel/metric v1.24.0 + go.opentelemetry.io/otel/sdk v1.24.0 + go.opentelemetry.io/otel/sdk/metric v1.24.0 gopkg.in/d4l3k/messagediff.v1 v1.2.1 gopkg.in/yaml.v3 v3.0.1 sigs.k8s.io/controller-runtime v0.15.1 @@ -69,32 +69,32 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/gookit/color v1.4.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect - go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index 6a3d0d4f5..ff33d4fc5 100644 --- a/go.sum +++ b/go.sum @@ -127,8 +127,9 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= @@ -194,8 +195,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -299,8 +300,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -357,23 +358,23 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/pseudomuto/protoc-gen-doc v1.3.2/go.mod h1:y5+P6n3iGrbKG+9O04V5ld71in3v/bX88wUwgt+U8EA= github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -455,18 +456,18 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= -go.opentelemetry.io/otel/exporters/prometheus v0.42.0 h1:jwV9iQdvp38fxXi8ZC+lNpxjK16MRcZlpDYvbuO1FiA= -go.opentelemetry.io/otel/exporters/prometheus v0.42.0/go.mod h1:f3bYiqNqhoPxkvI2LrXqQVC546K7BuRDL/kKuxkujhA= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k= -go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ= +go.opentelemetry.io/otel/exporters/prometheus v0.46.0/go.mod h1:ztwVUHe5DTR/1v7PeuGRnU5Bbd4QKYwApWmuutKsJSs= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= +go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -573,8 +574,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -636,8 +637,8 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/types.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/types.go index 712b2b5ae..ec075bfc0 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/types.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/types.go @@ -159,8 +159,8 @@ func (chk *ClickHouseKeeperInstallation) MergeFrom(from *ClickHouseKeeperInstall // ChkSpec defines spec section of ClickHouseKeeper resource type ChkSpec struct { - Configuration *ChkConfiguration `json:"configuration,omitempty" yaml:"configuration,omitempty"` - Templates *apiChi.ChiTemplates `json:"templates,omitempty" yaml:"templates,omitempty"` + Configuration *ChkConfiguration `json:"configuration,omitempty" yaml:"configuration,omitempty"` + Templates *apiChi.Templates `json:"templates,omitempty" yaml:"templates,omitempty"` } func (spec ChkSpec) GetConfiguration() *ChkConfiguration { @@ -174,7 +174,7 @@ func (spec ChkSpec) EnsureConfiguration() *ChkConfiguration { return spec.Configuration } -func (spec ChkSpec) GetTemplates() *apiChi.ChiTemplates { +func (spec ChkSpec) GetTemplates() *apiChi.Templates { return spec.Templates } diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go index 420f808ec..5df513df4 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/zz_generated.deepcopy.go @@ -105,7 +105,7 @@ func (in *ChkSpec) DeepCopyInto(out *ChkSpec) { } if in.Templates != nil { in, out := &in.Templates, &out.Templates - *out = new(clickhousealtinitycomv1.ChiTemplates) + *out = new(clickhousealtinitycomv1.Templates) (*in).DeepCopyInto(*out) } return diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go index ecfd9cb27..7c410e3b7 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go @@ -56,7 +56,7 @@ func (chi *ClickHouseInstallation) FillSelfCalculatedAddressInfo() { // What is the max number of Pods allowed per Node // TODO need to support multi-cluster maxNumberOfPodsPerNode := 0 - chi.WalkPodTemplates(func(template *ChiPodTemplate) { + chi.WalkPodTemplates(func(template *PodTemplate) { for i := range template.PodDistribution { podDistribution := &template.PodDistribution[i] if podDistribution.Type == deployment.PodDistributionMaxNumberPerNode { @@ -491,7 +491,7 @@ func (chi *ClickHouseInstallation) HostsCount() int { } // HostsCountAttributes counts hosts by attributes -func (chi *ClickHouseInstallation) HostsCountAttributes(a *ChiHostReconcileAttributes) int { +func (chi *ClickHouseInstallation) HostsCountAttributes(a *HostReconcileAttributes) int { count := 0 chi.WalkHosts(func(host *ChiHost) error { if host.GetReconcileAttributes().Any(a) { @@ -502,16 +502,16 @@ func (chi *ClickHouseInstallation) HostsCountAttributes(a *ChiHostReconcileAttri return count } -// GetHostTemplate gets ChiHostTemplate by name -func (chi *ClickHouseInstallation) GetHostTemplate(name string) (*ChiHostTemplate, bool) { +// GetHostTemplate gets HostTemplate by name +func (chi *ClickHouseInstallation) GetHostTemplate(name string) (*HostTemplate, bool) { if !chi.Spec.Templates.GetHostTemplatesIndex().Has(name) { return nil, false } return chi.Spec.Templates.GetHostTemplatesIndex().Get(name), true } -// GetPodTemplate gets ChiPodTemplate by name -func (chi *ClickHouseInstallation) GetPodTemplate(name string) (*ChiPodTemplate, bool) { +// GetPodTemplate gets PodTemplate by name +func (chi *ClickHouseInstallation) GetPodTemplate(name string) (*PodTemplate, bool) { if !chi.Spec.Templates.GetPodTemplatesIndex().Has(name) { return nil, false } @@ -519,12 +519,12 @@ func (chi *ClickHouseInstallation) GetPodTemplate(name string) (*ChiPodTemplate, } // WalkPodTemplates walks over all PodTemplates -func (chi *ClickHouseInstallation) WalkPodTemplates(f func(template *ChiPodTemplate)) { +func (chi *ClickHouseInstallation) WalkPodTemplates(f func(template *PodTemplate)) { chi.Spec.Templates.GetPodTemplatesIndex().Walk(f) } -// GetVolumeClaimTemplate gets ChiVolumeClaimTemplate by name -func (chi *ClickHouseInstallation) GetVolumeClaimTemplate(name string) (*ChiVolumeClaimTemplate, bool) { +// GetVolumeClaimTemplate gets VolumeClaimTemplate by name +func (chi *ClickHouseInstallation) GetVolumeClaimTemplate(name string) (*VolumeClaimTemplate, bool) { if chi.Spec.Templates.GetVolumeClaimTemplatesIndex().Has(name) { return chi.Spec.Templates.GetVolumeClaimTemplatesIndex().Get(name), true } @@ -532,23 +532,23 @@ func (chi *ClickHouseInstallation) GetVolumeClaimTemplate(name string) (*ChiVolu } // WalkVolumeClaimTemplates walks over all VolumeClaimTemplates -func (chi *ClickHouseInstallation) WalkVolumeClaimTemplates(f func(template *ChiVolumeClaimTemplate)) { +func (chi *ClickHouseInstallation) WalkVolumeClaimTemplates(f func(template *VolumeClaimTemplate)) { if chi == nil { return } chi.Spec.Templates.GetVolumeClaimTemplatesIndex().Walk(f) } -// GetServiceTemplate gets ChiServiceTemplate by name -func (chi *ClickHouseInstallation) GetServiceTemplate(name string) (*ChiServiceTemplate, bool) { +// GetServiceTemplate gets ServiceTemplate by name +func (chi *ClickHouseInstallation) GetServiceTemplate(name string) (*ServiceTemplate, bool) { if !chi.Spec.Templates.GetServiceTemplatesIndex().Has(name) { return nil, false } return chi.Spec.Templates.GetServiceTemplatesIndex().Get(name), true } -// GetCHIServiceTemplate gets ChiServiceTemplate of a CHI -func (chi *ClickHouseInstallation) GetCHIServiceTemplate() (*ChiServiceTemplate, bool) { +// GetCHIServiceTemplate gets ServiceTemplate of a CHI +func (chi *ClickHouseInstallation) GetCHIServiceTemplate() (*ServiceTemplate, bool) { if !chi.Spec.Defaults.Templates.HasServiceTemplate() { return nil, false } @@ -722,7 +722,7 @@ func (chi *ClickHouseInstallation) EnsureRuntime() *ClickHouseInstallationRuntim defer chi.runtimeCreatorMutex.Unlock() // Note that we have to check this property again to avoid a TOCTOU bug. if chi.runtime == nil { - chi.runtime = &ClickHouseInstallationRuntime{} + chi.runtime = newClickHouseInstallationRuntime() } return chi.runtime } @@ -823,3 +823,31 @@ func (chi *ClickHouseInstallation) FirstHost() *ChiHost { }) return result } + +func (chi *ClickHouseInstallation) GetName() string { + if chi == nil { + return "" + } + return chi.Name +} + +func (chi *ClickHouseInstallation) GetNamespace() string { + if chi == nil { + return "" + } + return chi.Namespace +} + +func (chi *ClickHouseInstallation) GetLabels() map[string]string { + if chi == nil { + return nil + } + return chi.Labels +} + +func (chi *ClickHouseInstallation) GetAnnotations() map[string]string { + if chi == nil { + return nil + } + return chi.Annotations +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go b/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go index 0ad29983b..2a58e7bb8 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go @@ -155,7 +155,7 @@ func (cluster *Cluster) InheritTemplatesFrom(chi *ClickHouseInstallation) { } // GetServiceTemplate returns service template, if exists -func (cluster *Cluster) GetServiceTemplate() (*ChiServiceTemplate, bool) { +func (cluster *Cluster) GetServiceTemplate() (*ServiceTemplate, bool) { if !cluster.Templates.HasClusterServiceTemplate() { return nil, false } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go index c3a0ce3b8..039681ce4 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_config_chop.go @@ -594,7 +594,7 @@ func (c *OperatorConfig) unlistCHITemplate(template *ClickHouseInstallation) { } // FindTemplate finds specified template within possibly specified namespace -func (c *OperatorConfig) FindTemplate(templateRef *ChiTemplateRef, fallbackNamespace string) *ClickHouseInstallation { +func (c *OperatorConfig) FindTemplate(templateRef *TemplateRef, fallbackNamespace string) *ClickHouseInstallation { c.Template.CHI.Runtime.mutex.RLock() defer c.Template.CHI.Runtime.mutex.RUnlock() diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_host.go b/pkg/apis/clickhouse.altinity.com/v1/type_host.go index c2d3ec7d3..4b6e34f16 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_host.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_host.go @@ -43,10 +43,10 @@ type ChiHost struct { type ChiHostRuntime struct { // Internal data - Address ChiHostAddress `json:"-" yaml:"-"` - Config ChiHostConfig `json:"-" yaml:"-"` - Version *swversion.SoftWareVersion `json:"-" yaml:"-"` - reconcileAttributes *ChiHostReconcileAttributes `json:"-" yaml:"-" testdiff:"ignore"` + Address ChiHostAddress `json:"-" yaml:"-"` + Config ChiHostConfig `json:"-" yaml:"-"` + Version *swversion.SoftWareVersion `json:"-" yaml:"-"` + reconcileAttributes *HostReconcileAttributes `json:"-" yaml:"-" testdiff:"ignore"` // CurStatefulSet is a current stateful set, fetched from k8s CurStatefulSet *apps.StatefulSet `json:"-" yaml:"-" testdiff:"ignore"` // DesiredStatefulSet is a desired stateful set - reconcile target @@ -55,7 +55,7 @@ type ChiHostRuntime struct { } // GetReconcileAttributes is an ensurer getter -func (host *ChiHost) GetReconcileAttributes() *ChiHostReconcileAttributes { +func (host *ChiHost) GetReconcileAttributes() *HostReconcileAttributes { if host == nil { return nil } @@ -88,7 +88,7 @@ func (host *ChiHost) InheritFilesFrom(shard *ChiShard, replica *ChiReplica) { } // InheritTemplatesFrom inherits templates from specified shard and replica -func (host *ChiHost) InheritTemplatesFrom(shard *ChiShard, replica *ChiReplica, template *ChiHostTemplate) { +func (host *ChiHost) InheritTemplatesFrom(shard *ChiShard, replica *ChiReplica, template *HostTemplate) { if shard != nil { host.Templates = host.Templates.MergeFrom(shard.Templates, MergeTypeFillEmptyValues) } @@ -136,7 +136,7 @@ func (host *ChiHost) MergeFrom(from *ChiHost) { } // GetHostTemplate gets host template -func (host *ChiHost) GetHostTemplate() (*ChiHostTemplate, bool) { +func (host *ChiHost) GetHostTemplate() (*HostTemplate, bool) { if !host.Templates.HasHostTemplate() { return nil, false } @@ -145,7 +145,7 @@ func (host *ChiHost) GetHostTemplate() (*ChiHostTemplate, bool) { } // GetPodTemplate gets pod template -func (host *ChiHost) GetPodTemplate() (*ChiPodTemplate, bool) { +func (host *ChiHost) GetPodTemplate() (*PodTemplate, bool) { if !host.Templates.HasPodTemplate() { return nil, false } @@ -154,7 +154,7 @@ func (host *ChiHost) GetPodTemplate() (*ChiPodTemplate, bool) { } // GetServiceTemplate gets service template -func (host *ChiHost) GetServiceTemplate() (*ChiServiceTemplate, bool) { +func (host *ChiHost) GetServiceTemplate() (*ServiceTemplate, bool) { if !host.Templates.HasReplicaServiceTemplate() { return nil, false } @@ -245,7 +245,7 @@ func (host *ChiHost) HasAncestorCHI() bool { } // WalkVolumeClaimTemplates walks VolumeClaimTemplate(s) -func (host *ChiHost) WalkVolumeClaimTemplates(f func(template *ChiVolumeClaimTemplate)) { +func (host *ChiHost) WalkVolumeClaimTemplates(f func(template *VolumeClaimTemplate)) { host.GetCHI().WalkVolumeClaimTemplates(f) } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go b/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go index 6b8586638..57a097b2b 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go @@ -29,8 +29,8 @@ const ( ObjectStatusUnknown ObjectStatus = "unknown" ) -// ChiHostReconcileAttributes defines host reconcile status and attributes -type ChiHostReconcileAttributes struct { +// HostReconcileAttributes defines host reconcile status and attributes +type HostReconcileAttributes struct { status ObjectStatus // Attributes are used by config generator @@ -39,15 +39,17 @@ type ChiHostReconcileAttributes struct { remove bool modify bool found bool + + exclude bool } // NewChiHostReconcileAttributes creates new reconcile attributes -func NewChiHostReconcileAttributes() *ChiHostReconcileAttributes { - return &ChiHostReconcileAttributes{} +func NewChiHostReconcileAttributes() *HostReconcileAttributes { + return &HostReconcileAttributes{} } // Equal checks whether reconcile attributes are equal -func (s *ChiHostReconcileAttributes) Equal(to ChiHostReconcileAttributes) bool { +func (s *HostReconcileAttributes) Equal(to HostReconcileAttributes) bool { if s == nil { return false } @@ -55,11 +57,12 @@ func (s *ChiHostReconcileAttributes) Equal(to ChiHostReconcileAttributes) bool { (s.add == to.add) && (s.remove == to.remove) && (s.modify == to.modify) && - (s.found == to.found) + (s.found == to.found) && + (s.exclude == to.exclude) } // Any checks whether any of the attributes is set -func (s *ChiHostReconcileAttributes) Any(of *ChiHostReconcileAttributes) bool { +func (s *HostReconcileAttributes) Any(of *HostReconcileAttributes) bool { if s == nil { return false } @@ -70,11 +73,12 @@ func (s *ChiHostReconcileAttributes) Any(of *ChiHostReconcileAttributes) bool { (s.add && of.add) || (s.remove && of.remove) || (s.modify && of.modify) || - (s.found && of.found) + (s.found && of.found) || + (s.exclude && of.exclude) } // SetStatus sets status -func (s *ChiHostReconcileAttributes) SetStatus(status ObjectStatus) *ChiHostReconcileAttributes { +func (s *HostReconcileAttributes) SetStatus(status ObjectStatus) *HostReconcileAttributes { if s == nil { return s } @@ -83,7 +87,7 @@ func (s *ChiHostReconcileAttributes) SetStatus(status ObjectStatus) *ChiHostReco } // GetStatus gets status -func (s *ChiHostReconcileAttributes) GetStatus() ObjectStatus { +func (s *HostReconcileAttributes) GetStatus() ObjectStatus { if s == nil { return ObjectStatus("") } @@ -91,7 +95,7 @@ func (s *ChiHostReconcileAttributes) GetStatus() ObjectStatus { } // SetAdd sets 'add' attribute -func (s *ChiHostReconcileAttributes) SetAdd() *ChiHostReconcileAttributes { +func (s *HostReconcileAttributes) SetAdd() *HostReconcileAttributes { if s == nil { return s } @@ -100,7 +104,7 @@ func (s *ChiHostReconcileAttributes) SetAdd() *ChiHostReconcileAttributes { } // UnsetAdd unsets 'add' attribute -func (s *ChiHostReconcileAttributes) UnsetAdd() *ChiHostReconcileAttributes { +func (s *HostReconcileAttributes) UnsetAdd() *HostReconcileAttributes { if s == nil { return s } @@ -108,8 +112,16 @@ func (s *ChiHostReconcileAttributes) UnsetAdd() *ChiHostReconcileAttributes { return s } +// IsAdd checks whether 'add' attribute is set +func (s *HostReconcileAttributes) IsAdd() bool { + if s == nil { + return false + } + return s.add +} + // SetRemove sets 'remove' attribute -func (s *ChiHostReconcileAttributes) SetRemove() *ChiHostReconcileAttributes { +func (s *HostReconcileAttributes) SetRemove() *HostReconcileAttributes { if s == nil { return s } @@ -117,8 +129,16 @@ func (s *ChiHostReconcileAttributes) SetRemove() *ChiHostReconcileAttributes { return s } +// IsRemove checks whether 'remove' attribute is set +func (s *HostReconcileAttributes) IsRemove() bool { + if s == nil { + return false + } + return s.remove +} + // SetModify sets 'modify' attribute -func (s *ChiHostReconcileAttributes) SetModify() *ChiHostReconcileAttributes { +func (s *HostReconcileAttributes) SetModify() *HostReconcileAttributes { if s == nil { return s } @@ -126,8 +146,16 @@ func (s *ChiHostReconcileAttributes) SetModify() *ChiHostReconcileAttributes { return s } +// IsModify checks whether 'modify' attribute is set +func (s *HostReconcileAttributes) IsModify() bool { + if s == nil { + return false + } + return s.modify +} + // SetFound sets 'found' attribute -func (s *ChiHostReconcileAttributes) SetFound() *ChiHostReconcileAttributes { +func (s *HostReconcileAttributes) SetFound() *HostReconcileAttributes { if s == nil { return s } @@ -135,51 +163,54 @@ func (s *ChiHostReconcileAttributes) SetFound() *ChiHostReconcileAttributes { return s } -// IsAdd checks whether 'add' attribute is set -func (s *ChiHostReconcileAttributes) IsAdd() bool { +// IsFound checks whether 'found' attribute is set +func (s *HostReconcileAttributes) IsFound() bool { if s == nil { return false } - return s.add + return s.found } -// IsRemove checks whether 'remove' attribute is set -func (s *ChiHostReconcileAttributes) IsRemove() bool { +// SetExclude sets 'exclude' attribute +func (s *HostReconcileAttributes) SetExclude() *HostReconcileAttributes { if s == nil { - return false + return s } - return s.remove + s.exclude = true + return s } -// IsModify checks whether 'modify' attribute is set -func (s *ChiHostReconcileAttributes) IsModify() bool { +// UnsetExclude unsets 'exclude' attribute +func (s *HostReconcileAttributes) UnsetExclude() *HostReconcileAttributes { if s == nil { - return false + return s } - return s.modify + s.exclude = false + return s } -// IsFound checks whether 'found' attribute is set -func (s *ChiHostReconcileAttributes) IsFound() bool { +// IsExclude checks whether 'exclude' attribute is set +func (s *HostReconcileAttributes) IsExclude() bool { if s == nil { return false } - return s.found + return s.exclude } // String returns string form -func (s *ChiHostReconcileAttributes) String() string { +func (s *HostReconcileAttributes) String() string { if s == nil { return "(nil)" } return fmt.Sprintf( - "status: %s, add: %t, remove: %t, modify: %t, found: %t", + "status: %s, add: %t, remove: %t, modify: %t, found: %t, exclude: %t", s.status, s.add, s.remove, s.modify, s.found, + s.exclude, ) } @@ -193,6 +224,8 @@ type ChiHostReconcileAttributesCounters struct { remove int modify int found int + + exclude int } // NewChiHostReconcileAttributesCounters creates new reconcile attributes @@ -202,8 +235,8 @@ func NewChiHostReconcileAttributesCounters() *ChiHostReconcileAttributesCounters } } -// Add adds to counters provided ChiHostReconcileAttributes -func (s *ChiHostReconcileAttributesCounters) Add(a *ChiHostReconcileAttributes) { +// Add adds to counters provided HostReconcileAttributes +func (s *ChiHostReconcileAttributesCounters) Add(a *HostReconcileAttributes) { if s == nil { return } @@ -228,6 +261,9 @@ func (s *ChiHostReconcileAttributesCounters) Add(a *ChiHostReconcileAttributes) if a.IsFound() { s.found++ } + if a.IsExclude() { + s.exclude++ + } } // GetAdd gets added @@ -261,3 +297,11 @@ func (s *ChiHostReconcileAttributesCounters) GetFound() int { } return s.found } + +// GetExclude gets exclude +func (s *ChiHostReconcileAttributesCounters) GetExclude() int { + if s == nil { + return 0 + } + return s.exclude +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_replica.go b/pkg/apis/clickhouse.altinity.com/v1/type_replica.go index fa0fd1cdf..426749fad 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_replica.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_replica.go @@ -31,7 +31,7 @@ func (replica *ChiReplica) InheritTemplatesFrom(cluster *Cluster) { } // GetServiceTemplate gets service template -func (replica *ChiReplica) GetServiceTemplate() (*ChiServiceTemplate, bool) { +func (replica *ChiReplica) GetServiceTemplate() (*ServiceTemplate, bool) { if !replica.Templates.HasReplicaServiceTemplate() { return nil, false } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_shard.go b/pkg/apis/clickhouse.altinity.com/v1/type_shard.go index 8f6be0fde..f714054f4 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_shard.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_shard.go @@ -31,7 +31,7 @@ func (shard *ChiShard) InheritTemplatesFrom(cluster *Cluster) { } // GetServiceTemplate gets service template -func (shard *ChiShard) GetServiceTemplate() (*ChiServiceTemplate, bool) { +func (shard *ChiShard) GetServiceTemplate() (*ServiceTemplate, bool) { if !shard.Templates.HasShardServiceTemplate() { return nil, false } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_status.go b/pkg/apis/clickhouse.altinity.com/v1/type_status.go index 70e9d27e7..2e869dcf5 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_status.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_status.go @@ -73,7 +73,7 @@ type ChiStatus struct { NormalizedCHI *ClickHouseInstallation `json:"normalized,omitempty" yaml:"normalized,omitempty"` NormalizedCHICompleted *ClickHouseInstallation `json:"normalizedCompleted,omitempty" yaml:"normalizedCompleted,omitempty"` HostsWithTablesCreated []string `json:"hostsWithTablesCreated,omitempty" yaml:"hostsWithTablesCreated,omitempty"` - UsedTemplates []*ChiTemplateRef `json:"usedTemplates,omitempty" yaml:"usedTemplates,omitempty"` + UsedTemplates []*TemplateRef `json:"usedTemplates,omitempty" yaml:"usedTemplates,omitempty"` mu sync.RWMutex `json:"-" yaml:"-"` } @@ -175,7 +175,7 @@ func (s *ChiStatus) SyncHostTablesCreated() { } // PushUsedTemplate pushes used template to the list of used templates -func (s *ChiStatus) PushUsedTemplate(templateRef *ChiTemplateRef) { +func (s *ChiStatus) PushUsedTemplate(templateRef *TemplateRef) { doWithWriteLock(s, func(s *ChiStatus) { s.UsedTemplates = append(s.UsedTemplates, templateRef) }) diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go b/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go index e370fd42a..3f69d82a3 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_template_indexes.go @@ -17,13 +17,13 @@ package v1 // HostTemplatesIndex describes index of host templates type HostTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*ChiHostTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*HostTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` } // NewHostTemplatesIndex creates new HostTemplatesIndex object func NewHostTemplatesIndex() *HostTemplatesIndex { return &HostTemplatesIndex{ - templates: make(map[string]*ChiHostTemplate), + templates: make(map[string]*HostTemplate), } } @@ -40,7 +40,7 @@ func (i *HostTemplatesIndex) Has(name string) bool { } // Get returns entity `name` from the index -func (i *HostTemplatesIndex) Get(name string) *ChiHostTemplate { +func (i *HostTemplatesIndex) Get(name string) *HostTemplate { if !i.Has(name) { return nil } @@ -48,7 +48,7 @@ func (i *HostTemplatesIndex) Get(name string) *ChiHostTemplate { } // Set sets named template into index -func (i *HostTemplatesIndex) Set(name string, entry *ChiHostTemplate) { +func (i *HostTemplatesIndex) Set(name string, entry *HostTemplate) { if i == nil { return } @@ -59,7 +59,7 @@ func (i *HostTemplatesIndex) Set(name string, entry *ChiHostTemplate) { } // Walk calls specified function over each entry in the index -func (i *HostTemplatesIndex) Walk(f func(template *ChiHostTemplate)) { +func (i *HostTemplatesIndex) Walk(f func(template *HostTemplate)) { if i == nil { return } @@ -71,13 +71,13 @@ func (i *HostTemplatesIndex) Walk(f func(template *ChiHostTemplate)) { // PodTemplatesIndex describes index of pod templates type PodTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*ChiPodTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*PodTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` } // NewPodTemplatesIndex creates new PodTemplatesIndex object func NewPodTemplatesIndex() *PodTemplatesIndex { return &PodTemplatesIndex{ - templates: make(map[string]*ChiPodTemplate), + templates: make(map[string]*PodTemplate), } } @@ -94,7 +94,7 @@ func (i *PodTemplatesIndex) Has(name string) bool { } // Get returns entity `name` from the index -func (i *PodTemplatesIndex) Get(name string) *ChiPodTemplate { +func (i *PodTemplatesIndex) Get(name string) *PodTemplate { if !i.Has(name) { return nil } @@ -102,7 +102,7 @@ func (i *PodTemplatesIndex) Get(name string) *ChiPodTemplate { } // Set sets named template into index -func (i *PodTemplatesIndex) Set(name string, entry *ChiPodTemplate) { +func (i *PodTemplatesIndex) Set(name string, entry *PodTemplate) { if i == nil { return } @@ -113,7 +113,7 @@ func (i *PodTemplatesIndex) Set(name string, entry *ChiPodTemplate) { } // Walk calls specified function over each entry in the index -func (i *PodTemplatesIndex) Walk(f func(template *ChiPodTemplate)) { +func (i *PodTemplatesIndex) Walk(f func(template *PodTemplate)) { if i == nil { return } @@ -125,13 +125,13 @@ func (i *PodTemplatesIndex) Walk(f func(template *ChiPodTemplate)) { // VolumeClaimTemplatesIndex describes index of volume claim templates type VolumeClaimTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*ChiVolumeClaimTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*VolumeClaimTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` } // NewVolumeClaimTemplatesIndex creates new VolumeClaimTemplatesIndex object func NewVolumeClaimTemplatesIndex() *VolumeClaimTemplatesIndex { return &VolumeClaimTemplatesIndex{ - templates: make(map[string]*ChiVolumeClaimTemplate), + templates: make(map[string]*VolumeClaimTemplate), } } @@ -148,7 +148,7 @@ func (i *VolumeClaimTemplatesIndex) Has(name string) bool { } // Get returns entity `name` from the index -func (i *VolumeClaimTemplatesIndex) Get(name string) *ChiVolumeClaimTemplate { +func (i *VolumeClaimTemplatesIndex) Get(name string) *VolumeClaimTemplate { if !i.Has(name) { return nil } @@ -156,7 +156,7 @@ func (i *VolumeClaimTemplatesIndex) Get(name string) *ChiVolumeClaimTemplate { } // Set sets named template into index -func (i *VolumeClaimTemplatesIndex) Set(name string, entry *ChiVolumeClaimTemplate) { +func (i *VolumeClaimTemplatesIndex) Set(name string, entry *VolumeClaimTemplate) { if i == nil { return } @@ -167,7 +167,7 @@ func (i *VolumeClaimTemplatesIndex) Set(name string, entry *ChiVolumeClaimTempla } // Walk calls specified function over each entry in the index -func (i *VolumeClaimTemplatesIndex) Walk(f func(template *ChiVolumeClaimTemplate)) { +func (i *VolumeClaimTemplatesIndex) Walk(f func(template *VolumeClaimTemplate)) { if i == nil { return } @@ -179,13 +179,13 @@ func (i *VolumeClaimTemplatesIndex) Walk(f func(template *ChiVolumeClaimTemplate // ServiceTemplatesIndex describes index of service templates type ServiceTemplatesIndex struct { // templates maps 'name of the template' -> 'template itself' - templates map[string]*ChiServiceTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` + templates map[string]*ServiceTemplate `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` } // NewServiceTemplatesIndex creates new ServiceTemplatesIndex object func NewServiceTemplatesIndex() *ServiceTemplatesIndex { return &ServiceTemplatesIndex{ - templates: make(map[string]*ChiServiceTemplate), + templates: make(map[string]*ServiceTemplate), } } @@ -202,7 +202,7 @@ func (i *ServiceTemplatesIndex) Has(name string) bool { } // Get returns entity `name` from the index -func (i *ServiceTemplatesIndex) Get(name string) *ChiServiceTemplate { +func (i *ServiceTemplatesIndex) Get(name string) *ServiceTemplate { if !i.Has(name) { return nil } @@ -210,7 +210,7 @@ func (i *ServiceTemplatesIndex) Get(name string) *ChiServiceTemplate { } // Set sets named template into index -func (i *ServiceTemplatesIndex) Set(name string, entry *ChiServiceTemplate) { +func (i *ServiceTemplatesIndex) Set(name string, entry *ServiceTemplate) { if i == nil { return } @@ -221,7 +221,7 @@ func (i *ServiceTemplatesIndex) Set(name string, entry *ChiServiceTemplate) { } // Walk calls specified function over each entry in the index -func (i *ServiceTemplatesIndex) Walk(f func(template *ChiServiceTemplate)) { +func (i *ServiceTemplatesIndex) Walk(f func(template *ServiceTemplate)) { if i == nil { return } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_templates.go b/pkg/apis/clickhouse.altinity.com/v1/type_templates.go index 21b514f48..7d72653ba 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_templates.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_templates.go @@ -18,33 +18,33 @@ import ( "github.com/imdario/mergo" ) -// NewChiTemplates creates new ChiTemplates object -func NewChiTemplates() *ChiTemplates { - return new(ChiTemplates) +// NewChiTemplates creates new Templates object +func NewChiTemplates() *Templates { + return new(Templates) } -func (templates *ChiTemplates) GetHostTemplates() []ChiHostTemplate { +func (templates *Templates) GetHostTemplates() []HostTemplate { if templates == nil { return nil } return templates.HostTemplates } -func (templates *ChiTemplates) GetPodTemplates() []ChiPodTemplate { +func (templates *Templates) GetPodTemplates() []PodTemplate { if templates == nil { return nil } return templates.PodTemplates } -func (templates *ChiTemplates) GetVolumeClaimTemplates() []ChiVolumeClaimTemplate { +func (templates *Templates) GetVolumeClaimTemplates() []VolumeClaimTemplate { if templates == nil { return nil } return templates.VolumeClaimTemplates } -func (templates *ChiTemplates) GetServiceTemplates() []ChiServiceTemplate { +func (templates *Templates) GetServiceTemplates() []ServiceTemplate { if templates == nil { return nil } @@ -52,7 +52,7 @@ func (templates *ChiTemplates) GetServiceTemplates() []ChiServiceTemplate { } // Len returns accumulated len of all templates -func (templates *ChiTemplates) Len() int { +func (templates *Templates) Len() int { if templates == nil { return 0 } @@ -65,13 +65,13 @@ func (templates *ChiTemplates) Len() int { } // MergeFrom merges from specified object -func (templates *ChiTemplates) MergeFrom(_from any, _type MergeType) *ChiTemplates { +func (templates *Templates) MergeFrom(_from any, _type MergeType) *Templates { // Typed from - var from *ChiTemplates + var from *Templates // Ensure type switch typed := _from.(type) { - case *ChiTemplates: + case *Templates: from = typed default: return templates @@ -98,7 +98,7 @@ func (templates *ChiTemplates) MergeFrom(_from any, _type MergeType) *ChiTemplat } // mergeHostTemplates merges host templates section -func (templates *ChiTemplates) mergeHostTemplates(from *ChiTemplates) { +func (templates *Templates) mergeHostTemplates(from *Templates) { if len(from.HostTemplates) == 0 { return } @@ -131,7 +131,7 @@ func (templates *ChiTemplates) mergeHostTemplates(from *ChiTemplates) { } // mergePodTemplates merges pod templates section -func (templates *ChiTemplates) mergePodTemplates(from *ChiTemplates) { +func (templates *Templates) mergePodTemplates(from *Templates) { if len(from.PodTemplates) == 0 { return } @@ -169,7 +169,7 @@ func (templates *ChiTemplates) mergePodTemplates(from *ChiTemplates) { } // mergeVolumeClaimTemplates merges volume claim templates section -func (templates *ChiTemplates) mergeVolumeClaimTemplates(from *ChiTemplates) { +func (templates *Templates) mergeVolumeClaimTemplates(from *Templates) { if len(from.VolumeClaimTemplates) == 0 { return } @@ -202,7 +202,7 @@ func (templates *ChiTemplates) mergeVolumeClaimTemplates(from *ChiTemplates) { } // mergeServiceTemplates merges service templates section -func (templates *ChiTemplates) mergeServiceTemplates(from *ChiTemplates) { +func (templates *Templates) mergeServiceTemplates(from *Templates) { if len(from.ServiceTemplates) == 0 { return } @@ -235,7 +235,7 @@ func (templates *ChiTemplates) mergeServiceTemplates(from *ChiTemplates) { } // GetHostTemplatesIndex returns index of host templates -func (templates *ChiTemplates) GetHostTemplatesIndex() *HostTemplatesIndex { +func (templates *Templates) GetHostTemplatesIndex() *HostTemplatesIndex { if templates == nil { return nil } @@ -243,7 +243,7 @@ func (templates *ChiTemplates) GetHostTemplatesIndex() *HostTemplatesIndex { } // EnsureHostTemplatesIndex ensures index exists -func (templates *ChiTemplates) EnsureHostTemplatesIndex() *HostTemplatesIndex { +func (templates *Templates) EnsureHostTemplatesIndex() *HostTemplatesIndex { if templates == nil { return nil } @@ -255,7 +255,7 @@ func (templates *ChiTemplates) EnsureHostTemplatesIndex() *HostTemplatesIndex { } // GetPodTemplatesIndex returns index of pod templates -func (templates *ChiTemplates) GetPodTemplatesIndex() *PodTemplatesIndex { +func (templates *Templates) GetPodTemplatesIndex() *PodTemplatesIndex { if templates == nil { return nil } @@ -263,7 +263,7 @@ func (templates *ChiTemplates) GetPodTemplatesIndex() *PodTemplatesIndex { } // EnsurePodTemplatesIndex ensures index exists -func (templates *ChiTemplates) EnsurePodTemplatesIndex() *PodTemplatesIndex { +func (templates *Templates) EnsurePodTemplatesIndex() *PodTemplatesIndex { if templates == nil { return nil } @@ -275,7 +275,7 @@ func (templates *ChiTemplates) EnsurePodTemplatesIndex() *PodTemplatesIndex { } // GetVolumeClaimTemplatesIndex returns index of VolumeClaim templates -func (templates *ChiTemplates) GetVolumeClaimTemplatesIndex() *VolumeClaimTemplatesIndex { +func (templates *Templates) GetVolumeClaimTemplatesIndex() *VolumeClaimTemplatesIndex { if templates == nil { return nil } @@ -283,7 +283,7 @@ func (templates *ChiTemplates) GetVolumeClaimTemplatesIndex() *VolumeClaimTempla } // EnsureVolumeClaimTemplatesIndex ensures index exists -func (templates *ChiTemplates) EnsureVolumeClaimTemplatesIndex() *VolumeClaimTemplatesIndex { +func (templates *Templates) EnsureVolumeClaimTemplatesIndex() *VolumeClaimTemplatesIndex { if templates == nil { return nil } @@ -295,7 +295,7 @@ func (templates *ChiTemplates) EnsureVolumeClaimTemplatesIndex() *VolumeClaimTem } // GetServiceTemplatesIndex returns index of Service templates -func (templates *ChiTemplates) GetServiceTemplatesIndex() *ServiceTemplatesIndex { +func (templates *Templates) GetServiceTemplatesIndex() *ServiceTemplatesIndex { if templates == nil { return nil } @@ -303,7 +303,7 @@ func (templates *ChiTemplates) GetServiceTemplatesIndex() *ServiceTemplatesIndex } // EnsureServiceTemplatesIndex ensures index exists -func (templates *ChiTemplates) EnsureServiceTemplatesIndex() *ServiceTemplatesIndex { +func (templates *Templates) EnsureServiceTemplatesIndex() *ServiceTemplatesIndex { if templates == nil { return nil } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go index e4c6684ce..0a5e60f5a 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_volume_claim_template.go @@ -15,16 +15,16 @@ package v1 import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// ChiVolumeClaimTemplate defines PersistentVolumeClaim Template -type ChiVolumeClaimTemplate struct { +// VolumeClaimTemplate defines PersistentVolumeClaim Template +type VolumeClaimTemplate struct { Name string `json:"name" yaml:"name"` StorageManagement - ObjectMeta metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` - Spec corev1.PersistentVolumeClaimSpec `json:"spec,omitempty" yaml:"spec,omitempty"` + ObjectMeta meta.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` + Spec core.PersistentVolumeClaimSpec `json:"spec,omitempty" yaml:"spec,omitempty"` } // PVCProvisioner defines PVC provisioner diff --git a/pkg/apis/clickhouse.altinity.com/v1/types.go b/pkg/apis/clickhouse.altinity.com/v1/types.go index d890ff4d3..c4ef772bd 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/types.go +++ b/pkg/apis/clickhouse.altinity.com/v1/types.go @@ -49,26 +49,28 @@ type ClickHouseInstallation struct { } type ClickHouseInstallationRuntime struct { - attributes *ComparableAttributes `json:"-" yaml:"-"` + attributes *ComparableAttributes `json:"-" yaml:"-"` + commonConfigMutex sync.Mutex `json:"-" yaml:"-"` } -func (runtime *ClickHouseInstallationRuntime) EnsureAttributes() *ComparableAttributes { - if runtime == nil { - return nil - } - - // Assume that most of the time, we'll see a non-nil value. - if runtime.attributes != nil { - return runtime.attributes +func newClickHouseInstallationRuntime() *ClickHouseInstallationRuntime { + return &ClickHouseInstallationRuntime{ + attributes: &ComparableAttributes{}, } +} - // Note that we have to check this property again to avoid a TOCTOU bug. - if runtime.attributes == nil { - runtime.attributes = &ComparableAttributes{} - } +func (runtime *ClickHouseInstallationRuntime) GetAttributes() *ComparableAttributes { return runtime.attributes } +func (runtime *ClickHouseInstallationRuntime) LockCommonConfig() { + runtime.commonConfigMutex.Lock() +} + +func (runtime *ClickHouseInstallationRuntime) UnlockCommonConfig() { + runtime.commonConfigMutex.Unlock() +} + // ComparableAttributes specifies CHI attributes that are comparable type ComparableAttributes struct { AdditionalEnvVars []core.EnvVar `json:"-" yaml:"-"` @@ -98,21 +100,21 @@ type ClickHouseOperatorConfiguration struct { // ChiSpec defines spec section of ClickHouseInstallation resource type ChiSpec struct { - TaskID *string `json:"taskID,omitempty" yaml:"taskID,omitempty"` - Stop *StringBool `json:"stop,omitempty" yaml:"stop,omitempty"` - Restart string `json:"restart,omitempty" yaml:"restart,omitempty"` - Troubleshoot *StringBool `json:"troubleshoot,omitempty" yaml:"troubleshoot,omitempty"` - NamespaceDomainPattern string `json:"namespaceDomainPattern,omitempty" yaml:"namespaceDomainPattern,omitempty"` - Templating *ChiTemplating `json:"templating,omitempty" yaml:"templating,omitempty"` - Reconciling *ChiReconciling `json:"reconciling,omitempty" yaml:"reconciling,omitempty"` - Defaults *ChiDefaults `json:"defaults,omitempty" yaml:"defaults,omitempty"` - Configuration *Configuration `json:"configuration,omitempty" yaml:"configuration,omitempty"` - Templates *ChiTemplates `json:"templates,omitempty" yaml:"templates,omitempty"` - UseTemplates []*ChiTemplateRef `json:"useTemplates,omitempty" yaml:"useTemplates,omitempty"` -} - -// ChiTemplateRef defines UseTemplate section of ClickHouseInstallation resource -type ChiTemplateRef struct { + TaskID *string `json:"taskID,omitempty" yaml:"taskID,omitempty"` + Stop *StringBool `json:"stop,omitempty" yaml:"stop,omitempty"` + Restart string `json:"restart,omitempty" yaml:"restart,omitempty"` + Troubleshoot *StringBool `json:"troubleshoot,omitempty" yaml:"troubleshoot,omitempty"` + NamespaceDomainPattern string `json:"namespaceDomainPattern,omitempty" yaml:"namespaceDomainPattern,omitempty"` + Templating *ChiTemplating `json:"templating,omitempty" yaml:"templating,omitempty"` + Reconciling *ChiReconciling `json:"reconciling,omitempty" yaml:"reconciling,omitempty"` + Defaults *ChiDefaults `json:"defaults,omitempty" yaml:"defaults,omitempty"` + Configuration *Configuration `json:"configuration,omitempty" yaml:"configuration,omitempty"` + Templates *Templates `json:"templates,omitempty" yaml:"templates,omitempty"` + UseTemplates []*TemplateRef `json:"useTemplates,omitempty" yaml:"useTemplates,omitempty"` +} + +// TemplateRef defines UseTemplate section of ClickHouseInstallation resource +type TemplateRef struct { Name string `json:"name,omitempty" yaml:"name,omitempty"` Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` UseType string `json:"useType,omitempty" yaml:"useType,omitempty"` @@ -659,15 +661,15 @@ type ChiReplicaAddress struct { ReplicaIndex int `json:"replicaIndex,omitempty" yaml:"replicaIndex,omitempty"` } -// ChiHostTemplate defines full Host Template -type ChiHostTemplate struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - PortDistribution []ChiPortDistribution `json:"portDistribution,omitempty" yaml:"portDistribution,omitempty"` - Spec ChiHost `json:"spec,omitempty" yaml:"spec,omitempty"` +// HostTemplate defines full Host Template +type HostTemplate struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + PortDistribution []PortDistribution `json:"portDistribution,omitempty" yaml:"portDistribution,omitempty"` + Spec ChiHost `json:"spec,omitempty" yaml:"spec,omitempty"` } -// ChiPortDistribution defines port distribution -type ChiPortDistribution struct { +// PortDistribution defines port distribution +type PortDistribution struct { Type string `json:"type,omitempty" yaml:"type,omitempty"` } @@ -678,13 +680,13 @@ type ChiHostConfig struct { FilesFingerprint string `json:"filesfingerprint" yaml:"filesfingerprint"` } -// ChiTemplates defines templates section of .spec -type ChiTemplates struct { +// Templates defines templates section of .spec +type Templates struct { // Templates - HostTemplates []ChiHostTemplate `json:"hostTemplates,omitempty" yaml:"hostTemplates,omitempty"` - PodTemplates []ChiPodTemplate `json:"podTemplates,omitempty" yaml:"podTemplates,omitempty"` - VolumeClaimTemplates []ChiVolumeClaimTemplate `json:"volumeClaimTemplates,omitempty" yaml:"volumeClaimTemplates,omitempty"` - ServiceTemplates []ChiServiceTemplate `json:"serviceTemplates,omitempty" yaml:"serviceTemplates,omitempty"` + HostTemplates []HostTemplate `json:"hostTemplates,omitempty" yaml:"hostTemplates,omitempty"` + PodTemplates []PodTemplate `json:"podTemplates,omitempty" yaml:"podTemplates,omitempty"` + VolumeClaimTemplates []VolumeClaimTemplate `json:"volumeClaimTemplates,omitempty" yaml:"volumeClaimTemplates,omitempty"` + ServiceTemplates []ServiceTemplate `json:"serviceTemplates,omitempty" yaml:"serviceTemplates,omitempty"` // Index maps template name to template itself HostTemplatesIndex *HostTemplatesIndex `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` @@ -693,32 +695,32 @@ type ChiTemplates struct { ServiceTemplatesIndex *ServiceTemplatesIndex `json:",omitempty" yaml:",omitempty" testdiff:"ignore"` } -// ChiPodTemplate defines full Pod Template, directly used by StatefulSet -type ChiPodTemplate struct { - Name string `json:"name" yaml:"name"` - GenerateName string `json:"generateName,omitempty" yaml:"generateName,omitempty"` - Zone ChiPodTemplateZone `json:"zone,omitempty" yaml:"zone,omitempty"` - PodDistribution []ChiPodDistribution `json:"podDistribution,omitempty" yaml:"podDistribution,omitempty"` - ObjectMeta meta.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` - Spec core.PodSpec `json:"spec,omitempty" yaml:"spec,omitempty"` +// PodTemplate defines full Pod Template, directly used by StatefulSet +type PodTemplate struct { + Name string `json:"name" yaml:"name"` + GenerateName string `json:"generateName,omitempty" yaml:"generateName,omitempty"` + Zone PodTemplateZone `json:"zone,omitempty" yaml:"zone,omitempty"` + PodDistribution []PodDistribution `json:"podDistribution,omitempty" yaml:"podDistribution,omitempty"` + ObjectMeta meta.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` + Spec core.PodSpec `json:"spec,omitempty" yaml:"spec,omitempty"` } -// ChiPodTemplateZone defines pod template zone -type ChiPodTemplateZone struct { +// PodTemplateZone defines pod template zone +type PodTemplateZone struct { Key string `json:"key,omitempty" yaml:"key,omitempty"` Values []string `json:"values,omitempty" yaml:"values,omitempty"` } -// ChiPodDistribution defines pod distribution -type ChiPodDistribution struct { +// PodDistribution defines pod distribution +type PodDistribution struct { Type string `json:"type,omitempty" yaml:"type,omitempty"` Scope string `json:"scope,omitempty" yaml:"scope,omitempty"` Number int `json:"number,omitempty" yaml:"number,omitempty"` TopologyKey string `json:"topologyKey,omitempty" yaml:"topologyKey,omitempty"` } -// ChiServiceTemplate defines CHI service template -type ChiServiceTemplate struct { +// ServiceTemplate defines CHI service template +type ServiceTemplate struct { Name string `json:"name" yaml:"name"` GenerateName string `json:"generateName,omitempty" yaml:"generateName,omitempty"` ObjectMeta meta.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` diff --git a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go index 013b84a2d..d457de29f 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go +++ b/pkg/apis/clickhouse.altinity.com/v1/zz_generated.deepcopy.go @@ -253,22 +253,6 @@ func (in *ChiHostConfig) DeepCopy() *ChiHostConfig { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiHostReconcileAttributes) DeepCopyInto(out *ChiHostReconcileAttributes) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiHostReconcileAttributes. -func (in *ChiHostReconcileAttributes) DeepCopy() *ChiHostReconcileAttributes { - if in == nil { - return nil - } - out := new(ChiHostReconcileAttributes) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ChiHostReconcileAttributesCounters) DeepCopyInto(out *ChiHostReconcileAttributesCounters) { *out = *in @@ -304,7 +288,7 @@ func (in *ChiHostRuntime) DeepCopyInto(out *ChiHostRuntime) { } if in.reconcileAttributes != nil { in, out := &in.reconcileAttributes, &out.reconcileAttributes - *out = new(ChiHostReconcileAttributes) + *out = new(HostReconcileAttributes) **out = **in } if in.CurStatefulSet != nil { @@ -335,28 +319,6 @@ func (in *ChiHostRuntime) DeepCopy() *ChiHostRuntime { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiHostTemplate) DeepCopyInto(out *ChiHostTemplate) { - *out = *in - if in.PortDistribution != nil { - in, out := &in.PortDistribution, &out.PortDistribution - *out = make([]ChiPortDistribution, len(*in)) - copy(*out, *in) - } - in.Spec.DeepCopyInto(&out.Spec) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiHostTemplate. -func (in *ChiHostTemplate) DeepCopy() *ChiHostTemplate { - if in == nil { - return nil - } - out := new(ChiHostTemplate) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ChiObjectsCleanup) DeepCopyInto(out *ChiObjectsCleanup) { *out = *in @@ -373,83 +335,6 @@ func (in *ChiObjectsCleanup) DeepCopy() *ChiObjectsCleanup { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiPodDistribution) DeepCopyInto(out *ChiPodDistribution) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiPodDistribution. -func (in *ChiPodDistribution) DeepCopy() *ChiPodDistribution { - if in == nil { - return nil - } - out := new(ChiPodDistribution) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiPodTemplate) DeepCopyInto(out *ChiPodTemplate) { - *out = *in - in.Zone.DeepCopyInto(&out.Zone) - if in.PodDistribution != nil { - in, out := &in.PodDistribution, &out.PodDistribution - *out = make([]ChiPodDistribution, len(*in)) - copy(*out, *in) - } - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiPodTemplate. -func (in *ChiPodTemplate) DeepCopy() *ChiPodTemplate { - if in == nil { - return nil - } - out := new(ChiPodTemplate) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiPodTemplateZone) DeepCopyInto(out *ChiPodTemplateZone) { - *out = *in - if in.Values != nil { - in, out := &in.Values, &out.Values - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiPodTemplateZone. -func (in *ChiPodTemplateZone) DeepCopy() *ChiPodTemplateZone { - if in == nil { - return nil - } - out := new(ChiPodTemplateZone) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiPortDistribution) DeepCopyInto(out *ChiPortDistribution) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiPortDistribution. -func (in *ChiPortDistribution) DeepCopy() *ChiPortDistribution { - if in == nil { - return nil - } - out := new(ChiPortDistribution) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ChiReconciling) DeepCopyInto(out *ChiReconciling) { *out = *in @@ -552,24 +437,6 @@ func (in *ChiReplicaRuntime) DeepCopy() *ChiReplicaRuntime { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiServiceTemplate) DeepCopyInto(out *ChiServiceTemplate) { - *out = *in - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiServiceTemplate. -func (in *ChiServiceTemplate) DeepCopy() *ChiServiceTemplate { - if in == nil { - return nil - } - out := new(ChiServiceTemplate) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ChiShard) DeepCopyInto(out *ChiShard) { *out = *in @@ -701,16 +568,16 @@ func (in *ChiSpec) DeepCopyInto(out *ChiSpec) { } if in.Templates != nil { in, out := &in.Templates, &out.Templates - *out = new(ChiTemplates) + *out = new(Templates) (*in).DeepCopyInto(*out) } if in.UseTemplates != nil { in, out := &in.UseTemplates, &out.UseTemplates - *out = make([]*ChiTemplateRef, len(*in)) + *out = make([]*TemplateRef, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] - *out = new(ChiTemplateRef) + *out = new(TemplateRef) **out = **in } } @@ -783,11 +650,11 @@ func (in *ChiStatus) DeepCopyInto(out *ChiStatus) { } if in.UsedTemplates != nil { in, out := &in.UsedTemplates, &out.UsedTemplates - *out = make([]*ChiTemplateRef, len(*in)) + *out = make([]*TemplateRef, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] - *out = new(ChiTemplateRef) + *out = new(TemplateRef) **out = **in } } @@ -822,86 +689,6 @@ func (in *ChiTemplateNames) DeepCopy() *ChiTemplateNames { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiTemplateRef) DeepCopyInto(out *ChiTemplateRef) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiTemplateRef. -func (in *ChiTemplateRef) DeepCopy() *ChiTemplateRef { - if in == nil { - return nil - } - out := new(ChiTemplateRef) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiTemplates) DeepCopyInto(out *ChiTemplates) { - *out = *in - if in.HostTemplates != nil { - in, out := &in.HostTemplates, &out.HostTemplates - *out = make([]ChiHostTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.PodTemplates != nil { - in, out := &in.PodTemplates, &out.PodTemplates - *out = make([]ChiPodTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.VolumeClaimTemplates != nil { - in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates - *out = make([]ChiVolumeClaimTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.ServiceTemplates != nil { - in, out := &in.ServiceTemplates, &out.ServiceTemplates - *out = make([]ChiServiceTemplate, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.HostTemplatesIndex != nil { - in, out := &in.HostTemplatesIndex, &out.HostTemplatesIndex - *out = new(HostTemplatesIndex) - (*in).DeepCopyInto(*out) - } - if in.PodTemplatesIndex != nil { - in, out := &in.PodTemplatesIndex, &out.PodTemplatesIndex - *out = new(PodTemplatesIndex) - (*in).DeepCopyInto(*out) - } - if in.VolumeClaimTemplatesIndex != nil { - in, out := &in.VolumeClaimTemplatesIndex, &out.VolumeClaimTemplatesIndex - *out = new(VolumeClaimTemplatesIndex) - (*in).DeepCopyInto(*out) - } - if in.ServiceTemplatesIndex != nil { - in, out := &in.ServiceTemplatesIndex, &out.ServiceTemplatesIndex - *out = new(ServiceTemplatesIndex) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiTemplates. -func (in *ChiTemplates) DeepCopy() *ChiTemplates { - if in == nil { - return nil - } - out := new(ChiTemplates) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ChiTemplating) DeepCopyInto(out *ChiTemplating) { *out = *in @@ -925,25 +712,6 @@ func (in *ChiTemplating) DeepCopy() *ChiTemplating { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChiVolumeClaimTemplate) DeepCopyInto(out *ChiVolumeClaimTemplate) { - *out = *in - out.StorageManagement = in.StorageManagement - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChiVolumeClaimTemplate. -func (in *ChiVolumeClaimTemplate) DeepCopy() *ChiVolumeClaimTemplate { - if in == nil { - return nil - } - out := new(ChiVolumeClaimTemplate) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ChiZookeeperConfig) DeepCopyInto(out *ChiZookeeperConfig) { *out = *in @@ -1068,6 +836,7 @@ func (in *ClickHouseInstallationRuntime) DeepCopyInto(out *ClickHouseInstallatio *out = new(ComparableAttributes) (*in).DeepCopyInto(*out) } + out.commonConfigMutex = in.commonConfigMutex return } @@ -1575,19 +1344,57 @@ func (in *HostAddress) DeepCopy() *HostAddress { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostReconcileAttributes) DeepCopyInto(out *HostReconcileAttributes) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostReconcileAttributes. +func (in *HostReconcileAttributes) DeepCopy() *HostReconcileAttributes { + if in == nil { + return nil + } + out := new(HostReconcileAttributes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostTemplate) DeepCopyInto(out *HostTemplate) { + *out = *in + if in.PortDistribution != nil { + in, out := &in.PortDistribution, &out.PortDistribution + *out = make([]PortDistribution, len(*in)) + copy(*out, *in) + } + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostTemplate. +func (in *HostTemplate) DeepCopy() *HostTemplate { + if in == nil { + return nil + } + out := new(HostTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HostTemplatesIndex) DeepCopyInto(out *HostTemplatesIndex) { *out = *in if in.templates != nil { in, out := &in.templates, &out.templates - *out = make(map[string]*ChiHostTemplate, len(*in)) + *out = make(map[string]*HostTemplate, len(*in)) for key, val := range *in { - var outVal *ChiHostTemplate + var outVal *HostTemplate if val == nil { (*out)[key] = nil } else { in, out := &val, &outVal - *out = new(ChiHostTemplate) + *out = new(HostTemplate) (*in).DeepCopyInto(*out) } (*out)[key] = outVal @@ -2153,19 +1960,80 @@ func (in *OperatorConfigWatch) DeepCopy() *OperatorConfigWatch { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDistribution) DeepCopyInto(out *PodDistribution) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDistribution. +func (in *PodDistribution) DeepCopy() *PodDistribution { + if in == nil { + return nil + } + out := new(PodDistribution) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodTemplate) DeepCopyInto(out *PodTemplate) { + *out = *in + in.Zone.DeepCopyInto(&out.Zone) + if in.PodDistribution != nil { + in, out := &in.PodDistribution, &out.PodDistribution + *out = make([]PodDistribution, len(*in)) + copy(*out, *in) + } + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplate. +func (in *PodTemplate) DeepCopy() *PodTemplate { + if in == nil { + return nil + } + out := new(PodTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodTemplateZone) DeepCopyInto(out *PodTemplateZone) { + *out = *in + if in.Values != nil { + in, out := &in.Values, &out.Values + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplateZone. +func (in *PodTemplateZone) DeepCopy() *PodTemplateZone { + if in == nil { + return nil + } + out := new(PodTemplateZone) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodTemplatesIndex) DeepCopyInto(out *PodTemplatesIndex) { *out = *in if in.templates != nil { in, out := &in.templates, &out.templates - *out = make(map[string]*ChiPodTemplate, len(*in)) + *out = make(map[string]*PodTemplate, len(*in)) for key, val := range *in { - var outVal *ChiPodTemplate + var outVal *PodTemplate if val == nil { (*out)[key] = nil } else { in, out := &val, &outVal - *out = new(ChiPodTemplate) + *out = new(PodTemplate) (*in).DeepCopyInto(*out) } (*out)[key] = outVal @@ -2184,6 +2052,22 @@ func (in *PodTemplatesIndex) DeepCopy() *PodTemplatesIndex { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortDistribution) DeepCopyInto(out *PortDistribution) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortDistribution. +func (in *PortDistribution) DeepCopy() *PortDistribution { + if in == nil { + return nil + } + out := new(PortDistribution) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchemaPolicy) DeepCopyInto(out *SchemaPolicy) { *out = *in @@ -2226,19 +2110,37 @@ func (in *ScopeAddress) DeepCopy() *ScopeAddress { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceTemplate) DeepCopyInto(out *ServiceTemplate) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceTemplate. +func (in *ServiceTemplate) DeepCopy() *ServiceTemplate { + if in == nil { + return nil + } + out := new(ServiceTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceTemplatesIndex) DeepCopyInto(out *ServiceTemplatesIndex) { *out = *in if in.templates != nil { in, out := &in.templates, &out.templates - *out = make(map[string]*ChiServiceTemplate, len(*in)) + *out = make(map[string]*ServiceTemplate, len(*in)) for key, val := range *in { - var outVal *ChiServiceTemplate + var outVal *ServiceTemplate if val == nil { (*out)[key] = nil } else { in, out := &val, &outVal - *out = new(ChiServiceTemplate) + *out = new(ServiceTemplate) (*in).DeepCopyInto(*out) } (*out)[key] = outVal @@ -2410,19 +2312,118 @@ func (in *StorageManagement) DeepCopy() *StorageManagement { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemplateRef) DeepCopyInto(out *TemplateRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemplateRef. +func (in *TemplateRef) DeepCopy() *TemplateRef { + if in == nil { + return nil + } + out := new(TemplateRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Templates) DeepCopyInto(out *Templates) { + *out = *in + if in.HostTemplates != nil { + in, out := &in.HostTemplates, &out.HostTemplates + *out = make([]HostTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodTemplates != nil { + in, out := &in.PodTemplates, &out.PodTemplates + *out = make([]PodTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]VolumeClaimTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ServiceTemplates != nil { + in, out := &in.ServiceTemplates, &out.ServiceTemplates + *out = make([]ServiceTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.HostTemplatesIndex != nil { + in, out := &in.HostTemplatesIndex, &out.HostTemplatesIndex + *out = new(HostTemplatesIndex) + (*in).DeepCopyInto(*out) + } + if in.PodTemplatesIndex != nil { + in, out := &in.PodTemplatesIndex, &out.PodTemplatesIndex + *out = new(PodTemplatesIndex) + (*in).DeepCopyInto(*out) + } + if in.VolumeClaimTemplatesIndex != nil { + in, out := &in.VolumeClaimTemplatesIndex, &out.VolumeClaimTemplatesIndex + *out = new(VolumeClaimTemplatesIndex) + (*in).DeepCopyInto(*out) + } + if in.ServiceTemplatesIndex != nil { + in, out := &in.ServiceTemplatesIndex, &out.ServiceTemplatesIndex + *out = new(ServiceTemplatesIndex) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Templates. +func (in *Templates) DeepCopy() *Templates { + if in == nil { + return nil + } + out := new(Templates) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeClaimTemplate) DeepCopyInto(out *VolumeClaimTemplate) { + *out = *in + out.StorageManagement = in.StorageManagement + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeClaimTemplate. +func (in *VolumeClaimTemplate) DeepCopy() *VolumeClaimTemplate { + if in == nil { + return nil + } + out := new(VolumeClaimTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VolumeClaimTemplatesIndex) DeepCopyInto(out *VolumeClaimTemplatesIndex) { *out = *in if in.templates != nil { in, out := &in.templates, &out.templates - *out = make(map[string]*ChiVolumeClaimTemplate, len(*in)) + *out = make(map[string]*VolumeClaimTemplate, len(*in)) for key, val := range *in { - var outVal *ChiVolumeClaimTemplate + var outVal *VolumeClaimTemplate if val == nil { (*out)[key] = nil } else { in, out := &val, &outVal - *out = new(ChiVolumeClaimTemplate) + *out = new(VolumeClaimTemplate) (*in).DeepCopyInto(*out) } (*out)[key] = outVal diff --git a/pkg/apis/metrics/prometheus_writer.go b/pkg/apis/metrics/prometheus_writer.go index 22551ba4e..bdecd3659 100644 --- a/pkg/apis/metrics/prometheus_writer.go +++ b/pkg/apis/metrics/prometheus_writer.go @@ -16,7 +16,7 @@ package metrics import ( "fmt" - "sort" + "github.com/altinity/clickhouse-operator/pkg/metrics" "strconv" "time" @@ -221,50 +221,17 @@ func (w *CHIPrometheusWriter) WriteOKFetch(fetchType string) { labelNames, labelValues) } -func (w *CHIPrometheusWriter) getCHILabels() (labels []string) { - for label := range w.chi.Labels { - labels = append(labels, label) - } - sort.Strings(labels) - return labels -} - -func (w *CHIPrometheusWriter) getCHILabelValue(label string) (string, bool) { - value, ok := w.chi.Labels[label] - return value, ok -} - -func (w *CHIPrometheusWriter) getCHILabelValues() []string { - var labelValues []string - labels := w.getCHILabels() - for _, label := range labels { - if labelValue, ok := w.getCHILabelValue(label); ok { - labelValues = append(labelValues, labelValue) - } - } - return labelValues -} - -func (w *CHIPrometheusWriter) getMandatoryLabels() (mandatoryLabels []string) { - mandatoryLabels = append(mandatoryLabels, "chi", "namespace", "hostname") - mandatoryLabels = append(mandatoryLabels, w.getCHILabels()...) - return mandatoryLabels +func (w *CHIPrometheusWriter) appendHostLabel(labels, values []string) ([]string, []string) { + return append(labels, "hostname"), append(values, w.host.Hostname) } -func (w *CHIPrometheusWriter) getMandatoryLabelValues() (mandatoryLabelValues []string) { - mandatoryLabelValues = append(mandatoryLabelValues, w.chi.Name, w.chi.Namespace, w.host.Hostname) - mandatoryLabelValues = append(mandatoryLabelValues, w.getCHILabelValues()...) - return mandatoryLabelValues -} +func (w *CHIPrometheusWriter) getMandatoryLabelsAndValues() (labelNames []string, labelValues []string) { + // Prepare mandatory set of labels + labelNames, labelValues = metrics.GetMandatoryLabelsAndValues(w.chi) + // Append current host label + labelNames, labelValues = w.appendHostLabel(labelNames, labelValues) -func (w *CHIPrometheusWriter) getMandatoryLabelsAndValues() (mandatoryLabels []string, mandatoryLabelValues []string) { - mandatoryLabels = w.getMandatoryLabels() - mandatoryLabelValues = w.getMandatoryLabelValues() - if len(mandatoryLabels) == len(mandatoryLabelValues) { - return mandatoryLabels, mandatoryLabelValues - } - log.Warningf("Unequal number of labels and values for %s/%s/%s", w.chi.Namespace, w.chi.Name, w.host.Hostname) - return nil, nil + return labelNames, labelValues } func (w *CHIPrometheusWriter) writeSingleMetricToPrometheus( diff --git a/pkg/apis/metrics/type_watched_chi.go b/pkg/apis/metrics/type_watched_chi.go index c9cda4dc0..bf042bee5 100644 --- a/pkg/apis/metrics/type_watched_chi.go +++ b/pkg/apis/metrics/type_watched_chi.go @@ -92,6 +92,34 @@ func (chi *WatchedCHI) walkHosts(f func(*WatchedCHI, *WatchedCluster, *WatchedHo } } +func (chi *WatchedCHI) GetName() string { + if chi == nil { + return "" + } + return chi.Name +} + +func (chi *WatchedCHI) GetNamespace() string { + if chi == nil { + return "" + } + return chi.Namespace +} + +func (chi *WatchedCHI) GetLabels() map[string]string { + if chi == nil { + return nil + } + return chi.Labels +} + +func (chi *WatchedCHI) GetAnnotations() map[string]string { + if chi == nil { + return nil + } + return chi.Annotations +} + // String is a stringifier func (chi *WatchedCHI) String() string { if chi == nil { diff --git a/pkg/controller/chi/metrics.go b/pkg/controller/chi/metrics.go index 8b21895f8..d8465cb46 100644 --- a/pkg/controller/chi/metrics.go +++ b/pkg/controller/chi/metrics.go @@ -17,39 +17,41 @@ package chi import ( "context" - otelApi "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/metrics" ) // Metrics is a set of metrics that are tracked by the operator type Metrics struct { // CHIReconcilesStarted is a number (counter) of started CHI reconciles - CHIReconcilesStarted otelApi.Int64Counter + CHIReconcilesStarted metric.Int64Counter // CHIReconcilesCompleted is a number (counter) of completed CHI reconciles. // In ideal world number of completed reconciles should be equal to CHIReconcilesStarted - CHIReconcilesCompleted otelApi.Int64Counter + CHIReconcilesCompleted metric.Int64Counter // CHIReconcilesAborted is a number (counter) of explicitly aborted CHI reconciles. // This counter does not includes reconciles that we not completed due to external rasons, such as operator restart - CHIReconcilesAborted otelApi.Int64Counter + CHIReconcilesAborted metric.Int64Counter // CHIReconcilesTimings is a histogram of durations of successfully completed CHI reconciles - CHIReconcilesTimings otelApi.Float64Histogram + CHIReconcilesTimings metric.Float64Histogram // HostReconcilesStarted is a number (counter) of started host reconciles - HostReconcilesStarted otelApi.Int64Counter + HostReconcilesStarted metric.Int64Counter // HostReconcilesCompleted is a number (counter) of completed host reconciles. // In ideal world number of completed reconciles should be equal to HostReconcilesStarted - HostReconcilesCompleted otelApi.Int64Counter + HostReconcilesCompleted metric.Int64Counter // HostReconcilesRestarts is a number (counter) of host restarts during reconcile - HostReconcilesRestarts otelApi.Int64Counter + HostReconcilesRestarts metric.Int64Counter // HostReconcilesErrors is a number (counter) of failed (non-completed) host reconciles. - HostReconcilesErrors otelApi.Int64Counter + HostReconcilesErrors metric.Int64Counter // HostReconcilesTimings is a histogram of durations of successfully completed host reconciles - HostReconcilesTimings otelApi.Float64Histogram + HostReconcilesTimings metric.Float64Histogram - PodAddEvents otelApi.Int64Counter - PodUpdateEvents otelApi.Int64Counter - PodDeleteEvents otelApi.Int64Counter + PodAddEvents metric.Int64Counter + PodUpdateEvents metric.Int64Counter + PodDeleteEvents metric.Int64Counter } var m *Metrics @@ -58,65 +60,65 @@ func createMetrics() *Metrics { // The unit u should be defined using the appropriate [UCUM](https://ucum.org) case-sensitive code. CHIReconcilesStarted, _ := metrics.Meter().Int64Counter( "clickhouse_operator_chi_reconciles_started", - otelApi.WithDescription("number of CHI reconciles started"), - otelApi.WithUnit("items"), + metric.WithDescription("number of CHI reconciles started"), + metric.WithUnit("items"), ) CHIReconcilesCompleted, _ := metrics.Meter().Int64Counter( "clickhouse_operator_chi_reconciles_completed", - otelApi.WithDescription("number of CHI reconciles completed successfully"), - otelApi.WithUnit("items"), + metric.WithDescription("number of CHI reconciles completed successfully"), + metric.WithUnit("items"), ) CHIReconcilesAborted, _ := metrics.Meter().Int64Counter( "clickhouse_operator_chi_reconciles_aborted", - otelApi.WithDescription("number of CHI reconciles aborted"), - otelApi.WithUnit("items"), + metric.WithDescription("number of CHI reconciles aborted"), + metric.WithUnit("items"), ) CHIReconcilesTimings, _ := metrics.Meter().Float64Histogram( "clickhouse_operator_chi_reconciles_timings", - otelApi.WithDescription("timings of CHI reconciles completed successfully"), - otelApi.WithUnit("s"), + metric.WithDescription("timings of CHI reconciles completed successfully"), + metric.WithUnit("s"), ) HostReconcilesStarted, _ := metrics.Meter().Int64Counter( "clickhouse_operator_host_reconciles_started", - otelApi.WithDescription("number of host reconciles started"), - otelApi.WithUnit("items"), + metric.WithDescription("number of host reconciles started"), + metric.WithUnit("items"), ) HostReconcilesCompleted, _ := metrics.Meter().Int64Counter( "clickhouse_operator_host_reconciles_completed", - otelApi.WithDescription("number of host reconciles completed successfully"), - otelApi.WithUnit("items"), + metric.WithDescription("number of host reconciles completed successfully"), + metric.WithUnit("items"), ) HostReconcilesRestarts, _ := metrics.Meter().Int64Counter( "clickhouse_operator_host_reconciles_restarts", - otelApi.WithDescription("number of host restarts during reconciles"), - otelApi.WithUnit("items"), + metric.WithDescription("number of host restarts during reconciles"), + metric.WithUnit("items"), ) HostReconcilesErrors, _ := metrics.Meter().Int64Counter( "clickhouse_operator_host_reconciles_errors", - otelApi.WithDescription("number of host reconciles errors"), - otelApi.WithUnit("items"), + metric.WithDescription("number of host reconciles errors"), + metric.WithUnit("items"), ) HostReconcilesTimings, _ := metrics.Meter().Float64Histogram( "clickhouse_operator_host_reconciles_timings", - otelApi.WithDescription("timings of host reconciles completed successfully"), - otelApi.WithUnit("s"), + metric.WithDescription("timings of host reconciles completed successfully"), + metric.WithUnit("s"), ) PodAddEvents, _ := metrics.Meter().Int64Counter( "clickhouse_operator_pod_add_events", - otelApi.WithDescription("number PodAdd events"), - otelApi.WithUnit("items"), + metric.WithDescription("number PodAdd events"), + metric.WithUnit("items"), ) PodUpdateEvents, _ := metrics.Meter().Int64Counter( "clickhouse_operator_pod_update_events", - otelApi.WithDescription("number PodUpdate events"), - otelApi.WithUnit("items"), + metric.WithDescription("number PodUpdate events"), + metric.WithUnit("items"), ) PodDeleteEvents, _ := metrics.Meter().Int64Counter( "clickhouse_operator_pod_delete_events", - otelApi.WithDescription("number PodDelete events"), - otelApi.WithUnit("items"), + metric.WithDescription("number PodDelete events"), + metric.WithUnit("items"), ) return &Metrics{ @@ -144,33 +146,44 @@ func ensureMetrics() *Metrics { return m } -func metricsCHIReconcilesStarted(ctx context.Context) { - ensureMetrics().CHIReconcilesStarted.Add(ctx, 1) +func prepareLabels(chi *api.ClickHouseInstallation) (attributes []attribute.KeyValue) { + labels, values := metrics.GetMandatoryLabelsAndValues(chi) + for i := range labels { + label := labels[i] + value := values[i] + attributes = append(attributes, attribute.String(label, value)) + } + + return attributes +} + +func metricsCHIReconcilesStarted(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().CHIReconcilesStarted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsCHIReconcilesCompleted(ctx context.Context) { - ensureMetrics().CHIReconcilesCompleted.Add(ctx, 1) +func metricsCHIReconcilesCompleted(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().CHIReconcilesCompleted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsCHIReconcilesAborted(ctx context.Context) { - ensureMetrics().CHIReconcilesAborted.Add(ctx, 1) +func metricsCHIReconcilesAborted(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().CHIReconcilesAborted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsCHIReconcilesTimings(ctx context.Context, seconds float64) { - ensureMetrics().CHIReconcilesTimings.Record(ctx, seconds) +func metricsCHIReconcilesTimings(ctx context.Context, chi *api.ClickHouseInstallation, seconds float64) { + ensureMetrics().CHIReconcilesTimings.Record(ctx, seconds, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsHostReconcilesStarted(ctx context.Context) { - ensureMetrics().HostReconcilesStarted.Add(ctx, 1) +func metricsHostReconcilesStarted(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().HostReconcilesStarted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsHostReconcilesCompleted(ctx context.Context) { - ensureMetrics().HostReconcilesCompleted.Add(ctx, 1) +func metricsHostReconcilesCompleted(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().HostReconcilesCompleted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsHostReconcilesRestart(ctx context.Context) { - ensureMetrics().HostReconcilesRestarts.Add(ctx, 1) +func metricsHostReconcilesRestart(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().HostReconcilesRestarts.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsHostReconcilesErrors(ctx context.Context) { - ensureMetrics().HostReconcilesErrors.Add(ctx, 1) +func metricsHostReconcilesErrors(ctx context.Context, chi *api.ClickHouseInstallation) { + ensureMetrics().HostReconcilesErrors.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) } -func metricsHostReconcilesTimings(ctx context.Context, seconds float64) { - ensureMetrics().HostReconcilesTimings.Record(ctx, seconds) +func metricsHostReconcilesTimings(ctx context.Context, chi *api.ClickHouseInstallation, seconds float64) { + ensureMetrics().HostReconcilesTimings.Record(ctx, seconds, metric.WithAttributes(prepareLabels(chi)...)) } func metricsPodAdd(ctx context.Context) { diff --git a/pkg/controller/chi/worker-chi-reconciler.go b/pkg/controller/chi/worker-chi-reconciler.go index e9579a91c..bde05f053 100644 --- a/pkg/controller/chi/worker-chi-reconciler.go +++ b/pkg/controller/chi/worker-chi-reconciler.go @@ -59,7 +59,7 @@ func (w *worker) reconcileCHI(ctx context.Context, old, new *api.ClickHouseInsta w.a.M(new).S().P() defer w.a.M(new).E().P() - metricsCHIReconcilesStarted(ctx) + metricsCHIReconcilesStarted(ctx, new) startTime := time.Now() w.a.M(new).F().Info("Changing OLD to Normalized COMPLETED: %s/%s", new.Namespace, new.Name) @@ -112,7 +112,7 @@ func (w *worker) reconcileCHI(ctx context.Context, old, new *api.ClickHouseInsta Error("FAILED to reconcile CHI err: %v", err) w.markReconcileCompletedUnsuccessfully(ctx, new, err) if errors.Is(err, errCRUDAbort) { - metricsCHIReconcilesAborted(ctx) + metricsCHIReconcilesAborted(ctx, new) } } else { // Reconcile successful @@ -127,8 +127,8 @@ func (w *worker) reconcileCHI(ctx context.Context, old, new *api.ClickHouseInsta w.waitForIPAddresses(ctx, new) w.finalizeReconcileAndMarkCompleted(ctx, new) - metricsCHIReconcilesCompleted(ctx) - metricsCHIReconcilesTimings(ctx, time.Now().Sub(startTime).Seconds()) + metricsCHIReconcilesCompleted(ctx, new) + metricsCHIReconcilesTimings(ctx, new, time.Now().Sub(startTime).Seconds()) } return nil @@ -187,10 +187,12 @@ func (w *worker) reconcileCHIAuxObjectsPreliminary(ctx context.Context, chi *api defer w.a.V(2).M(chi).E().P() // CHI common ConfigMap without added hosts - options := w.options() - if err := w.reconcileCHIConfigMapCommon(ctx, chi, options); err != nil { + chi.EnsureRuntime().LockCommonConfig() + if err := w.reconcileCHIConfigMapCommon(ctx, chi, w.options()); err != nil { w.a.F().Error("failed to reconcile config map common. err: %v", err) } + chi.EnsureRuntime().UnlockCommonConfig() + // 3. CHI users ConfigMap if err := w.reconcileCHIConfigMapUsers(ctx, chi); err != nil { w.a.F().Error("failed to reconcile config map users. err: %v", err) @@ -229,7 +231,7 @@ func (w *worker) reconcileCHIServiceFinal(ctx context.Context, chi *api.ClickHou } // reconcileCHIAuxObjectsFinal reconciles CHI global objects -func (w *worker) reconcileCHIAuxObjectsFinal(ctx context.Context, chi *api.ClickHouseInstallation) error { +func (w *worker) reconcileCHIAuxObjectsFinal(ctx context.Context, chi *api.ClickHouseInstallation) (err error) { if util.IsContextDone(ctx) { log.V(2).Info("task is done") return nil @@ -239,7 +241,10 @@ func (w *worker) reconcileCHIAuxObjectsFinal(ctx context.Context, chi *api.Click defer w.a.V(2).M(chi).E().P() // CHI ConfigMaps with update - return w.reconcileCHIConfigMapCommon(ctx, chi, nil) + chi.EnsureRuntime().LockCommonConfig() + err = w.reconcileCHIConfigMapCommon(ctx, chi, nil) + chi.EnsureRuntime().UnlockCommonConfig() + return err } // reconcileCHIConfigMapCommon reconciles all CHI's common ConfigMap @@ -411,7 +416,7 @@ func (w *worker) reconcileHostStatefulSet(ctx context.Context, host *api.ChiHost w.a.V(1).M(host).F().Info("Reconcile host: %s. Shutting host down due to force restart", host.GetName()) w.prepareHostStatefulSetWithStatus(ctx, host, true) _ = w.reconcileStatefulSet(ctx, host, false) - metricsHostReconcilesRestart(ctx) + metricsHostReconcilesRestart(ctx, host.GetCHI()) // At this moment StatefulSet has 0 replicas. // First stage of RollingUpdate completed. } @@ -662,7 +667,7 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.ChiHost) error { w.a.V(2).M(host).S().P() defer w.a.V(2).M(host).E().P() - metricsHostReconcilesStarted(ctx) + metricsHostReconcilesStarted(ctx, host.GetCHI()) startTime := time.Now() if host.IsFirst() { @@ -689,7 +694,7 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.ChiHost) error { w.prepareHostStatefulSetWithStatus(ctx, host, false) if err := w.excludeHost(ctx, host); err != nil { - metricsHostReconcilesErrors(ctx) + metricsHostReconcilesErrors(ctx, host.GetCHI()) w.a.V(1). M(host).F(). Warning("Reconcile Host interrupted with an error 1. Host: %s Err: %v", host.GetName(), err) @@ -699,7 +704,7 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.ChiHost) error { _ = w.completeQueries(ctx, host) if err := w.reconcileHostConfigMap(ctx, host); err != nil { - metricsHostReconcilesErrors(ctx) + metricsHostReconcilesErrors(ctx, host.GetCHI()) w.a.V(1). M(host).F(). Warning("Reconcile Host interrupted with an error 2. Host: %s Err: %v", host.GetName(), err) @@ -726,7 +731,7 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.ChiHost) error { } if err := w.reconcileHostStatefulSet(ctx, host, reconcileHostStatefulSetOpts); err != nil { - metricsHostReconcilesErrors(ctx) + metricsHostReconcilesErrors(ctx, host.GetCHI()) w.a.V(1). M(host).F(). Warning("Reconcile Host interrupted with an error 3. Host: %s Err: %v", host.GetName(), err) @@ -754,7 +759,7 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.ChiHost) error { _ = w.migrateTables(ctx, host, migrateTableOpts) if err := w.includeHost(ctx, host); err != nil { - metricsHostReconcilesErrors(ctx) + metricsHostReconcilesErrors(ctx, host.GetCHI()) w.a.V(1). M(host).F(). Warning("Reconcile Host interrupted with an error 4. Host: %s Err: %v", host.GetName(), err) @@ -797,8 +802,8 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.ChiHost) error { }, }) - metricsHostReconcilesCompleted(ctx) - metricsHostReconcilesTimings(ctx, time.Now().Sub(startTime).Seconds()) + metricsHostReconcilesCompleted(ctx, host.GetCHI()) + metricsHostReconcilesTimings(ctx, host.GetCHI(), time.Now().Sub(startTime).Seconds()) return nil } @@ -1194,7 +1199,7 @@ func (w *worker) fetchPVC( volumeMount *core.VolumeMount, ) ( pvc *core.PersistentVolumeClaim, - vct *api.ChiVolumeClaimTemplate, + vct *api.VolumeClaimTemplate, isModelCreated bool, err error, ) { @@ -1254,7 +1259,7 @@ func (w *worker) reconcilePVC( ctx context.Context, pvc *core.PersistentVolumeClaim, host *api.ChiHost, - template *api.ChiVolumeClaimTemplate, + template *api.VolumeClaimTemplate, ) (*core.PersistentVolumeClaim, error) { if pvc == nil { w.a.V(2).M(host).F().Info("nil PVC, nothing to reconcile") diff --git a/pkg/controller/chi/worker-deleter.go b/pkg/controller/chi/worker-deleter.go index 18c96a54a..d908bd384 100644 --- a/pkg/controller/chi/worker-deleter.go +++ b/pkg/controller/chi/worker-deleter.go @@ -648,7 +648,7 @@ func (w *worker) deleteCHI(ctx context.Context, old, new *api.ClickHouseInstalla _ = w.deleteCHIProtocol(ctx, new) } else { - new.EnsureRuntime().EnsureAttributes().SkipOwnerRef = true + new.EnsureRuntime().GetAttributes().SkipOwnerRef = true _ = w.reconcileCHI(ctx, old, new) } diff --git a/pkg/controller/chi/worker.go b/pkg/controller/chi/worker.go index 787f655de..df7b64998 100644 --- a/pkg/controller/chi/worker.go +++ b/pkg/controller/chi/worker.go @@ -825,27 +825,23 @@ func (w *worker) walkHosts(ctx context.Context, chi *api.ClickHouseInstallation, }) } -// baseRemoteServersGeneratorOptions build base set of RemoteServersGeneratorOptions +// getRemoteServersGeneratorOptions build base set of RemoteServersGeneratorOptions // which are applied on each of `remote_servers` reconfiguration during reconcile cycle -func (w *worker) baseRemoteServersGeneratorOptions() *model.RemoteServersGeneratorOptions { - opts := model.NewRemoteServersGeneratorOptions() - opts.ExcludeReconcileAttributes( - api.NewChiHostReconcileAttributes().SetAdd(), +func (w *worker) getRemoteServersGeneratorOptions() *model.RemoteServersGeneratorOptions { + // Base model.RemoteServersGeneratorOptions specifies to exclude: + // 1. all newly added hosts + // 2. all explicitly excluded hosts + return model.NewRemoteServersGeneratorOptions().ExcludeReconcileAttributes( + api.NewChiHostReconcileAttributes(). + SetAdd(). + SetExclude(), ) - - return opts } // options build ClickHouseConfigFilesGeneratorOptions -func (w *worker) options(excludeHosts ...*api.ChiHost) *model.ClickHouseConfigFilesGeneratorOptions { - // Stringify - str := "" - for _, host := range excludeHosts { - str += fmt.Sprintf("name: '%s' sts: '%s'", host.GetName(), host.Runtime.Address.StatefulSet) - } - - opts := w.baseRemoteServersGeneratorOptions().ExcludeHosts(excludeHosts...) - w.a.Info("RemoteServersGeneratorOptions: %s, excluded host(s): %s", opts, str) +func (w *worker) options() *model.ClickHouseConfigFilesGeneratorOptions { + opts := w.getRemoteServersGeneratorOptions() + w.a.Info("RemoteServersGeneratorOptions: %s", opts) return model.NewClickHouseConfigFilesGeneratorOptions().SetRemoteServersGeneratorOptions(opts) } @@ -1098,19 +1094,22 @@ func (w *worker) excludeHostFromClickHouseCluster(ctx context.Context, host *api return } - // Remove host from cluster config and wait for ClickHouse to pick-up the change - if w.shouldWaitExcludeHost(host) { - w.a.V(1). - M(host).F(). - Info("going to exclude host %d shard %d cluster %s", - host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName) + w.a.V(1). + M(host).F(). + Info("going to exclude host %d shard %d cluster %s", + host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName) + + // Specify in options to exclude this host from ClickHouse config file + host.GetCHI().EnsureRuntime().LockCommonConfig() + host.GetReconcileAttributes().SetExclude() + _ = w.reconcileCHIConfigMapCommon(ctx, host.GetCHI(), w.options()) + host.GetCHI().EnsureRuntime().UnlockCommonConfig() - // Specify in options to exclude host from ClickHouse config file - options := w.options(host) - _ = w.reconcileCHIConfigMapCommon(ctx, host.GetCHI(), options) - // Wait for ClickHouse to pick-up the change - _ = w.waitHostNotInCluster(ctx, host) + if !w.shouldWaitExcludeHost(host) { + return } + // Wait for ClickHouse to pick-up the change + _ = w.waitHostNotInCluster(ctx, host) } // includeHostIntoClickHouseCluster includes host into ClickHouse configuration @@ -1120,9 +1119,16 @@ func (w *worker) includeHostIntoClickHouseCluster(ctx context.Context, host *api return } - // Add host to the cluster config - options := w.options() - _ = w.reconcileCHIConfigMapCommon(ctx, host.GetCHI(), options) + w.a.V(1). + M(host).F(). + Info("going to include host %d shard %d cluster %s", + host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName) + + // Specify in options to add this host into ClickHouse config file + host.GetCHI().EnsureRuntime().LockCommonConfig() + host.GetReconcileAttributes().UnsetExclude() + _ = w.reconcileCHIConfigMapCommon(ctx, host.GetCHI(), w.options()) + host.GetCHI().EnsureRuntime().UnlockCommonConfig() if !w.shouldWaitIncludeHost(host) { return @@ -1181,20 +1187,20 @@ func (w *worker) shouldWaitExcludeHost(host *api.ChiHost) bool { case host.GetCHI().GetReconciling().IsReconcilingPolicyWait(): w.a.V(1). M(host).F(). - Info("IsReconcilingPolicyWait() need to exclude host %d shard %d cluster %s", + Info("IsReconcilingPolicyWait() need to wait to exclude host %d shard %d cluster %s", host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName) return true case host.GetCHI().GetReconciling().IsReconcilingPolicyNoWait(): w.a.V(1). M(host).F(). - Info("IsReconcilingPolicyNoWait() need NOT to exclude host %d shard %d cluster %s", + Info("IsReconcilingPolicyNoWait() need NOT to wait to exclude host %d shard %d cluster %s", host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName) return false } w.a.V(1). M(host).F(). - Info("fallback to operator's settings. host %d shard %d cluster %s", + Info("wait to exclude host fallback to operator's settings. host %d shard %d cluster %s", host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName) return chop.Config().Reconcile.Host.Wait.Exclude.Value() } @@ -1760,7 +1766,7 @@ func (w *worker) recreateStatefulSet(ctx context.Context, host *api.ChiHost, reg // applyPVCResourcesRequests func (w *worker) applyPVCResourcesRequests( pvc *core.PersistentVolumeClaim, - template *api.ChiVolumeClaimTemplate, + template *api.VolumeClaimTemplate, ) bool { return w.applyResourcesList(pvc.Spec.Resources.Requests, template.Spec.Resources.Requests) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 450740968..ed0c6f18b 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -23,11 +23,12 @@ import ( otelApi "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/sdk/metric" otelResource "go.opentelemetry.io/otel/sdk/resource" - semconv "go.opentelemetry.io/otel/semconv/v1.21.0" + semconv "go.opentelemetry.io/otel/semconv/v1.24.0" log "github.com/altinity/clickhouse-operator/pkg/announcer" "github.com/altinity/clickhouse-operator/pkg/apis/deployment" "github.com/altinity/clickhouse-operator/pkg/chop" + "github.com/altinity/clickhouse-operator/pkg/util" "github.com/altinity/clickhouse-operator/pkg/version" ) @@ -96,11 +97,53 @@ func Meter() otelApi.Meter { } func serveMetrics(addr, path string) { - fmt.Printf("serving metrics at %s%s", addr, path) + fmt.Printf("start serving metrics at: %s%s\n", addr, path) http.Handle(path, promhttp.Handler()) err := http.ListenAndServe(addr, nil) if err != nil { fmt.Printf("error serving http: %v", err) - return } + fmt.Printf("end serving metrics at: %s%s\n", addr, path) +} + +type BaseInfoGetter interface { + GetName() string + GetNamespace() string + GetLabels() map[string]string + GetAnnotations() map[string]string +} + +func getLabelsFromName(chi BaseInfoGetter) (labels []string, values []string) { + return []string{"chi", "namespace"}, []string{chi.GetName(), chi.GetNamespace()} +} + +func getLabelsFromLabels(chi BaseInfoGetter) (labels []string, values []string) { + return util.MapGetSortedKeysAndValues(chi.GetLabels()) +} + +func getLabelsFromAnnotations(chi BaseInfoGetter) (labels []string, values []string) { + return util.MapGetSortedKeysAndValues( + // Exclude skipped annotations + util.CopyMapFilter( + chi.GetAnnotations(), + nil, + util.ListSkippedAnnotations(), + ), + ) +} + +func GetMandatoryLabelsAndValues(chi BaseInfoGetter) (labels []string, values []string) { + labelsFromNames, valuesFromNames := getLabelsFromName(chi) + labels = append(labels, labelsFromNames...) + values = append(values, valuesFromNames...) + + labelsFromLabels, valuesFromLabels := getLabelsFromLabels(chi) + labels = append(labels, labelsFromLabels...) + values = append(values, valuesFromLabels...) + + labelsFromAnnotations, valuesFromAnnotations := getLabelsFromAnnotations(chi) + labels = append(labels, labelsFromAnnotations...) + values = append(values, valuesFromAnnotations...) + + return labels, values } diff --git a/pkg/model/chi/action_plan.go b/pkg/model/chi/action_plan.go index c25d066ce..9c25308bc 100644 --- a/pkg/model/chi/action_plan.go +++ b/pkg/model/chi/action_plan.go @@ -57,21 +57,21 @@ func NewActionPlan(old, new *api.ClickHouseInstallation) *ActionPlan { ap.deletionTimestampEqual = ap.timestampEqual(ap.old.DeletionTimestamp, ap.new.DeletionTimestamp) ap.deletionTimestampDiff, _ = messagediff.DeepDiff(ap.old.DeletionTimestamp, ap.new.DeletionTimestamp) ap.finalizersDiff, ap.finalizersEqual = messagediff.DeepDiff(ap.old.Finalizers, ap.new.Finalizers) - ap.attributesDiff, ap.attributesEqual = messagediff.DeepDiff(ap.old.EnsureRuntime().EnsureAttributes(), ap.new.EnsureRuntime().EnsureAttributes()) + ap.attributesDiff, ap.attributesEqual = messagediff.DeepDiff(ap.old.EnsureRuntime().GetAttributes(), ap.new.EnsureRuntime().GetAttributes()) } else if old == nil { ap.specDiff, ap.specEqual = messagediff.DeepDiff(nil, ap.new.Spec) ap.labelsDiff, ap.labelsEqual = messagediff.DeepDiff(nil, ap.new.Labels) ap.deletionTimestampEqual = ap.timestampEqual(nil, ap.new.DeletionTimestamp) ap.deletionTimestampDiff, _ = messagediff.DeepDiff(nil, ap.new.DeletionTimestamp) ap.finalizersDiff, ap.finalizersEqual = messagediff.DeepDiff(nil, ap.new.Finalizers) - ap.attributesDiff, ap.attributesEqual = messagediff.DeepDiff(nil, ap.new.EnsureRuntime().EnsureAttributes()) + ap.attributesDiff, ap.attributesEqual = messagediff.DeepDiff(nil, ap.new.EnsureRuntime().GetAttributes()) } else if new == nil { ap.specDiff, ap.specEqual = messagediff.DeepDiff(ap.old.Spec, nil) ap.labelsDiff, ap.labelsEqual = messagediff.DeepDiff(ap.old.Labels, nil) ap.deletionTimestampEqual = ap.timestampEqual(ap.old.DeletionTimestamp, nil) ap.deletionTimestampDiff, _ = messagediff.DeepDiff(ap.old.DeletionTimestamp, nil) ap.finalizersDiff, ap.finalizersEqual = messagediff.DeepDiff(ap.old.Finalizers, nil) - ap.attributesDiff, ap.attributesEqual = messagediff.DeepDiff(ap.old.EnsureRuntime().EnsureAttributes(), nil) + ap.attributesDiff, ap.attributesEqual = messagediff.DeepDiff(ap.old.EnsureRuntime().GetAttributes(), nil) } else { // Both are nil ap.specDiff = nil diff --git a/pkg/model/chi/affinity.go b/pkg/model/chi/affinity.go index 45b07eff9..8b1360caf 100644 --- a/pkg/model/chi/affinity.go +++ b/pkg/model/chi/affinity.go @@ -26,7 +26,7 @@ import ( ) // NewAffinity creates new Affinity struct -func NewAffinity(template *api.ChiPodTemplate) *core.Affinity { +func NewAffinity(template *api.PodTemplate) *core.Affinity { // Pod node affinity scheduling rules. nodeAffinity := newNodeAffinity(template) // Pod affinity scheduling rules. Ex.: co-locate this pod in the same node, zone, etc @@ -75,7 +75,7 @@ func MergeAffinity(dst *core.Affinity, src *core.Affinity) *core.Affinity { } // newNodeAffinity -func newNodeAffinity(template *api.ChiPodTemplate) *core.NodeAffinity { +func newNodeAffinity(template *api.PodTemplate) *core.NodeAffinity { if template.Zone.Key == "" { return nil } @@ -229,7 +229,7 @@ func mergeNodeAffinity(dst *core.NodeAffinity, src *core.NodeAffinity) *core.Nod } // newPodAffinity -func newPodAffinity(template *api.ChiPodTemplate) *core.PodAffinity { +func newPodAffinity(template *api.PodTemplate) *core.PodAffinity { // Return podAffinity only in case something was added into it added := false podAffinity := &core.PodAffinity{} @@ -450,7 +450,7 @@ func mergePodAffinity(dst *core.PodAffinity, src *core.PodAffinity) *core.PodAff // newMatchLabels func newMatchLabels( - podDistribution *api.ChiPodDistribution, + podDistribution *api.PodDistribution, matchLabels map[string]string, ) map[string]string { var scopeLabels map[string]string @@ -493,7 +493,7 @@ func newMatchLabels( } // newPodAntiAffinity -func newPodAntiAffinity(template *api.ChiPodTemplate) *core.PodAntiAffinity { +func newPodAntiAffinity(template *api.PodTemplate) *core.PodAntiAffinity { // Return podAntiAffinity only in case something was added into it added := false podAntiAffinity := &core.PodAntiAffinity{} @@ -740,7 +740,7 @@ func mergePodAntiAffinity(dst *core.PodAntiAffinity, src *core.PodAntiAffinity) // newPodAffinityTermWithMatchLabels func newPodAffinityTermWithMatchLabels( - podDistribution *api.ChiPodDistribution, + podDistribution *api.PodDistribution, matchLabels map[string]string, ) core.PodAffinityTerm { return core.PodAffinityTerm{ @@ -767,7 +767,7 @@ func newPodAffinityTermWithMatchLabels( // newPodAffinityTermWithMatchExpressions func newPodAffinityTermWithMatchExpressions( - podDistribution *api.ChiPodDistribution, + podDistribution *api.PodDistribution, matchExpressions []meta.LabelSelectorRequirement, ) core.PodAffinityTerm { return core.PodAffinityTerm{ @@ -794,7 +794,7 @@ func newPodAffinityTermWithMatchExpressions( // newWeightedPodAffinityTermWithMatchLabels is an enhanced append() func newWeightedPodAffinityTermWithMatchLabels( weight int32, - podDistribution *api.ChiPodDistribution, + podDistribution *api.PodDistribution, matchLabels map[string]string, ) core.WeightedPodAffinityTerm { return core.WeightedPodAffinityTerm{ @@ -823,7 +823,7 @@ func newWeightedPodAffinityTermWithMatchLabels( } // PrepareAffinity -func PrepareAffinity(podTemplate *api.ChiPodTemplate, host *api.ChiHost) { +func PrepareAffinity(podTemplate *api.PodTemplate, host *api.ChiHost) { switch { case podTemplate == nil: return diff --git a/pkg/model/chi/annotator.go b/pkg/model/chi/annotator.go index 651555385..a01ef80af 100644 --- a/pkg/model/chi/annotator.go +++ b/pkg/model/chi/annotator.go @@ -133,7 +133,7 @@ func (a *Annotator) GetPV(pv *core.PersistentVolume, host *api.ChiHost) map[stri func (a *Annotator) GetPVC( pvc *core.PersistentVolumeClaim, host *api.ChiHost, - template *api.ChiVolumeClaimTemplate, + template *api.VolumeClaimTemplate, ) map[string]string { annotations := util.MergeStringMapsOverwrite(pvc.Annotations, template.ObjectMeta.Annotations) return util.MergeStringMapsOverwrite(annotations, a.GetHostScope(host)) diff --git a/pkg/model/chi/ch_config_generator.go b/pkg/model/chi/ch_config_generator.go index e9b1340f0..99aa86d53 100644 --- a/pkg/model/chi/ch_config_generator.go +++ b/pkg/model/chi/ch_config_generator.go @@ -167,7 +167,7 @@ func (c *ClickHouseConfigGenerator) GetHostZookeeper(host *api.ChiHost) string { // RemoteServersGeneratorOptions specifies options for remote-servers generator type RemoteServersGeneratorOptions struct { exclude struct { - attributes *api.ChiHostReconcileAttributes + attributes *api.HostReconcileAttributes hosts []*api.ChiHost } } @@ -198,7 +198,7 @@ func (o *RemoteServersGeneratorOptions) ExcludeHosts(hosts ...*api.ChiHost) *Rem } // ExcludeReconcileAttributes specifies to exclude reconcile attributes -func (o *RemoteServersGeneratorOptions) ExcludeReconcileAttributes(attrs *api.ChiHostReconcileAttributes) *RemoteServersGeneratorOptions { +func (o *RemoteServersGeneratorOptions) ExcludeReconcileAttributes(attrs *api.HostReconcileAttributes) *RemoteServersGeneratorOptions { if (o == nil) || (attrs == nil) { return o } diff --git a/pkg/model/chi/creator/host.go b/pkg/model/chi/creator/host.go index a69c727df..72cea3f71 100644 --- a/pkg/model/chi/creator/host.go +++ b/pkg/model/chi/creator/host.go @@ -20,10 +20,10 @@ import ( ) // NewDefaultHostTemplate returns default Host Template to be used with StatefulSet -func NewDefaultHostTemplate(name string) *api.ChiHostTemplate { - return &api.ChiHostTemplate{ +func NewDefaultHostTemplate(name string) *api.HostTemplate { + return &api.HostTemplate{ Name: name, - PortDistribution: []api.ChiPortDistribution{ + PortDistribution: []api.PortDistribution{ { Type: deployment.PortDistributionUnspecified, }, @@ -41,10 +41,10 @@ func NewDefaultHostTemplate(name string) *api.ChiHostTemplate { } // NewDefaultHostTemplateForHostNetwork -func NewDefaultHostTemplateForHostNetwork(name string) *api.ChiHostTemplate { - return &api.ChiHostTemplate{ +func NewDefaultHostTemplateForHostNetwork(name string) *api.HostTemplate { + return &api.HostTemplate{ Name: name, - PortDistribution: []api.ChiPortDistribution{ + PortDistribution: []api.PortDistribution{ { Type: deployment.PortDistributionClusterScopeIndex, }, diff --git a/pkg/model/chi/creator/owner_reference.go b/pkg/model/chi/creator/owner_reference.go index bc11ad106..49ac2ccaa 100644 --- a/pkg/model/chi/creator/owner_reference.go +++ b/pkg/model/chi/creator/owner_reference.go @@ -21,7 +21,7 @@ import ( ) func getOwnerReferences(chi *api.ClickHouseInstallation) []meta.OwnerReference { - if chi.EnsureRuntime().EnsureAttributes().SkipOwnerRef { + if chi.EnsureRuntime().GetAttributes().SkipOwnerRef { return nil } return []meta.OwnerReference{ diff --git a/pkg/model/chi/creator/pvc.go b/pkg/model/chi/creator/pvc.go index ba8abb93c..2d0b41738 100644 --- a/pkg/model/chi/creator/pvc.go +++ b/pkg/model/chi/creator/pvc.go @@ -26,7 +26,7 @@ import ( func (c *Creator) PreparePersistentVolumeClaim( pvc *core.PersistentVolumeClaim, host *api.ChiHost, - template *api.ChiVolumeClaimTemplate, + template *api.VolumeClaimTemplate, ) *core.PersistentVolumeClaim { pvc.Labels = model.Macro(host).Map(c.labels.GetPVC(pvc, host, template)) pvc.Annotations = model.Macro(host).Map(c.annotations.GetPVC(pvc, host, template)) @@ -77,6 +77,6 @@ func (c *Creator) CreatePVC(name string, host *api.ChiHost, spec *core.Persisten } // OperatorShouldCreatePVC checks whether operator should create PVC for specified volumeCLimaTemplate -func OperatorShouldCreatePVC(host *api.ChiHost, volumeClaimTemplate *api.ChiVolumeClaimTemplate) bool { +func OperatorShouldCreatePVC(host *api.ChiHost, volumeClaimTemplate *api.VolumeClaimTemplate) bool { return model.GetPVCProvisioner(host, volumeClaimTemplate) == api.PVCProvisionerOperator } diff --git a/pkg/model/chi/creator/service.go b/pkg/model/chi/creator/service.go index 58c995dc3..d46dfb626 100644 --- a/pkg/model/chi/creator/service.go +++ b/pkg/model/chi/creator/service.go @@ -178,9 +178,9 @@ func appendServicePorts(service *core.Service, host *api.ChiHost) { ) } -// createServiceFromTemplate create Service from ChiServiceTemplate and additional info +// createServiceFromTemplate create Service from ServiceTemplate and additional info func (c *Creator) createServiceFromTemplate( - template *api.ChiServiceTemplate, + template *api.ServiceTemplate, namespace string, name string, labels map[string]string, diff --git a/pkg/model/chi/creator/stateful_set.go b/pkg/model/chi/creator/stateful_set.go index e6722014c..d391e6d61 100644 --- a/pkg/model/chi/creator/stateful_set.go +++ b/pkg/model/chi/creator/stateful_set.go @@ -90,7 +90,7 @@ func setupEnvVars(statefulSet *apps.StatefulSet, host *api.ChiHost) { return } - container.Env = append(container.Env, host.GetCHI().EnsureRuntime().EnsureAttributes().AdditionalEnvVars...) + container.Env = append(container.Env, host.GetCHI().EnsureRuntime().GetAttributes().AdditionalEnvVars...) } // ensureMainContainerSpecified is a unification wrapper @@ -211,7 +211,7 @@ func (c *Creator) setupLogContainer(statefulSet *apps.StatefulSet, host *api.Chi } // getPodTemplate gets Pod Template to be used to create StatefulSet -func (c *Creator) getPodTemplate(host *api.ChiHost) *api.ChiPodTemplate { +func (c *Creator) getPodTemplate(host *api.ChiHost) *api.PodTemplate { // Which pod template should be used - either explicitly defined or a default one podTemplate, ok := host.GetPodTemplate() if ok { @@ -269,14 +269,14 @@ func (c *Creator) statefulSetSetupVolumesForSecrets(statefulSet *apps.StatefulSe // Add all additional Volumes k8s.StatefulSetAppendVolumes( statefulSet, - host.GetCHI().EnsureRuntime().EnsureAttributes().AdditionalVolumes..., + host.GetCHI().EnsureRuntime().GetAttributes().AdditionalVolumes..., ) // And reference these Volumes in each Container via VolumeMount // So Pod will have additional volumes mounted as Volumes k8s.StatefulSetAppendVolumeMounts( statefulSet, - host.GetCHI().EnsureRuntime().EnsureAttributes().AdditionalVolumeMounts..., + host.GetCHI().EnsureRuntime().GetAttributes().AdditionalVolumeMounts..., ) } @@ -326,14 +326,14 @@ func (c *Creator) setupStatefulSetVolumeClaimTemplates(statefulSet *apps.Statefu c.statefulSetAppendUsedPVCTemplates(statefulSet, host) } -// statefulSetApplyPodTemplate fills StatefulSet.Spec.Template with data from provided ChiPodTemplate +// statefulSetApplyPodTemplate fills StatefulSet.Spec.Template with data from provided PodTemplate func (c *Creator) statefulSetApplyPodTemplate( statefulSet *apps.StatefulSet, - template *api.ChiPodTemplate, + template *api.PodTemplate, host *api.ChiHost, ) { - // StatefulSet's pod template is not directly compatible with ChiPodTemplate, - // we need to extract some fields from ChiPodTemplate and apply on StatefulSet + // StatefulSet's pod template is not directly compatible with PodTemplate, + // we need to extract some fields from PodTemplate and apply on StatefulSet statefulSet.Spec.Template = core.PodTemplateSpec{ ObjectMeta: meta.ObjectMeta{ Name: template.Name, @@ -387,11 +387,11 @@ func ensureNamedPortsSpecified(statefulSet *apps.StatefulSet, host *api.ChiHost) ) } -// statefulSetAppendPVCTemplate appends to StatefulSet.Spec.VolumeClaimTemplates new entry with data from provided 'src' ChiVolumeClaimTemplate +// statefulSetAppendPVCTemplate appends to StatefulSet.Spec.VolumeClaimTemplates new entry with data from provided 'src' VolumeClaimTemplate func (c *Creator) statefulSetAppendPVCTemplate( statefulSet *apps.StatefulSet, host *api.ChiHost, - volumeClaimTemplate *api.ChiVolumeClaimTemplate, + volumeClaimTemplate *api.VolumeClaimTemplate, ) { // Since we have the same names for PVs produced from both VolumeClaimTemplates and Volumes, // we need to check naming for all of them @@ -432,13 +432,13 @@ func (c *Creator) statefulSetAppendPVCTemplate( } // newDefaultPodTemplate is a unification wrapper -func newDefaultPodTemplate(host *api.ChiHost) *api.ChiPodTemplate { +func newDefaultPodTemplate(host *api.ChiHost) *api.PodTemplate { return newDefaultClickHousePodTemplate(host) } // newDefaultClickHousePodTemplate returns default Pod Template to be used with StatefulSet -func newDefaultClickHousePodTemplate(host *api.ChiHost) *api.ChiPodTemplate { - podTemplate := &api.ChiPodTemplate{ +func newDefaultClickHousePodTemplate(host *api.ChiHost) *api.PodTemplate { + podTemplate := &api.PodTemplate{ Name: model.CreateStatefulSetName(host), Spec: core.PodSpec{ Containers: []core.Container{}, diff --git a/pkg/model/chi/creator/volume.go b/pkg/model/chi/creator/volume.go index 19ac4c3b2..84ef1bd1f 100644 --- a/pkg/model/chi/creator/volume.go +++ b/pkg/model/chi/creator/volume.go @@ -57,7 +57,7 @@ func newVolumeMount(name, mountPath string) core.VolumeMount { } } -func getVolumeClaimTemplate(volumeMount *core.VolumeMount, host *api.ChiHost) (*api.ChiVolumeClaimTemplate, bool) { +func getVolumeClaimTemplate(volumeMount *core.VolumeMount, host *api.ChiHost) (*api.VolumeClaimTemplate, bool) { volumeClaimTemplateName := volumeMount.Name volumeClaimTemplate, ok := host.GetCHI().GetVolumeClaimTemplate(volumeClaimTemplateName) diff --git a/pkg/model/chi/deleter.go b/pkg/model/chi/deleter.go index 56029b8d1..0ade99a91 100644 --- a/pkg/model/chi/deleter.go +++ b/pkg/model/chi/deleter.go @@ -48,7 +48,7 @@ func HostCanDeletePVC(host *api.ChiHost, pvcName string) bool { // HostCanDeleteAllPVCs checks whether all PVCs can be deleted func HostCanDeleteAllPVCs(host *api.ChiHost) bool { canDeleteAllPVCs := true - host.GetCHI().WalkVolumeClaimTemplates(func(template *api.ChiVolumeClaimTemplate) { + host.GetCHI().WalkVolumeClaimTemplates(func(template *api.VolumeClaimTemplate) { if getPVCReclaimPolicy(host, template) == api.PVCReclaimPolicyRetain { // At least one template wants to keep its PVC canDeleteAllPVCs = false diff --git a/pkg/model/chi/labeler.go b/pkg/model/chi/labeler.go index 48a6b81cd..077298a1c 100644 --- a/pkg/model/chi/labeler.go +++ b/pkg/model/chi/labeler.go @@ -264,7 +264,7 @@ func (l *Labeler) GetHostScopeReady(host *api.ChiHost, applySupplementaryService } // getHostScopeReclaimPolicy gets host scope labels with PVCReclaimPolicy from template -func (l *Labeler) getHostScopeReclaimPolicy(host *api.ChiHost, template *api.ChiVolumeClaimTemplate, applySupplementaryServiceLabels bool) map[string]string { +func (l *Labeler) getHostScopeReclaimPolicy(host *api.ChiHost, template *api.VolumeClaimTemplate, applySupplementaryServiceLabels bool) map[string]string { return util.MergeStringMapsOverwrite(l.GetHostScope(host, applySupplementaryServiceLabels), map[string]string{ LabelPVCReclaimPolicyName: getPVCReclaimPolicy(host, template).String(), }) @@ -279,7 +279,7 @@ func (l *Labeler) GetPV(pv *core.PersistentVolume, host *api.ChiHost) map[string func (l *Labeler) GetPVC( pvc *core.PersistentVolumeClaim, host *api.ChiHost, - template *api.ChiVolumeClaimTemplate, + template *api.VolumeClaimTemplate, ) map[string]string { // Prepare main labels based on template labels := util.MergeStringMapsOverwrite(pvc.Labels, template.ObjectMeta.Labels) diff --git a/pkg/model/chi/namer.go b/pkg/model/chi/namer.go index 6a5a4f4bc..54ae1121a 100644 --- a/pkg/model/chi/namer.go +++ b/pkg/model/chi/namer.go @@ -718,7 +718,7 @@ func createPodNamesOfCHI(chi *api.ClickHouseInstallation) (names []string) { } // CreatePVCNameByVolumeClaimTemplate creates PVC name -func CreatePVCNameByVolumeClaimTemplate(host *api.ChiHost, volumeClaimTemplate *api.ChiVolumeClaimTemplate) string { +func CreatePVCNameByVolumeClaimTemplate(host *api.ChiHost, volumeClaimTemplate *api.VolumeClaimTemplate) string { return createPVCName(host, volumeClaimTemplate.Name) } diff --git a/pkg/model/chi/normalizer/normalizer.go b/pkg/model/chi/normalizer/normalizer.go index 64726fdaf..a597913e9 100644 --- a/pkg/model/chi/normalizer/normalizer.go +++ b/pkg/model/chi/normalizer/normalizer.go @@ -147,7 +147,7 @@ func (n *Normalizer) fillCHIAddressInfo() { } // getHostTemplate gets Host Template to be used to normalize Host -func (n *Normalizer) getHostTemplate(host *api.ChiHost) *api.ChiHostTemplate { +func (n *Normalizer) getHostTemplate(host *api.ChiHost) *api.HostTemplate { // Which host template would be used - either explicitly defined in or a default one hostTemplate, ok := host.GetHostTemplate() if ok { @@ -178,7 +178,7 @@ func (n *Normalizer) getHostTemplate(host *api.ChiHost) *api.ChiHostTemplate { } // hostApplyHostTemplate -func hostApplyHostTemplate(host *api.ChiHost, template *api.ChiHostTemplate) { +func hostApplyHostTemplate(host *api.ChiHost, template *api.HostTemplate) { if host.GetName() == "" { host.Name = template.Spec.Name } @@ -413,7 +413,7 @@ func (n *Normalizer) normalizeConfigurationAllSettingsBasedSections(conf *api.Co } // normalizeTemplates normalizes .spec.templates -func (n *Normalizer) normalizeTemplates(templates *api.ChiTemplates) *api.ChiTemplates { +func (n *Normalizer) normalizeTemplates(templates *api.Templates) *api.Templates { if templates == nil { //templates = api.NewChiTemplates() return nil @@ -505,39 +505,39 @@ func (n *Normalizer) normalizeCleanup(str *string, value string) { } } -func (n *Normalizer) normalizeHostTemplates(templates *api.ChiTemplates) { +func (n *Normalizer) normalizeHostTemplates(templates *api.Templates) { for i := range templates.HostTemplates { n.normalizeHostTemplate(&templates.HostTemplates[i]) } } -func (n *Normalizer) normalizePodTemplates(templates *api.ChiTemplates) { +func (n *Normalizer) normalizePodTemplates(templates *api.Templates) { for i := range templates.PodTemplates { n.normalizePodTemplate(&templates.PodTemplates[i]) } } -func (n *Normalizer) normalizeVolumeClaimTemplates(templates *api.ChiTemplates) { +func (n *Normalizer) normalizeVolumeClaimTemplates(templates *api.Templates) { for i := range templates.VolumeClaimTemplates { n.normalizeVolumeClaimTemplate(&templates.VolumeClaimTemplates[i]) } } -func (n *Normalizer) normalizeServiceTemplates(templates *api.ChiTemplates) { +func (n *Normalizer) normalizeServiceTemplates(templates *api.Templates) { for i := range templates.ServiceTemplates { n.normalizeServiceTemplate(&templates.ServiceTemplates[i]) } } // normalizeHostTemplate normalizes .spec.templates.hostTemplates -func (n *Normalizer) normalizeHostTemplate(template *api.ChiHostTemplate) { +func (n *Normalizer) normalizeHostTemplate(template *api.HostTemplate) { templatesNormalizer.NormalizeHostTemplate(template) // Introduce HostTemplate into Index n.ctx.GetTarget().Spec.Templates.EnsureHostTemplatesIndex().Set(template.Name, template) } // normalizePodTemplate normalizes .spec.templates.podTemplates -func (n *Normalizer) normalizePodTemplate(template *api.ChiPodTemplate) { +func (n *Normalizer) normalizePodTemplate(template *api.PodTemplate) { // TODO need to support multi-cluster replicasCount := 1 if len(n.ctx.GetTarget().Spec.Configuration.Clusters) > 0 { @@ -549,21 +549,21 @@ func (n *Normalizer) normalizePodTemplate(template *api.ChiPodTemplate) { } // normalizeVolumeClaimTemplate normalizes .spec.templates.volumeClaimTemplates -func (n *Normalizer) normalizeVolumeClaimTemplate(template *api.ChiVolumeClaimTemplate) { +func (n *Normalizer) normalizeVolumeClaimTemplate(template *api.VolumeClaimTemplate) { templatesNormalizer.NormalizeVolumeClaimTemplate(template) // Introduce VolumeClaimTemplate into Index n.ctx.GetTarget().Spec.Templates.EnsureVolumeClaimTemplatesIndex().Set(template.Name, template) } // normalizeServiceTemplate normalizes .spec.templates.serviceTemplates -func (n *Normalizer) normalizeServiceTemplate(template *api.ChiServiceTemplate) { +func (n *Normalizer) normalizeServiceTemplate(template *api.ServiceTemplate) { templatesNormalizer.NormalizeServiceTemplate(template) // Introduce ServiceClaimTemplate into Index n.ctx.GetTarget().Spec.Templates.EnsureServiceTemplatesIndex().Set(template.Name, template) } // normalizeUseTemplates is a wrapper to hold the name of normalized section -func (n *Normalizer) normalizeUseTemplates(templates []*api.ChiTemplateRef) []*api.ChiTemplateRef { +func (n *Normalizer) normalizeUseTemplates(templates []*api.TemplateRef) []*api.TemplateRef { return templatesNormalizer.NormalizeTemplatesList(templates) } @@ -800,14 +800,14 @@ func (n *Normalizer) appendAdditionalEnvVar(envVar core.EnvVar) { return } - for _, existingEnvVar := range n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalEnvVars { + for _, existingEnvVar := range n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalEnvVars { if existingEnvVar.Name == envVar.Name { // Such a variable already exists return } } - n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalEnvVars = append(n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalEnvVars, envVar) + n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalEnvVars = append(n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalEnvVars, envVar) } func (n *Normalizer) appendAdditionalVolume(volume core.Volume) { @@ -816,14 +816,14 @@ func (n *Normalizer) appendAdditionalVolume(volume core.Volume) { return } - for _, existingVolume := range n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalVolumes { + for _, existingVolume := range n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalVolumes { if existingVolume.Name == volume.Name { // Such a variable already exists return } } - n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalVolumes = append(n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalVolumes, volume) + n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalVolumes = append(n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalVolumes, volume) } func (n *Normalizer) appendAdditionalVolumeMount(volumeMount core.VolumeMount) { @@ -832,14 +832,14 @@ func (n *Normalizer) appendAdditionalVolumeMount(volumeMount core.VolumeMount) { return } - for _, existingVolumeMount := range n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalVolumeMounts { + for _, existingVolumeMount := range n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalVolumeMounts { if existingVolumeMount.Name == volumeMount.Name { // Such a variable already exists return } } - n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalVolumeMounts = append(n.ctx.GetTarget().EnsureRuntime().EnsureAttributes().AdditionalVolumeMounts, volumeMount) + n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalVolumeMounts = append(n.ctx.GetTarget().EnsureRuntime().GetAttributes().AdditionalVolumeMounts, volumeMount) } var ErrSecretValueNotFound = fmt.Errorf("secret value not found") diff --git a/pkg/model/chi/normalizer/templates/chi.go b/pkg/model/chi/normalizer/templates/chi.go index 01eaf36c9..b1624917a 100644 --- a/pkg/model/chi/normalizer/templates/chi.go +++ b/pkg/model/chi/normalizer/templates/chi.go @@ -27,7 +27,7 @@ const ( ) // prepareListOfTemplates prepares list of CHI templates to be used by the CHI -func prepareListOfTemplates(chi *api.ClickHouseInstallation) (templates []*api.ChiTemplateRef) { +func prepareListOfTemplates(chi *api.ClickHouseInstallation) (templates []*api.TemplateRef) { // 1. Get list of auto templates available templates = append(templates, prepareListOfAutoTemplates(chi)...) // 2. Append templates which are explicitly requested by the CHI @@ -39,7 +39,7 @@ func prepareListOfTemplates(chi *api.ClickHouseInstallation) (templates []*api.C return templates } -func prepareListOfAutoTemplates(chi *api.ClickHouseInstallation) (templates []*api.ChiTemplateRef) { +func prepareListOfAutoTemplates(chi *api.ClickHouseInstallation) (templates []*api.TemplateRef) { // 1. Get list of auto templates available if autoTemplates := chop.Config().GetAutoTemplates(); len(autoTemplates) > 0 { log.V(1).M(chi).F().Info("Found auto-templates num: %d", len(autoTemplates)) @@ -47,7 +47,7 @@ func prepareListOfAutoTemplates(chi *api.ClickHouseInstallation) (templates []*a log.V(1).M(chi).F().Info( "Adding auto-template to the list of applicable templates: %s/%s ", template.Namespace, template.Name) - templates = append(templates, &api.ChiTemplateRef{ + templates = append(templates, &api.TemplateRef{ Name: template.Name, Namespace: template.Namespace, UseType: UseTypeMerge, @@ -58,7 +58,7 @@ func prepareListOfAutoTemplates(chi *api.ClickHouseInstallation) (templates []*a return templates } -func prepareListOfManualTemplates(chi *api.ClickHouseInstallation) (templates []*api.ChiTemplateRef) { +func prepareListOfManualTemplates(chi *api.ClickHouseInstallation) (templates []*api.TemplateRef) { if len(chi.Spec.UseTemplates) > 0 { log.V(1).M(chi).F().Info("Found manual-templates num: %d", len(chi.Spec.UseTemplates)) templates = append(templates, chi.Spec.UseTemplates...) @@ -68,7 +68,7 @@ func prepareListOfManualTemplates(chi *api.ClickHouseInstallation) (templates [] } // ApplyCHITemplates applies templates over target n.ctx.chi -func ApplyCHITemplates(target, chi *api.ClickHouseInstallation) (appliedTemplates []*api.ChiTemplateRef) { +func ApplyCHITemplates(target, chi *api.ClickHouseInstallation) (appliedTemplates []*api.TemplateRef) { // Prepare list of templates to be applied to the CHI templates := prepareListOfTemplates(chi) @@ -85,7 +85,7 @@ func ApplyCHITemplates(target, chi *api.ClickHouseInstallation) (appliedTemplate // applyTemplate applies a template over target n.ctx.chi // `chi *api.ClickHouseInstallation` is used to determine whether the template should be applied or not only -func applyTemplate(target *api.ClickHouseInstallation, templateRef *api.ChiTemplateRef, chi *api.ClickHouseInstallation) bool { +func applyTemplate(target *api.ClickHouseInstallation, templateRef *api.TemplateRef, chi *api.ClickHouseInstallation) bool { if templateRef == nil { log.Warning("unable to apply template - nil templateRef provided") // Template is not applied @@ -158,15 +158,15 @@ func mergeFromTemplate(target, template *api.ClickHouseInstallation) *api.ClickH } // NormalizeTemplatesList normalizes list of templates use specifications -func NormalizeTemplatesList(templates []*api.ChiTemplateRef) []*api.ChiTemplateRef { +func NormalizeTemplatesList(templates []*api.TemplateRef) []*api.TemplateRef { for i := range templates { templates[i] = normalizeTemplateRef(templates[i]) } return templates } -// normalizeTemplateRef normalizes ChiTemplateRef -func normalizeTemplateRef(templateRef *api.ChiTemplateRef) *api.ChiTemplateRef { +// normalizeTemplateRef normalizes TemplateRef +func normalizeTemplateRef(templateRef *api.TemplateRef) *api.TemplateRef { // Check Name if templateRef.Name == "" { // This is strange diff --git a/pkg/model/chi/normalizer/templates/host.go b/pkg/model/chi/normalizer/templates/host.go index 554c2601f..77a715fe1 100644 --- a/pkg/model/chi/normalizer/templates/host.go +++ b/pkg/model/chi/normalizer/templates/host.go @@ -21,14 +21,14 @@ import ( ) // NormalizeHostTemplate normalizes .spec.templates.hostTemplates -func NormalizeHostTemplate(template *api.ChiHostTemplate) { +func NormalizeHostTemplate(template *api.HostTemplate) { // Name // PortDistribution if template.PortDistribution == nil { // In case no PortDistribution provided - setup default one - template.PortDistribution = []api.ChiPortDistribution{ + template.PortDistribution = []api.PortDistribution{ { Type: deployment.PortDistributionUnspecified, }, diff --git a/pkg/model/chi/normalizer/templates/pod.go b/pkg/model/chi/normalizer/templates/pod.go index d697f23c0..7d08b79ee 100644 --- a/pkg/model/chi/normalizer/templates/pod.go +++ b/pkg/model/chi/normalizer/templates/pod.go @@ -23,7 +23,7 @@ import ( ) // NormalizePodTemplate normalizes .spec.templates.podTemplates -func NormalizePodTemplate(replicasCount int, template *api.ChiPodTemplate) { +func NormalizePodTemplate(replicasCount int, template *api.PodTemplate) { // Name // GenerateName // No normalization so far for these @@ -45,7 +45,7 @@ func NormalizePodTemplate(replicasCount int, template *api.ChiPodTemplate) { } } -func normalizePodTemplateZone(template *api.ChiPodTemplate) { +func normalizePodTemplateZone(template *api.PodTemplate) { switch { case len(template.Zone.Values) == 0: // In case no values specified - no key is reasonable @@ -60,7 +60,7 @@ func normalizePodTemplateZone(template *api.ChiPodTemplate) { } } -func normalizePodTemplateDistribution(replicasCount int, template *api.ChiPodTemplate) { +func normalizePodTemplateDistribution(replicasCount int, template *api.PodTemplate) { for i := range template.PodDistribution { if additionalPodDistributions := normalizePodDistribution(replicasCount, &template.PodDistribution[i]); additionalPodDistributions != nil { template.PodDistribution = append(template.PodDistribution, additionalPodDistributions...) @@ -70,7 +70,7 @@ func normalizePodTemplateDistribution(replicasCount int, template *api.ChiPodTem const defaultTopologyKey = core.LabelHostname -func normalizePodDistribution(replicasCount int, podDistribution *api.ChiPodDistribution) []api.ChiPodDistribution { +func normalizePodDistribution(replicasCount int, podDistribution *api.PodDistribution) []api.PodDistribution { // Ensure topology key if podDistribution.TopologyKey == "" { podDistribution.TopologyKey = defaultTopologyKey @@ -122,7 +122,7 @@ func normalizePodDistribution(replicasCount int, podDistribution *api.ChiPodDist } // Expand shortcut - return []api.ChiPodDistribution{ + return []api.PodDistribution{ { Type: deployment.PodDistributionShardAntiAffinity, Scope: podDistribution.Scope, diff --git a/pkg/model/chi/normalizer/templates/service.go b/pkg/model/chi/normalizer/templates/service.go index b3f95bbe4..bd3e3c318 100644 --- a/pkg/model/chi/normalizer/templates/service.go +++ b/pkg/model/chi/normalizer/templates/service.go @@ -17,7 +17,7 @@ package templates import api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" // NormalizeServiceTemplate normalizes .spec.templates.volumeClaimTemplates -func NormalizeServiceTemplate(template *api.ChiServiceTemplate) { +func NormalizeServiceTemplate(template *api.ServiceTemplate) { // Check name // Check GenerateName // Check ObjectMeta diff --git a/pkg/model/chi/normalizer/templates/volume_claim.go b/pkg/model/chi/normalizer/templates/volume_claim.go index d59f5b8cd..ffd6b1f05 100644 --- a/pkg/model/chi/normalizer/templates/volume_claim.go +++ b/pkg/model/chi/normalizer/templates/volume_claim.go @@ -17,7 +17,7 @@ package templates import api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" // NormalizeVolumeClaimTemplate normalizes .spec.templates.volumeClaimTemplates -func NormalizeVolumeClaimTemplate(template *api.ChiVolumeClaimTemplate) { +func NormalizeVolumeClaimTemplate(template *api.VolumeClaimTemplate) { // Check name // Skip for now diff --git a/pkg/model/chi/registry.go b/pkg/model/chi/registry.go index 937bbc1f1..91145f510 100644 --- a/pkg/model/chi/registry.go +++ b/pkg/model/chi/registry.go @@ -58,6 +58,14 @@ type objectMetaIdentity struct { namespace string } +// newObjectMetaIdentity creates new objectMetaIdentity from an ObjectMeta +func newObjectMetaIdentity(obj *meta.ObjectMeta) objectMetaIdentity { + return objectMetaIdentity{ + name: obj.Name, + namespace: obj.Namespace, + } +} + // objectMetaSet is an internal collections type used for efficient lookups of object metadata by identity, // defined in this context as the combination of namespace and name. A set is expected to correspond to a single // entity type, but set additions to not validate this property to avoid the introduction of error results onto @@ -69,6 +77,14 @@ type objectMetaSet struct { sync.RWMutex } +// newObjectMetaSet constructor for a set holding ObjectMeta with reader/writer synchronization +func newObjectMetaSet(entityType EntityType) *objectMetaSet { + return &objectMetaSet{ + entityType: entityType, + contents: make(map[objectMetaIdentity]meta.ObjectMeta), + } +} + // NewRegistry creates new registry func NewRegistry() *Registry { return &Registry{ @@ -78,23 +94,23 @@ func NewRegistry() *Registry { // Len return len of the whole registry or specified entity types // Note that this is unsafe to call recursively, including in calls to other synchronized Registry functions -// like Walk (and therefore in the "work" function passed into iterators like Walk and WalkEntityType). -func (r *Registry) Len(_what ...EntityType) int { +// like Walk (and therefore in the "work" function passed into iterators like Walk and walkEntityType). +func (r *Registry) Len(what ...EntityType) int { if r == nil { return 0 } // Avoid coarse grained locking when we will just return the number of entity types in the registry. - if len(_what) == 0 { + if len(what) == 0 { r.mu.RLock() defer r.mu.RUnlock() return len(r.r) } result := 0 - for _, entityType := range _what { - os := r.ensureObjectSetForType(entityType) - result += os.len() + for _, entityType := range what { + set := r.ensureObjectSetForType(entityType) + result += set.len() } return result } @@ -110,15 +126,15 @@ func (r *Registry) Walk(f func(entityType EntityType, meta meta.ObjectMeta)) { r.mu.RLock() defer r.mu.RUnlock() - for et, os := range r.r { - os.walk(func(meta meta.ObjectMeta) { - f(et, meta) + for _type, set := range r.r { + set.walk(func(meta meta.ObjectMeta) { + f(_type, meta) }) } } -// WalkEntityType walks over registry -func (r *Registry) WalkEntityType(entityType EntityType, f func(meta meta.ObjectMeta)) { +// walkEntityType walks over registry +func (r *Registry) walkEntityType(entityType EntityType, f func(meta meta.ObjectMeta)) { if r == nil { return } @@ -132,11 +148,11 @@ func (r *Registry) String() string { if r == nil { return "" } - s := "" + str := "" r.Walk(func(entityType EntityType, meta meta.ObjectMeta) { - s += fmt.Sprintf("%s: %s/%s\n", entityType, meta.Namespace, meta.Name) + str += fmt.Sprintf("%s: %s/%s\n", entityType, meta.Namespace, meta.Name) }) - return s + return str } // registerEntity register entity @@ -178,7 +194,7 @@ func (r *Registry) NumStatefulSet() int { // WalkStatefulSet walk over specified entity types func (r *Registry) WalkStatefulSet(f func(meta meta.ObjectMeta)) { - r.WalkEntityType(StatefulSet, f) + r.walkEntityType(StatefulSet, f) } // RegisterConfigMap register ConfigMap @@ -198,7 +214,7 @@ func (r *Registry) NumConfigMap() int { // WalkConfigMap walk over specified entity types func (r *Registry) WalkConfigMap(f func(meta meta.ObjectMeta)) { - r.WalkEntityType(ConfigMap, f) + r.walkEntityType(ConfigMap, f) } // RegisterService register Service @@ -218,7 +234,7 @@ func (r *Registry) NumService() int { // WalkService walk over specified entity types func (r *Registry) WalkService(f func(meta meta.ObjectMeta)) { - r.WalkEntityType(Service, f) + r.walkEntityType(Service, f) } // RegisterSecret register Secret @@ -238,7 +254,7 @@ func (r *Registry) NumSecret() int { // WalkSecret walk over specified entity types func (r *Registry) WalkSecret(f func(meta meta.ObjectMeta)) { - r.WalkEntityType(Secret, f) + r.walkEntityType(Secret, f) } // RegisterPVC register PVC @@ -258,7 +274,7 @@ func (r *Registry) NumPVC() int { // WalkPVC walk over specified entity types func (r *Registry) WalkPVC(f func(meta meta.ObjectMeta)) { - r.WalkEntityType(PVC, f) + r.walkEntityType(PVC, f) } // Comment out PV @@ -279,7 +295,7 @@ func (r *Registry) WalkPVC(f func(meta meta.ObjectMeta)) { // //// WalkPV walk over specified entity types //func (r *Registry) WalkPV(f func(meta meta.ObjectMeta)) { -// r.WalkEntityType(PV, f) +// r.walkEntityType(PV, f) //} // RegisterPDB register PDB @@ -299,7 +315,7 @@ func (r *Registry) NumPDB() int { // WalkPDB walk over specified entity types func (r *Registry) WalkPDB(f func(meta meta.ObjectMeta)) { - r.WalkEntityType(PDB, f) + r.walkEntityType(PDB, f) } // Subtract subtracts specified registry from main @@ -330,11 +346,6 @@ func (r *Registry) hasEntity(entityType EntityType, meta meta.ObjectMeta) bool { return setForType.contains(&meta) } -// isEqual -func (r *Registry) isEqual(a, b meta.ObjectMeta) bool { - return (a.Namespace == b.Namespace) && (a.Name == b.Name) -} - // deleteEntity func (r *Registry) deleteEntity(entityType EntityType, meta meta.ObjectMeta) bool { // Try to minimize coarse grained locking at the registry level. Immediately getOrCreate for the entity type @@ -372,41 +383,33 @@ func (r *Registry) ensureObjectSetForType(entityType EntityType) *objectMetaSet return newSet } -// objIdentity derives a objectMetaIdentity from an ObjectMeta -func (s *objectMetaSet) objIdentity(obj *meta.ObjectMeta) objectMetaIdentity { - return objectMetaIdentity{ - name: obj.Name, - namespace: obj.Namespace, - } -} - // maybeAdd adds an ObjectMeta to the set if an object with an equivalent identity is not already present -func (s *objectMetaSet) maybeAdd(o *meta.ObjectMeta) bool { +func (s *objectMetaSet) maybeAdd(meta *meta.ObjectMeta) bool { s.Lock() defer s.Unlock() - if _, ok := s.contents[s.objIdentity(o)]; ok { + if _, ok := s.contents[newObjectMetaIdentity(meta)]; ok { return false } - s.contents[s.objIdentity(o)] = *o + s.contents[newObjectMetaIdentity(meta)] = *meta return true } // remove deletes an ObjectMeta from the set, matching only on identity -func (s *objectMetaSet) remove(o *meta.ObjectMeta) bool { +func (s *objectMetaSet) remove(meta *meta.ObjectMeta) bool { s.Lock() defer s.Unlock() - if _, ok := s.contents[s.objIdentity(o)]; !ok { + if _, ok := s.contents[newObjectMetaIdentity(meta)]; !ok { return false } - delete(s.contents, s.objIdentity(o)) + delete(s.contents, newObjectMetaIdentity(meta)) return true } // contains determines if an ObjectMeta exists in the set (based on identity only) -func (s *objectMetaSet) contains(o *meta.ObjectMeta) bool { +func (s *objectMetaSet) contains(meta *meta.ObjectMeta) bool { s.RLock() defer s.RUnlock() - _, ok := s.contents[s.objIdentity(o)] + _, ok := s.contents[newObjectMetaIdentity(meta)] return ok } @@ -429,11 +432,3 @@ func (s *objectMetaSet) len() int { return len(s.contents) } - -// newObjectMetaSet constructor for a set holding ObjectMeta with reader/writer synchronization -func newObjectMetaSet(entityType EntityType) *objectMetaSet { - return &objectMetaSet{ - entityType: entityType, - contents: make(map[objectMetaIdentity]meta.ObjectMeta), - } -} diff --git a/pkg/model/chi/volumer.go b/pkg/model/chi/volumer.go index 9d90f620a..c4bb21d3b 100644 --- a/pkg/model/chi/volumer.go +++ b/pkg/model/chi/volumer.go @@ -20,7 +20,7 @@ import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" ) -func GetVolumeClaimTemplate(host *api.ChiHost, volumeMount *core.VolumeMount) (*api.ChiVolumeClaimTemplate, bool) { +func GetVolumeClaimTemplate(host *api.ChiHost, volumeMount *core.VolumeMount) (*api.VolumeClaimTemplate, bool) { volumeClaimTemplateName := volumeMount.Name volumeClaimTemplate, ok := host.GetCHI().GetVolumeClaimTemplate(volumeClaimTemplateName) // Sometimes it is impossible to find VolumeClaimTemplate related to specified volumeMount. @@ -28,7 +28,7 @@ func GetVolumeClaimTemplate(host *api.ChiHost, volumeMount *core.VolumeMount) (* return volumeClaimTemplate, ok } -func getPVCReclaimPolicy(host *api.ChiHost, template *api.ChiVolumeClaimTemplate) api.PVCReclaimPolicy { +func getPVCReclaimPolicy(host *api.ChiHost, template *api.VolumeClaimTemplate) api.PVCReclaimPolicy { // Order by priority // VolumeClaimTemplate.PVCReclaimPolicy, in case specified @@ -44,7 +44,7 @@ func getPVCReclaimPolicy(host *api.ChiHost, template *api.ChiVolumeClaimTemplate return api.PVCReclaimPolicyDelete } -func GetPVCProvisioner(host *api.ChiHost, template *api.ChiVolumeClaimTemplate) api.PVCProvisioner { +func GetPVCProvisioner(host *api.ChiHost, template *api.VolumeClaimTemplate) api.PVCProvisioner { // Order by priority // VolumeClaimTemplate.PVCProvisioner, in case specified diff --git a/pkg/model/chk/normalizer.go b/pkg/model/chk/normalizer.go index 66a8eb889..23071c703 100644 --- a/pkg/model/chk/normalizer.go +++ b/pkg/model/chk/normalizer.go @@ -130,7 +130,7 @@ func (n *Normalizer) normalizeConfiguration(conf *apiChk.ChkConfiguration) *apiC } // normalizeTemplates normalizes .spec.templates -func (n *Normalizer) normalizeTemplates(templates *apiChi.ChiTemplates) *apiChi.ChiTemplates { +func (n *Normalizer) normalizeTemplates(templates *apiChi.Templates) *apiChi.Templates { if templates == nil { //templates = apiChi.NewChiTemplates() return nil @@ -155,7 +155,7 @@ func (n *Normalizer) normalizeTemplates(templates *apiChi.ChiTemplates) *apiChi. } // normalizePodTemplate normalizes .spec.templates.podTemplates -func (n *Normalizer) normalizePodTemplate(template *apiChi.ChiPodTemplate) { +func (n *Normalizer) normalizePodTemplate(template *apiChi.PodTemplate) { // TODO need to support multi-cluster replicasCount := 1 if len(n.ctx.chk.Spec.Configuration.Clusters) > 0 { @@ -167,14 +167,14 @@ func (n *Normalizer) normalizePodTemplate(template *apiChi.ChiPodTemplate) { } // normalizeVolumeClaimTemplate normalizes .spec.templates.volumeClaimTemplates -func (n *Normalizer) normalizeVolumeClaimTemplate(template *apiChi.ChiVolumeClaimTemplate) { +func (n *Normalizer) normalizeVolumeClaimTemplate(template *apiChi.VolumeClaimTemplate) { templatesNormalizer.NormalizeVolumeClaimTemplate(template) // Introduce VolumeClaimTemplate into Index n.ctx.chk.Spec.Templates.EnsureVolumeClaimTemplatesIndex().Set(template.Name, template) } // normalizeServiceTemplate normalizes .spec.templates.serviceTemplates -func (n *Normalizer) normalizeServiceTemplate(template *apiChi.ChiServiceTemplate) { +func (n *Normalizer) normalizeServiceTemplate(template *apiChi.ServiceTemplate) { templatesNormalizer.NormalizeServiceTemplate(template) // Introduce ServiceClaimTemplate into Index n.ctx.chk.Spec.Templates.EnsureServiceTemplatesIndex().Set(template.Name, template) diff --git a/pkg/model/chk/templates.go b/pkg/model/chk/templates.go index f5dd6486f..f601a3b8a 100644 --- a/pkg/model/chk/templates.go +++ b/pkg/model/chk/templates.go @@ -21,9 +21,9 @@ import ( apiChi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" ) -func getPodTemplate(chk *apiChk.ClickHouseKeeperInstallation) apiChi.ChiPodTemplate { +func getPodTemplate(chk *apiChk.ClickHouseKeeperInstallation) apiChi.PodTemplate { if len(chk.Spec.GetTemplates().GetPodTemplates()) < 1 { - return apiChi.ChiPodTemplate{} + return apiChi.PodTemplate{} } return chk.Spec.GetTemplates().GetPodTemplates()[0] } diff --git a/pkg/util/k8s.go b/pkg/util/k8s.go index 32701ed61..c37b3fa7f 100644 --- a/pkg/util/k8s.go +++ b/pkg/util/k8s.go @@ -43,7 +43,5 @@ func IsAnnotationToBeSkipped(annotation string) bool { // ListSkippedAnnotations provides list of annotations that should be skipped func ListSkippedAnnotations() []string { - return []string{ - "kubectl.kubernetes.io/last-applied-configuration", - } + return AnnotationsTobeSkipped } diff --git a/pkg/util/map.go b/pkg/util/map.go index 4932da42c..2d4d2e1f2 100644 --- a/pkg/util/map.go +++ b/pkg/util/map.go @@ -240,3 +240,27 @@ func Map2String(name string, m map[string]string) string { return b.String() } + +func MapGetSortedKeys(m map[string]string) (keys []string) { + if m == nil { + return nil + } + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func MapGetSortedKeysAndValues(m map[string]string) (keys []string, values []string) { + if m == nil { + return nil, nil + } + keys = MapGetSortedKeys(m) + for _, key := range keys { + if value, ok := m[key]; ok { + values = append(values, value) + } + } + return keys, values +} diff --git a/release b/release index 40a6dfede..f6de00174 100644 --- a/release +++ b/release @@ -1 +1 @@ -0.23.4 +0.23.5 diff --git a/releases b/releases index e2b10d252..d6472dba7 100644 --- a/releases +++ b/releases @@ -1,3 +1,4 @@ +0.23.4 0.23.3 0.23.2 0.23.1 diff --git a/tests/e2e/manifests/chi/test-017-multi-version.yaml b/tests/e2e/manifests/chi/test-017-multi-version.yaml index 215c42595..b6998a1ac 100644 --- a/tests/e2e/manifests/chi/test-017-multi-version.yaml +++ b/tests/e2e/manifests/chi/test-017-multi-version.yaml @@ -3,7 +3,9 @@ kind: ClickHouseInstallation metadata: name: test-017-multi-version labels: - clickhouse.altinity.com/chi: test-017-multi-version + clickhouse.altinity.com/chi: test-017-multi-version + annotations: + clickhouse.altinity.com/email: "myname@mydomain.com, yourname@yourdoman.com" spec: templates: podTemplates: diff --git a/tests/e2e/manifests/chi/test-046-0-clickhouse-operator-metrics.yaml b/tests/e2e/manifests/chi/test-046-0-clickhouse-operator-metrics.yaml new file mode 100644 index 000000000..34b62af68 --- /dev/null +++ b/tests/e2e/manifests/chi/test-046-0-clickhouse-operator-metrics.yaml @@ -0,0 +1,12 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-046-operator-metrics +spec: + useTemplates: + - name: clickhouse-version + configuration: + clusters: + - name: default + layout: + replicasCount: 2 \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-046-1-clickhouse-operator-metrics.yaml b/tests/e2e/manifests/chi/test-046-1-clickhouse-operator-metrics.yaml new file mode 100644 index 000000000..301705786 --- /dev/null +++ b/tests/e2e/manifests/chi/test-046-1-clickhouse-operator-metrics.yaml @@ -0,0 +1,13 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-046-operator-metrics +spec: + taskID: "run reconcile" + useTemplates: + - name: clickhouse-version + configuration: + clusters: + - name: default + layout: + replicasCount: 2 \ No newline at end of file diff --git a/tests/e2e/manifests/chi/test-046-2-clickhouse-operator-metrics.yaml b/tests/e2e/manifests/chi/test-046-2-clickhouse-operator-metrics.yaml new file mode 100644 index 000000000..19e7b55eb --- /dev/null +++ b/tests/e2e/manifests/chi/test-046-2-clickhouse-operator-metrics.yaml @@ -0,0 +1,21 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-046-operator-metrics + +spec: + configuration: + clusters: + - name: default + layout: + replicasCount: 2 + templates: + podTemplates: + - name: clickhouse-new + spec: + containers: + - name: clickhouse-pod + image: clickhouse/clickhouse-server:22.8-broken + defaults: + templates: + podTemplate: clickhouse-new \ No newline at end of file diff --git a/tests/e2e/steps.py b/tests/e2e/steps.py index 9ec1e9ede..d1c9adf7e 100644 --- a/tests/e2e/steps.py +++ b/tests/e2e/steps.py @@ -4,10 +4,13 @@ import e2e.util as util import uuid import os +import re import yaml +import time import inspect import pathlib from testflows.core import current +from testflows.asserts import error import e2e.kubectl as kubectl @@ -142,3 +145,38 @@ def create_shell_namespace_clickhouse_template(self): kubectl.apply( util.get_full_path(current().context.clickhouse_template, lookup_in_host=False), ) + + +@TestStep(Then) +def check_metrics_monitoring( + self, + operator_namespace, + operator_pod, + expect_pattern, + container="metrics-exporter", + port="8888", + max_retries=7 +): + with Then(f"metrics-exporter /metrics endpoint result should contain {expect_pattern}"): + for i in range(1, max_retries): + url_cmd = util.make_http_get_request("127.0.0.1", port, "/metrics") + out = kubectl.launch( + f"exec {operator_pod} -c {container} -- {url_cmd}", + ns=operator_namespace, + ) + # print(out) + + rx = re.compile(expect_pattern, re.MULTILINE) + matches = rx.findall(out) + expected_pattern_found = False + + if matches: + expected_pattern_found = True + + if expected_pattern_found: + break + + with Then("Not ready. Wait for " + str(i * 5) + " seconds"): + time.sleep(i * 5) + + assert expected_pattern_found, error() \ No newline at end of file diff --git a/tests/e2e/test_metrics_exporter.py b/tests/e2e/test_metrics_exporter.py index 6234d49e1..4e6f4c52c 100644 --- a/tests/e2e/test_metrics_exporter.py +++ b/tests/e2e/test_metrics_exporter.py @@ -98,7 +98,7 @@ def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, ma "namespace": "test", "name": "test-017-multi-version", "labels": {"clickhouse.altinity.com/chi": "test-017-multi-version"}, - "annotations": {}, + "annotations": {"clickhouse.altinity.com/email": "myname@mydomain.com, yourname@yourdoman.com"}, "clusters": [ { "name": "default", @@ -122,6 +122,11 @@ def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, ma ] with Then("Check both pods are monitored"): check_monitoring_chi(self.context.operator_namespace, operator_pod, expected_chi) + labels = ','.join([ + 'chi="test-017-multi-version"', + 'clickhouse_altinity_com_chi="test-017-multi-version"', + 'clickhouse_altinity_com_email="myname@mydomain.com, yourname@yourdoman.com"' + ]) with Then("Check not empty /metrics"): check_monitoring_metrics( self.context.operator_namespace, @@ -129,8 +134,8 @@ def check_monitoring_metrics(operator_namespace, operator_pod, expect_result, ma expect_result={ "# HELP chi_clickhouse_metric_VersionInteger": True, "# TYPE chi_clickhouse_metric_VersionInteger gauge": True, - 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",clickhouse_altinity_com_chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-0-0': True, - 'chi_clickhouse_metric_VersionInteger{chi="test-017-multi-version",clickhouse_altinity_com_chi="test-017-multi-version",hostname="chi-test-017-multi-version-default-1-0': True, + "chi_clickhouse_metric_VersionInteger{" + labels +",hostname=\"chi-test-017-multi-version-default-0-0": True, + "chi_clickhouse_metric_VersionInteger{" + labels +",hostname=\"chi-test-017-multi-version-default-1-0": True, }, ) diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 2fa0af0a7..07a8dd387 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -3471,26 +3471,6 @@ def test_034(self): chopconf_file = "manifests/chopconf/test-034-chopconf.yaml" operator_namespace = current().context.operator_namespace - def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, max_retries=7): - with Then(f"metrics-exporter /metrics endpoint result should contain pattern: '{expect_pattern}'"): - for i in range(1, max_retries): - url_cmd = util.make_http_get_request("127.0.0.1", "8888", "/metrics") - out = kubectl.launch( - f"exec {operator_pod} -c metrics-exporter -- {url_cmd}", - ns=operator_namespace, - ) - rx = re.compile(expect_pattern, re.MULTILINE) - matches = rx.findall(out) - expected_pattern_found = False - if matches: - expected_pattern_found = True - - if expected_pattern_found: - break - with Then("Not ready. Wait for " + str(i * 5) + " seconds"): - time.sleep(i * 5) - assert expected_pattern_found, error() - with When("create the chi without secure connection"): manifest = "manifests/chi/test-034-http.yaml" chi = yaml_manifest.get_chi_name(util.get_full_path(manifest)) @@ -3516,8 +3496,8 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m out = kubectl.launch("get pods -l app=clickhouse-operator", ns=operator_namespace).splitlines()[1] operator_pod = re.split(r"[\t\r\n\s]+", out)[0] check_metrics_monitoring( - operator_namespace, - operator_pod, + operator_namespace=operator_namespace, + operator_pod=operator_pod, expect_pattern="^chi_clickhouse_metric_fetch_errors{(.*?)} 0$", ) @@ -3531,8 +3511,8 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m with Then("check for `chi_clickhouse_metric_fetch_errors` is not zero"): check_metrics_monitoring( - operator_namespace, - operator_pod, + operator_namespace=operator_namespace, + operator_pod=operator_pod, expect_pattern="^chi_clickhouse_metric_fetch_errors{(.*?)} 1$", ) @@ -3546,8 +3526,8 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m with Then("check for `chi_clickhouse_metric_fetch_errors` is zero [2]"): check_metrics_monitoring( - operator_namespace, - operator_pod, + operator_namespace=operator_namespace, + operator_pod=operator_pod, expect_pattern="^chi_clickhouse_metric_fetch_errors{(.*?)} 0$", ) @@ -3608,8 +3588,8 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m with Then("check for `chi_clickhouse_metric_fetch_errors` is zero [3]"): check_metrics_monitoring( - operator_namespace, - operator_pod, + operator_namespace=operator_namespace, + operator_pod=operator_pod, expect_pattern="^chi_clickhouse_metric_fetch_errors{(.*?)} 0$", ) @@ -3624,8 +3604,8 @@ def check_metrics_monitoring(operator_namespace, operator_pod, expect_pattern, m # 0.21.2+ with Then("check for `chi_clickhouse_metric_fetch_errors` is zero [4]"): check_metrics_monitoring( - operator_namespace, - operator_pod, + operator_namespace=operator_namespace, + operator_pod=operator_pod, expect_pattern="^chi_clickhouse_metric_fetch_errors{(.*?)} 0$", ) @@ -4462,6 +4442,119 @@ def test_045_2(self): test_045(manifest=f"manifests/chi/test-045-2-wait-query-finish.yaml") +@TestScenario +@Name("test_046. Metrics for clickhouse-operator") +def test_046(self): + """Check that clickhouse-operator creates metrics for reconcile and other clickhouse-operator events.""" + create_shell_namespace_clickhouse_template() + cluster = "default" + manifest = f"manifests/chi/test-046-0-clickhouse-operator-metrics.yaml" + chi = yaml_manifest.get_chi_name(util.get_full_path(manifest)) + operator_namespace = current().context.operator_namespace + out = kubectl.launch("get pods -l app=clickhouse-operator", ns=current().context.operator_namespace).splitlines()[1] + operator_pod = re.split(r"[\t\r\n\s]+", out)[0] + + with Given("CHI with 1 replica is installed"): + kubectl.create_and_check( + manifest=manifest, + check={ + "pod_count": 2, + "do_not_delete": 1, + }, + ) + + def check_metrics(metric_names): + for metric_name in metric_names: + with Then(f"I check {metric_name} metric for clickhouse-operator exists"): + check_metrics_monitoring( + operator_namespace=operator_namespace, + operator_pod=operator_pod, + container="clickhouse-operator", + port="9999", + expect_pattern=metric_name, + ) + + with Then(f"Check clickhouse-operator exposes clickhouse_operator_chi_reconciles_* metrics"): + check_metrics([ + "clickhouse_operator_chi_reconciles_started{.*chi=\"test-046-operator-metrics\".*} 1", + "clickhouse_operator_chi_reconciles_completed{.*chi=\"test-046-operator-metrics\".*} 1" + ]) + + with Then("I update CHI manifest to trigger reconcile"): + with By("adding taskID to CHI"): + kubectl.create_and_check( + manifest="manifests/chi/test-046-1-clickhouse-operator-metrics.yaml", + check={ + "pod_count": 2, + "do_not_delete": 1, + }, + ) + + with Then(f"Check clickhouse-operator exposes clickhouse_operator_chi_reconciles_* metrics"): + check_metrics([ + "clickhouse_operator_chi_reconciles_started{.*chi=\"test-046-operator-metrics\".*} 2", + "clickhouse_operator_chi_reconciles_completed{.*chi=\"test-046-operator-metrics\".*} 2" + ]) + + with Then("I update CHI manifest with wrong clickhouse version"): + kubectl.create_and_check( + manifest="manifests/chi/test-046-2-clickhouse-operator-metrics.yaml", + check={ + "pod_count": 2, + "do_not_delete": 1, + "chi_status": "InProgress", + }, + ) + + with Then("ClickHouse image can not be retrieved"): + kubectl.wait_field( + "pod", + "chi-test-046-operator-metrics-default-0-0-0", + ".status.containerStatuses[0].state.waiting.reason", + "ImagePullBackOff", + ) + + with Then("Wait until operator aborts"): + kubectl.wait_chi_status(chi, "Aborted") + + with Then(f"Check clickhouse-operator exposes clickhouse_operator_chi_reconciles_aborted metric"): + check_metrics([ + "clickhouse_operator_chi_reconciles_started{.*chi=\"test-046-operator-metrics\".*} 3", + "clickhouse_operator_chi_reconciles_completed{.*chi=\"test-046-operator-metrics\".*} 2", + "clickhouse_operator_chi_reconciles_aborted{.*chi=\"test-046-operator-metrics\".*} 1", + "clickhouse_operator_host_reconciles_errors{.*chi=\"test-046-operator-metrics\".*} 1", + ]) + + with Then("I restore the correct version"): + kubectl.create_and_check( + manifest="manifests/chi/test-046-0-clickhouse-operator-metrics.yaml", + check={ + "pod_count": 2, + "do_not_delete": 1, + }, + ) + + with Then(f"Check all chi and host reconciles metrics"): + check_metrics([ + "clickhouse_operator_chi_reconciles_started{.*chi=\"test-046-operator-metrics\".*} 4", + "clickhouse_operator_chi_reconciles_completed{.*chi=\"test-046-operator-metrics\".*} 3", + "clickhouse_operator_chi_reconciles_aborted{.*chi=\"test-046-operator-metrics\".*} 1", + "clickhouse_operator_chi_reconciles_timings.*chi=\"test-046-operator-metrics\".*", + # TODO: add proper counts for host reconciles + "clickhouse_operator_host_reconciles_started.*chi=\"test-046-operator-metrics\".*", + "clickhouse_operator_host_reconciles_completed.*chi=\"test-046-operator-metrics\".*", +# "clickhouse_operator_host_reconciles_restarts.*chi=\"test-046-operator-metrics\".*", + "clickhouse_operator_host_reconciles_errors.*chi=\"test-046-operator-metrics\".*", + "clickhouse_operator_host_reconciles_timings.*chi=\"test-046-operator-metrics\".*", + ]) + + with Finally("I clean up"): + with By("deleting chi"): + kubectl.delete_chi(chi) + with And("deleting test namespace"): + delete_test_namespace() + + @TestScenario @Requirements(RQ_SRS_026_ClickHouseOperator_CustomResource_Spec_Configuration_Clusters_Cluster_Layout_Shards_Weight("1.0")) @Name("test_047. Zero weighted shard")