Merge pull request #27 from IBM/prajwal-jarali-202509241308

Unification of Data format for API response and Bootstrap file

Author: saikumar1607

.cra/.fileignore

@@ -0,0 +1 @@
+*

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2025-04-25T05:22:50Z",
+  "generated_at": "2025-09-24T13:09:33Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"

ibm_appconfiguration/configurations/configuration_handler.py

@@ -15,11 +15,14 @@
 """
 Internal class to handle the configuration.
 """
+import json
 import os
 from typing import Dict, List, Any
 from threading import Timer, Thread
 from ibm_appconfiguration.configurations.internal.common import config_messages, config_constants
+from ibm_appconfiguration.version import __version__
 from .internal.utils.logger import Logger
+from .internal.utils.parser import extract_configurations, format_config
 from .internal.utils.validators import Validators
 from .models import Feature
 from .models import SegmentRules
@@ -147,7 +150,9 @@ def load_data(self):
             self.__persistent_data = FileManager.read_files(
                 file_path=os.path.join(self.__persistent_cache_dir, 'appconfiguration.json'))
             if self.__persistent_data is not None:
-                self.__load_configurations(self.__persistent_data)
+                self.__load_configurations(
+                    extract_configurations(self.__persistent_data, self.__environment_id, self.__collection_id)
+                )
             if not os.access(self.__persistent_cache_dir, os.W_OK):
                 Logger.error(config_messages.ERROR_NO_WRITE_PERMISSION)
                 return
@@ -156,11 +161,13 @@ def load_data(self):
                 if self.__persistent_data is None or len(self.__persistent_data) == 0:
                     bootstrap_file_data = FileManager.read_files(file_path=self.__bootstrap_file)
                     if bootstrap_file_data is not None:
-                        self.__load_configurations(bootstrap_file_data)
+                        configurations = extract_configurations(bootstrap_file_data, self.__environment_id, self.__collection_id)
+                        self.__load_configurations(configurations)
+                        self.__write_to_persistent_storage(format_config(configurations, self.__environment_id, self.__collection_id),
+                            self.__persistent_cache_dir)
                     else:
                         Logger.error("Error reading bootstrap file data")
                         return
-                    self.__write_to_persistent_storage(bootstrap_file_data, self.__persistent_cache_dir)
                     if self.__configuration_update_listener and callable(self.__configuration_update_listener):
                         self.__configuration_update_listener()
                 else:
@@ -169,7 +176,9 @@ def load_data(self):
             else:
                 bootstrap_file_data = FileManager.read_files(file_path=self.__bootstrap_file)
                 if bootstrap_file_data is not None:
-                    self.__load_configurations(bootstrap_file_data)
+                    self.__load_configurations(
+                        extract_configurations(bootstrap_file_data, self.__environment_id, self.__collection_id)
+                    )
                 else:
                     Logger.error("Error reading bootstrap file data")
                     return
@@ -271,7 +280,8 @@ def __fetch_config_data(self):
     def __start_web_socket(self):
         bearer_token = URLBuilder.get_iam_authenticator().token_manager.get_token()
         headers = {
-            'Authorization': 'Bearer ' + bearer_token
+            'Authorization': 'Bearer ' + bearer_token,
+            'User-Agent': '{0}/{1}'.format(config_constants.SDK_NAME, __version__)
         }
         if self.__socket:
             self.__socket.cancel()
@@ -478,8 +488,8 @@ def __parse_rules(self, segment_rules: List) -> dict:
                 Logger.debug(err)
         return rule_map
 
-    def __write_to_persistent_storage(self, json: dict, file_path: str):
-        FileManager.store_files(json, os.path.join(file_path, 'appconfiguration.json'))
+    def __write_to_persistent_storage(self, data: str, file_path: str):
+        FileManager.store_files(json.dumps(json.loads(data), indent=2), os.path.join(file_path, 'appconfiguration.json'))
 
     def __fetch_from_api(self):
         if self.__is_initialized:
@@ -506,16 +516,16 @@ def __fetch_from_api(self):
                 Logger.info(config_messages.CONFIGURATIONS_FETCH_SUCCESS)
                 response_data = response.get_result()
                 try:
-                    response_data = dict(response_data)
-                    self.__load_configurations(response_data)  # load response to cache maps
+                    configurations = extract_configurations(json.dumps(response_data), self.__environment_id, self.__collection_id)
+                    self.__load_configurations(configurations)  # load response to cache maps
                     if self.__configuration_update_listener and callable(self.__configuration_update_listener):
                         self.__configuration_update_listener()
                     # we have already loaded the configurations to feature & property dicts.
                     # it is okay to "detach" the job of "writing to persistent location" from the main thread and finish the job using another thread.
                     # But the thread shouldn't be a daemon thread, because the writing should complete even if the main thread has terminated.
                     if self.__persistent_cache_dir:
                         file_write_thread = Thread(target=self.__write_to_persistent_storage,
-                                                   args=(response_data, self.__persistent_cache_dir,))
+                            args=(format_config(configurations, self.__environment_id, self.__collection_id), self.__persistent_cache_dir))
                         file_write_thread.start()
                 except Exception as exception:
                     Logger.error(f'error while while fetching {exception}')

ibm_appconfiguration/configurations/internal/utils/file_manager.py

@@ -18,8 +18,7 @@
 """
 
 import fcntl
-import json
-from typing import Optional, Any
+from typing import Optional
 
 from .logger import Logger
 
@@ -28,25 +27,25 @@ class FileManager:
     """FileManager to handle the cache"""
 
     @classmethod
-    def store_files(cls, json_data: {}, file_path: str) -> bool:
+    def store_files(cls, data: str, file_path: str) -> bool:
         """Store the file
 
         Args:
-            json_data: Data to be stored.
+            data: Data to be stored.
             file_path: File path for the cache.
         """
         try:
             with open(file_path, 'w') as cache:
                 fcntl.flock(cache, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                json.dump(json_data, cache)
+                cache.write(data)
                 fcntl.flock(cache, fcntl.LOCK_UN)
                 return True
         except Exception as err:
             Logger.error(err)
             return False
 
     @classmethod
-    def read_files(cls, file_path: str) -> Optional[Any]:
+    def read_files(cls, file_path: str) -> Optional[str]:
         """
         Read the data from the given path.
 
@@ -59,9 +58,9 @@ def read_files(cls, file_path: str) -> Optional[Any]:
         try:
             with open(file_path, 'r') as file:
                 fcntl.flock(file, fcntl.LOCK_EX | fcntl.LOCK_NB)
-                data = json.load(file)
+                data = file.read()
                 fcntl.flock(file, fcntl.LOCK_UN)
-                return data
+                return data if len(data) > 0 else None
         except Exception as err:
             Logger.error(err)
             return None

ibm_appconfiguration/configurations/internal/utils/parser.py

@@ -0,0 +1,149 @@
+import json
+from typing import Any, Dict, List, Set
+
+
+def extract_environment_data(data: Dict[str, Any], environment_id: str) -> Dict[str, Any]:
+    """
+    Prepares config data for extraction with validation.
+
+    :param data: The full JSON configuration as a dictionary.
+    :param environment_id: The environment ID to extract.
+    :return: A dictionary containing 'features', 'properties', and 'segments'.
+    :raises: Exception if the format is invalid or environment not found.
+    """
+    if not isinstance(data.get("segments"), list) or not isinstance(data.get("environments"), list):
+        raise Exception("Improper Data format present in configuration")
+
+    for environment in data["environments"]:
+        if environment.get("environment_id") == environment_id:
+            return {
+                "features": environment.get("features", []),
+                "properties": environment.get("properties", []),
+                "segments": data["segments"]
+            }
+
+    raise Exception("Matching environment not found in configuration")
+
+
+def validate_resource(resource: Dict[str, Any], collection: str) -> bool:
+    """
+    Validates if the feature/property belongs to the given collection.
+
+    :param resource: The feature or property dictionary.
+    :param collection: The collection ID to match.
+    :return: True if valid, False otherwise.
+    :raises: Exception if collection format is invalid.
+    """
+    if "collections" not in resource:
+        return True
+
+    collections = resource["collections"]
+    if not isinstance(collections, list):
+        raise Exception("Improper collection format in resource data")
+
+    for col in collections:
+        if col.get("collection_id") == collection:
+            return True
+
+    return False
+
+
+def append_segment_ids(resource: Dict[str, Any], segment_ids: Set[str]):
+    """
+    Appends segment IDs from the resource's segment rules into the given set.
+
+    :param resource: The feature or property dictionary.
+    :param segment_ids: A set to accumulate segment IDs.
+    """
+    for segment_rule in resource.get("segment_rules", []):
+        for rule in segment_rule.get("rules", []):
+            for segment_id in rule.get("segments", []):
+                segment_ids.add(segment_id)
+
+
+def extract_resources(resource_data: Dict[str, Any], collection: str) -> Dict[str, List[Any]]:
+    """
+    Extracts features, properties, and segments after validation.
+
+    :param resource_data: The environment-specific data.
+    :param collection: The collection ID to validate against.
+    :return: A dictionary with keys 'features', 'properties', and 'segments'.
+    :raises: Exception if any required segment is missing.
+    """
+    features = []
+    properties = []
+    segments = []
+    required_segment_ids = set()
+
+    for feature in resource_data.get("features", []):
+        if validate_resource(feature, collection):
+            append_segment_ids(feature, required_segment_ids)
+            features.append(feature)
+
+    for property_ in resource_data.get("properties", []):
+        if validate_resource(property_, collection):
+            append_segment_ids(property_, required_segment_ids)
+            properties.append(property_)
+
+    available_segments = resource_data.get("segments", [])
+    for segment in available_segments:
+        if segment.get("segment_id") in required_segment_ids:
+            segments.append(segment)
+            required_segment_ids.remove(segment.get("segment_id"))
+
+    if len(required_segment_ids) > 0:
+        raise Exception(f"Required segment doesn't exist in provided segments")
+
+    return {
+        "features": features,
+        "properties": properties,
+        "segments": segments
+    }
+
+
+def extract_configurations(data: str, environment: str, collection: str) -> Dict[str, List[Any]]:
+    """
+    Unified parser for app-config data for new SDK/export/promote format.
+
+    :param data: Raw JSON string of the config.
+    :param environment: The environment ID.
+    :param collection: The collection ID.
+    :return: A dictionary with 'features', 'properties', and 'segments'.
+    :raises: Exception on any validation or format error.
+    """
+    try:
+        configurations = json.loads(data)
+
+        if "collections" not in configurations or not isinstance(configurations["collections"], list):
+            raise Exception("Improper/Missing collections in configuration")
+
+        if not any(col.get("collection_id") == collection for col in configurations["collections"]):
+            raise Exception("Required collection not found in collections")
+
+        config_data = extract_environment_data(configurations, environment)
+        return extract_resources(config_data, collection)
+
+    except Exception as e:
+        raise Exception(f"Extraction of configurations failed with error:\n {str(e)}")
+
+
+def format_config(res: Dict[str, List[Any]], environment_id: str, collection_id: str) -> str:
+    """
+    Formats the extracted resources into unified config format.
+
+    :param res: The extracted config (from `extract_configurations`).
+    :param environment_id: The environment ID to include.
+    :param collection_id: The collection ID to include.
+    :return: A formatted configuration dictionary.
+    """
+    return json.dumps({
+        "environments": [
+            {
+                "environment_id": environment_id,
+                "features": res.get("features", []),
+                "properties": res.get("properties", [])
+            }
+        ],
+        "collections": [{"collection_id": collection_id}],
+        "segments": res.get("segments", [])
+    })

ibm_appconfiguration/configurations/internal/utils/url_builder.py

@@ -84,12 +84,12 @@ def init_with_collection_id(cls, collection_id='', environment_id='', region='',
                     cls.__iam_url = "https://iam.cloud.ibm.com"
                     cls.__web_socket_base = cls.__wss + region + cls.__base_url
 
-            cls.__config_path = '{0}{1}{2}/collections/{3}/{4}?environment_id={5}'.format(
+            cls.__config_path = '{0}{1}{2}/{3}?action=sdkConfig&collection_id={4}&environment_id={5}'.format(
                 cls.__service,
                 cls.__feature_path,
                 guid,
-                collection_id,
                 cls.__config,
+                collection_id,
                 environment_id)
             cls.__metering_path = '{0}{1}{2}/usage'.format(cls.__service,
                                                            cls.__events_path,

ibm_appconfiguration/version.py

@@ -15,4 +15,4 @@
 """
 Version of ibm-appconfiguration-python-sdk
 """
-__version__ = '0.3.8'
+__version__ = '0.3.9'

setup.py

@@ -12,7 +12,7 @@
 from setuptools import setup, find_packages
 
 NAME = "ibm-appconfiguration-python-sdk"
-VERSION = "0.3.8"
+VERSION = "0.3.9"
 # To install the library, run the following
 #
 # python setup.py install