added video-ads implementation

singer-io · Apr 14, 2024 · e93cb50 · e93cb50
1 parent 93d6cf5
commit e93cb50
Show file tree

Hide file tree

Showing 4 changed files with 216 additions and 17 deletions.
diff --git a/tap_linkedin_ads/schemas/video_ads.json b/tap_linkedin_ads/schemas/video_ads.json
@@ -11,6 +11,186 @@
         "string"
       ]
     },
+    "lifecycle_state": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "visibility": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "published_at": {
+      "type": [
+        "null",
+        "string"
+      ],
+      "format": "date-time"
+    },
+    "author": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "content_call_to_action_label": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "distribution": {
+      "type": [
+        "null",
+        "object"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "feed_distribution": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "third_party_distribution_channels": {
+          "type": [
+            "null",
+            "array"
+          ],
+          "items": {
+            "type": [
+              "null",
+              "string"
+            ]
+          }
+        }
+      }
+    },
+    "content": {
+      "type": [
+        "null",
+        "object"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "media": {
+          "type": [
+            "null",
+            "object"
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "title": {
+              "type": [
+                "null",
+                "string"
+              ]
+            },
+            "id": {
+              "type": [
+                "null",
+                "string"
+              ]
+            }
+          }
+        }
+      }
+    },
+    "content_landing_page": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "lifecycle_state_info": {
+      "type": [
+        "null",
+        "object"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "is_edited_by_author": {
+          "type": [
+            "null",
+            "boolean"
+          ]
+        }
+      }
+    },
+    "is_reshare_disabled_by_author": {
+      "type": [
+        "null",
+        "boolean"
+      ]
+    },
+    "created_at": {
+      "type": [
+        "null",
+        "string"
+      ],
+      "format": "date-time"
+    },
+    "last_modified_at": {
+      "type": [
+        "null",
+        "string"
+      ],
+      "format": "date-time"
+    },
+    "id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "commentary": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "ad_context": {
+      "type": [
+        "null",
+        "object"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "dsc_status": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "dsc_name": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "dsc_ad_type": {
+          "type": [
+            "null",
+            "string"
+          ]
+        },
+        "is_dsc": {
+          "type": [
+            "null",
+            "boolean"
+          ]
+        },
+        "dsc_ad_account": {
+          "type": [
+            "null",
+            "string"
+          ]
+        }
+      }
+    },
     "account_id": {
       "type": [
         "null",

diff --git a/tap_linkedin_ads/streams.py b/tap_linkedin_ads/streams.py
@@ -347,7 +347,10 @@ def sync_endpoint(self,
                 url = f"{BASE_URL}/adAccounts/{account}/{self.path}?{querystring}"
                 urllist.append((account, url))
         else:
-            url = 'https://api.linkedin.com/rest/{}?{}'.format(self.path, querystring)
+            if self.path == 'posts':
+                url = '{}/{}?{}&dscAdAccount=urn%3Ali%3AsponsoredAccount%3A{}'.format(BASE_URL, self.path, querystring, parent_id)
+            else:
+                url = '{}/{}?{}'.format(BASE_URL, self.path, querystring)
             urllist.append((None, url))
 
         for acct_id, next_url in urllist:
@@ -405,12 +408,6 @@ def sync_endpoint(self,
                                 account = 'urn:li:sponsoredAccount:{}'.format(parent_id)
                                 owner_id = record.get('reference_organization_id', None)
                                 owner = 'urn:li:organization:{}'.format(owner_id)
-                                if child_stream_name == 'video_ads' and owner_id is not None:
-                                    child_stream_params['account'] = account
-                                    child_stream_params['owner'] = owner
-                                else:
-                                    LOGGER.warning("Skipping video_ads call for %s account as reference_organization_id is not found.", account)
-                                    continue
                             elif self.tap_stream_id == 'campaigns':
                                 campaign = 'urn:li:sponsoredCampaign:{}'.format(parent_id)
                                 if child_stream_name == 'creatives':
@@ -616,12 +613,14 @@ class VideoAds(LinkedInAds):
     replication_method = "INCREMENTAL"
     key_properties = ["content_reference"]
     foreign_key = "id"
-    path = "adDirectSponsoredContents"
+    path = "posts"
     data_key = "elements"
     parent = "accounts"
     params = {
-        "q": "account"
+        "q": "dscAdAccount",
+        "dscAdTypes": "List(VIDEO)"
     }
+    headers = {'X-Restli-Protocol-Version': "2.0.0"}
 
 class AccountUsers(LinkedInAds):
     """
@@ -736,7 +735,7 @@ class AdAnalyticsByCreative(LinkedInAds):
 # Dictionary of the stream classes
 STREAMS = {
     "accounts": Accounts,
-    # "video_ads": VideoAds,
+    "video_ads": VideoAds,
     "account_users": AccountUsers,
     "campaign_groups": CampaignGroups,
     "campaigns": Campaigns,

diff --git a/tap_linkedin_ads/transform.py b/tap_linkedin_ads/transform.py
@@ -294,6 +294,24 @@ def transform_urn(data_dict):
     return data_dict
 
 
+def transform_video_ads(data_dict):
+    # TODO: To be removed in next major version release
+    if 'author' in data_dict:
+        data_dict['owner'] = data_dict["author"]
+    if 'id' in data_dict:
+        data_dict['content_reference'] = data_dict["id"]
+    if 'ad_context' in data_dict:
+        if 'dsc_name' in data_dict['ad_context']:
+            data_dict['name'] = data_dict["ad_context"]['dsc_name']
+        if 'dsc_ad_type' in data_dict['ad_context']:
+            data_dict['type'] = data_dict["ad_context"]['dsc_ad_type']
+        if 'dsc_ad_account' in data_dict['ad_context']:
+            data_dict['account'] = data_dict["ad_context"]['dsc_ad_account']
+    if 'last_modified_at' in data_dict:
+        data_dict['last_modified_time'] = data_dict["last_modified_at"]
+    if 'created_at' in data_dict:
+        data_dict['created_time'] = data_dict["created_at"]
+    return data_dict
 
 def transform_data(data_dict, stream_name):
     new_dict = data_dict
@@ -308,6 +326,8 @@ def transform_data(data_dict, stream_name):
             this_dict = transform_campaigns(this_dict)
         elif stream_name == 'creatives':
             this_dict = transform_creatives(this_dict)
+        elif stream_name == 'video_ads':
+            this_dict = transform_video_ads(this_dict)
         this_dict = transform_urn(this_dict)
         this_dict = transform_audit_fields(this_dict)
 

diff --git a/tests/base.py b/tests/base.py
@@ -81,7 +81,7 @@ def get_credentials(self):
     def expected_check_streams():
         return {
             'accounts',
-            # 'video_ads',
+            'video_ads',
             'account_users',
             'campaign_groups',
             'campaigns',
@@ -100,12 +100,12 @@ def expected_metadata(self):
                 self.OBEYS_START_DATE: True,
                 self.REPLICATION_KEYS: {'last_modified_time'}
             },
-            # 'video_ads': {
-            #     self.PRIMARY_KEYS: {'content_reference'},
-            #     self.REPLICATION_METHOD: self.INCREMENTAL,
-            #     self.OBEYS_START_DATE: True,
-            #     self.REPLICATION_KEYS: {'last_modified_time'}
-            # },
+            'video_ads': {
+                self.PRIMARY_KEYS: {'content_reference'},
+                self.REPLICATION_METHOD: self.INCREMENTAL,
+                self.OBEYS_START_DATE: True,
+                self.REPLICATION_KEYS: {'last_modified_time'}
+            },
             'account_users': {
                 self.PRIMARY_KEYS: {'account_id', 'user_person_id'},
                 self.REPLICATION_METHOD: self.INCREMENTAL,