Skip to content

Commit

Permalink
Fixed streams, sync & transform unit tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@shantanu73
shantanu73 committed Nov 8, 2023
1 parent e19aa0e commit 57276b1
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 51 deletions.
60 changes: 32 additions & 28 deletions tests/unittests/test_streams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,32 +190,32 @@ def test_merge_responses_no_overlap(self):
"""
expected_output = {
('urn:li:sponsoredCampaign:123456789', '2020-10-1') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}},
'a': 1, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'a': 1, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-2') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}},
'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-3') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}},
'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-4') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}},
'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-5') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}},
'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-6') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}},
'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
}

data = [
[{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}},
'a': 1, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'a': 1, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}},
'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}},
'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},],
'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}],
[{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}},
'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}},
'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}},
'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},],
'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}],
]

actual_output = merge_responses(data)
Expand All @@ -228,34 +228,34 @@ def test_merge_responses_with_overlap(self):
"""
data = [
[{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}},
'a': 1, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'a': 1, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}},
'b': 7, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'b': 7, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}},
'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}},
'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},],
'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}],
[{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}},
'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}},
'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
{'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}},
'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},],
'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']}],
]

expected_output = {
('urn:li:sponsoredCampaign:123456789', '2020-10-1') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 1}},
'a': 1, 'b': 7, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'a': 1, 'b': 7, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-2') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 2}},
'b': 2, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'b': 2, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-3') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 3}},
'c': 3, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'c': 3, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-4') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 4}},
'd': 4, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'd': 4, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-5') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 5}},
'e': 5, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'e': 5, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
('urn:li:sponsoredCampaign:123456789', '2020-10-6') : {'dateRange': {'start': {'year': 2020, 'month': 10, 'day': 6}},
'f': 6, 'pivotValue': 'urn:li:sponsoredCampaign:123456789'},
'f': 6, 'pivotValues': ['urn:li:sponsoredCampaign:123456789']},
}

actual_output = merge_responses(data)
Expand Down Expand Up @@ -327,11 +327,12 @@ def test_process_records(self, name, stream_obj, records, replication_key, expec
])
@mock.patch("tap_linkedin_ads.streams.LinkedInAds.sync_ad_analytics", return_value=(1, "2019-07-31T15:07:00.000000Z"))
@mock.patch("tap_linkedin_ads.streams.LinkedInAds.get_bookmark", return_value = "2019-07-31T15:07:00.000000Z")
@mock.patch("tap_linkedin_ads.client.LinkedinClient.get")
@mock.patch("tap_linkedin_ads.client.LinkedinClient.request")
@mock.patch("tap_linkedin_ads.streams.LinkedInAds.process_records")
@mock.patch("tap_linkedin_ads.streams.LinkedInAds.write_schema")
def test_sync_endpoint(self, name, selected_streams, stream_obj, mock_response, expected_write_schema_count, mock_record_count,
mock_write_schema,mock_process_records,mock_client,mock_get_bookmark, mock_sync_ad_analytics):
mock_write_schema,mock_process_records,mock_client,mock_get,mock_get_bookmark, mock_sync_ad_analytics):
"""
Test sync_endpoint function for parent and child streams.
"""
Expand All @@ -341,9 +342,11 @@ def test_sync_endpoint(self, name, selected_streams, stream_obj, mock_response,
page_size = 100
date_window_size = 7

mock_get.side_effect = [{"elements": [{"1": "a"}]}]
mock_client.side_effect = mock_response
config = {"accounts": "123"}
mock_process_records.return_value = "2019-07-31T15:07:00.000000Z",1
actual_total_record, actual_max_bookmark = stream_obj.sync_endpoint(client, CATALOG, state, page_size, start_date, selected_streams, date_window_size)
actual_total_record, actual_max_bookmark = stream_obj.sync_endpoint(client, CATALOG, config, state, page_size, start_date, selected_streams, date_window_size)

# Verify total no of records
self.assertEqual(actual_total_record, mock_record_count)
Expand All @@ -368,10 +371,11 @@ def test_sync_endpoint_for_reference_organization_id_is_None(self, mock_write_sc
page_size = 100
date_window_size = 7
selected_streams = ['accounts', 'video_ads']
config = {"accounts": "123"}

mock_client.side_effect = [{'paging': {'start': 0, 'count': 100, 'links': [], 'total': 1},'elements': [{'changeAuditStamps': {'created': {'time': 1564585620000}, 'lastModified': {'time': 1564585620000}}, 'id': 1}]}]
mock_process_records.return_value = "2019-07-31T15:07:00.000000Z",1
ACCOUNT_OBJ.sync_endpoint(client, CATALOG, state, page_size, start_date, selected_streams, date_window_size)
ACCOUNT_OBJ.sync_endpoint(client, CATALOG, config, state, page_size, start_date, selected_streams, date_window_size)

mock_warning.assert_called_with('Skipping video_ads call for %s account as reference_organization_id is not found.', 'urn:li:sponsoredAccount:1')

Expand Down
3 changes: 2 additions & 1 deletion tests/unittests/test_sync.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,8 @@ def test_sync(self, name, config, expected_date_window, mock_sync_endpoint):

sync(client, config, CATALOG, state)
mock_sync_endpoint.assert_called_with(client=client,
catalog=CATALOG,
catalog=CATALOG,
config=config,
state=state,
page_size=100,
start_date="2019-06-01T00:00:00Z",
Expand Down
47 changes: 25 additions & 22 deletions tests/unittests/test_transform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from parameterized import parameterized
from tap_linkedin_ads.transform import (convert, snake_case_to_camel_case, convert_array, convert_json,
transform_accounts, transform_analytics, transform_json,
transform_campaigns, transform_creatives, transform_audit_fields,
transform_campaigns, transform_creatives, transform_ad_context_fields,
transform_urn, transform_data, string_to_decimal)


Expand Down Expand Up @@ -337,34 +337,38 @@ def test_transform_creatives(self, test_dict_1, expected_dict):
self.assertEqual(transformed_dict, expected_dict)


class TestTransformAuditFields(unittest.TestCase):
class TestTransformAdContextFields(unittest.TestCase):
"""
Test `transform_audit_fields` function.
Test `transform_ad_context_fields` function.
"""

test_dict_1 = {"reference": "urn:li:organization:20111635"}

test_dict_2 = {
"change_audit_stamps": {
"created": {"time": 1563562455000},
"last_modified": {"time": 1626169039381}
"ad_context": {
"dsc_status": "ACTIVE",
"dsc_name": "Stitch Tableau",
"dsc_ad_type": "VIDEO",
"dsc_ad_account": "urn:li:sponsoredAccount:503498742"
}
}
added_fields_2 = {
"created_time": 1563562455000,
"last_modified_time": 1626169039381,
"status": "ACTIVE",
"name": "Stitch Tableau",
"type": "VIDEO",
"account": "urn:li:sponsoredAccount:503498742"
}

@parameterized.expand([
(test_dict_1, {**test_dict_1}),
(test_dict_2, {**test_dict_2, **added_fields_2}),
])
def test_transform_audit_fields(self, test_dict, expected_dict):
def test_transform_ad_context_fields(self, test_dict, expected_dict):
"""
Test that time fields are added to first level.
"""

transformed_dict = transform_audit_fields(test_dict)
transformed_dict = transform_ad_context_fields(test_dict)

# Verify returned dict is expected
self.assertEqual(transformed_dict, expected_dict)
Expand Down Expand Up @@ -398,8 +402,7 @@ def test_transform_urn(self, test_dict, expected_dict):
self.assertEqual(transformed_dict, expected_dict)


@mock.patch("tap_linkedin_ads.transform.transform_urn")
@mock.patch("tap_linkedin_ads.transform.transform_audit_fields")
@mock.patch("tap_linkedin_ads.transform.transform_ad_context_fields")
class TestTransformData(unittest.TestCase):
"""
Test `transform_data` function that it calls other transform function respective to stream_name
Expand All @@ -408,47 +411,47 @@ class TestTransformData(unittest.TestCase):
test_dict = {"type": "BUSINESS", "id": 503491473}

@mock.patch("tap_linkedin_ads.transform.transform_accounts")
def test_accounts_stream(self, mock_transform_accounts, mock_audit_fields, mock_transform_urn):
def test_accounts_stream(self, mock_transform_accounts, mock_ad_context_fields):
"""
Test for `accounts` stream `transform_accounts` is called.
"""
mock_audit_fields.return_value = self.test_dict
mock_ad_context_fields.return_value = self.test_dict
transformed_dict = transform_data({"elements": [self.test_dict]*3}, "accounts")

# Verify transform function called for each element
self.assertEqual(mock_transform_accounts.call_count, 3)
self.assertEqual(transformed_dict, {"elements": [self.test_dict]*3})

@mock.patch("tap_linkedin_ads.transform.transform_campaigns")
def test_campaigns_stream(self, mock_transform_campaigns, mock_audit_fields, mock_transform_urn):
def test_campaigns_stream(self, mock_transform_campaigns, mock_ad_context_fields):
"""
Test for `campaigns` stream `transform_campaigns` is called.
"""
mock_audit_fields.return_value = self.test_dict
mock_ad_context_fields.return_value = self.test_dict
transformed_dict = transform_data({"elements": [self.test_dict]*4}, "campaigns")

# Verify transform function called for each element
self.assertEqual(mock_transform_campaigns.call_count, 4)
self.assertEqual(transformed_dict, {"elements": [self.test_dict]*4})

@mock.patch("tap_linkedin_ads.transform.transform_analytics")
def test_analytics_stream(self, mock_transform_analytics, mock_audit_fields, mock_transform_urn):
def test_analytics_stream(self, mock_transform_analytics, mock_ad_context_fields):
"""
Test for any analytics stream `transform_analytics` is called.
"""
mock_audit_fields.return_value = self.test_dict
mock_ad_context_fields.return_value = self.test_dict
transformed_dict = transform_data({"elements": [self.test_dict]*4}, "ad_analytics_by_creatives")

# Verify transform function called for each element
self.assertEqual(mock_transform_analytics.call_count, 4)
self.assertEqual(transformed_dict, {"elements": [self.test_dict]*4})

@mock.patch("tap_linkedin_ads.transform.transform_creatives")
def test_creatives_stream(self, mock_transform_creatives, mock_audit_fields, mock_transform_urn):
def test_creatives_stream(self, mock_transform_creatives, mock_ad_context_fields):
"""
Test for `creatives` stream `transform_creatives` is called.
"""
mock_audit_fields.return_value = self.test_dict
mock_ad_context_fields.return_value = self.test_dict
transformed_dict = transform_data({"elements": [self.test_dict]*4}, "creatives")

# Verify transform function called for each element
Expand All @@ -459,11 +462,11 @@ def test_creatives_stream(self, mock_transform_creatives, mock_audit_fields, moc
@mock.patch("tap_linkedin_ads.transform.transform_campaigns")
@mock.patch("tap_linkedin_ads.transform.transform_accounts")
@mock.patch("tap_linkedin_ads.transform.transform_creatives")
def test_other_streams(self, transform_creatives, transform_accounts, transform_campaigns, transform_analytics, mock_audit_fields, mock_transform_urn):
def test_other_streams(self, transform_creatives, transform_accounts, transform_campaigns, transform_analytics, mock_ad_context_fields):
"""
Test for any other streams transformed dictionary is returned.
"""
mock_audit_fields.return_value = self.test_dict
mock_ad_context_fields.return_value = self.test_dict
transformed_dict = transform_data({"elements": [self.test_dict]*4}, "video_ads")

# Verify any transform function specific to a stream was not called
Expand Down

0 comments on commit 57276b1

Please sign in to comment.