Skip to content

Commit 9ba46ce

Browse files
kornelregiuskornelregius
authored
Simplified Cloud AI API integration. (#932)
* Cloud Translation, Natural language, Vision, Video intelligence GCP integrations.
1 parent c462fc1 commit 9ba46ce

10 files changed

+70
-100
lines changed

‎dev.Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,5 @@ ADD patches/kaggle_gcp.py /root/.local/lib/python3.7/site-packages/kaggle_gcp.py
1919
ADD patches/kaggle_secrets.py /root/.local/lib/python3.7/site-packages/kaggle_secrets.py
2020
ADD patches/kaggle_session.py /root/.local/lib/python3.7/site-packages/kaggle_session.py
2121
ADD patches/kaggle_web_client.py /root/.local/lib/python3.7/site-packages/kaggle_web_client.py
22-
ADD patches/kaggle_datasets.py /root/.local/lib/python3.7/site-packages/kaggle_datasets.py
22+
ADD patches/kaggle_datasets.py /root/.local/lib/python3.7/site-packages/kaggle_datasets.py
23+
ADD patches/sitecustomize.py /root/.local/lib/python3.7/site-packages/sitecustomize.py

‎patches/kaggle_gcp.py

Lines changed: 17 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -42,20 +42,9 @@ def has_bigquery(self):
4242
def has_gcs(self):
4343
return GcpTarget.GCS in self.integrations
4444

45-
def has_automl(self):
46-
return GcpTarget.AUTOML in self.integrations
47-
48-
def has_translation(self):
49-
return GcpTarget.TRANSLATION in self.integrations
50-
51-
def has_natural_language(self):
52-
return GcpTarget.NATURAL_LANGUAGE in self.integrations
53-
54-
def has_video_intelligence(self):
55-
return GcpTarget.VIDEO_INTELLIGENCE in self.integrations
56-
57-
def has_vision(self):
58-
return GcpTarget.VISION in self.integrations
45+
def has_cloudai(self):
46+
return GcpTarget.CLOUDAI in self.integrations or \
47+
GcpTarget.AUTOML in self.integrations
5948

6049
class KaggleKernelCredentials(credentials.Credentials):
6150
"""Custom Credentials used to authenticate using the Kernel's connected OAuth account.
@@ -74,16 +63,8 @@ def refresh(self, request):
7463
self.token, self.expiry = client.get_bigquery_access_token()
7564
elif self.target == GcpTarget.GCS:
7665
self.token, self.expiry = client._get_gcs_access_token()
77-
elif self.target == GcpTarget.AUTOML:
78-
self.token, self.expiry = client._get_automl_access_token()
79-
elif self.target == GcpTarget.TRANSLATION:
80-
self.token, self.expiry = client._get_translation_access_token()
81-
elif self.target == GcpTarget.NATURAL_LANGUAGE:
82-
self.token, self.expiry = client._get_natural_language_access_token()
83-
elif self.target == GcpTarget.VIDEO_INTELLIGENCE:
84-
self.token, self.expiry = client._get_video_intelligence_access_token()
85-
elif self.target == GcpTarget.VISION:
86-
self.token, self.expiry = client._get_vision_access_token()
66+
elif self.target == GcpTarget.CLOUDAI:
67+
self.token, self.expiry = client._get_cloudai_access_token()
8768
except ConnectionError as e:
8869
Log.error(f"Connection error trying to refresh access token: {e}")
8970
print("There was a connection error trying to fetch the access token. "
@@ -262,12 +243,12 @@ def init_automl():
262243
return
263244

264245
from kaggle_gcp import get_integrations
265-
if not get_integrations().has_automl():
246+
if not get_integrations().has_cloudai():
266247
return
267248

268249
from kaggle_secrets import GcpTarget
269250
from kaggle_gcp import KaggleKernelCredentials
270-
kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.AUTOML)
251+
kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
271252

272253
# Patch the 2 GA clients: AutoMlClient and PreditionServiceClient
273254
monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials)
@@ -293,10 +274,10 @@ def init_translation_v2():
293274
return translate_v2
294275

295276
from kaggle_gcp import get_integrations
296-
if not get_integrations().has_translation():
277+
if not get_integrations().has_cloudai():
297278
return translate_v2
298279
from kaggle_secrets import GcpTarget
299-
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION)
280+
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
300281
monkeypatch_client(translate_v2.Client, kernel_credentials)
301282
return translate_v2
302283

@@ -307,10 +288,10 @@ def init_translation_v3():
307288
return translate_v3
308289

309290
from kaggle_gcp import get_integrations
310-
if not get_integrations().has_translation():
291+
if not get_integrations().has_cloudai():
311292
return translate_v3
312293
from kaggle_secrets import GcpTarget
313-
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION)
294+
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
314295
monkeypatch_client(translate_v3.TranslationServiceClient, kernel_credentials)
315296
return translate_v3
316297

@@ -320,11 +301,11 @@ def init_natural_language():
320301
return language
321302

322303
from kaggle_gcp import get_integrations
323-
if not get_integrations().has_natural_language():
304+
if not get_integrations().has_cloudai():
324305
return language
325306

326307
from kaggle_secrets import GcpTarget
327-
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.NATURAL_LANGUAGE)
308+
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
328309
monkeypatch_client(language.LanguageServiceClient, kernel_credentials)
329310
monkeypatch_client(language.LanguageServiceAsyncClient, kernel_credentials)
330311
return language
@@ -335,11 +316,11 @@ def init_video_intelligence():
335316
return videointelligence
336317

337318
from kaggle_gcp import get_integrations
338-
if not get_integrations().has_video_intelligence():
319+
if not get_integrations().has_cloudai():
339320
return videointelligence
340321

341322
from kaggle_secrets import GcpTarget
342-
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VIDEO_INTELLIGENCE)
323+
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
343324
monkeypatch_client(
344325
videointelligence.VideoIntelligenceServiceClient,
345326
kernel_credentials)
@@ -354,11 +335,11 @@ def init_vision():
354335
return vision
355336

356337
from kaggle_gcp import get_integrations
357-
if not get_integrations().has_vision():
338+
if not get_integrations().has_cloudai():
358339
return vision
359340

360341
from kaggle_secrets import GcpTarget
361-
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VISION)
342+
kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI)
362343
monkeypatch_client(vision.ImageAnnotatorClient, kernel_credentials)
363344
monkeypatch_client(vision.ImageAnnotatorAsyncClient, kernel_credentials)
364345
return vision

‎patches/kaggle_secrets.py

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,9 @@ class GcpTarget(Enum):
2323
"""Enum class to store GCP targets."""
2424
BIGQUERY = (1, "BigQuery")
2525
GCS = (2, "Google Cloud Storage")
26+
# Old name, should remove later.
2627
AUTOML = (3, "Cloud AutoML")
27-
TRANSLATION = (4, "Cloud Translation")
28-
NATURAL_LANGUAGE = (5, "Cloud Natural Language")
29-
VIDEO_INTELLIGENCE = (6, "Cloud Video Intelligence")
30-
VISION = (7, "Cloud Vision")
28+
CLOUDAI = (3, "Google Cloud AI Platform")
3129

3230
def __init__(self, target, service):
3331
self._target = target
@@ -155,20 +153,8 @@ def _write_gsutil_credentials_file(self, credentials) -> str:
155153
def _get_gcs_access_token(self) -> Tuple[str, Optional[datetime]]:
156154
return self._get_access_token(GcpTarget.GCS)
157155

158-
def _get_automl_access_token(self) -> Tuple[str, Optional[datetime]]:
159-
return self._get_access_token(GcpTarget.AUTOML)
160-
161-
def _get_translation_access_token(self) -> Tuple[str, Optional[datetime]]:
162-
return self._get_access_token(GcpTarget.TRANSLATION)
163-
164-
def _get_natural_language_access_token(self) -> Tuple[str, Optional[datetime]]:
165-
return self._get_access_token(GcpTarget.NATURAL_LANGUAGE)
166-
167-
def _get_video_intelligence_access_token(self) -> Tuple[str, Optional[datetime]]:
168-
return self._get_access_token(GcpTarget.VIDEO_INTELLIGENCE)
169-
170-
def _get_vision_access_token(self) -> Tuple[str, Optional[datetime]]:
171-
return self._get_access_token(GcpTarget.VISION)
156+
def _get_cloudai_access_token(self) -> Tuple[str, Optional[datetime]]:
157+
return self._get_access_token(GcpTarget.CLOUDAI)
172158

173159
def _get_access_token(self, target: GcpTarget) -> Tuple[str, Optional[datetime]]:
174160
request_body = {

‎tests/test_automl.py

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ def test_user_provided_credentials(self):
2727
credentials = _make_credentials()
2828
env = EnvironmentVarGuard()
2929
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
30-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
30+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
3131
with env:
3232
init_automl()
3333
client = automl.AutoMlClient(credentials=credentials)
@@ -48,7 +48,7 @@ def test_tables_client_credentials(self):
4848
credentials = _make_credentials()
4949
env = EnvironmentVarGuard()
5050
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
51-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
51+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
5252
with env:
5353
init_automl()
5454
tables_client = automl_v1beta1.TablesClient(credentials=credentials)
@@ -58,7 +58,7 @@ def test_tables_client_credentials(self):
5858
def test_default_credentials_automl_client(self):
5959
env = EnvironmentVarGuard()
6060
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
61-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
61+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
6262
with env:
6363
init_automl()
6464
automl_client = automl.AutoMlClient()
@@ -70,7 +70,7 @@ def test_default_credentials_automl_client(self):
7070
def test_default_credentials_automl_v1beta1_client(self):
7171
env = EnvironmentVarGuard()
7272
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
73-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
73+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
7474
with env:
7575
init_automl()
7676
automl_client = automl_v1beta1.AutoMlClient()
@@ -82,7 +82,7 @@ def test_default_credentials_automl_v1beta1_client(self):
8282
def test_default_credentials_tables_client(self):
8383
env = EnvironmentVarGuard()
8484
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
85-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
85+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
8686
with env:
8787
init_automl()
8888
tables_client = automl_v1beta1.TablesClient()
@@ -94,7 +94,7 @@ def test_default_credentials_tables_client(self):
9494
def test_default_credentials_prediction_client(self):
9595
env = EnvironmentVarGuard()
9696
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
97-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
97+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
9898
with env:
9999
prediction_client = automl.PredictionServiceClient()
100100
self.assertIsNotNone(prediction_client.credentials)
@@ -105,7 +105,7 @@ def test_default_credentials_prediction_client(self):
105105
def test_default_credentials_prediction_v1beta1_client(self):
106106
env = EnvironmentVarGuard()
107107
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
108-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
108+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
109109
with env:
110110
prediction_client = automl_v1beta1.PredictionServiceClient()
111111
self.assertIsNotNone(prediction_client.credentials)
@@ -115,9 +115,23 @@ def test_default_credentials_prediction_v1beta1_client(self):
115115
def test_monkeypatching_idempotent(self):
116116
env = EnvironmentVarGuard()
117117
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
118-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
118+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
119119
with env:
120120
client1 = automl.AutoMlClient.__init__
121121
init_automl()
122122
client2 = automl.AutoMlClient.__init__
123123
self.assertEqual(client1, client2)
124+
125+
@patch("google.cloud.automl_v1beta1.PredictionServiceClient", new=FakeClient)
126+
def test_legacy_AUTOML_variable_v1beta1_client(self):
127+
"""
128+
Tests previous KAGGLE_KERNEL_INTEGRATIONS="AUTOML" environment setting
129+
"""
130+
env = EnvironmentVarGuard()
131+
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
132+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
133+
with env:
134+
prediction_client = automl_v1beta1.PredictionServiceClient()
135+
self.assertIsNotNone(prediction_client.credentials)
136+
self.assertIsInstance(prediction_client.credentials, KaggleKernelCredentials)
137+
self.assertTrue(prediction_client._connection.user_agent.startswith("kaggle-gcp-client/1.0"))

‎tests/test_natural_language.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ def __init__(self, user_agent):
2626
def test_default_credentials(self):
2727
env = EnvironmentVarGuard()
2828
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
29-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
29+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
3030
with env:
3131
init_natural_language()
3232
client = language.LanguageServiceClient()
@@ -38,7 +38,7 @@ def test_user_provided_credentials(self):
3838
credentials = _make_credentials()
3939
env = EnvironmentVarGuard()
4040
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
41-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
41+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
4242
with env:
4343
init_natural_language()
4444
client = language.LanguageServiceClient(credentials=credentials)
@@ -49,7 +49,7 @@ def test_user_provided_credentials(self):
4949
def test_monkeypatching_succeed(self):
5050
env = EnvironmentVarGuard()
5151
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
52-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
52+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
5353
with env:
5454
init_natural_language()
5555
client = language.LanguageServiceClient.__init__
@@ -58,7 +58,7 @@ def test_monkeypatching_succeed(self):
5858
def test_monkeypatching_idempotent(self):
5959
env = EnvironmentVarGuard()
6060
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
61-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE')
61+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
6262
with env:
6363
init_natural_language()
6464
client1 = language.LanguageServiceClient.__init__

‎tests/test_tensorflow_credentials.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs):
1313
credential = '{"client_id":"fake_client_id",' \
1414
'"client_secret":"fake_client_secret",' \
1515
'"refresh_token":"not a refresh token",' \
16-
'"type":"authorized_user"}';
16+
'"type":"authorized_user"}'
1717

1818
env = EnvironmentVarGuard()
1919
env.set('HOME', '/tmp')
@@ -22,7 +22,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs):
2222
# These need to be set to make UserSecretsClient happy, but aren't
2323
# pertinent to this test.
2424
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
25-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML')
25+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
2626

2727
user_secrets = UserSecretsClient()
2828
user_secrets.set_tensorflow_credential(credential)

‎tests/test_translation.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def __init__(self, user_agent):
2828
def test_default_credentials_v2(self):
2929
env = EnvironmentVarGuard()
3030
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
31-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
31+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
3232
with env:
3333
init_translation_v2()
3434
client = translate_v2.Client()
@@ -41,7 +41,7 @@ def test_user_provided_credentials_v2(self):
4141
credentials = _make_credentials()
4242
env = EnvironmentVarGuard()
4343
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
44-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
44+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
4545
with env:
4646
init_translation_v2()
4747
client = translate_v2.Client(credentials=credentials)
@@ -52,7 +52,7 @@ def test_user_provided_credentials_v2(self):
5252
def test_default_credentials_v3(self):
5353
env = EnvironmentVarGuard()
5454
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
55-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
55+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
5656
with env:
5757
init_translation_v3()
5858
client = translate.TranslationServiceClient()
@@ -65,7 +65,7 @@ def test_user_provided_credentials_v3(self):
6565
credentials = _make_credentials()
6666
env = EnvironmentVarGuard()
6767
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
68-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
68+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
6969
with env:
7070
init_translation_v3()
7171
client = translate.TranslationServiceClient(credentials=credentials)
@@ -76,7 +76,7 @@ def test_user_provided_credentials_v3(self):
7676
def test_monkeypatching_succeed(self):
7777
env = EnvironmentVarGuard()
7878
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
79-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
79+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
8080
with env:
8181
init_translation_v2()
8282
init_translation_v3()
@@ -90,7 +90,7 @@ def test_monkeypatching_succeed(self):
9090
def test_monkeypatching_idempotent(self):
9191
env = EnvironmentVarGuard()
9292
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
93-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
93+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
9494
with env:
9595
init_translation_v2()
9696
init_translation_v3()
@@ -116,7 +116,7 @@ def test_client_credential_uniqueness_v3(self):
116116
credentials = _make_credentials()
117117
env = EnvironmentVarGuard()
118118
env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar')
119-
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION')
119+
env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI')
120120
with env:
121121
init_translation_v3()
122122
client1 = translate.TranslationServiceClient()

0 commit comments

Comments
 (0)