From 285ae7ccdff44a5b99fa183ddd614ae2bc94b350 Mon Sep 17 00:00:00 2001 From: Dimytch Date: Sat, 13 May 2017 20:41:10 +0300 Subject: [PATCH 1/5] renamed api schema back to v1 --- install_api.sql | 4 ++-- install_api_vars.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/install_api.sql b/install_api.sql index 42f2cc5..fa4870f 100644 --- a/install_api.sql +++ b/install_api.sql @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION v2.translate_array(source CHAR(2), target CHAR(2), q JSON) +CREATE OR REPLACE FUNCTION v1.translate_array(source CHAR(2), target CHAR(2), q JSON) RETURNS TEXT[] AS $BODY$ DECLARE rez TEXT[]; @@ -14,4 +14,4 @@ BEGIN END; $BODY$ LANGUAGE PLPGSQL SECURITY DEFINER; -GRANT EXECUTE ON FUNCTION v2.translate_array(CHAR(2), CHAR(2), JSON) TO apiuser; +GRANT EXECUTE ON FUNCTION v1.translate_array(CHAR(2), CHAR(2), JSON) TO apiuser; diff --git a/install_api_vars.sql b/install_api_vars.sql index 0192e8f..2606d41 100644 --- a/install_api_vars.sql +++ b/install_api_vars.sql @@ -1,6 +1,6 @@ ALTER DATABASE DBNAME SET translation_proxy.api.current_engine = 'CURRENT_API_ENGINE'; -CREATE SCHEMA IF NOT EXISTS v2; +CREATE SCHEMA IF NOT EXISTS v1; DO $$ BEGIN @@ -14,4 +14,4 @@ BEGIN END $$; -GRANT USAGE ON SCHEMA v2 TO apiuser; +GRANT USAGE ON SCHEMA v1 TO apiuser; From ba4e3579b52e220ee4874f5ff90baaac57a41b31 Mon Sep 17 00:00:00 2001 From: Dimytch Date: Tue, 16 May 2017 16:49:37 +0300 Subject: [PATCH 2/5] comments --- install_promt_core.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/install_promt_core.sql b/install_promt_core.sql index 573bcf4..9982455 100644 --- a/install_promt_core.sql +++ b/install_promt_core.sql @@ -9,6 +9,7 @@ -- Dumb functions for login, logout, translate and detect lnaguage -- authorizes on Promt API, writes cookie to db and returns it (or NULL) for next queries +-- curl -X POST -d 'username=startupturbo' -d 'password=Startupturb0' -d 'isPersistent=false' https://nombox.csd.promt.ru/pts/Services/auth/rest.svc/Login CREATE OR REPLACE FUNCTION translation_proxy._promt_login() RETURNS TEXT AS $$ import pycurl from StringIO import StringIO From 5d7309691a989b1446069ba32bd8ec78a5b9851f Mon Sep 17 00:00:00 2001 From: Dimytch Date: Tue, 16 May 2017 17:20:27 +0300 Subject: [PATCH 3/5] added noverify ssl option to curl --- install_promt_core.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/install_promt_core.sql b/install_promt_core.sql index 9982455..67e9625 100644 --- a/install_promt_core.sql +++ b/install_promt_core.sql @@ -31,6 +31,8 @@ CREATE OR REPLACE FUNCTION translation_proxy._promt_login() RETURNS TEXT AS $$ curl = pycurl.Curl() curl.setopt( curl.URL, server_url ) curl.setopt( pycurl.HTTPHEADER, ['Accept: application/json', 'Content-Type: application/json'] ) + curl.setopt(pycurl.SSL_VERIFYPEER, 0) + # curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt( pycurl.POST, 1 ) curl.setopt( pycurl.POSTFIELDS, json.dumps( @@ -66,6 +68,8 @@ RETURNS VOID AS $$ curl = pycurl.Curl() curl.setopt( pycurl.HTTPHEADER, [ 'Accept: application/json' ] ) + curl.setopt(pycurl.SSL_VERIFYPEER, 0) + # curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt( pycurl.COOKIELIST, cookie ) cursor = plpy.cursor( """ SELECT id, source, target, q, profile @@ -163,6 +167,8 @@ RETURNS CHAR(10) AS $$ curl.setopt( pycurl.URL, server_url + '?' + urlencode({ 'text': qs } )) curl.setopt( pycurl.WRITEDATA, buffer ) curl.setopt( pycurl.COOKIELIST, cookie ) + curl.setopt(pycurl.SSL_VERIFYPEER, 0) + # curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.perform() answer_code = curl.getinfo( pycurl.RESPONSE_CODE ) curl.close() @@ -215,6 +221,8 @@ CREATE OR REPLACE FUNCTION translation_proxy._promt_logout() RETURNS BOOLEAN AS curl = pycurl.Curl() curl.setopt( pycurl.URL, server_url ) + curl.setopt(pycurl.SSL_VERIFYPEER, 0) + # curl.setopt(pycurl.SSL_VERIFYHOST, 0) curl.setopt( pycurl.WRITEDATA, buffer ) curl.setopt( pycurl.COOKIELIST, cookie ) curl.perform() From 5b0ecac9cd7bfb40971db6ab04110c3e0d7f6aa7 Mon Sep 17 00:00:00 2001 From: Dimytch Date: Wed, 17 May 2017 08:19:38 +0300 Subject: [PATCH 4/5] unicode processing for urls --- install_api.sql | 2 +- install_global_core.sql | 53 +++++++++++++++++++++++++++++++++++++---- install_promt_core.sql | 3 ++- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/install_api.sql b/install_api.sql index fa4870f..d387e51 100644 --- a/install_api.sql +++ b/install_api.sql @@ -8,7 +8,7 @@ BEGIN WHEN 'google' THEN translation_proxy.google_translate_array( source, target, q ) WHEN 'promt' THEN - translation_proxy.promt_translate_array( source, target, array_agg( json_array_elements_text(q) ) ) + translation_proxy.promt_translate_array( source, target, ARRAY( SELECT json_array_elements_text(q) ) ) END INTO rez; RETURN rez; END; diff --git a/install_global_core.sql b/install_global_core.sql index 5522225..418d419 100644 --- a/install_global_core.sql +++ b/install_global_core.sql @@ -21,11 +21,27 @@ CREATE UNIQUE INDEX u_cache_q_source_target ON translation_proxy.cache CREATE INDEX cache_created ON translation_proxy.cache ( created ); COMMENT ON TABLE translation_proxy.cache IS 'The cache for API calls of the Translation proxy'; +--- Disclaimer: this urlencode is unusual -- it doesn't touch most chars (incl. multibytes) +--- to avoid reaching 2K limit for URL in Google API calls. +--- "Regular" urlencode() with multibyte chars support is shown above (commented out block of code). -- trigger, that URLencodes query in cache, when no translation is given CREATE OR REPLACE FUNCTION translation_proxy._urlencode_fields() RETURNS TRIGGER AS $BODY$ - from urllib import quote_plus - TD['new']['encoded'] = quote_plus( TD['new']['q'] ) + import StringIO + body = unicode(TD['new']['q'], 'utf-8') + o = StringIO.StringIO() + for c in body: + if c in u"+\]\[%&#\n\r": + o.write('%%%s' % c.encode('hex').upper()) + elif ord(c) in ( range(0x7f,0xa5) + [0xa0] ): + o.write('+') + elif ord(c) in range(0x00,0x19): + continue + else: + o.write(c) + + TD['new']['encoded'] = o.getvalue() + o.close() return 'MODIFY' $BODY$ LANGUAGE plpython2u; @@ -86,7 +102,21 @@ $$ LANGUAGE plpgsql; -- adding new parameter to url until it exceeds the limit of 2000 bytes CREATE OR REPLACE FUNCTION translation_proxy._urladd( url TEXT, a TEXT ) RETURNS TEXT AS $$ from urllib import quote_plus - r = url + quote_plus( a ) + import StringIO + body = unicode(a, 'utf-8') + o = StringIO.StringIO() + for c in body: + if c in u"+\]\[%&#\n\r": + o.write('%%%s' % c.encode('hex').upper()) + elif ord(c) in ( range(0x7f,0xa5) + [0xa0] ): + o.write('+') + elif ord(c) in range(0x00,0x19): + continue + else: + o.write(c) + + r = url + o.getvalue() + o.close() if len(r) > 1999 : plpy.error('URL length is over, time to fetch.', sqlstate = 'EOURL') return r @@ -95,6 +125,19 @@ $$ LANGUAGE plpython2u; -- urlencoding utility CREATE OR REPLACE FUNCTION translation_proxy._urlencode(q TEXT) RETURNS TEXT AS $BODY$ - from urllib import quote_plus - return quote_plus( q ) +import StringIO +body = unicode(q, 'utf-8') +o = StringIO.StringIO() +for c in body: + if c in u"+\]\[%&#\n\r": + o.write('%%%s' % c.encode('hex').upper()) + elif ord(c) in ( range(0x7f,0xa5) + [0xa0] ): + o.write('+') + elif ord(c) in range(0x00,0x19): + continue + else: + o.write(c) + body = o.getvalue() + o.close() + return body $BODY$ LANGUAGE plpython2u; diff --git a/install_promt_core.sql b/install_promt_core.sql index 67e9625..b1223d5 100644 --- a/install_promt_core.sql +++ b/install_promt_core.sql @@ -145,7 +145,8 @@ END; $BODY$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION translation_proxy.promt_translate( - src CHAR(2), dst CHAR(2), qs TEXT, api_profile text DEFAULT '') RETURNS TEXT[] AS $BODY$ + src CHAR(2), dst CHAR(2), qs TEXT, api_profile text DEFAULT '') +RETURNS TEXT[] AS $BODY$ BEGIN SELECT translation_proxy.promt_translate_array( src, dst, ARRAY[qs], api_profile); END; From e1e2fad52563e9ff94e7695df678cda4b834c114 Mon Sep 17 00:00:00 2001 From: Dimytch Date: Thu, 18 May 2017 01:20:33 +0300 Subject: [PATCH 5/5] Changed Promt translation method from GET to POST --- install_promt_core.sql | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/install_promt_core.sql b/install_promt_core.sql index b1223d5..e362d8a 100644 --- a/install_promt_core.sql +++ b/install_promt_core.sql @@ -64,12 +64,13 @@ RETURNS VOID AS $$ "UPDATE translation_proxy.cache SET result = $1, encoded = NULL WHERE id = $2", [ 'text', 'bigint' ] ) cookie = plpy.execute( "SELECT translation_proxy._promt_login()" )[0]['_promt_login'] - server_url = plpy.execute( "SELECT current_setting('translation_proxy.promt.server_url')" )[0]['current_setting'] + '/Services/v1/rest.svc/TranslateText?' + server_url = plpy.execute( "SELECT current_setting('translation_proxy.promt.server_url')" )[0]['current_setting'] + '/Services/v1/rest.svc/TranslateText' curl = pycurl.Curl() - curl.setopt( pycurl.HTTPHEADER, [ 'Accept: application/json' ] ) - curl.setopt(pycurl.SSL_VERIFYPEER, 0) + curl.setopt( pycurl.HTTPHEADER, [ 'Accept: application/json', 'Content-Type: application/json' ] ) + curl.setopt( pycurl.SSL_VERIFYPEER, 0) # curl.setopt(pycurl.SSL_VERIFYHOST, 0) + curl.setopt( pycurl.POST, 1 ) curl.setopt( pycurl.COOKIELIST, cookie ) cursor = plpy.cursor( """ SELECT id, source, target, q, profile @@ -85,12 +86,13 @@ RETURNS VOID AS $$ break buffer = StringIO() curl.setopt( pycurl.WRITEDATA, buffer ) - curl.setopt( pycurl.URL, server_url + - urlencode({ 'from': row[0]['source'], - 'to': row[0]['target'], - 'text': row[0]['q'], - 'profile': row[0]['profile'] }) ) + curl.setopt( pycurl.URL, server_url ) + j = json.dumps({ 'from': row[0]['source'], 'to': row[0]['target'], 'text': row[0]['q'], 'profile': row[0]['profile'] }) + output = StringIO(j) + curl.setopt( pycurl.POSTFIELDSIZE, output.len ) + curl.setopt( pycurl.READDATA, output ) curl.perform() + output.close() answer_code = curl.getinfo( pycurl.RESPONSE_CODE ) if answer_code != 200 : plpy.error( "Promt API returned %s\nBody is: %s" % ( answer_code, buffer.getvalue() ))