Dmius ssl fix. Merge from deemytch/postgrest-translation-proxy

install_api.sql

@@ -1,4 +1,4 @@
-CREATE OR REPLACE FUNCTION v2.translate_array(source CHAR(2), target CHAR(2), q JSON)
+CREATE OR REPLACE FUNCTION v1.translate_array(source CHAR(2), target CHAR(2), q JSON)
 RETURNS TEXT[] AS $BODY$
 DECLARE
   rez TEXT[];
@@ -8,10 +8,10 @@ BEGIN
       WHEN 'google' THEN
         translation_proxy.google_translate_array( source, target, q )
       WHEN 'promt' THEN
-        translation_proxy.promt_translate_array( source, target, array_agg( json_array_elements_text(q) ) )
+        translation_proxy.promt_translate_array( source, target, ARRAY( SELECT json_array_elements_text(q) ) )
     END INTO rez;
     RETURN rez;
 END;
 $BODY$ LANGUAGE PLPGSQL SECURITY DEFINER;
 
-GRANT EXECUTE ON FUNCTION v2.translate_array(CHAR(2), CHAR(2), JSON) TO apiuser;
+GRANT EXECUTE ON FUNCTION v1.translate_array(CHAR(2), CHAR(2), JSON) TO apiuser;

install_api_vars.sql

@@ -1,6 +1,6 @@
 ALTER DATABASE DBNAME SET translation_proxy.api.current_engine = 'CURRENT_API_ENGINE';
 
-CREATE SCHEMA IF NOT EXISTS v2;
+CREATE SCHEMA IF NOT EXISTS v1;
 DO
 $$
 BEGIN
@@ -14,4 +14,4 @@ BEGIN
 END
 $$;
 
-GRANT USAGE ON SCHEMA v2 TO apiuser;
+GRANT USAGE ON SCHEMA v1 TO apiuser;

install_global_core.sql

@@ -21,11 +21,27 @@ CREATE UNIQUE INDEX u_cache_q_source_target ON translation_proxy.cache
 CREATE INDEX cache_created ON translation_proxy.cache ( created );
 COMMENT ON TABLE translation_proxy.cache IS 'The cache for API calls of the Translation proxy';
 
+--- Disclaimer: this urlencode is unusual -- it doesn't touch most chars (incl. multibytes)
+--- to avoid reaching 2K limit for URL in Google API calls.
+--- "Regular" urlencode() with multibyte chars support is shown above (commented out block of code).
 -- trigger, that URLencodes query in cache, when no translation is given
 CREATE OR REPLACE FUNCTION translation_proxy._urlencode_fields()
 RETURNS TRIGGER AS $BODY$
-  from urllib import quote_plus
-  TD['new']['encoded'] =  quote_plus( TD['new']['q'] )
+  import StringIO
+  body = unicode(TD['new']['q'], 'utf-8')
+  o = StringIO.StringIO()
+  for c in body:
+    if c in u"+\]\[%&#\n\r":
+      o.write('%%%s' % c.encode('hex').upper())
+    elif ord(c) in ( range(0x7f,0xa5) + [0xa0] ):
+      o.write('+')
+    elif ord(c) in range(0x00,0x19):
+      continue
+    else:
+      o.write(c)
+
+  TD['new']['encoded'] = o.getvalue()
+  o.close()
   return 'MODIFY'
 $BODY$ LANGUAGE plpython2u;
 
@@ -86,7 +102,21 @@ $$ LANGUAGE plpgsql;
 -- adding new parameter to url until it exceeds the limit of 2000 bytes
 CREATE OR REPLACE FUNCTION translation_proxy._urladd( url TEXT, a TEXT ) RETURNS TEXT AS $$
   from urllib import quote_plus
-  r = url + quote_plus( a )
+  import StringIO
+  body = unicode(a, 'utf-8')
+  o = StringIO.StringIO()
+  for c in body:
+    if c in u"+\]\[%&#\n\r":
+      o.write('%%%s' % c.encode('hex').upper())
+    elif ord(c) in ( range(0x7f,0xa5) + [0xa0] ):
+      o.write('+')
+    elif ord(c) in range(0x00,0x19):
+      continue
+    else:
+      o.write(c)
+
+  r = url + o.getvalue()
+  o.close()
   if len(r) > 1999 :
     plpy.error('URL length is over, time to fetch.', sqlstate = 'EOURL')
   return r
@@ -95,6 +125,19 @@ $$ LANGUAGE plpython2u;
 -- urlencoding utility
 CREATE OR REPLACE FUNCTION translation_proxy._urlencode(q TEXT)
 RETURNS TEXT AS $BODY$
-  from urllib import quote_plus
-  return quote_plus( q )
+import StringIO
+body = unicode(q, 'utf-8')
+o = StringIO.StringIO()
+for c in body:
+  if c in u"+\]\[%&#\n\r":
+    o.write('%%%s' % c.encode('hex').upper())
+  elif ord(c) in ( range(0x7f,0xa5) + [0xa0] ):
+    o.write('+')
+  elif ord(c) in range(0x00,0x19):
+    continue
+  else:
+    o.write(c)
+  body = o.getvalue()
+  o.close()
+  return body
 $BODY$ LANGUAGE plpython2u;

install_promt_core.sql

@@ -9,6 +9,7 @@
 
 -- Dumb functions for login, logout, translate and detect lnaguage
 -- authorizes on Promt API, writes cookie to db and returns it (or NULL) for next queries
+-- curl -X POST -d 'username=startupturbo' -d 'password=Startupturb0' -d 'isPersistent=false' https://nombox.csd.promt.ru/pts/Services/auth/rest.svc/Login
 CREATE OR REPLACE FUNCTION translation_proxy._promt_login() RETURNS TEXT AS $$
   import pycurl
   from StringIO import StringIO
@@ -30,6 +31,8 @@ CREATE OR REPLACE FUNCTION translation_proxy._promt_login() RETURNS TEXT AS $$
   curl = pycurl.Curl()
   curl.setopt( curl.URL, server_url )
   curl.setopt( pycurl.HTTPHEADER, ['Accept: application/json', 'Content-Type: application/json'] )
+  curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+  # curl.setopt(pycurl.SSL_VERIFYHOST, 0)
   curl.setopt( pycurl.POST, 1 )
   curl.setopt( pycurl.POSTFIELDS,
     json.dumps(
@@ -61,10 +64,13 @@ RETURNS VOID AS $$
       "UPDATE translation_proxy.cache SET result = $1, encoded = NULL WHERE id = $2",
       [ 'text', 'bigint' ] )
   cookie = plpy.execute( "SELECT translation_proxy._promt_login()" )[0]['_promt_login']
-  server_url = plpy.execute( "SELECT current_setting('translation_proxy.promt.server_url')" )[0]['current_setting'] + '/Services/v1/rest.svc/TranslateText?'
+  server_url = plpy.execute( "SELECT current_setting('translation_proxy.promt.server_url')" )[0]['current_setting'] + '/Services/v1/rest.svc/TranslateText'
 
   curl = pycurl.Curl()
-  curl.setopt( pycurl.HTTPHEADER, [ 'Accept: application/json' ] )
+  curl.setopt( pycurl.HTTPHEADER, [ 'Accept: application/json', 'Content-Type: application/json' ] )
+  curl.setopt( pycurl.SSL_VERIFYPEER, 0)
+  # curl.setopt(pycurl.SSL_VERIFYHOST, 0)
+  curl.setopt( pycurl.POST, 1 )
   curl.setopt( pycurl.COOKIELIST, cookie )
   cursor = plpy.cursor( """
     SELECT id, source, target, q, profile
@@ -80,12 +86,13 @@ RETURNS VOID AS $$
       break
     buffer = StringIO()
     curl.setopt( pycurl.WRITEDATA, buffer )
-    curl.setopt( pycurl.URL, server_url +
-        urlencode({ 'from': row[0]['source'],
-          'to': row[0]['target'],
-          'text': row[0]['q'],
-          'profile': row[0]['profile'] }) )
+    curl.setopt( pycurl.URL, server_url )
+    j = json.dumps({ 'from': row[0]['source'], 'to': row[0]['target'], 'text': row[0]['q'], 'profile': row[0]['profile'] })
+    output = StringIO(j)
+    curl.setopt( pycurl.POSTFIELDSIZE, output.len )
+    curl.setopt( pycurl.READDATA, output )
     curl.perform()
+    output.close()
     answer_code = curl.getinfo( pycurl.RESPONSE_CODE )
     if answer_code != 200 :
       plpy.error( "Promt API returned %s\nBody is: %s" % ( answer_code, buffer.getvalue() ))
@@ -140,7 +147,8 @@ END;
 $BODY$ LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION translation_proxy.promt_translate(
-    src CHAR(2), dst CHAR(2), qs TEXT, api_profile text DEFAULT '') RETURNS TEXT[] AS $BODY$
+    src CHAR(2), dst CHAR(2), qs TEXT, api_profile text DEFAULT '')
+RETURNS TEXT[] AS $BODY$
 BEGIN
   SELECT translation_proxy.promt_translate_array( src, dst, ARRAY[qs], api_profile);
 END;
@@ -162,6 +170,8 @@ RETURNS CHAR(10) AS $$
   curl.setopt( pycurl.URL, server_url + '?' + urlencode({ 'text': qs } ))
   curl.setopt( pycurl.WRITEDATA, buffer )
   curl.setopt( pycurl.COOKIELIST, cookie )
+  curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+  # curl.setopt(pycurl.SSL_VERIFYHOST, 0)
   curl.perform()
   answer_code = curl.getinfo( pycurl.RESPONSE_CODE )
   curl.close()
@@ -214,6 +224,8 @@ CREATE OR REPLACE FUNCTION translation_proxy._promt_logout() RETURNS BOOLEAN AS
 
   curl = pycurl.Curl()
   curl.setopt( pycurl.URL, server_url )
+  curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+  # curl.setopt(pycurl.SSL_VERIFYHOST, 0)
   curl.setopt( pycurl.WRITEDATA, buffer )
   curl.setopt( pycurl.COOKIELIST, cookie )
   curl.perform()