Skip to content

ctlib: Implement full Unicode support for TDS 7+ (MSSQL). #576

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

ctlib: Implement full Unicode support for TDS 7+ (MSSQL). #576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion include/cspublic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -613,9 +613,11 @@ enum
#define CS_IMAGELOCATOR_TYPE TDS_STATIC_CAST(CS_INT, 38)
#define CS_UNITEXTLOCATOR_TYPE TDS_STATIC_CAST(CS_INT, 39)
#define CS_UNIQUE_TYPE TDS_STATIC_CAST(CS_INT, 40)
#define CS_NVARCHAR_TYPE TDS_STATIC_CAST(CS_INT, 41)
#define CS_NLONGCHAR_TYPE TDS_STATIC_CAST(CS_INT, 42)

#define CS_MIN_SYBTYPE CS_CHAR_TYPE
#define CS_MAX_SYBTYPE CS_UNIQUE_TYPE
#define CS_MAX_SYBTYPE CS_NLONGCHAR_TYPE

#define CS_USER_TYPE TDS_STATIC_CAST(CS_INT, 100)
/* cs_dt_info type values */
Expand Down
12 changes: 10 additions & 2 deletions src/ctlib/cs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,9 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat
return CS_FAIL;
}
src_len = srcfmt->maxlength;
if (datatype == CS_VARCHAR_TYPE || datatype == CS_VARBINARY_TYPE) {
if (datatype == CS_VARCHAR_TYPE ||
datatype == CS_NVARCHAR_TYPE ||
datatype == CS_VARBINARY_TYPE) {
CS_VARCHAR *vc = (CS_VARCHAR *) srcdata;
src_len = vc->len;
srcdata = vc->str;
Expand All @@ -562,7 +564,9 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat
return CS_FAIL;
}
destlen = destfmt->maxlength;
if (datatype == CS_VARCHAR_TYPE || datatype == CS_VARBINARY_TYPE) {
if (datatype == CS_VARCHAR_TYPE ||
datatype == CS_NVARCHAR_TYPE ||
datatype == CS_VARBINARY_TYPE) {
destvc = (CS_VARCHAR *) destdata;
destlen = sizeof(destvc->str);
destdata = destvc->str;
Expand Down Expand Up @@ -622,6 +626,8 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat
case SYBCHAR:
case SYBVARCHAR:
case SYBTEXT:
case SYBNVARCHAR:
case SYBNTEXT:
tdsdump_log(TDS_DBG_FUNC, "cs_convert() desttype = character\n");

memcpy(dest, srcdata, minlen);
Expand Down Expand Up @@ -824,6 +830,8 @@ _cs_convert(CS_CONTEXT * ctx, const CS_DATAFMT_COMMON * srcfmt, CS_VOID * srcdat
case SYBCHAR:
case SYBVARCHAR:
case SYBTEXT:
case SYBNVARCHAR:
case SYBNTEXT:
ret = CS_SUCCEED;
if (len > destlen) {
tdsdump_log(TDS_DBG_FUNC, "Data-conversion resulted in overflow\n");
Expand Down
41 changes: 39 additions & 2 deletions src/ctlib/ct.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2179,6 +2179,7 @@ _ct_get_server_type(TDSSOCKET *tds, int datatype)
case CS_BIT_TYPE: return SYBBIT;
case CS_CHAR_TYPE: return SYBCHAR;
case CS_VARCHAR_TYPE: return SYBVARCHAR;
case CS_NVARCHAR_TYPE: return SYBNVARCHAR;
case CS_LONG_TYPE:
case CS_UBIGINT_TYPE:
if (!tds || tds_capability_has_req(tds->conn, TDS_REQ_DATA_UINT8))
Expand Down Expand Up @@ -2208,11 +2209,12 @@ _ct_get_server_type(TDSSOCKET *tds, int datatype)
case CS_TEXT_TYPE: return SYBTEXT;
case CS_UNIQUE_TYPE: return SYBUNIQUE;
case CS_LONGBINARY_TYPE: return SYBLONGBINARY;
case CS_UNICHAR_TYPE: return SYBVARCHAR;
case CS_UNICHAR_TYPE: return SYBNVARCHAR;
case CS_LONGCHAR_TYPE:
if (!tds || IS_TDS7_PLUS(tds->conn))
return SYBVARCHAR;
return SYBLONGCHAR;
case CS_NLONGCHAR_TYPE: return SYBNVARCHAR;
case CS_DATE_TYPE:
if (!tds || tds_capability_has_req(tds->conn, TDS_REQ_DATA_DATE))
return SYBDATE;
Expand Down Expand Up @@ -3383,6 +3385,8 @@ ct_param(CS_COMMAND * cmd, CS_DATAFMT * datafmt_arg, CS_VOID * data, CS_INT data
CS_PARAM *param;
const CS_DATAFMT_LARGE *datafmt;
CS_DATAFMT_LARGE datafmt_buf;
bool promote = (datafmt_arg->datatype == CS_VARCHAR_TYPE ||
datafmt_arg->datatype == CS_LONGCHAR_TYPE);

tdsdump_log(TDS_DBG_FUNC, "ct_param(%p, %p, %p, %d, %hd)\n", cmd, datafmt_arg, data, datalen, indicator);

Expand All @@ -3391,6 +3395,30 @@ ct_param(CS_COMMAND * cmd, CS_DATAFMT * datafmt_arg, CS_VOID * data, CS_INT data

datafmt = _ct_datafmt_conv_in(cmd->con->ctx, datafmt_arg, &datafmt_buf);

if (promote && IS_TDS7_PLUS(cmd->con->tds_socket->conn)) {
/* Actually promote only if non-ASCII characters are present */
CS_INT i;

promote = 0;
for (i = 0; i < datalen; ++i) {
if (((const char*)data)[i] & 0x80) {
promote = 1;
break;
}
}

if ( !promote ) {
/* pure ASCII, leave as is */
} else if (datafmt->datatype == CS_VARCHAR_TYPE) {
datafmt_arg->datatype = CS_NVARCHAR_TYPE;
datafmt_buf.datatype = CS_NVARCHAR_TYPE;
} else if (datafmt->datatype == CS_LONGCHAR_TYPE) {
datafmt_arg->datatype = CS_NLONGCHAR_TYPE;
datafmt_buf.datatype = CS_NLONGCHAR_TYPE;
}
}


switch (cmd->command_type) {
case CS_RPC_CMD:
if (!cmd->rpc) {
Expand Down Expand Up @@ -4203,7 +4231,9 @@ paraminfoalloc(TDSSOCKET * tds, CS_PARAM * first_param)
temp_datalen = *(p->datalen);
}

if (temp_type == CS_VARCHAR_TYPE || temp_type == CS_VARBINARY_TYPE) {
if (temp_type == CS_VARCHAR_TYPE
|| temp_type == CS_NVARCHAR_TYPE
|| temp_type == CS_VARBINARY_TYPE) {
CS_VARCHAR *vc = (CS_VARCHAR *) temp_value;

if (vc) {
Expand All @@ -4230,6 +4260,13 @@ paraminfoalloc(TDSSOCKET * tds, CS_PARAM * first_param)
if (p->maxlen < 0) {
tds_free_param_results(params);
return NULL;
} else if (p->maxlen == 0
&& is_unicode_type(tds_type)) {
/*
* Auto-detect; account for possible
* conversion to UCS-2.
*/
p->maxlen = temp_datalen * 2;
}
pcol->on_server.column_size = pcol->column_size = p->maxlen;
pcol->column_cur_size = temp_value ? temp_datalen : -1;
Expand Down
7 changes: 4 additions & 3 deletions src/tds/convert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,11 @@ binary_to_result(int desttype, const void *data, size_t len, CONV_RESULT * cr)
return (TDS_INT)len;
}

/* "N" versions are safe to list due to iconv calls elsewhere. */
#define CASE_ALL_CHAR \
SYBCHAR: case SYBVARCHAR: case SYBTEXT: case XSYBCHAR: case XSYBVARCHAR
SYBCHAR: case SYBVARCHAR: case SYBTEXT: case XSYBCHAR: \
case XSYBVARCHAR: case SYBNVARCHAR: case SYBNTEXT: case XSYBNCHAR: \
case XSYBNVARCHAR
#define CASE_ALL_BINARY \
SYBBINARY: case SYBVARBINARY: case SYBIMAGE: case XSYBBINARY: case XSYBVARBINARY: \
case SYBLONGBINARY: case TDS_CONVERT_BINARY
Expand Down Expand Up @@ -1986,8 +1989,6 @@ tds_convert(const TDSCONTEXT *tds_ctx, int srctype, const void *src, TDS_UINT sr
case SYBUNIQUE:
length = tds_convert_unique(src, desttype, cr);
break;
case SYBNVARCHAR:
case SYBNTEXT:
case SYBMSTABLE:
default:
return TDS_CONVERT_NOAVAIL;
Expand Down
3 changes: 3 additions & 0 deletions src/tds/data.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,9 @@ tds_set_param_type(TDSCONNECTION * conn, TDSCOLUMN * curcol, TDS_SERVER_TYPE typ
{
if (IS_TDS7_PLUS(conn)) {
switch (type) {
case SYBNVARCHAR:
type = XSYBNVARCHAR;
break;
case SYBVARCHAR:
type = XSYBVARCHAR;
break;
Expand Down
31 changes: 17 additions & 14 deletions src/tds/tds_willconvert.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ ($)
return qw(DATETIME DATETIME4 DATE TIME MSDATE MSTIME MSDATETIME2 MSDATETIMEOFFSET 5BIGDATETIME 5BIGTIME) if $_ eq 'DATETIMEx';
return qw(BIT BITN) if $_ eq 'BITx';
return qw(BINARY VARBINARY IMAGE LONGBINARY XBINARY XVARBINARY) if $_ eq 'BINARYx';
return qw(CHAR VARCHAR XCHAR XVARCHAR) if $_ eq 'CHARx';
return qw(CHAR VARCHAR XCHAR XNCHAR XNVARCHAR XVARCHAR) if $_ eq 'CHARx';
return qw(TEXT NTEXT) if $_ eq 'TEXT';
return ($_);
}

Expand Down Expand Up @@ -129,21 +130,23 @@ ($)
}
print "};\n";

# NB: SYBSENSITIVITY (Sybase) == 103 (0x67) == SYBNVARCHAR (MS)

__DATA__
To
From
CHARx TEXT BINARYx INTx FLTx NUMERIC DECIMAL BITx MONEYx DATETIMEx BOUNDARY UNIQUE SENSITIVITY MSTABLE
CHARx T T T T T T T T T T T T t F
TEXT T T T T T T T T T T T T t F
BINARYx T T T T T F F F T F F F F F
INTx T T T T T T T T T F F F F F
FLTx T T T T T T T T T F F F F F
NUMERIC T T T T T T T T T F F F F F
DECIMAL T T T T T T T T T F F F F F
BITx T T T T T T T T T F F F F F
MONEYx T T T T T T T T T F F F F F
DATETIMEx T T T F F F F F F T F F F F
BOUNDARY T T F F F F F F F F T F F F
UNIQUE T T T F F F F F F F F T F F
SENSITIVITY t t F F F F F F F F F F t F
CHARx T T T T T T T T T T T T T F
TEXT T T T T T T T T T T T T T F
BINARYx T T T T T F F F T F F F T F
INTx T T T T T T T T T F F F T F
FLTx T T T T T T T T T F F F T F
NUMERIC T T T T T T T T T F F F T F
DECIMAL T T T T T T T T T F F F T F
BITx T T T T T T T T T F F F T F
MONEYx T T T T T T T T T F F F T F
DATETIMEx T T T F F F F F F T F F T F
BOUNDARY T T F F F F F F F F T F T F
UNIQUE T T T F F F F F F F F T T F
SENSITIVITY T T T T T T T T T T T T T F
MSTABLE F F F F F F F F F F F F F T
9 changes: 9 additions & 0 deletions src/tds/unittests/convert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ free_convert(int type, CONV_RESULT *cr)
{
switch (type) {
case SYBCHAR: case SYBVARCHAR: case SYBTEXT: case XSYBCHAR: case XSYBVARCHAR:
case SYBNVARCHAR: case SYBNTEXT: case XSYBNCHAR: case XSYBNVARCHAR:
case SYBBINARY: case SYBVARBINARY: case SYBIMAGE: case XSYBBINARY: case XSYBVARBINARY:
case SYBLONGBINARY:
free(cr->c);
Expand Down Expand Up @@ -157,10 +158,18 @@ main(int argc, char **argv)
case XSYBVARBINARY:
case XSYBCHAR:
case XSYBVARCHAR:
case SYBNTEXT:
case SYBNVARCHAR:
case XSYBNCHAR:
case XSYBNVARCHAR:
switch (desttype) {
case SYBCHAR:
case SYBVARCHAR:
case SYBTEXT:
case SYBNTEXT:
case SYBNVARCHAR:
case XSYBNCHAR:
case XSYBNVARCHAR:
case SYBDATETIME:
case SYBDATETIME4:
src = "Jan 1, 1999";
Expand Down