Merge branch 'main' of github.com:mos9527/PyCriCodecs

Author: mos9527

.github/workflows/tests.yml

@@ -0,0 +1,19 @@
+name: Tests
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  Test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: FedericoCarboni/setup-ffmpeg@v3        
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          pip install -U pytest
+          pip install .[usm]
+      - name: Run tests
+        run: |
+          pytest -v -s

.gitignore

@@ -7,4 +7,5 @@ temp_*
 *.aprof
 *.pdb
 dist/
-_*
+_*
+*.so

.vscode/launch.json

@@ -5,13 +5,40 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "(gdb) Launch USMDecode",
+            "name": "(gdb) Launch CRILAYLA",
             "type": "cppdbg",
             "request": "launch",
             "program": "/opt/miniconda3/bin/python",
             "args": [
                 "-m",
-                "Tests.test_USMDecode"
+                "Tests.test_CRILAYLA"
+            ],
+            "stopAtEntry": false,
+            "cwd": "${workspaceFolder}",            
+            "environment": [],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                },
+                {
+                    "description": "Set Disassembly Flavor to Intel",
+                    "text": "-gdb-set disassembly-flavor intel",
+                    "ignoreFailures": true
+                }
+            ]
+        },  
+        {
+            "name": "(gdb) Launch CPK",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/opt/miniconda3/bin/python",
+            "args": [
+                "-m",
+                "Tests.test_CPK"
             ],
             "stopAtEntry": false,
             "cwd": "${workspaceFolder}",
@@ -30,8 +57,7 @@
                     "ignoreFailures": true
                 }
             ]
-        },
-        
+        },              
         {
             "name": "Test: ACB",
             "type": "debugpy",

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+    "files.associations": {
+        "format": "cpp",
+        "__locale": "cpp",
+        "ios": "cpp",
+        "array": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "ranges": "cpp",
+        "span": "cpp",
+        "vector": "cpp"
+    }
+}

CriCodecsEx/adx.cpp

@@ -542,7 +542,7 @@ static PyObject* AdxEncode(PyObject* self, PyObject* args){
         PyAdxSetError(AdxErrorCode);
         return NULL;
     }
-    unsigned int len = adx.size;
+    Py_ssize_t len = adx.size;
     return Py_BuildValue("y#", adxdata, len);
 }
 
@@ -557,6 +557,6 @@ static PyObject* AdxDecode(PyObject* self, PyObject* args){
         return NULL;
     }
     unsigned char *out = wav.WAVEBuffer;
-    unsigned int len = wav.wav.size+8;
+    Py_ssize_t len = wav.wav.size+8;
     return Py_BuildValue("y#", out, len);
 }

CriCodecsEx/crilayla.cpp

@@ -1,13 +1,19 @@
-/* 
-    CRI layla decompression.
-	written by tpu. (https://forum.xentax.com/viewtopic.php?f=21&t=5137&p=44220&hilit=CRILAYLA#p44220)
-	Python wrapper by me (and modification).
+/* CRILAYLA Encoder/Decoder
 
-	CRIcompress method by KenTse
-    Taken from wmltogether's fork of CriPakTools.
-    Python wrapper by me.
 
-	Note: I have no idea how this compression technique works. I just made the wrapper.
+CRI layla decompression.
+written by tpu. (https://forum.xentax.com/viewtopic.php?f=21&t=5137&p=44220&hilit=CRILAYLA#p44220)
+Python wrapper by https://github.com/Youjose/PyCriCodecs (and modification).
+
+CRIcompress method by KenTse
+Taken from wmltogether's fork of CriPakTools.
+Python wrapper by https://github.com/Youjose/PyCriCodecs.    
+TODO: This implementation may produce larger output - which shouldn't be
+possible with LZ-based compression. Investigate and fix.
+For now, if compression fails, the original data is returned.
+See also:
+    - https://github.com/FanTranslatorsInternational/Kuriimu2/blob/imgui/src/lib/Kompression/Encoder/CrilaylaEncoder.cs
+    - https://glinscott.github.io/lz/index.html
 */
 #define PY_SSIZE_T_CLEAN
 #pragma once
@@ -22,32 +28,30 @@ struct crilayla_header{
     unsigned int compressed_size;
 };
 
-
-unsigned char *sbuf;
-unsigned int bitcnt;
-unsigned int bitdat;
-unsigned int get_bits(unsigned int n){
-	unsigned int data, mask;
-
-    if (bitcnt<n){
-	  data = ((24-bitcnt)>>3)+1;
-	  bitcnt += data*8;
-      while(data) {
-		bitdat = (bitdat<<8) | (*sbuf--);
-		data--;
-      }
-    }
-
-	data = bitdat>>(bitcnt-n);
-	bitcnt -= n;
-	mask = (1<<n)-1;
-	data &= mask;
-	return data;
-}
-
 unsigned int llcp_dec(unsigned char *src, unsigned int src_len, unsigned char *dst, unsigned int dst_len){
-	unsigned char *dbuf, *pbuf;
+    unsigned char *dbuf, *pbuf;
 	unsigned int plen, poffset, byte;
+    unsigned char *sbuf;
+    unsigned int bitcnt;
+    unsigned int bitdat;
+    auto get_bits = [&](unsigned int n){
+        unsigned int data, mask;
+    
+        if (bitcnt<n){
+          data = ((24-bitcnt)>>3)+1;
+          bitcnt += data*8;
+          while(data) {
+            bitdat = (bitdat<<8) | (*sbuf--);
+            data--;
+          }
+        }
+    
+        data = bitdat>>(bitcnt-n);
+        bitcnt -= n;
+        mask = (1<<n)-1;
+        data &= mask;
+        return data;
+    };
 
 	sbuf = src+src_len-1;
 	dbuf = dst+dst_len-1;
@@ -108,40 +112,41 @@ unsigned char* layla_decomp(unsigned char* data, crilayla_header header){
 	return dst;
 }
 
-unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned char* src, unsigned int srcLen){
-    unsigned int n = srcLen - 1, m = *destLen - 0x1, T = 0, d = 0, p, q, i, j, k;
-    unsigned char* odest = dest;
+unsigned int layla_comp(unsigned char *dest, int *destLen, unsigned char *src, int srcLen)
+{
+    int n = srcLen - 1, m = *destLen - 0x1, T = 0, d = 0, p, q, i, j, k;
+    unsigned char *odest = dest;
     for (; n >= 0x100;)
     {
         j = n + 3 + 0x2000;
-        if (j > srcLen) j = srcLen;
-        for (i = n + 3, p = 0; i < j; i++)
+        if (j>srcLen) j = srcLen;
+        for (i = n + 3, p = 0; i<j; i++)
         {
             for (k = 0; k <= n - 0x100; k++)
             {
                 if (*(src + n - k) != *(src + i - k)) break;
             }
-            if (k > p)
+            if (k>p)
             {
                 q = i - n - 3; p = k;
             }
         }
-        if (p < 3)
+        if (p<3)
         {
             d = (d << 9) | (*(src + n--)); T += 9;
         }
         else
         {
             d = (((d << 1) | 1) << 13) | q; T += 14; n -= p;
-            if (p < 6)
+            if (p<6)
             {
                 d = (d << 2) | (p - 3); T += 2;
             }
-            else if (p < 13)
+            else if (p<13)
             {
                 d = (((d << 2) | 3) << 3) | (p - 6); T += 5;
             }
-            else if (p < 44)
+            else if (p<44)
             {
                 d = (((d << 5) | 0x1f) << 5) | (p - 13); T += 10;
             }
@@ -150,45 +155,50 @@ unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned cha
                 d = ((d << 10) | 0x3ff); T += 10; p -= 44;
                 for (;;)
                 {
-                    for (; T >= 8;)
+                    for (; m > 0 && T >= 8;)
                     {
-                        *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d & ((1 << T) - 1);
+                        *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d&((1 << T) - 1);
                     }
-                    if (p < 255) break;
+                    if (p<255) break;
                     d = (d << 8) | 0xff; T += 8; p = p - 0xff;
                 }
                 d = (d << 8) | p; T += 8;
             }
         }
-        for (; T >= 8;)
+        for (; m > 0 && T >= 8;)
         {
-            *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d & ((1 << T) - 1);
+            *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d&((1 << T) - 1);
         }
     }
-    if (T != 0)
+    if (m > 0 && T != 0)
     {
         *(dest + m--) = d << (8 - T);
     }
-    *(dest + m--) = 0; *(dest + m) = 0;
-    for (;;)
-    {
-        if (((*destLen - m) & 3) == 0) break;
-        *(dest + m--) = 0;
+    if (m > 0) {
+        *(dest + m--) = 0; *(dest + m) = 0;
+        for (;;)
+        {
+            if (((*destLen - m) & 3) == 0) break;
+            *(dest + m--) = 0;
+        }
     }
+    if (m <= 0) 
+        return 0; // Underflow    
     *destLen = *destLen - m; dest += m;
-    unsigned int l[] = { 0x4c495243,0x414c5941,srcLen - 0x100,*destLen };
-    for (j = 0; j < 4; j++)
+    // CRIL AYLA srcLen-0x100 destLen
+    int l[] = { 0x4c495243,0x414c5941,srcLen - 0x100,*destLen };
+    for (j = 0; j<4; j++)
     {
-        for (i = 0; i < 4; i++)
+        for (i = 0; i<4; i++)
         {
             *(odest + i + j * 4) = l[j] & 0xff; l[j] >>= 8;
         }
     }
-    for (j = 0, odest += 0x10; j < *destLen; j++)
+    for (j = 0, odest += 0x10; j<*destLen; j++)
     {
         *(odest++) = *(dest + j);
     }
-    for (j = 0; j < 0x100; j++)
+    for (j = 0; j<0x100; j++)
     {
         *(odest++) = *(src + j);
     }
@@ -200,6 +210,11 @@ PyObject* CriLaylaDecompress(PyObject* self, PyObject* d){
 	unsigned char *data = (unsigned char *)PyBytes_AsString(d);
 	crilayla_header header = *(crilayla_header*)data;
 
+    if (header.crilayla != 0x414c59434152494cULL) {
+        PyErr_SetString(PyExc_ValueError, "Invalid CRILAYLA header.");
+        return NULL;
+    }
+    
     unsigned char *out;
     Py_BEGIN_ALLOW_THREADS
 	out = layla_decomp((data+16), header);
@@ -218,18 +233,26 @@ unsigned char* CriLaylaDecompress(unsigned char* d){
 
 PyObject* CriLaylaCompress(PyObject* self, PyObject* args){
 	unsigned char *data;
-	unsigned int data_size;
+	Py_ssize_t data_size;
     if(!PyArg_ParseTuple(args, "y#", &data, &data_size)){
         return NULL;
     }
-    unsigned char *buf = new unsigned char[data_size];
-    memset(buf, 0, data_size);
 
+    unsigned char *buf = new unsigned char[data_size + 0x200];
+
+    int compressed_size = data_size;    
+    int res = 0;
     Py_BEGIN_ALLOW_THREADS
-    layla_comp(buf, &data_size, data, data_size);
+    res = layla_comp(buf, &compressed_size, data, compressed_size);
     Py_END_ALLOW_THREADS
-
-	PyObject* bufObj = Py_BuildValue("y#", buf, data_size);
+    PyObject* bufObj;
+    if (res && res < data_size) {
+        bufObj = Py_BuildValue("y#", buf, compressed_size);
+    } else {        
+        // FIXME
+        PyErr_SetString(PyExc_RuntimeError, "Compression failure.");
+        return NULL;
+    }
     delete[] buf;
     return bufObj;
 }

CriCodecsEx/hca.cpp

@@ -3296,7 +3296,7 @@ static PyObject* HcaCrypt(PyObject* self, PyObject* args){
     }
 
     unsigned char *buffer = (unsigned char *)view.buf;
-    unsigned int length = view.len;
+    Py_ssize_t length = view.len;
     PyBuffer_Release(&view);
 
     clHCA* hca = (clHCA*)malloc(sizeof(clHCA));
@@ -3345,7 +3345,7 @@ static PyObject* HcaDecode(PyObject* self, PyObject* args){
     data *wavData = &wav.wav.chunks.WAVEdata;
 
     unsigned char *data;
-    unsigned int data_size;
+    Py_ssize_t data_size;
     unsigned int header_size;
     unsigned long long keycode;
     unsigned short subkey;

PyCriCodecsEx/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.3"
+__version__ = "0.0.4"

PyCriCodecsEx/acb.py

@@ -221,7 +221,7 @@ def set_waveforms(self, value: List[HCACodec | ADXCodec | Tuple[AcbEncodeTypes,
 
         Input item may be a codec (if known), or a tuple of (Codec ID, Channel Count, Sample Count, Sample Rate, Raw data).
 
-        NOTE: Cue duration is not set. You need to change that manually.
+        NOTE: Cue duration is not set. You need to change that manually - this is usually unecessary as the player will just play until the end of the waveform.
         """
         WAVEFORM = self.view.WaveformTable[0]._payload.copy()
         encoded = []