Extremely large memory use when saving to file or writing to memory

[Jaster1999]

I am trying to convert between numpy and vips image formats, using vips rotate function then converting back to numpy.

I have a function to convert from numpy to vips, based off the documentation

# Function to convert an image from numpy format to vips
def convert_to_vips(np_image):
    dtype_to_format = {
    'uint8': 'uchar',
    'int8': 'char',
    'uint16': 'ushort',
    'int16': 'short',
    'uint32': 'uint',
    'int32': 'int',
    'float32': 'float',
    'float64': 'double',
    'complex64': 'complex',
    'complex128': 'dpcomplex',
    }
    height, width, bands = np_image.shape
    linear = np_image.reshape(width * height * bands)
    vips_image = pyvips.Image.new_from_memory(linear.data, width, height, bands, dtype_to_format[str(np_image.dtype)])
    return vips_image

However when I ry to convert from vips to numpy I get massive ram usage (16gb computer vips uses 12gb+ then the process kills). My conversion function is

# Function to convert from vips format to numpy
def convert_to_numpy(vips_image):
    format_to_dtype = {
    'uchar': np.uint8,
    'char': np.int8,
    'ushort': np.uint16,
    'short': np.int16,
    'uint': np.uint32,
    'int': np.int32,
    'float': np.float32,
    'double': np.float64,
    'complex': np.complex64,
    'dpcomplex': np.complex128,
    }
    # Loading a vips image into memory causes a massive memory spike, in excess of all 16g's of ram so... dont know why it does this 
    #buffer=vips_image.write_to_memory()
    #np_image = np.ndarray(buffer, dtype=format_to_dtype[vips_image.format], shape=[vips_image.height, vips_image.width, vips_image.bands])
    vips_image.write_to_file("temp.tif", )
    np_image = imread("temp.tif")
    os.remove("temp.tif")
    return np_image

Note that I was using write_to_memory() thought that was the issue but it also does this on write_to_file()

My full program code is:

# Function to convert from vips format to numpy
def convert_to_numpy(vips_image):
    format_to_dtype = {
    'uchar': np.uint8,
    'char': np.int8,
    'ushort': np.uint16,
    'short': np.int16,
    'uint': np.uint32,
    'int': np.int32,
    'float': np.float32,
    'double': np.float64,
    'complex': np.complex64,
    'dpcomplex': np.complex128,
    }
    # Loading a vips image into memory causes a massive memory spike, in excess of all 16g's of ram so... dont know why it does this 
    #buffer=vips_image.write_to_memory()
    #np_image = np.ndarray(buffer, dtype=format_to_dtype[vips_image.format], shape=[vips_image.height, vips_image.width, vips_image.bands])
    vips_image.write_to_file("temp.tif", )
    np_image = imread("temp.tif")
    os.remove("temp.tif")
    return np_image

# Function to convert an image from numpy format to vips
def convert_to_vips(np_image):
    dtype_to_format = {
    'uint8': 'uchar',
    'int8': 'char',
    'uint16': 'ushort',
    'int16': 'short',
    'uint32': 'uint',
    'int32': 'int',
    'float32': 'float',
    'float64': 'double',
    'complex64': 'complex',
    'complex128': 'dpcomplex',
    }
    height, width, bands = np_image.shape
    linear = np_image.reshape(width * height * bands)
    vips_image = pyvips.Image.new_from_memory(linear.data, width, height, bands, dtype_to_format[str(np_image.dtype)])
    return vips_image

def large_img_rotate(image, angle):
    img = convert_to_vips(image)
    img_rotated = img.rotate(angle, interpolate=pyvips.Interpolate.new("nearest"))
    img = convert_to_numpy(img_rotated)
    return img


img = imread('photo.tif')
rotated_img = large_img_rotate(img, 346.2)
cv2.imwrite('rotated_photo.tif', rotated_img)

[jcupitt]

Hi @Jaster1999,

You'd need to share more details about the image, but I think you're probably just seeing expected behaviour with numpy. It keeps images as huge memory arrays, so you will see very high memory usage with large images. There's no way round it if you use numpy.

I made a runnable version of your test:

#!/usr/bin/python3

import sys
import os
import psutil

import pyvips
import numpy as np
import cv2

format_to_dtype = {
    'uchar': np.uint8,
    'char': np.int8,
    'ushort': np.uint16,
    'short': np.int16,
    'uint': np.uint32,
    'int': np.int32,
    'float': np.float32,
    'double': np.float64,
    'complex': np.complex64,
    'dpcomplex': np.complex128,
}

dtype_to_format = {
    'uint8': 'uchar',
    'int8': 'char',
    'uint16': 'ushort',
    'int16': 'short',
    'uint32': 'uint',
    'int32': 'int',
    'float32': 'float',
    'float64': 'double',
    'complex64': 'complex',
    'complex128': 'dpcomplex',
}

process = psutil.Process(os.getpid())

def pyvips_to_numpy(image):
    return np.ndarray(buffer=image.write_to_memory(),
                      dtype=format_to_dtype[image.format],
                      shape=[image.height, image.width, image.bands])

def numpy_to_pyvips(npimage):
    height, width, bands = npimage.shape
    linear = npimage.reshape(width * height * bands)
    return pyvips.Image.new_from_memory(linear.data, width, height, bands,
                                        dtype_to_format[str(npimage.dtype)])

def process_size():
    print(f"process size: {process.memory_info().rss / (1024 * 1024):.2f} MB")

print("loading with numpy ...")
npimage = cv2.imread(sys.argv[1])
process_size()

print("converting to pyvips ...")
image = numpy_to_pyvips(npimage)
process_size()

print("rotating with pyvips and creating new numpy image ...")
image = image.rotate(346.2)
npimage2 = pyvips_to_numpy(image)
process_size()

print("writing to output ...")
cv2.imwrite(sys.argv[2], npimage2)
process_size()

If I run it on a 10kx10k RGB JPG image I see:

john@yingna ~/try $ ./vips-numpy-rotate.py ~/pics/wtc.jpg x.tif
loading with numpy ...
process size: 332.16 MB
converting to pyvips ...
process size: 332.16 MB
rotating with pyvips and creating new numpy image ...
process size: 702.04 MB
writing to output ...
process size: 702.04 MB

Looking at the numbers:

1. The initial load with cv2 takes ~300mb of memory: 10,000 * 10,000 * 3 bytes, plus some extra for python.
2. Converting to pyvips uses no memory, since pyvips will reuse the buffer from numpy.
3. pyvips rotation plus write_to_memory adds another 11,337 * 11,337 * 3 bytes since a second copy of the slightly larger rotated image now needs to be held as a numpy array. numpy will reuse the buffer from pyvips.
4. The save adds no more memory since cv2 will just write from the second numpy array.

If you do everything with pyvips, you can avoid the huge memory arrays since pyvips is able to stream pixels. For example:

#!/usr/bin/python3
  
import sys
import os
import psutil

import pyvips

process = psutil.Process(os.getpid())

def process_size():
    print(f"process size: {process.memory_info().rss / (1024 * 1024):.2f} MB")

print("loading with pyvips ...")
image = pyvips.Image.new_from_file(sys.argv[1])
process_size()

print("rotating with pyvips ...")
image = image.rotate(346.2)
process_size()

print("writing to output ...")
image.write_to_file(sys.argv[2], )
process_size()

I see:

$ ./vips-rotate.py ~/pics/wtc.jpg x.tif
loading with pyvips ...
process size: 41.02 MB
rotating with pyvips ...
process size: 41.21 MB
writing to output ...
process size: 77.23 MB