decompress_RLE

#include <stdio.h>
#include <stdint.h>
#include <assert.h>
#include <stdlib.h>

struct Psd{
	int64_t width;
	int64_t height;
};

struct Channel{
	uint8_t *pointer;
	int64_t ID;
};

struct Layer{
  	uint64_t channel_count;
  	Channel channel[4]; // max 4 at them moment
	int64_t top;
	int64_t bottom;
	int64_t left;
	int64_t right;
};

struct Clamp{
	int64_t x;
	int64_t y;
	int64_t w;
	int64_t h;
};

uint8_t *decompress_RLE(
  Psd *psd,
  Layer *layer,
  uint8_t *file_data)
{
  	uint64_t total_height = layer->bottom - layer->top;
  
  	Clamp clamp = {};
  	if(layer->left < 0) clamp.x = layer->left;
  	if(layer->top < 0) clamp.y = layer->top;
  	if(layer->right > psd->width) clamp.w = psd->width - layer->left;
  	else clamp.w = layer->right - layer->left;
  	if(layer->bottom > psd->height) clamp.h = psd->height - layer->top;
  	else clamp.h = layer->bottom - layer->top;
  
  	int64_t data_size = clamp.w * clamp.h * 4;
  	if(data_size < 0) data_size = 0;
  	
  	
  	uint8_t *image_data;
  	if(data_size != 0)
    {
    	image_data = (uint8_t *)malloc(data_size);
    }
  
  	for(size_t CC = 0; CC < layer->channel_count; ++CC)
    {
    	file_data = layer->channel[CC].pointer;
      	uint16_t compression = (uint16_t)file_data[0] * 256 + (uint16_t)file_data[1];
      	file_data += 2;
      
      	double opacity = 1;
      	int64_t pixel_pos_c;
      	if(layer->channel[CC].ID == -1) pixel_pos_c = 3;
      	else if(layer->channel[CC].ID == 0) pixel_pos_c = 0;
      	else if(layer->channel[CC].ID == 1) pixel_pos_c = 1;
      	else if(layer->channel[CC].ID == 2) pixel_pos_c = 2;
      	//else assert(false);
      	
      	if(compression == 1)
        {
        	uint16_t *line_info = (uint16_t *)malloc(total_height * sizeof(*line_info));
          	for(size_t LINE = 0; LINE < total_height; ++LINE)
            {
            	line_info[LINE] = (uint16_t)file_data[0] * 256 + (uint16_t)file_data[1];
              	file_data += sizeof(*line_info);
            }
  			
          
          	int64_t pixel_pos_y = clamp.y;
          	for(size_t LINE = 0; LINE < total_height; ++LINE)
            {
                uint8_t *layer_end = file_data + line_info[LINE];
              	if(pixel_pos_y >= 0)
                {
                	if(pixel_pos_y < clamp.h + clamp.y)
                    {
                      	int64_t pixel_pos_x = clamp.x;
                      	while(file_data < layer_end)
                        {
                        	int8_t head = *file_data;
                          	++file_data;
                          	if(head >= 0)
                            {
                            	for(size_t i = 0; i < 1 + head; ++i)
                                {
                                	uint8_t pixel = *file_data * opacity;
                                  	++file_data;
                                  	if(pixel_pos_x >= 0 && pixel_pos_x < clamp.x + clamp.w)
                                    {
                                    	image_data[pixel_pos_c + (pixel_pos_x*4) + (pixel_pos_y*(clamp.w+clamp.x)*4)] = pixel;
                                    }
                                  	
                                  	++pixel_pos_x;
                                }
                            }
                          	else if(head > -128)
                            {
                            	uint8_t pixel = *file_data * opacity;
                                ++file_data;
                              	for(size_t i = 0; i < 1 - head; ++i)
                                {
                                  	if(pixel_pos_x >= 0 && pixel_pos_x < clamp.x + clamp.y)
                                    {
                                    	image_data[pixel_pos_c + (pixel_pos_x*4) + (pixel_pos_y*(clamp.w+clamp.x)*4)] = pixel;
                                    }
                                  	
                                  	++pixel_pos_x;
                                }
                            }
                          	else
                            {
                            	++file_data;
                            }
                        }
                    }
                  	else
                    {
                    	break;
                    }
                }
              	else
                {
                	file_data = layer_end;
                }
              	++pixel_pos_y;
              	free(line_info);
            }
        }
      	else if(compression == 0)
        {
          	//todo
        }
    }
  	return image_data;
}