Testing complete Mode 1 decompression is BIT PERFECT on all cases of SPL4 and MDH, on 100% of the overdumped gfx data.
Code:
// feoez.cpp : Defines the entry point for the console application.
//
#include "stdafx.h"
#include <stdio.h>
#include <stdlib.h>
typedef unsigned char uint8;
uint8 SeenEvolution[53][2]=
{
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0},
{0,0}
};
uint8 EvolutionTable[53][4]=
{
//prob, nextlps, nextmps, toggle invert
{0x5a, 1, 1,1}, //0 l,m
{0x25, 6, 2,0}, //1 l,m
{0x11, 8, 3,0}, //2 l,m
{0x08, 10, 4,0}, //3 ,m
{0x03, 12, 5,0}, //4 ,m
{0x01, 15, 5,0}, //5 ,m
{0x5a, 7, 7,1}, //6 l,
{0x3f, 19, 8,0}, //7 l,m
{0x2c, 21, 9,0}, //8 l,m
{0x20, 22, 10,0}, //9 ,m
{0x17, 23, 11,0}, //10 ,m
{0x11, 25, 12,0}, //11 ,m
{0x0c, 26, 13,0}, //12 ,m
{0x09, 28, 14,0}, //13 ,m
{0x07, 29, 15,0}, //14 ,m
{0x05, 31, 16,0}, //15 ,m
{0x04, 32, 17,0}, //16 ,m
{0x03, 34, 18,0}, //17 ,m
{0x02, 35, 5,0}, //18 ,m
{0x5a, 20, 20,1}, //19 l,m
{0x48, 39, 21,0}, //20 l,m
{0x3a, 40, 22,0}, //21 l,m
{0x2e, 42, 23,0}, //22 l,m
{0x26, 44, 24,0}, //23 l,m
{0x1f, 45, 25,0}, //24 l,m
{0x19, 46, 26,0}, //25 l,m
{0x15, 25, 27,0}, //26 l,m
{0x11, 26, 28,0}, //27 l,m
{0x0e, 26, 29,0}, //28 l,m
{0x0b, 27, 30,0}, //29 ,m
{0x09, 28, 31,0}, //30 ,m
{0x08, 29, 32,0}, //31 l,m
{0x07, 30, 33,0}, //32 l,m
{0x05, 31, 34,0}, //33 l,m <--- changed lps
{0x04, 33, 35,0}, //34 ,m ... this is NOT skipped
{0x04, 33, 36,0}, //35 ,m
{0x03, 34, 37,0}, //36 ,m
{0x02, 35, 38,0}, //37 ,m ... this is NOT skipped
{0x02, 36, 5,0}, //38 ,m
{0x58, 39, 40,1}, //39 l,m
{0x4d, 47, 41,0}, //40 l,m
{0x43, 48, 42,0}, //41 ,m
{0x3b, 49, 43,0}, //42 ,m
{0x34, 50, 44,0}, //43 l,m
{0x2e, 51, 45,0}, //44 l,m
{0x29, 44, 46,0}, //45 l,m
{0x25, 45, 24,0}, //46 ,m
{0x56, 47, 48,1}, //47 l,m
{0x4f, 47, 49,0}, //48 l,m
{0x47, 48, 50,0}, //49 l,m
{0x41, 49, 51,0}, //50 l,m
{0x3c, 50, 52,0}, //51 l,m
{0x37, 51, 43,0} //52 ,m
};
#define PROB(x) EvolutionTable[Contexts[x].index][0]
#define NEXT_LPS(x) EvolutionTable[Contexts[x].index][1]
#define NEXT_MPS(x) EvolutionTable[Contexts[x].index][2]
#define TOGGLE_INVERT(x) EvolutionTable[Contexts[x].index][3]
typedef struct {
uint8 index;
uint8 invert;
} context_state;
#define NUM_CONTEXTS 30
context_state Contexts[NUM_CONTEXTS];
#define BIT(x,y) ((x>>y)&1)
void DecompressMode0(uint8 *datain, uint8 *dataout, int len)
{
uint8 top,val;
uint8 con,mps,prob;
uint8 flag_lps,shift,mask;
int out=0;
int inverts=0;
int lps=0;
unsigned char in;
int in_count;
int i,bit;
//setup
top=0xFF;
val=*datain;
datain++;
in=*datain;
datain++;
in_count=8;
//reset context states
for(i=0;i<NUM_CONTEXTS;i++)
{
Contexts[i].index=0;
Contexts[i].invert=0;
}
for(i=0;i<len;i++)
{
if(i==-1800)
{
int k;
printf("\nEvolution table:\n");
for(k=0;k<53;k++)
printf(" %d,%d //%d\n",SeenEvolution[k][0],SeenEvolution[k][1],k);
}
for(bit=0;bit<8;bit++)
{
//get context
mask = (1<<(bit&3)) - 1;
con = mask + ((inverts&mask)^(lps&mask));
if(bit>3)
con+=15;
//get PROB and MPS
prob = PROB(con);
mps = (BIT(out,15) ^ Contexts[con].invert);
if(i>=15 && i<=18 && 0)
printf("byte %d bit %d: val=%.2X top=%.2X prob=%.2X mps=%d con=%d state=%d\n",
i,bit,val,top,prob,mps,con,Contexts[con].index);
//get bit
if (val <= top-prob)
{
//mps
top = top - prob;
out = (out << 1) + mps;
flag_lps=0;
}
else
{
//lps
val = val - (top - (prob - 1));
top = prob - 1;
out = (out << 1) + 1-mps;
flag_lps=1;
}
// renormalize
shift=0;
while(top<0x7F) // NOTE: not 0x80, it's a strange border case
{
shift++;
top = (top<<1)+1;
val = (val<<1)+(in>>7);
in = (in<<1);
if(--in_count==0)
{
in=*datain;
datain++;
in_count=8;
}
}
//update processing info
lps = (lps<<1) + flag_lps;
inverts = (inverts<<1) + Contexts[con].invert;
//update context state
if(flag_lps & TOGGLE_INVERT(con))
Contexts[con].invert ^= 1;
if(flag_lps)
{
SeenEvolution[Contexts[con].index][0]=1;
Contexts[con].index = NEXT_LPS(con);
}
else if(shift)
{
SeenEvolution[Contexts[con].index][1]=1;
Contexts[con].index = NEXT_MPS(con);
}
}
//save byte
*dataout = (out & 0xFF);
dataout++;
}
}
int DecompressMode1(uint8 *datain, uint8 *dataout, int len)
{
int pixelorder[4]={0,1,2,3};
int realorder[4];
int a,b,c;
int m,n;
uint8 top,val;
uint8 con,prob;
uint8 flag_lps,shift;
int out=0;
int inverts=0;
int lps=0;
unsigned char in;
int in_count;
int in_len=0;
int i,j,pixel;
//setup
top=0xFF;
val=datain[in_len++];
in=datain[in_len++];
in_count=8;
//reset context states
for(i=0;i<NUM_CONTEXTS;i++)
{
Contexts[i].index=0;
Contexts[i].invert=0;
}
for(i=0;i<len;i+=2)
{
if(i!=0)
{
//turn pixel data into bitplanes
//and save as output
*dataout = (BIT(out,15)<<7) + (BIT(out,13)<<6) + (BIT(out,11)<<5) + (BIT(out,9)<<4)
+ (BIT(out,7)<<3) + (BIT(out,5)<<2) + (BIT(out,3)<<1) + BIT(out,1);
dataout++;
*dataout = (BIT(out,14)<<7) + (BIT(out,12)<<6) + (BIT(out,10)<<5) + (BIT(out,8)<<4)
+ (BIT(out,6)<<3) + (BIT(out,4)<<2) + (BIT(out,2)<<1) + BIT(out,0);
dataout++;
}
for(pixel=0;pixel<8;pixel++)
{
//get first symbol context
a = ((out >> (1*2)) & 0x3);
b = ((out >> (7*2)) & 0x3);
c = ((out >> (8*2)) & 0x3);
if(a==b && b==c)
con=0;
else if (a==b && b!=c)
con=1;
else if (a!=b && b==c)
con=2;
else if (a==c && b!=c)
con=3;
else
con=4;
//update pixel order
for(m=0;m<4;m++)
if(pixelorder[m]==a)
break;
for(n=m;n>0;n--)
{
j=pixelorder[n-1];
pixelorder[n-1]=pixelorder[n];
pixelorder[n]=j;
}
//get PROB
prob = PROB(con);
//get symbol
if (val <= top-prob)
{
//mps
top = top - prob;
flag_lps=0;
}
else
{
//lps
val = val - (top - (prob - 1));
top = prob - 1;
flag_lps=1;
}
// renormalize
shift=0;
while(top<0x7F)
{
shift++;
top = (top<<1)+1;
val = (val<<1)+(in>>7);
in = (in<<1);
if(--in_count==0)
{
in=datain[in_len++];
in_count=8;
}
}
//update processing info
lps = (lps<<1) + flag_lps;
inverts = (inverts<<1) + Contexts[con].invert;
//update context state
if(flag_lps & TOGGLE_INVERT(con))
Contexts[con].invert ^= 1;
if(flag_lps)
Contexts[con].index = NEXT_LPS(con);
else if(shift)
Contexts[con].index = NEXT_MPS(con);
//get context of second symbol
con = 5 + con*2 + ((lps^inverts)&1);
//get PROB
prob = PROB(con);
//get symbol
if (val <= top-prob)
{
//mps
top = top - prob;
flag_lps=0;
}
else
{
//lps
val = val - (top - (prob - 1));
top = prob - 1;
flag_lps=1;
}
// renormalize
shift=0;
while(top<0x7F)
{
shift++;
top = (top<<1)+1;
val = (val<<1)+(in>>7);
in = (in<<1);
if(--in_count==0)
{
in=datain[in_len++];
in_count=8;
}
}
//calculate the real pixel order
for(m=0;m<4;m++)
realorder[m]=pixelorder[m];
//shift refence pixel c value to top
for(m=0;m<4;m++)
if(realorder[m]==c)
break;
for(n=m;n>0;n--)
{
j=realorder[n-1];
realorder[n-1]=realorder[n];
realorder[n]=j;
}
//shift refence pixel b value to top
for(m=0;m<4;m++)
if(realorder[m]==b)
break;
for(n=m;n>0;n--)
{
j=realorder[n-1];
realorder[n-1]=realorder[n];
realorder[n]=j;
}
//shift refence pixel a value to top
for(m=0;m<4;m++)
if(realorder[m]==a)
break;
for(n=m;n>0;n--)
{
j=realorder[n-1];
realorder[n-1]=realorder[n];
realorder[n]=j;
}
//update processing info
lps = (lps<<1) + flag_lps;
inverts = (inverts<<1) + Contexts[con].invert;
//update context state
if(flag_lps & TOGGLE_INVERT(con))
Contexts[con].invert ^= 1;
if(flag_lps)
Contexts[con].index = NEXT_LPS(con);
else if(shift)
Contexts[con].index = NEXT_MPS(con);
//get pixel
b=realorder[(lps^inverts)&3];
out = (out<<2) + b;
}
}
//turn pixel data into bitplanes
//and save as output.. BUT don't save second byte unless asked to
*dataout = (BIT(out,15)<<7) + (BIT(out,13)<<6) + (BIT(out,11)<<5) + (BIT(out,9)<<4)
+ (BIT(out,7)<<3) + (BIT(out,5)<<2) + (BIT(out,3)<<1) + BIT(out,1);
dataout++;
if((len&1)==0)
{
*dataout = (BIT(out,14)<<7) + (BIT(out,12)<<6) + (BIT(out,10)<<5) + (BIT(out,8)<<4)
+ (BIT(out,6)<<3) + (BIT(out,4)<<2) + (BIT(out,2)<<1) + BIT(out,0);
dataout++;
}
if(in_count==8)
in_len--;
//printf("Used %d bytes of input.\n",in_len);
return in_len;
}
// I'm lazy, so here's static allocation
//plenty of room for all the files we'll be looking at
uint8 datain[0x10000];
uint8 dataout[0x10000];
uint8 realout[0x10000];
uint8 datarom[0x400000];
int htoi(char *str)
{
int i=0,j=0;
while(str[i]!=0)
{
j<<=4;
if((str[i]>='0')&&(str[i]<='9'))
j+=str[i]-'0';
if((str[i]>='A')&&(str[i]<='F'))
j+=str[i]-'A'+10;
if((str[i]>='a')&&(str[i]<='f'))
j+=str[i]-'a'+10;
i++;
}
return j;
}
int main(int argc, char* argv[])
{
FILE *fp;
char filename[260];
int i,j,k,l=0,m=0,len,count;
//for(i=0;i<0x10000;i++)
// datain[i]=0xFF;
//datain[0x2000]=0x7F;
//DecompressMode0(datain,dataout,0x10000);
sprintf(filename,"%s",argv[2]);
fp=fopen(filename,"rb");
if(fp==NULL)
{
printf("ERROR: Can't open file [%s] for reading.\n",filename);
return 1;
}
fread(datarom,1,0x400000,fp);
fclose(fp);
fp=NULL;
sprintf(filename,"%s.bin",argv[1]);
fp=fopen(filename,"rb");
if(fp==NULL)
{
printf("ERROR: Can't open file [%s] for reading.\n",filename);
return 1;
}
fseek(fp,0,SEEK_END);
len=ftell(fp);
fseek(fp,0,SEEK_SET);
printf("File [%s] opened\n",filename);
printf("%d entries in Table\n",len / 0x8000);
len = 0x8000; //All Raw GFX pack decompressions are 0x8000 bytes in size.
k=0;
i=0;
while(filename[i++]!='/'); //Extract Table Address from bin file name.
filename[i+6]=0;
j=htoi(&filename[i]);
sprintf(filename,"%s.bin",argv[1]);
while(fread(realout,1,0x8000,fp)==0x8000) //For each full readout, decompress the data, and compare.
{
printf("Table %.6X-%.2X-%d read --- ",j,k,datarom[j+(k*4)]&3);
if((datarom[j+(k*4)]&2)==2) //Decompression type 2 not supported, Type 3 is invalid.
{
printf("Skipped\n",datarom[j+(k*4)]);
k++;
continue;
}
l=datarom[j+(k*4)+1]<<16;
l+=datarom[j+(k*4)+2]<<8;
l+=datarom[j+(k*4)+3]; //Get pointer to compressed data.
if((datarom[j+(k*4)]&3)==0)
DecompressMode0(&datarom[l],dataout,len);
else if ((datarom[j+(k*4)]&3)==1)
DecompressMode1(&datarom[l],dataout,len);
//else if ((datarom[j+(k*4)]&3)==2)
// DecompressMode2(&datarom[l],dataout,len);
//compare
for(i=0;i<len;i++)
{
if(dataout[i]!=realout[i])
break;
}
if(i==len)
printf("Success! No mismatches.\n");
else
{
printf("****FAIL**** mismatch on byte %d - ",i);
int j=i;
for(;i<len;i++)
{
if(realout[j]!=realout[i])
break;
}
if(i==len)
printf("Real Decompressor crashed\n");
else
printf("Error in either decompressor or Raw Data\n");
/*
int j;
printf("****FAIL**** mismatch on byte %d\n",i);
j=i-3;
if(j<0)
j=0;
for(;j<=i;j++)
printf("byte %d: decomp[%.2X]->real[%.2X]\n",j,dataout[j],realout[j]);
if(0)
{
printf("\nEvolution table:\n");
for(i=0;i<53;i++)
printf(" %d,%d //%d\n",SeenEvolution[i][0],SeenEvolution[i][1],i);
}*/
}
k++;
}
fclose(fp);
return 0;
}
Now just waiting for Mode 2 to be figured out, so that it can be dropped into the above code, and completely tested.