Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

unpackdat: Update algorithm #8

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

unpackdat: Update algorithm #8

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 151 additions & 42 deletions cli_utils/unpackdat.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
/* (c) fenugrec 2015-2016
* GPLv3
* unpack nissan .dat repro files.
* currently recognizes only one chunk format: "FMT SRC TGT LEN 34 82 AH AM AL CL <CL bytes of data> <CRC1> <CRC2> <CKS>"
* not very clever about possible variants, plenty of hardcoded junk. Also doesn't check CRC and cks.
* unpack nissan .dat, .kwp and other repro files.
* currently recognizes two chunk formats:
* extended "FMT SRC TGT LEN 34 82 AH AM AL CL <CL bytes of data> <CRC1> <CRC2> <CKS>"
* short "FMT_LEN SRC TGT 34 82 AH AM AL CL <CL bytes of data> <CRC1> <CRC2> <CKS>"
* Also doesn't check CRC and cks.
*
*/

#include <stdint.h>
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h> //malloc etc
#include <string.h> //memcpy

Expand All @@ -16,15 +19,60 @@

FILE *dbg_stream;

#define FMT_CHUNK_EXT 0x80
#define CRC_TAIL_LEN 3
#define MAX_DAT_OFFSET 0xffffff
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume MAX_DAT_OFFSET is to cover the maximum "AH:AM:AL" value ?


struct datfile {
FILE *hf;
u32 siz; //size of .dat, in bytes
u32 buf_size; //size of .dat, in bytes
uint8_t *buf; //copied here

u32 dbuf_size; //size of .bin, in bytes
uint8_t *dbuf; //unpacked data
};

typedef struct {
uint8_t sign[3]; // 0xA1 XX XX

char part_number[5];
uint32_t date; //?
char part_number2[5];

char ver_str[6]; // e.g. LBC ZE1/ VCM

uint32_t off17; // 00 00 00 80
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I count properly, this field is not 4-byte aligned in the (packed) struct, correct ?

char zeroes[5];

uint32_t off20; // 82 00 00 0x
uint8_t cks; //?
} dat_header_t;

typedef struct {
uint8_t fmt; // 0x80 if exist extra byte ckslen
// or 0x80 + len (ckslen missed)
uint8_t src;
uint8_t tgt;

uint8_t ckslen; // data for cks from cmd upto cks.
// byte missed if fmt != 0x80
} chunk_header_t;

typedef struct {
uint16_t cmd; // 34 82

uint8_t offset_hi;
uint8_t offset_mid;
uint8_t offset_low;

uint8_t chunklen;
uint8_t payload[]; // chunklen size

// CRC at the end
//uint16_t crc;
//uint8_t cks;
} chunk_t;


//load ROM to a new buffer
//ret 0 if OK
Expand All @@ -45,7 +93,7 @@ static int open_dat(struct datfile *rf, const char *fname) {
if ((!file_len) || (file_len > 3*1024*1024L)) {
printf("empty, or huge file (length %lu)\n", (unsigned long) file_len);
}
rf->siz = file_len;
rf->buf_size = file_len;

buf = malloc(file_len);
if (!buf) {
Expand All @@ -54,7 +102,9 @@ static int open_dat(struct datfile *rf, const char *fname) {
return -1;
}

rf->dbuf = malloc(file_len);
// Allocate max size for output according to offset field dimension
rf->dbuf_size = MAX_DAT_OFFSET;
rf->dbuf = malloc(rf->dbuf_size);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, but maybe the size should be +1 : if the last chunk has len=0x80 starting at 0xff ff80 then the last byte would overflow.

if (!rf->dbuf) {
printf("malloc choke\n");
free(buf);
Expand Down Expand Up @@ -90,47 +140,105 @@ void close_dat(struct datfile *rf) {
return;
}

#define DCHUNK_SIZE 0x8D //chunk size is very happy
#define DCHUNK_DLEN 0x80 //bytes of payload
// "FMT SRC TGT LEN 34 82 AH AM AL CL <CL bytes of data> <CRC1> <CRC2> <CKS>"
void unpack_dat(FILE *outf, uint8_t *dest, const uint8_t *src, u32 siz) {
// 1- search for "88 34 82"
const uint8_t pm[]={0x88, 0x34, 0x82};
void print_hex(const void *src, int len) {
const uint8_t *ptr = src;
for (int i = 0; i < len; i++) {
printf(" %02x", ptr[i]);
}
printf("\n");
}

void unpack_dat(FILE *outf, struct datfile *rf, bool trim) {
Copy link
Owner

@fenugrec fenugrec Apr 20, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, passing in a struct * is good
Can you add a comment for argument "trim"

const uint8_t *src = rf->buf;
const u32 src_size = rf->buf_size;
const uint8_t pm[] = {0x34, 0x82};
const uint8_t *psrc; //misc pointer inside src buf
uint8_t *orig_dest = dest;
uint32_t curaddr = 0;
uint8_t *dest = NULL;

const chunk_header_t *ch_header = NULL;

unsigned long plsiz = 0; //total payload bytes
uint8_t chklen = 0;
uint32_t offset = 0;
uint32_t firstchaddr = 0;
unsigned long outsize = 0;
unsigned pl_numc = 0; //# of chunks

psrc = u8memstr(src, siz, pm, 3);
if (!psrc) return;

//2- point to "AH AM AL..." and start
psrc += 3;
while ( psrc < &src[siz - 1] ) {
// parse AH:AM:AL
uint32_t temp_addr = reconst_32(psrc);
temp_addr >>= 8;
if (temp_addr != (curaddr + DCHUNK_DLEN)) {
printf("addr skip @ %lX\n", (unsigned long) temp_addr);
const dat_header_t *dath = (const dat_header_t *)src;
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't like casting a raw buffer to a struct like this, especially if the struct has unaligned bytes. It also leads to assuming host endianness.
Either at least memcpy to your struct (and mark the struct with __attribute packed etc), or copy the elements one by one with reconst_XX() as required.


printf("SIGN:");
print_hex(dath->sign, 3);
printf("PART1: %.*s\n", 5, dath->part_number);
printf("DATE:");
print_hex(&dath->date, sizeof(dath->date));
printf("PART2: %.*s", 5, dath->part_number2);
print_hex(dath->part_number2, 5);
printf("VER: %.*s", 6, dath->ver_str);
print_hex(dath->ver_str, 6);

// TODO get rid of search. The first chunk is located at a constant offset 0x25
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, we need still to need search : I have at least 9HA7D.dat where the first chunk is at offset 0x73.

psrc = u8memstr(src, src_size, pm, 2);
if (!psrc) {
printf("Pattern 34 82 not found\n");
return;
}

ch_header = (const chunk_header_t *)(psrc - sizeof(chunk_header_t));
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

here too, casting directly to struct.
And because you're backing up by a fixed size this may break with my weird 9HA7D.dat as I mentioned

if (ch_header->fmt != FMT_CHUNK_EXT) {
// fallback to short chunk
ch_header = (const chunk_header_t *)(psrc - offsetof(chunk_header_t, ckslen));
}

printf("SRC: %#x\n", ch_header->src);
printf("TGT: %#x\n", ch_header->tgt);

while (true) {
const chunk_t *ch = NULL;
if (ch_header->fmt == FMT_CHUNK_EXT) {
// 80 SRC TGT LEN 34 82
ch = (chunk_t*)((uint8_t*)ch_header + sizeof(chunk_header_t));
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

casting to struct

} else {
// XX SRC TGT 34 82
ch = (chunk_t*)((uint8_t*)ch_header + offsetof(chunk_header_t, ckslen));
}
// TODO check whether ch buffer is not go beyond the end of file

if (ch->cmd != 0x8234) {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here, you're assuming host endianness. Why not memcmp() with pm[] ?

printf("Unexpected cmd pattern %#x at offset %#llx\n",
ch->cmd, ((uint8_t*)ch - src + offsetof(chunk_t, cmd)));
break;
}

offset = ch->offset_hi << 16 | ch->offset_mid << 8 | ch->offset_low;
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer either using a truncated reconst_32() call like I did, or better, adding a reconst_24() to nislib.c+h

if (pl_numc == 0) {
if (trim) firstchaddr = offset;
printf("Trim gap in .bin before first chunk offset 0x%06x\n", offset);

printf("Chunks:\n");
printf("# : offset @ size\n");
}
curaddr = temp_addr;
psrc += 3; //seek to "CL"
if (*psrc != DCHUNK_DLEN) {
printf("bad chunk len ?\n");
return;

pl_numc++;
chklen = ch->chunklen;
printf("#%04d: 0x%06x @ %d\n", pl_numc, offset, chklen);
dest = rf->dbuf + offset;
memcpy(dest, ch->payload, chklen);

ch_header = (const chunk_header_t *)((uint8_t*)ch + offsetof(chunk_t, payload) + chklen + CRC_TAIL_LEN);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

casting to struct

if (((uint8_t*)ch_header - src) >= src_size) {
// This is the end of file
break;
}
psrc += 1; //seek to payload data
memcpy(dest, psrc, DCHUNK_DLEN);
pl_numc += 1;
dest += DCHUNK_DLEN;
plsiz += DCHUNK_DLEN;
psrc += (DCHUNK_SIZE - 4); //realign to AH
}

outsize = offset + chklen - firstchaddr;
dest = rf->dbuf + firstchaddr;
printf("total: %u chunks; %lu (0x%06lX) bytes in PL.\n",
pl_numc, plsiz, plsiz);
fwrite(orig_dest, 1, plsiz, outf);
pl_numc, outsize, outsize);

if (pl_numc > 0)
fwrite(dest, outsize, 1, outf);
else
printf("No recognized chunks to save in file\n");
return;
}

Expand All @@ -140,9 +248,10 @@ int main(int argc, char *argv[])
bool dbg_file; //flag if dbgstream is a real file
FILE *pl0;
struct datfile rf = {0};
bool trim = true;

if (argc !=3) {
printf("%s <file.dat> <out.dat> : unpack payload to <out.dat> etc\n",argv[0]);
if (argc < 3) {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what was wrong with !=3 ?

printf("%s <file.dat> <out.dat>: unpack payload to <out.dat>\n",argv[0]);
return 0;
}

Expand All @@ -167,7 +276,7 @@ int main(int argc, char *argv[])

/* add header to dbg log */
fprintf(dbg_stream, "\n********************\n**** Started analyzing %s\n", argv[1]);
unpack_dat(pl0, rf.dbuf, rf.buf, rf.siz);
unpack_dat(pl0, &rf, trim);

printf("\n");
close_dat(&rf);
Expand Down