From edf90d250f71bc480f34a3eedd520835e1ece8de Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 10:14:51 -0700 Subject: [PATCH 01/11] task_database: fix memory leak in error path --- tasks/task_database_cue.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index 94e7e5e5a0..81350e1a58 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -415,8 +415,7 @@ int find_first_data_track(const char *cue_path, fd = intfstream_init(&info); if (!fd) { - free(tmp_token); - return -errno; + goto error; } if (!intfstream_open(fd, cue_path, RFILE_MODE_READ, -1)) From 5b20c13f92a9a4867a861569448bef9dbc4f445d Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 14:20:32 -0700 Subject: [PATCH 02/11] chd_stream: support PRIMARY special track --- libretro-common/include/streams/chd_stream.h | 4 ++ libretro-common/streams/chd_stream.c | 76 ++++++++++++++++---- 2 files changed, 65 insertions(+), 15 deletions(-) diff --git a/libretro-common/include/streams/chd_stream.h b/libretro-common/include/streams/chd_stream.h index 3febc8e0ae..9abece73e3 100644 --- a/libretro-common/include/streams/chd_stream.h +++ b/libretro-common/include/streams/chd_stream.h @@ -32,8 +32,12 @@ RETRO_BEGIN_DECLS typedef struct chdstream chdstream_t; +// First data track #define CHDSTREAM_TRACK_FIRST_DATA (-1) +// Last track #define CHDSTREAM_TRACK_LAST (-2) +// Primary (largest) data track, used for CRC identification purposes +#define CHDSTREAM_TRACK_PRIMARY (-3) chdstream_t *chdstream_open(const char *path, int32_t track); diff --git a/libretro-common/streams/chd_stream.c b/libretro-common/streams/chd_stream.c index fb77a07201..0704e1b5a9 100644 --- a/libretro-common/streams/chd_stream.c +++ b/libretro-common/streams/chd_stream.c @@ -82,10 +82,12 @@ chdstream_get_meta(chd_file *chd, int idx, metadata_t *md) { char meta[256]; uint32_t meta_size = 0; - chd_error err = chd_get_metadata( - chd, CDROM_TRACK_METADATA2_TAG, idx, meta, - sizeof(meta), &meta_size, NULL, NULL); + chd_error err; + memset(md, 0, sizeof(*md)); + + err = chd_get_metadata(chd, CDROM_TRACK_METADATA2_TAG, idx, meta, + sizeof(meta), &meta_size, NULL, NULL); if (err == CHDERR_NONE) { sscanf(meta, CDROM_TRACK_METADATA2_FORMAT, @@ -111,40 +113,84 @@ chdstream_get_meta(chd_file *chd, int idx, metadata_t *md) } static bool -chdstream_find_track(chd_file *fd, int32_t track, metadata_t *meta) +chdstream_find_track_number(chd_file *fd, int32_t track, metadata_t *meta) { uint32_t i; - - memset(meta, 0, sizeof(*meta)); + uint32_t frame_offset = 0; for (i = 0; true; ++i) { if (!chdstream_get_meta(fd, i, meta)) { - if (track != CHDSTREAM_TRACK_LAST) - return false; + return false; + } - meta->frame_offset -= meta->frames + meta->extra; + if (track == meta->track) + { + meta->frame_offset = frame_offset; return true; } - if ((track == CHDSTREAM_TRACK_FIRST_DATA && - strcmp(meta->type, "AUDIO")) || - (track > 0 && track == meta->track)) - return true; + frame_offset += meta->frames + meta->extra; + } +} - meta->frame_offset += meta->frames + meta->extra; +static bool +chdstream_find_special_track(chd_file *fd, int32_t track, metadata_t *meta) +{ + int32_t i; + metadata_t iter; + int32_t largest_track = 0; + uint32_t largest_size = 0; + + for (i = 1; true; ++i) + { + if (!chdstream_find_track_number(fd, i, &iter)) { + if (track == CHDSTREAM_TRACK_LAST && i > 1) { + *meta = iter; + return true; + } else if (track == CHDSTREAM_TRACK_PRIMARY && largest_track != 0) { + return chdstream_find_track_number(fd, largest_track, meta); + } + } + + switch (track) { + case CHDSTREAM_TRACK_FIRST_DATA: + if (strcmp(iter.type, "AUDIO")) { + *meta = iter; + return true; + } + break; + case CHDSTREAM_TRACK_PRIMARY: + if (strcmp(iter.type, "AUDIO") && iter.frames > largest_size) { + largest_size = iter.frames; + largest_track = iter.track; + } + break; + default: + break; + } + } +} + +static bool +chdstream_find_track(chd_file *fd, int32_t track, metadata_t *meta) +{ + if (track < 0) { + return chdstream_find_special_track(fd, track, meta); + } else { + return chdstream_find_track_number(fd, track, meta); } } chdstream_t *chdstream_open(const char *path, int32_t track) { - metadata_t meta; uint32_t pregap = 0; const chd_header *hd = NULL; chdstream_t *stream = NULL; chd_file *chd = NULL; chd_error err = chd_open(path, CHD_OPEN_READ, NULL, &chd); + metadata_t meta; if (err != CHDERR_NONE) goto error; From fa6d0fac6731185bbf612fc3829b95f27855f5ae Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 14:26:06 -0700 Subject: [PATCH 03/11] task_database: more scanning rework - Use the primary (largest data) track for CRC identification, as it seems least likely to be shared among multiple discs - CRC the primary track when importing a loose .cue file. - If multiple tracks are in the same file, CRC the correct chunk --- tasks/task_database.c | 280 +++++++++++++++++++++++++++----------- tasks/task_database_cue.c | 135 ++++++++++++------ 2 files changed, 294 insertions(+), 121 deletions(-) diff --git a/tasks/task_database.c b/tasks/task_database.c index ae8697c8ca..9b49c11cec 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -68,8 +68,8 @@ typedef struct db_handle bool scan_started; } db_handle_t; -int find_first_data_track(const char* cue_path, - int32_t* offset, char* track_path, size_t max_len); +int find_track(const char *cue_path, bool first, size_t *offset, size_t *size, + char *track_path, size_t max_len); int detect_system(intfstream_t *fd, const char** system_name); @@ -99,7 +99,34 @@ static intfstream_t* open_file(const char *path) return fd; } -static intfstream_t *open_chd_track(const char *path, int32_t track) +static intfstream_t* +open_memory(void *data, size_t size) +{ + intfstream_info_t info; + intfstream_t *fd = NULL; + + info.type = INTFSTREAM_MEMORY; + info.memory.buf.data = data; + info.memory.buf.size = size; + info.memory.writable = false; + + fd = intfstream_init(&info); + if (!fd) + { + return NULL; + } + + if (!intfstream_open(fd, NULL, RFILE_MODE_READ, -1)) + { + intfstream_close(fd); + return NULL; + } + + return fd; +} + +static intfstream_t* +open_chd_track(const char *path, int32_t track) { intfstream_info_t info; intfstream_t *fd = NULL; @@ -186,75 +213,98 @@ static int task_database_iterate_start(database_info_handle_t *db, return 0; } -static int stream_get_serial(database_state_handle_t *db_state, - database_info_handle_t *db, intfstream_t *fd, char* serial) +static int stream_get_serial(intfstream_t *fd, char *serial) { - const char* system_name = NULL; + const char *system_name = NULL; - /* Check if the system was not auto-detected. */ - if (detect_system(fd, &system_name) < 0) - { - /* Attempt to read an ASCII serial, like Wii. */ - if (detect_serial_ascii_game(fd, serial)) - { - /* ASCII serial (Wii) was detected. */ - RARCH_LOG("%s '%s'\n", - msg_hash_to_str(MSG_FOUND_DISK_LABEL), serial); - return 0; - } - - /* Any other non-system specific detection methods? */ - return 0; - } - - if (string_is_equal_fast(system_name, "psp", 3)) - { - if (detect_psp_game(fd, serial) == 0) - return 0; + /* Check if the system was not auto-detected. */ + if (detect_system(fd, &system_name) < 0) + { + /* Attempt to read an ASCII serial, like Wii. */ + if (detect_serial_ascii_game(fd, serial)) + { + /* ASCII serial (Wii) was detected. */ RARCH_LOG("%s '%s'\n", msg_hash_to_str(MSG_FOUND_DISK_LABEL), serial); - } - else if (string_is_equal_fast(system_name, "ps1", 3)) - { - if (detect_ps1_game(fd, serial) == 0) - return 0; - RARCH_LOG("%s '%s'\n", msg_hash_to_str(MSG_FOUND_DISK_LABEL), serial); - } - else - { return 0; - } + } - return 1; + /* Any other non-system specific detection methods? */ + return 0; + } + + if (string_is_equal_fast(system_name, "psp", 3)) + { + if (detect_psp_game(fd, serial) == 0) + return 0; + RARCH_LOG("%s '%s'\n", msg_hash_to_str(MSG_FOUND_DISK_LABEL), serial); + } + else if (string_is_equal_fast(system_name, "ps1", 3)) + { + if (detect_ps1_game(fd, serial) == 0) + return 0; + RARCH_LOG("%s '%s'\n", msg_hash_to_str(MSG_FOUND_DISK_LABEL), serial); + } + else { + return 0; + } + + return 1; } -static int iso_get_serial(database_state_handle_t *db_state, - database_info_handle_t *db, const char *name, char* serial) +static bool file_get_serial(const char *name, size_t offset, size_t size, char *serial) { intfstream_t *fd = open_file(name); int rv; + uint8_t *data = NULL; + ssize_t file_size = -1; if (!fd) return 0; - rv = stream_get_serial(db_state, db, fd, serial); + intfstream_seek(fd, 0, SEEK_END); + file_size = intfstream_tell(fd); + intfstream_seek(fd, 0, SEEK_SET); + if (file_size < 0) { + intfstream_close(fd); + return 0; + } + + if (offset != 0 || size < (size_t) file_size) + { + data = malloc(size); + intfstream_seek(fd, offset, SEEK_SET); + if (intfstream_read(fd, data, size) != (ssize_t) size) + { + intfstream_close(fd); + free(data); + return 0; + } + intfstream_close(fd); + fd = open_memory(data, size); + if (!fd) { + free(data); + return 0; + } + } + + rv = stream_get_serial(fd, serial); intfstream_close(fd); - free(fd); + free(data); return rv; } -static int cue_get_serial(database_state_handle_t *db_state, - database_info_handle_t *db, const char *name, char* serial) +static int cue_get_serial(const char *name, char* serial) { char *track_path = (char*)malloc(PATH_MAX_LENGTH * sizeof(char)); int ret = 0; - int32_t offset = 0; + size_t offset = 0; + size_t size = 0; int rv = 0; track_path[0] = '\0'; - rv = find_first_data_track(name, - &offset, track_path, PATH_MAX_LENGTH); + rv = find_track(name, true, &offset, &size, track_path, PATH_MAX_LENGTH); if (rv < 0) { @@ -267,14 +317,13 @@ static int cue_get_serial(database_state_handle_t *db_state, RARCH_LOG("%s\n", msg_hash_to_str(MSG_READING_FIRST_DATA_TRACK)); - ret = iso_get_serial(db_state, db, track_path, serial); + ret = file_get_serial(track_path, offset, size, serial); free(track_path); return ret; } -static int chd_get_serial(database_state_handle_t *db_state, - database_info_handle_t *db, const char *name, char* serial) +static int chd_get_serial(const char *name, char* serial) { intfstream_t *fd = NULL; int result; @@ -285,54 +334,125 @@ static int chd_get_serial(database_state_handle_t *db_state, return 0; } - result = stream_get_serial(db_state, db, fd, serial); + result = stream_get_serial(fd, serial); intfstream_close(fd); return result; } -static bool file_get_crc(database_state_handle_t *db_state, - const char *name, uint32_t *crc) +static int stream_get_crc(intfstream_t *fd, uint32_t *crc) { - ssize_t ret; - int read_from = filestream_read_file( - name, (void**)&db_state->buf, &ret); + size_t read = 0; + uint32_t acc = 0; + uint8_t buffer[4096]; - if (read_from != 1 || ret <= 0) + while ((read = intfstream_read(fd, buffer, sizeof(buffer))) > 0) + { + acc = encoding_crc32(acc, buffer, read); + } + + if (read < 0) + { return 0; + } - *crc = encoding_crc32(0, db_state->buf, ret); + *crc = acc; return 1; } -static bool chd_get_crc(database_state_handle_t *db_state, - const char *name, uint32_t *crc) +static bool file_get_crc(const char *name, size_t offset, size_t size, uint32_t *crc) { - intfstream_t *fd = NULL; - uint32_t acc = 0; - uint8_t buffer[4096]; - ssize_t size; + intfstream_t *fd = open_file(name); + int rv; + uint8_t *data = NULL; + ssize_t file_size = -1; - fd = open_chd_track(name, CHDSTREAM_TRACK_FIRST_DATA); if (!fd) { return 0; } - while ((size = intfstream_read(fd, buffer, sizeof(buffer))) > 0) - { - acc = encoding_crc32(acc, buffer, size); + intfstream_seek(fd, 0, SEEK_END); + file_size = intfstream_tell(fd); + intfstream_seek(fd, 0, SEEK_SET); + if (file_size < 0) { + intfstream_close(fd); + return 0; } - if (size < 0) + + if (offset != 0 || size < (size_t) file_size) { + data = malloc(size); + intfstream_seek(fd, offset, SEEK_SET); + if (intfstream_read(fd, data, size) != (ssize_t) size) { + intfstream_close(fd); + free(data); + return 0; + } + intfstream_close(fd); + fd = open_memory(data, size); + if (!fd) { + free(data); + return 0; + } + } + + rv = stream_get_crc(fd, crc); + intfstream_close(fd); + free(data); + return rv; +} + +static int cue_get_crc(const char *name, uint32_t *crc) +{ + char *track_path = (char *)malloc(PATH_MAX_LENGTH); + int ret = 0; + size_t offset = 0; + size_t size = 0; + int rv = 0; + + track_path[0] = '\0'; + + rv = find_track(name, false, &offset, &size, track_path, PATH_MAX_LENGTH); + + if (rv < 0) { + RARCH_LOG("%s: %s\n", msg_hash_to_str(MSG_COULD_NOT_FIND_VALID_DATA_TRACK), + strerror(-rv)); + free(track_path); + return 0; + } + + RARCH_LOG("CUE '%s' primary track: %s\n (%Zu, %Zu)", name, track_path, offset, size); + + RARCH_LOG("%s\n", msg_hash_to_str(MSG_READING_FIRST_DATA_TRACK)); + + rv = file_get_crc(track_path, offset, size, crc); + if (rv == 1) { + RARCH_LOG("CUE '%s' crc: %x\n", name, *crc); + } + free(track_path); + return rv; +} + +static bool chd_get_crc(const char *name, uint32_t *crc) +{ + intfstream_t *fd = NULL; + int rv; + uint32_t acc = 0; + uint8_t buffer[4096]; + ssize_t size; + + fd = open_chd_track(name, CHDSTREAM_TRACK_PRIMARY); + if (!fd) { return 0; } - RARCH_LOG("CHD '%s' crc: %x\n", name, acc); - - *crc = acc; - - return 1; + rv = stream_get_crc(fd, crc); + if (rv == 1) { + RARCH_LOG("CHD '%s' crc: %x\n", name, *crc); + } + intfstream_close(fd); + return rv; } static int task_database_iterate_playlist( @@ -345,37 +465,37 @@ static int task_database_iterate_playlist( #ifdef HAVE_COMPRESSION database_info_set_type(db, DATABASE_TYPE_CRC_LOOKUP); /* first check crc of archive itself */ - return file_get_crc(db_state, name, &db_state->archive_crc); + return file_get_crc(name, 0, SIZE_MAX, &db_state->archive_crc); #else break; #endif case FILE_TYPE_CUE: db_state->serial[0] = '\0'; - if (cue_get_serial(db_state, db, name, db_state->serial)) + if (cue_get_serial(name, db_state->serial)) { database_info_set_type(db, DATABASE_TYPE_SERIAL_LOOKUP); } else { database_info_set_type(db, DATABASE_TYPE_CRC_LOOKUP); - return file_get_crc(db_state, name, &db_state->crc); + return cue_get_crc(name, &db_state->crc); } break; case FILE_TYPE_ISO: db_state->serial[0] = '\0'; - iso_get_serial(db_state, db, name, db_state->serial); + file_get_serial(name, 0, SIZE_MAX, db_state->serial); database_info_set_type(db, DATABASE_TYPE_SERIAL_LOOKUP); break; case FILE_TYPE_CHD: db_state->serial[0] = '\0'; - if (chd_get_serial(db_state, db, name, db_state->serial)) + if (chd_get_serial(name, db_state->serial)) { database_info_set_type(db, DATABASE_TYPE_SERIAL_LOOKUP); } else { database_info_set_type(db, DATABASE_TYPE_CRC_LOOKUP); - return chd_get_crc(db_state, name, &db_state->crc); + return chd_get_crc(name, &db_state->crc); } break; case FILE_TYPE_LUTRO: @@ -383,7 +503,7 @@ static int task_database_iterate_playlist( break; default: database_info_set_type(db, DATABASE_TYPE_CRC_LOOKUP); - return file_get_crc(db_state, name, &db_state->crc); + return file_get_crc(name, 0, SIZE_MAX, &db_state->crc); } return 1; diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index 81350e1a58..fed2ff865e 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -124,29 +124,6 @@ static ssize_t get_token(intfstream_t *fd, char *token, size_t max_len) } } -static int find_token(intfstream_t *fd, const char *token) -{ - int tmp_len = (int)strlen(token); - char *tmp_token = (char*)calloc(tmp_len+1, 1); - - if (!tmp_token) - return -1; - - while (strncmp(tmp_token, token, tmp_len) != 0) - { - if (get_token(fd, tmp_token, tmp_len) <= 0) - { - free(tmp_token); - return -1; - } - } - - free(tmp_token); - - return 0; -} - - static int detect_ps1_game_sub(intfstream_t *fp, char *game_id, int sub_channel_mixed) { @@ -402,13 +379,51 @@ clean: return rv; } -int find_first_data_track(const char *cue_path, - int32_t *offset, char *track_path, size_t max_len) +static ssize_t get_file_size(const char *path) +{ + ssize_t rv; + RFILE *fd = filestream_open(path, RFILE_MODE_READ, -1); + if (fd == NULL) { + return -1; + } + rv = filestream_get_size(fd); + filestream_close(fd); + return rv; +} + +static bool update_cand(ssize_t *cand_index, ssize_t *last_index, + size_t *largest, char *last_file, size_t *offset, + size_t *size, char *track_path, size_t max_len) +{ + if (*cand_index != -1) { + if (*last_index - *cand_index > *largest) { + *largest = *last_index - *cand_index; + strlcpy(track_path, last_file, max_len); + *offset = *cand_index; + *size = *largest; + *cand_index = -1; + return true; + } + *cand_index = -1; + } + return false; +} + +int find_track(const char *cue_path, bool first, + size_t *offset, size_t *size, char *track_path, size_t max_len) { int rv; + char *tmp_token = malloc(MAX_TOKEN_LEN); + char *last_file = malloc(PATH_MAX_LENGTH + 1); intfstream_info_t info; intfstream_t *fd = NULL; - char * tmp_token = malloc(MAX_TOKEN_LEN * sizeof(char)); + ssize_t last_index = -1; + ssize_t cand_index = -1; + int32_t cand_track = -1; + int32_t track = 0; + size_t largest = 0; + ssize_t volatile file_size = -1; + bool is_data = false; info.type = INTFSTREAM_FILE; @@ -429,56 +444,94 @@ int find_first_data_track(const char *cue_path, tmp_token[0] = '\0'; + rv = -EINVAL; + while (get_token(fd, tmp_token, MAX_TOKEN_LEN) > 0) { if (string_is_equal(tmp_token, "FILE")) { - char *cue_dir = (char*)malloc(PATH_MAX_LENGTH * sizeof(char)); - + char *cue_dir = (char*)malloc(PATH_MAX_LENGTH); cue_dir[0] = '\0'; - fill_pathname_basedir(cue_dir, cue_path, PATH_MAX_LENGTH * sizeof(char)); + /* Set last index to last EOF */ + if (file_size != -1) { + last_index = file_size; + } + + /* We're changing files since the candidate, update it */ + if (update_cand(&cand_index, &last_index, &largest, last_file, offset, + size, track_path, max_len)) { + rv = 0; + if (first) { + goto clean; + } + } + + fill_pathname_basedir(cue_dir, cue_path, PATH_MAX_LENGTH); get_token(fd, tmp_token, MAX_TOKEN_LEN); - fill_pathname_join(track_path, cue_dir, tmp_token, max_len); + fill_pathname_join(last_file, cue_dir, tmp_token, PATH_MAX_LENGTH); + + file_size = get_file_size(last_file); free(cue_dir); + get_token(fd, tmp_token, MAX_TOKEN_LEN); + } else if (string_is_equal(tmp_token, "TRACK")) { + get_token(fd, tmp_token, MAX_TOKEN_LEN); + get_token(fd, tmp_token, MAX_TOKEN_LEN); + is_data = !string_is_equal(tmp_token, "AUDIO"); + ++track; + } else if (string_is_equal(tmp_token, "INDEX")) { int m, s, f; get_token(fd, tmp_token, MAX_TOKEN_LEN); get_token(fd, tmp_token, MAX_TOKEN_LEN); - if (string_is_equal(tmp_token, "AUDIO")) - continue; - - find_token(fd, "INDEX"); - get_token(fd, tmp_token, MAX_TOKEN_LEN); - get_token(fd, tmp_token, MAX_TOKEN_LEN); - if (sscanf(tmp_token, "%02d:%02d:%02d", &m, &s, &f) < 3) { RARCH_LOG("Error parsing time stamp '%s'\n", tmp_token); goto error; } - *offset = ((m * 60) * (s * 75) * f) * 25; + last_index = (size_t) (((m * 60 + s) * 75) + f) * 2352; - rv = 0; - goto clean; + /* If we've changed tracks since the candidate, update it */ + if (cand_track != -1 && track != cand_track && + update_cand(&cand_index, &last_index, &largest, last_file, offset, + size, track_path, max_len)) { + rv = 0; + if (first) { + goto clean; + } + } + + if (!is_data) { + continue; + } + + if (cand_index == -1) { + cand_index = last_index; + cand_track = track; + } } } - rv = -EINVAL; + if (update_cand(&cand_index, &last_index, &largest, last_file, offset, + size, track_path, max_len)) { + rv = 0; + } clean: free(tmp_token); + free(last_file); intfstream_close(fd); return rv; error: free(tmp_token); + free(last_file); if (fd) intfstream_close(fd); return -errno; From 7061cb14fb0af8dc049721400be175a22cc733d6 Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 14:22:37 -0700 Subject: [PATCH 04/11] chd_stream: support CHDs from GDI files --- libretro-common/streams/chd_stream.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libretro-common/streams/chd_stream.c b/libretro-common/streams/chd_stream.c index 0704e1b5a9..adce6d71d4 100644 --- a/libretro-common/streams/chd_stream.c +++ b/libretro-common/streams/chd_stream.c @@ -103,8 +103,19 @@ chdstream_get_meta(chd_file *chd, int idx, metadata_t *md) sizeof(meta), &meta_size, NULL, NULL); if (err == CHDERR_NONE) { - sscanf(meta, CDROM_TRACK_METADATA_FORMAT, - &md->track, md->type, md->subtype, &md->frames); + sscanf(meta, CDROM_TRACK_METADATA_FORMAT, &md->track, md->type, + md->subtype, &md->frames); + md->extra = padding_frames(md->frames); + return true; + } + + err = chd_get_metadata(chd, GDROM_TRACK_METADATA_TAG, idx, meta, + sizeof(meta), &meta_size, NULL, NULL); + if (err == CHDERR_NONE) + { + sscanf(meta, GDROM_TRACK_METADATA_FORMAT, &md->track, md->type, + md->subtype, &md->frames, &md->pad, &md->pregap, md->pgtype, + md->pgsub, &md->postgap); md->extra = padding_frames(md->frames); return true; } From 70ff0aa0c193198bfb217224d945b2a1c2a18734 Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 17:40:04 -0700 Subject: [PATCH 05/11] msg_hash: support GDI files --- msg_hash.c | 5 +++++ msg_hash.h | 1 + 2 files changed, 6 insertions(+) diff --git a/msg_hash.c b/msg_hash.c index 5c4609d892..1ce3f6bc33 100644 --- a/msg_hash.c +++ b/msg_hash.c @@ -231,6 +231,8 @@ uint32_t msg_hash_calculate(const char *s) #define HASH_EXTENSION_ZIP_UPP 0x0b883b78U #define HASH_EXTENSION_CUE 0x0b886782U #define HASH_EXTENSION_CUE_UPPERCASE 0x0b87db22U +#define HASH_EXTENSION_GDI 0x00b887659 +#define HASH_EXTENSION_GDI_UPPERCASE 0x00b87e9f9 #define HASH_EXTENSION_ISO 0x0b8880d0U #define HASH_EXTENSION_ISO_UPPERCASE 0x0b87f470U #define HASH_EXTENSION_LUTRO 0x0fe37b7bU @@ -376,6 +378,9 @@ enum msg_file_type msg_hash_to_file_type(uint32_t hash) case HASH_EXTENSION_CUE: case HASH_EXTENSION_CUE_UPPERCASE: return FILE_TYPE_CUE; + case HASH_EXTENSION_GDI: + case HASH_EXTENSION_GDI_UPPERCASE: + return FILE_TYPE_GDI; case HASH_EXTENSION_ISO: case HASH_EXTENSION_ISO_UPPERCASE: return FILE_TYPE_ISO; diff --git a/msg_hash.h b/msg_hash.h index ae39bff8b5..dbf741f3de 100644 --- a/msg_hash.h +++ b/msg_hash.h @@ -132,6 +132,7 @@ enum msg_file_type FILE_TYPE_XM, FILE_TYPE_CUE, + FILE_TYPE_GDI, FILE_TYPE_ISO, FILE_TYPE_LUTRO, FILE_TYPE_CHD, From 6c1044a6994faf450618f1a0ff9df33eaafb589f Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 17:39:41 -0700 Subject: [PATCH 06/11] task_database: support GDI files --- tasks/task_database.c | 81 +++++++++++++++++++++++-- tasks/task_database_cue.c | 121 +++++++++++++++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 5 deletions(-) diff --git a/tasks/task_database.c b/tasks/task_database.c index 9b49c11cec..6c831dbbff 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -68,9 +68,11 @@ typedef struct db_handle bool scan_started; } db_handle_t; -int find_track(const char *cue_path, bool first, size_t *offset, size_t *size, +int cue_find_track(const char *cue_path, bool first, size_t *offset, size_t *size, char *track_path, size_t max_len); +int gdi_find_track(const char *gdi_path, bool first, char *track_path, size_t max_len); + int detect_system(intfstream_t *fd, const char** system_name); int detect_ps1_game(intfstream_t *fd, char *game_id); @@ -304,7 +306,7 @@ static int cue_get_serial(const char *name, char* serial) track_path[0] = '\0'; - rv = find_track(name, true, &offset, &size, track_path, PATH_MAX_LENGTH); + rv = cue_find_track(name, true, &offset, &size, track_path, PATH_MAX_LENGTH); if (rv < 0) { @@ -323,6 +325,34 @@ static int cue_get_serial(const char *name, char* serial) return ret; } +static int gdi_get_serial(const char *name, char* serial) +{ + char *track_path = (char*)malloc(PATH_MAX_LENGTH + * sizeof(char)); + int ret = 0; + int rv = 0; + + track_path[0] = '\0'; + + rv = gdi_find_track(name, true, track_path, PATH_MAX_LENGTH); + + if (rv < 0) + { + RARCH_LOG("%s: %s\n", + msg_hash_to_str(MSG_COULD_NOT_FIND_VALID_DATA_TRACK), + strerror(-rv)); + free(track_path); + return 0; + } + + RARCH_LOG("%s\n", msg_hash_to_str(MSG_READING_FIRST_DATA_TRACK)); + + ret = file_get_serial(track_path, 0, SIZE_MAX, serial); + free(track_path); + + return ret; +} + static int chd_get_serial(const char *name, char* serial) { intfstream_t *fd = NULL; @@ -412,7 +442,7 @@ static int cue_get_crc(const char *name, uint32_t *crc) track_path[0] = '\0'; - rv = find_track(name, false, &offset, &size, track_path, PATH_MAX_LENGTH); + rv = cue_find_track(name, false, &offset, &size, track_path, PATH_MAX_LENGTH); if (rv < 0) { RARCH_LOG("%s: %s\n", msg_hash_to_str(MSG_COULD_NOT_FIND_VALID_DATA_TRACK), @@ -433,6 +463,35 @@ static int cue_get_crc(const char *name, uint32_t *crc) return rv; } +static int gdi_get_crc(const char *name, uint32_t *crc) +{ + char *track_path = (char *)malloc(PATH_MAX_LENGTH); + int ret = 0; + int rv = 0; + + track_path[0] = '\0'; + + rv = gdi_find_track(name, false, track_path, PATH_MAX_LENGTH); + + if (rv < 0) { + RARCH_LOG("%s: %s\n", msg_hash_to_str(MSG_COULD_NOT_FIND_VALID_DATA_TRACK), + strerror(-rv)); + free(track_path); + return 0; + } + + RARCH_LOG("GDI '%s' primary track: %s\n", name, track_path); + + RARCH_LOG("%s\n", msg_hash_to_str(MSG_READING_FIRST_DATA_TRACK)); + + rv = file_get_crc(track_path, 0, SIZE_MAX, crc); + if (rv == 1) { + RARCH_LOG("GDI '%s' crc: %x\n", name, *crc); + } + free(track_path); + return rv; +} + static bool chd_get_crc(const char *name, uint32_t *crc) { intfstream_t *fd = NULL; @@ -481,6 +540,18 @@ static int task_database_iterate_playlist( return cue_get_crc(name, &db_state->crc); } break; + case FILE_TYPE_GDI: + db_state->serial[0] = '\0'; + if (gdi_get_serial(name, db_state->serial)) + { + database_info_set_type(db, DATABASE_TYPE_SERIAL_LOOKUP); + } + else + { + database_info_set_type(db, DATABASE_TYPE_CRC_LOOKUP); + return gdi_get_crc(name, &db_state->crc); + } + break; case FILE_TYPE_ISO: db_state->serial[0] = '\0'; file_get_serial(name, 0, SIZE_MAX, db_state->serial); @@ -488,7 +559,9 @@ static int task_database_iterate_playlist( break; case FILE_TYPE_CHD: db_state->serial[0] = '\0'; - if (chd_get_serial(name, db_state->serial)) + /* There are no serial databases, so don't bother with + serials at the moment */ + if (0 && chd_get_serial(name, db_state->serial)) { database_info_set_type(db, DATABASE_TYPE_SERIAL_LOOKUP); } diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index fed2ff865e..3481a3c12d 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -409,7 +409,7 @@ static bool update_cand(ssize_t *cand_index, ssize_t *last_index, return false; } -int find_track(const char *cue_path, bool first, +int cue_find_track(const char *cue_path, bool first, size_t *offset, size_t *size, char *track_path, size_t max_len) { int rv; @@ -536,3 +536,122 @@ error: intfstream_close(fd); return -errno; } + +int gdi_find_track(const char *gdi_path, bool first, char *track_path, size_t max_len) +{ + int rv; + char *tmp_token = malloc(MAX_TOKEN_LEN); + char *last_file = malloc(PATH_MAX_LENGTH + 1); + intfstream_info_t info; + intfstream_t *fd = NULL; + int32_t track = 0; + size_t largest = 0; + int size = -1; + int mode = -1; + ssize_t file_size = -1; + char *gdi_dir = (char*)malloc(PATH_MAX_LENGTH); + gdi_dir[0] = '\0'; + + fill_pathname_basedir(gdi_dir, gdi_path, PATH_MAX_LENGTH); + + info.type = INTFSTREAM_FILE; + + fd = intfstream_init(&info); + if (!fd) + { + goto error; + } + + if (!intfstream_open(fd, gdi_path, RFILE_MODE_READ, -1)) + { + RARCH_LOG("Could not open GDI file '%s': %s\n", gdi_path, + strerror(errno)); + goto error; + } + + RARCH_LOG("Parsing GDI file '%s'...\n", gdi_path); + + tmp_token[0] = '\0'; + + rv = -EINVAL; + + /* Skip track count */ + get_token(fd, tmp_token, MAX_TOKEN_LEN); + + /* Track number */ + while (get_token(fd, tmp_token, MAX_TOKEN_LEN) > 0) + { + /* Offset */ + if (get_token(fd, tmp_token, MAX_TOKEN_LEN) <= 0) + { + errno = EINVAL; + goto error; + } + + /* Mode */ + if (get_token(fd, tmp_token, MAX_TOKEN_LEN) <= 0) + { + errno = EINVAL; + goto error; + } + mode = atoi(tmp_token); + + /* Sector size */ + if (get_token(fd, tmp_token, MAX_TOKEN_LEN) <= 0) + { + errno = EINVAL; + goto error; + } + size = atoi(tmp_token); + + /* File name */ + if (get_token(fd, tmp_token, MAX_TOKEN_LEN) <= 0) + { + errno = EINVAL; + goto error; + } + + /* Check for data track */ + if (!(mode == 0 && size == 2352)) + { + fill_pathname_join(last_file, gdi_dir, tmp_token, PATH_MAX_LENGTH); + file_size = get_file_size(last_file); + if (file_size < 0) + { + goto error; + } + if (file_size > largest) + { + strlcpy(track_path, last_file, max_len); + rv = 0; + largest = file_size; + if (first) + { + goto clean; + } + } + } + + /* Disc offset (not used?) */ + if (get_token(fd, tmp_token, MAX_TOKEN_LEN) <= 0) + { + errno = EINVAL; + goto error; + } + } + +clean: + free(gdi_dir); + free(tmp_token); + free(last_file); + intfstream_close(fd); + return rv; + +error: + free(gdi_dir); + free(tmp_token); + free(last_file); + if (fd) + intfstream_close(fd); + return -errno; +} From c6c090052e0bb987fb4fe14ff3716cffd2a653cf Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 17:46:02 -0700 Subject: [PATCH 07/11] task_database: fix memory leak --- tasks/task_database_cue.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index 3481a3c12d..ffbae57940 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -424,6 +424,10 @@ int cue_find_track(const char *cue_path, bool first, size_t largest = 0; ssize_t volatile file_size = -1; bool is_data = false; + char *cue_dir = (char*)malloc(PATH_MAX_LENGTH); + cue_dir[0] = '\0'; + + fill_pathname_basedir(cue_dir, cue_path, PATH_MAX_LENGTH); info.type = INTFSTREAM_FILE; @@ -450,9 +454,6 @@ int cue_find_track(const char *cue_path, bool first, { if (string_is_equal(tmp_token, "FILE")) { - char *cue_dir = (char*)malloc(PATH_MAX_LENGTH); - cue_dir[0] = '\0'; - /* Set last index to last EOF */ if (file_size != -1) { last_index = file_size; @@ -467,14 +468,11 @@ int cue_find_track(const char *cue_path, bool first, } } - fill_pathname_basedir(cue_dir, cue_path, PATH_MAX_LENGTH); - get_token(fd, tmp_token, MAX_TOKEN_LEN); fill_pathname_join(last_file, cue_dir, tmp_token, PATH_MAX_LENGTH); file_size = get_file_size(last_file); - free(cue_dir); get_token(fd, tmp_token, MAX_TOKEN_LEN); } @@ -524,12 +522,14 @@ int cue_find_track(const char *cue_path, bool first, } clean: + free(cue_dir); free(tmp_token); free(last_file); intfstream_close(fd); return rv; error: + free(cue_dir); free(tmp_token); free(last_file); if (fd) From 46e78e9d1ba8d292bb43897712fb981bb3b0809b Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 17:52:11 -0700 Subject: [PATCH 08/11] task_database: fix parsing CUEs with one track --- tasks/task_database_cue.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index ffbae57940..027066996b 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -516,6 +516,10 @@ int cue_find_track(const char *cue_path, bool first, } } + if (file_size != -1) { + last_index = file_size; + } + if (update_cand(&cand_index, &last_index, &largest, last_file, offset, size, track_path, max_len)) { rv = 0; From 93d05665c2ccd64523aa0b24852f3f46a061684a Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 19:17:10 -0700 Subject: [PATCH 09/11] database_info: prioritize scanning cue/gdi files Scan them before the tracks so we can avoid scanning large files we don't need to. --- database_info.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/database_info.c b/database_info.c index b4ed7a12ac..522369ed5b 100644 --- a/database_info.c +++ b/database_info.c @@ -398,6 +398,31 @@ static int database_cursor_close(libretrodb_t *db, libretrodb_cursor_t *cur) return 0; } +static bool type_is_prioritized(enum msg_file_type type) +{ + return (type == FILE_TYPE_CUE || type == FILE_TYPE_GDI); +} + +static enum msg_file_type file_type(const char *path) +{ + return msg_hash_to_file_type(msg_hash_calculate(path_get_extension(path))); +} + +static int dir_entry_compare(const void *left, const void *right) +{ + const struct string_list_elem *le = left; + const struct string_list_elem *re = right; + bool l = type_is_prioritized(file_type(le->data)); + bool r = type_is_prioritized(file_type(re->data)); + + return (int) r - (int) l; +} + +static void dir_list_prioritize(struct string_list *list) +{ + qsort(list->elems, list->size, sizeof(*list->elems), dir_entry_compare); +} + database_info_handle_t *database_info_dir_init(const char *dir, enum database_type type, retro_task_t *task) { @@ -413,6 +438,8 @@ database_info_handle_t *database_info_dir_init(const char *dir, if (!db->list) goto error; + dir_list_prioritize(db->list); + db->list_ptr = 0; db->status = DATABASE_STATUS_ITERATE; db->type = type; From 84edc8ffb022dba39870f64df03e27a15e714a57 Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 19:18:50 -0700 Subject: [PATCH 10/11] task_database: prune files referenced from cue/gdi files Since we use cue/gdi files to find the right track file to scan, don't bother scanning them separately. --- tasks/task_database.c | 64 ++++++++++++++++++++++++++++++++++++ tasks/task_database_cue.c | 69 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/tasks/task_database.c b/tasks/task_database.c index 6c831dbbff..b1de616503 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -71,8 +71,12 @@ typedef struct db_handle int cue_find_track(const char *cue_path, bool first, size_t *offset, size_t *size, char *track_path, size_t max_len); +bool cue_next_file(intfstream_t *fd, const char *cue_path, char *path, size_t max_len); + int gdi_find_track(const char *gdi_path, bool first, char *track_path, size_t max_len); +bool gdi_next_file(intfstream_t *fd, const char *gdi_path, char *path, size_t max_len); + int detect_system(intfstream_t *fd, const char** system_name); int detect_ps1_game(intfstream_t *fd, char *game_id); @@ -180,6 +184,12 @@ static const char *database_info_get_current_element_name( { if (!handle || !handle->list) return NULL; + /* Skip pruned entries */ + while (handle->list->elems[handle->list_ptr].data == NULL) + { + if (++handle->list_ptr >= handle->list->size) + return NULL; + } return handle->list->elems[handle->list_ptr].data; } @@ -514,6 +524,58 @@ static bool chd_get_crc(const char *name, uint32_t *crc) return rv; } +static void cue_prune(database_info_handle_t *db, const char *name) +{ + char *path = (char *)malloc(PATH_MAX_LENGTH + 1); + intfstream_t *fd = open_file(name); + size_t i; + + if (!fd) + { + free(path); + return; + } + + while (cue_next_file(fd, name, path, PATH_MAX_LENGTH)) + { + for (i = db->list_ptr; i < db->list->size; ++i) + { + if (db->list->elems[i].data && !strcmp(path, db->list->elems[i].data)) + { + RARCH_LOG("Pruning file referenced by cue: %s\n", path); + free(db->list->elems[i].data); + db->list->elems[i].data = NULL; + } + } + } +} + +static void gdi_prune(database_info_handle_t *db, const char *name) +{ + char *path = (char *)malloc(PATH_MAX_LENGTH + 1); + intfstream_t *fd = open_file(name); + size_t i; + + if (!fd) + { + free(path); + return; + } + + while (gdi_next_file(fd, name, path, PATH_MAX_LENGTH)) + { + for (i = db->list_ptr; i < db->list->size; ++i) + { + if (db->list->elems[i].data && !strcmp(path, db->list->elems[i].data)) + { + RARCH_LOG("Pruning file referenced by gdi: %s\n", path); + free(db->list->elems[i].data); + db->list->elems[i].data = NULL; + } + } + } +} + static int task_database_iterate_playlist( database_state_handle_t *db_state, database_info_handle_t *db, const char *name) @@ -529,6 +591,7 @@ static int task_database_iterate_playlist( break; #endif case FILE_TYPE_CUE: + cue_prune(db, name); db_state->serial[0] = '\0'; if (cue_get_serial(name, db_state->serial)) { @@ -541,6 +604,7 @@ static int task_database_iterate_playlist( } break; case FILE_TYPE_GDI: + gdi_prune(db, name); db_state->serial[0] = '\0'; if (gdi_get_serial(name, db_state->serial)) { diff --git a/tasks/task_database_cue.c b/tasks/task_database_cue.c index 027066996b..8a9c7fc8b7 100644 --- a/tasks/task_database_cue.c +++ b/tasks/task_database_cue.c @@ -541,6 +541,34 @@ error: return -errno; } +bool cue_next_file(intfstream_t *fd, const char *cue_path, char *path, size_t max_len) +{ + bool rv = false; + char *tmp_token = malloc(MAX_TOKEN_LEN); + ssize_t volatile file_size = -1; + char *cue_dir = (char*)malloc(PATH_MAX_LENGTH); + cue_dir[0] = '\0'; + + fill_pathname_basedir(cue_dir, cue_path, PATH_MAX_LENGTH); + + tmp_token[0] = '\0'; + + while (get_token(fd, tmp_token, MAX_TOKEN_LEN) > 0) + { + if (string_is_equal(tmp_token, "FILE")) + { + get_token(fd, tmp_token, MAX_TOKEN_LEN); + fill_pathname_join(path, cue_dir, tmp_token, max_len); + rv = true; + break; + } + } + + free(cue_dir); + free(tmp_token); + return rv; +} + int gdi_find_track(const char *gdi_path, bool first, char *track_path, size_t max_len) { int rv; @@ -659,3 +687,44 @@ error: intfstream_close(fd); return -errno; } + +bool gdi_next_file(intfstream_t *fd, const char *gdi_path, char *path, size_t max_len) +{ + bool rv = false; + char *tmp_token = malloc(MAX_TOKEN_LEN); + ssize_t offset = -1; + char *gdi_dir = (char*)malloc(PATH_MAX_LENGTH); + + gdi_dir[0] = '\0'; + tmp_token[0] = '\0'; + + fill_pathname_basedir(gdi_dir, gdi_path, PATH_MAX_LENGTH); + + /* Skip initial track count */ + offset = intfstream_tell(fd); + if (offset == 0) + { + get_token(fd, tmp_token, MAX_TOKEN_LEN); + } + + /* Track number */ + get_token(fd, tmp_token, MAX_TOKEN_LEN); + /* Offset */ + get_token(fd, tmp_token, MAX_TOKEN_LEN); + /* Mode */ + get_token(fd, tmp_token, MAX_TOKEN_LEN); + /* Sector size */ + get_token(fd, tmp_token, MAX_TOKEN_LEN); + /* File name */ + if (get_token(fd, tmp_token, MAX_TOKEN_LEN) > 0) + { + fill_pathname_join(path, gdi_dir, tmp_token, max_len); + rv = true; + /* Disc offset */ + get_token(fd, tmp_token, MAX_TOKEN_LEN); + } + + free(gdi_dir); + free(tmp_token); + return rv; +} From d4beeeda33768ed2f978e195b351bd5c53417a5a Mon Sep 17 00:00:00 2001 From: Brian Koropoff Date: Sun, 17 Sep 2017 21:04:01 -0700 Subject: [PATCH 11/11] task_database: skip serials for the right file type! --- tasks/task_database.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/task_database.c b/tasks/task_database.c index b1de616503..605d4c757f 100644 --- a/tasks/task_database.c +++ b/tasks/task_database.c @@ -606,7 +606,9 @@ static int task_database_iterate_playlist( case FILE_TYPE_GDI: gdi_prune(db, name); db_state->serial[0] = '\0'; - if (gdi_get_serial(name, db_state->serial)) + /* There are no serial databases, so don't bother with + serials at the moment */ + if (0 && gdi_get_serial(name, db_state->serial)) { database_info_set_type(db, DATABASE_TYPE_SERIAL_LOOKUP); } @@ -623,9 +625,7 @@ static int task_database_iterate_playlist( break; case FILE_TYPE_CHD: db_state->serial[0] = '\0'; - /* There are no serial databases, so don't bother with - serials at the moment */ - if (0 && chd_get_serial(name, db_state->serial)) + if (chd_get_serial(name, db_state->serial)) { database_info_set_type(db, DATABASE_TYPE_SERIAL_LOOKUP); }