diff --git a/src/libduc/buffer.c b/src/libduc/buffer.c index dcce82d..d9c6046 100644 --- a/src/libduc/buffer.c +++ b/src/libduc/buffer.c @@ -47,8 +47,8 @@ void buffer_free(struct buffer *b) // Add item to buffer, but grow by doubling if needed static int buffer_put(struct buffer *b, const void *data, size_t len) { - if(b->ptr + len <= b->len) { - while(b->len + len > b->max) { + if(b->ptr + len > b->max) { + while(b->ptr + len > b->max) { b->max *= 2; } b->data = duc_realloc(b->data, b->max); @@ -271,7 +271,8 @@ void buffer_get_index_report(struct buffer *b, struct duc_index_report *report) buffer_get_varint(b, &length); buffer_get_string(b, &vs); report->topn_array[i] = duc_malloc0(sizeof(duc_topn_file)); - strncpy(report->topn_array[i]->name, vs, strlen(vs)); + strncpy(report->topn_array[i]->name, vs, DUC_PATH_MAX - 1); + report->topn_array[i]->name[DUC_PATH_MAX - 1] = '\0'; buffer_get_varint(b, &vi); report->topn_array[i]->size = vi; } } diff --git a/src/libduc/canonicalize.c b/src/libduc/canonicalize.c index e598394..b81fa45 100644 --- a/src/libduc/canonicalize.c +++ b/src/libduc/canonicalize.c @@ -232,7 +232,7 @@ char *duc_canonicalize_path(const char *in) if(n == 0) utstring_printf(&out, "/"); - free(s.cs); + duc_free(s.cs); return utstring_body(&out); } diff --git a/src/libduc/db-tkrzw.c b/src/libduc/db-tkrzw.c index 90840d3..1514ee3 100644 --- a/src/libduc/db-tkrzw.c +++ b/src/libduc/db-tkrzw.c @@ -50,41 +50,45 @@ duc_errno tkrzwdb_to_errno(TkrzwDBM *hdb) } } +static int options_append(char *options, size_t *options_len, size_t options_size, const char *fragment) +{ + size_t flen = strlen(fragment); + if (*options_len + flen >= options_size) + return -1; + memcpy(options + *options_len, fragment, flen + 1); + *options_len += flen; + return 0; +} + struct db *db_open(const char *path_db, int flags, duc_errno *e) { struct db *db; int compress = 0; int writeable = 0; char options[256] = "dbm=HashDBM,file=StdFile,offset_width=5"; + size_t options_len = strlen(options); - if (flags & DUC_OPEN_FORCE) { - char trunc[] = ",truncate=true"; - strcat(options,trunc); - } + if (flags & DUC_OPEN_FORCE) + options_append(options, &options_len, sizeof(options), ",truncate=true"); // Ideally we would know the filesystem here so we can scale things properly, but this is a major re-work of API, so for now just define some new DUC_FS_*" factors... - if (flags & DUC_FS_BIG) { - char big[] = ",num_buckets=100000000"; - strcat(options,big); - } + if (flags & DUC_FS_BIG) + options_append(options, &options_len, sizeof(options), ",num_buckets=100000000"); - if (flags & DUC_FS_BIGGER) { - char bigger[] = ",num_buckets=1000000000"; - strcat(options,bigger); - } + if (flags & DUC_FS_BIGGER) + options_append(options, &options_len, sizeof(options), ",num_buckets=1000000000"); - if (flags & DUC_FS_BIGGEST) { - char biggest[] = ",num_buckets=10000000000"; - strcat(options,biggest); - } + if (flags & DUC_FS_BIGGEST) + options_append(options, &options_len, sizeof(options), ",num_buckets=10000000000"); if (flags & DUC_OPEN_RW) writeable = 1; if (flags & DUC_OPEN_COMPRESS) { - /* Do no compression for now, need to update configure tests first */ - char comp[64]; - sprintf(comp,",record_comp_mode=%s",DUC_TKRZW_REC_COMP); - printf("opening tkzrw DB with compression: %s\n",DUC_TKRZW_REC_COMP); - strcat(options,comp); + /* Do no compression for now, need to update configure tests first */ + char comp[64]; + int r = snprintf(comp, sizeof(comp), ",record_comp_mode=%s", DUC_TKRZW_REC_COMP); + printf("opening tkzrw DB with compression: %s\n",DUC_TKRZW_REC_COMP); + if (r > 0 && (size_t)r < sizeof(comp)) + options_append(options, &options_len, sizeof(options), comp); } db = duc_malloc(sizeof *db); diff --git a/src/libduc/db.c b/src/libduc/db.c index c6abbdb..9355c5d 100644 --- a/src/libduc/db.c +++ b/src/libduc/db.c @@ -47,18 +47,24 @@ duc_errno db_write_report(duc *duc, const struct duc_index_report *report) sizeof(report->histogram)); } - /* write topn array, FIXME to really work... */ - char str[] = "duc_index_topn_info"; - int str_len = sizeof(str); - tmp = db_get(duc->db, str, str_len , &tmpl); - if (tmp) { - tmp = duc_realloc(tmp, tmpl + sizeof(report->topn_array)); - memcpy(tmp + tmpl, report->topn_array, sizeof(report->topn_array)); - db_put(duc->db, str, str_len, tmp, - tmpl + sizeof(report->topn_array)); - } else { - db_put(duc->db, str, str_len, report->topn_array, - sizeof(report->topn_array)); + /* write topn array: flatten pointer array into contiguous struct data */ + if (report->topn_cnt > 0) { + char str[] = "duc_index_topn_info"; + int str_len = sizeof(str); + size_t topn_size = (size_t)report->topn_cnt * sizeof(duc_topn_file); + char *topn_flat = duc_malloc(topn_size); + for (int i = 0; i < report->topn_cnt; i++) + memcpy(topn_flat + (size_t)i * sizeof(duc_topn_file), report->topn_array[i], sizeof(duc_topn_file)); + char *topn_prev = db_get(duc->db, str, str_len, &tmpl); + if (topn_prev) { + topn_prev = duc_realloc(topn_prev, tmpl + topn_size); + memcpy(topn_prev + tmpl, topn_flat, topn_size); + db_put(duc->db, str, str_len, topn_prev, tmpl + topn_size); + duc_free(topn_prev); + } else { + db_put(duc->db, str, str_len, topn_flat, topn_size); + } + duc_free(topn_flat); } } else { diff --git a/src/libduc/index.c b/src/libduc/index.c index 5c71f1e..e545ed8 100644 --- a/src/libduc/index.c +++ b/src/libduc/index.c @@ -102,34 +102,34 @@ int duc_index_req_free(duc_index_req *req) HASH_ITER(hh, req->hard_link_map, h, hn) { HASH_DEL(req->hard_link_map, h); - free(h); + duc_free(h); } HASH_ITER(hh, req->fstypes_mounted, f, fn) { duc_free(f->type); duc_free(f->path); HASH_DEL(req->fstypes_mounted, f); - free(f); + duc_free(f); } HASH_ITER(hh, req->fstypes_include, f, fn) { duc_free(f->type); HASH_DEL(req->fstypes_include, f); - free(f); + duc_free(f); } HASH_ITER(hh, req->fstypes_exclude, f, fn) { duc_free(f->type); HASH_DEL(req->fstypes_exclude, f); - free(f); + duc_free(f); } LL_FOREACH_SAFE(req->exclude_list, e, en) { - free(e->name); - free(e); + duc_free(e->name); + duc_free(e); } - free(req); + duc_free(req); return 0; } @@ -442,7 +442,7 @@ static struct scanner *scanner_new(struct duc *duc, struct scanner *scanner_pare err: if(scanner->d) closedir(scanner->d); - if(scanner) free(scanner); + if(scanner) duc_free(scanner); return NULL; } @@ -564,12 +564,15 @@ static void scanner_scan(struct scanner *scanner_dir) i = (int) floor(log(st_ent.st_size) / log(2)); } - /* clamp size of histogram even if we run into monster sized file */ - if (i >= report->histogram_buckets) { - i = report->histogram_buckets; - duc_log(duc, DUC_LOG_WRN, "File sizes large enough we ran out of histogram buckets %d, please increase the number of buckets and re-run your indexing.",report->histogram_buckets); + /* Only use histogram if buckets > 0 */ + if (report->histogram_buckets > 0) { + /* clamp size of histogram even if we run into monster sized file */ + if (i >= report->histogram_buckets) { + i = report->histogram_buckets - 1; + duc_log(duc, DUC_LOG_WRN, "File sizes large enough we ran out of histogram buckets %d, please increase the number of buckets and re-run your indexing.",report->histogram_buckets); + } + report->histogram[i]++; } - report->histogram[i]++; duc_log(duc, DUC_LOG_DMP, " %c %jd %jd %s", duc_file_type_char(ent.type), ent.size.apparent, ent.size.actual, name); @@ -587,7 +590,8 @@ static void scanner_scan(struct scanner *scanner_dir) } report->topn_array[0]->size = st_ent.st_size; - strncpy(report->topn_array[0]->name,path_full,sizeof(path_full)); + strncpy(report->topn_array[0]->name, path_full, DUC_PATH_MAX - 1); + report->topn_array[0]->name[DUC_PATH_MAX - 1] = '\0'; qsort(report->topn_array, req->topn_cnt, sizeof(struct duc_topn_file *), topn_comp); } } @@ -761,7 +765,7 @@ struct duc_index_report *duc_index(duc_index_req *req, const char *path, duc_ind db_write_report(duc, report); } - free(path_canon); + duc_free(path_canon); return report; } @@ -770,7 +774,7 @@ struct duc_index_report *duc_index(duc_index_req *req, const char *path, duc_ind int duc_index_report_free(struct duc_index_report *rep) { - free(rep); + duc_free(rep); return 0; }