Skip to content

[obsolete] Fix crashes on indexing -> replaced by MR 360 #351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
caco3 wants to merge 1 commit into
base: v1.5.0-rc2
Choose a base branch
from
Open

[obsolete] Fix crashes on indexing -> replaced by MR 360 #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions src/libduc/buffer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ void buffer_free(struct buffer *b)
// Add item to buffer, but grow by doubling if needed
static int buffer_put(struct buffer *b, const void *data, size_t len)
{
if(b->ptr + len <= b->len) {
while(b->len + len > b->max) {
if(b->ptr + len > b->max) {
while(b->ptr + len > b->max) {
b->max *= 2;
}
b->data = duc_realloc(b->data, b->max);
Expand Down Expand Up @@ -271,7 +271,8 @@ void buffer_get_index_report(struct buffer *b, struct duc_index_report *report)
buffer_get_varint(b, &length);
buffer_get_string(b, &vs);
report->topn_array[i] = duc_malloc0(sizeof(duc_topn_file));
strncpy(report->topn_array[i]->name, vs, strlen(vs));
strncpy(report->topn_array[i]->name, vs, DUC_PATH_MAX - 1);
report->topn_array[i]->name[DUC_PATH_MAX - 1] = '\0';
buffer_get_varint(b, &vi); report->topn_array[i]->size = vi;
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/libduc/canonicalize.c
View file
Open in desktop
Copy link
Copy Markdown
Collaborator

@l8gravely l8gravely Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks good.

Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ char *duc_canonicalize_path(const char *in)

if(n == 0) utstring_printf(&out, "/");

free(s.cs);
duc_free(s.cs);
return utstring_body(&out);
}

Expand Down
27 changes: 21 additions & 6 deletions src/libduc/db-tkrzw.c
View file
Open in desktop
Copy link
Copy Markdown
Collaborator

@l8gravely l8gravely Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All these option parsing should be handled in a function which has proper error reporting. Just because someone passes in too many options and causes a string overflow, we shouldn't crash without exiting cleanly with an error here. Or at least warning that this option was skipped because we ran out of space.

Original file line number Diff line number Diff line change
Expand Up @@ -56,35 +56,50 @@ struct db *db_open(const char *path_db, int flags, duc_errno *e)
int compress = 0;
int writeable = 0;
char options[256] = "dbm=HashDBM,file=StdFile,offset_width=5";
size_t options_len = strlen(options);

if (flags & DUC_OPEN_FORCE) {
char trunc[] = ",truncate=true";
strcat(options,trunc);
if(options_len + sizeof(trunc) < sizeof(options)) {
strcat(options,trunc);
options_len += sizeof(trunc) - 1;
}
}

// Ideally we would know the filesystem here so we can scale things properly, but this is a major re-work of API, so for now just define some new DUC_FS_*" factors...
if (flags & DUC_FS_BIG) {
char big[] = ",num_buckets=100000000";
strcat(options,big);
if(options_len + sizeof(big) < sizeof(options)) {
strcat(options,big);
options_len += sizeof(big) - 1;
}
}

if (flags & DUC_FS_BIGGER) {
char bigger[] = ",num_buckets=1000000000";
strcat(options,bigger);
if(options_len + sizeof(bigger) < sizeof(options)) {
strcat(options,bigger);
options_len += sizeof(bigger) - 1;
}
}

if (flags & DUC_FS_BIGGEST) {
char biggest[] = ",num_buckets=10000000000";
strcat(options,biggest);
if(options_len + sizeof(biggest) < sizeof(options)) {
strcat(options,biggest);
options_len += sizeof(biggest) - 1;
}
}

if (flags & DUC_OPEN_RW) writeable = 1;
if (flags & DUC_OPEN_COMPRESS) {
/* Do no compression for now, need to update configure tests first */
char comp[64];
sprintf(comp,",record_comp_mode=%s",DUC_TKRZW_REC_COMP);
int r = snprintf(comp, sizeof(comp), ",record_comp_mode=%s", DUC_TKRZW_REC_COMP);
printf("opening tkzrw DB with compression: %s\n",DUC_TKRZW_REC_COMP);
strcat(options,comp);
if(r > 0 && options_len + r < sizeof(options)) {
strcat(options,comp);
}
}

db = duc_malloc(sizeof *db);
Expand Down
16 changes: 9 additions & 7 deletions src/libduc/db.c
View file
Open in desktop
Copy link
Copy Markdown
Collaborator

@l8gravely l8gravely Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This chunk being commented out isn't what I'd like to do, rather let's fix any crash happening here. When you comment something out, please say why you did so and point to an example.

Original file line number Diff line number Diff line change
Expand Up @@ -47,19 +47,21 @@ duc_errno db_write_report(duc *duc, const struct duc_index_report *report)
sizeof(report->histogram));
}

/* write topn array, FIXME to really work... */
/* write topn array, FIXME to really work... DISABLED FOR NOW */
/*
char str[] = "duc_index_topn_info";
int str_len = sizeof(str);
tmp = db_get(duc->db, str, str_len , &tmpl);
if (tmp) {
tmp = duc_realloc(tmp, tmpl + sizeof(report->topn_array));
memcpy(tmp + tmpl, report->topn_array, sizeof(report->topn_array));
db_put(duc->db, str, str_len, tmp,
tmpl + sizeof(report->topn_array));
size_t topn_size = report->topn_cnt * sizeof(duc_topn_file *);
tmp = duc_realloc(tmp, tmpl + topn_size);
memcpy(tmp + tmpl, report->topn_array, topn_size);
db_put(duc->db, str, str_len, tmp, tmpl + topn_size);
} else {
db_put(duc->db, str, str_len, report->topn_array,
sizeof(report->topn_array));
size_t topn_size = report->topn_cnt * sizeof(duc_topn_file *);
db_put(duc->db, str, str_len, report->topn_array, topn_size);
}
*/

} else {
free(tmp);
Expand Down
36 changes: 20 additions & 16 deletions src/libduc/index.c
View file
Open in desktop
Copy link
Copy Markdown
Collaborator

@l8gravely l8gravely Apr 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are all good.

Original file line number Diff line number Diff line change
Expand Up @@ -102,34 +102,34 @@ int duc_index_req_free(duc_index_req *req)

HASH_ITER(hh, req->hard_link_map, h, hn) {
HASH_DEL(req->hard_link_map, h);
free(h);
duc_free(h);
}

HASH_ITER(hh, req->fstypes_mounted, f, fn) {
duc_free(f->type);
duc_free(f->path);
HASH_DEL(req->fstypes_mounted, f);
free(f);
duc_free(f);
}

HASH_ITER(hh, req->fstypes_include, f, fn) {
duc_free(f->type);
HASH_DEL(req->fstypes_include, f);
free(f);
duc_free(f);
}

HASH_ITER(hh, req->fstypes_exclude, f, fn) {
duc_free(f->type);
HASH_DEL(req->fstypes_exclude, f);
free(f);
duc_free(f);
}

LL_FOREACH_SAFE(req->exclude_list, e, en) {
free(e->name);
free(e);
duc_free(e->name);
duc_free(e);
}

free(req);
duc_free(req);

return 0;
}
Expand Down Expand Up @@ -442,7 +442,7 @@ static struct scanner *scanner_new(struct duc *duc, struct scanner *scanner_pare

err:
if(scanner->d) closedir(scanner->d);
if(scanner) free(scanner);
if(scanner) duc_free(scanner);
return NULL;
}

Expand Down Expand Up @@ -564,12 +564,15 @@ static void scanner_scan(struct scanner *scanner_dir)
i = (int) floor(log(st_ent.st_size) / log(2));
}

/* clamp size of histogram even if we run into monster sized file */
if (i >= report->histogram_buckets) {
i = report->histogram_buckets;
duc_log(duc, DUC_LOG_WRN, "File sizes large enough we ran out of histogram buckets %d, please increase the number of buckets and re-run your indexing.",report->histogram_buckets);
/* Only use histogram if buckets > 0 */
if (report->histogram_buckets > 0) {
/* clamp size of histogram even if we run into monster sized file */
if (i >= report->histogram_buckets) {
i = report->histogram_buckets - 1;
duc_log(duc, DUC_LOG_WRN, "File sizes large enough we ran out of histogram buckets %d, please increase the number of buckets and re-run your indexing.",report->histogram_buckets);
}
report->histogram[i]++;
}
report->histogram[i]++;

duc_log(duc, DUC_LOG_DMP, " %c %jd %jd %s",
duc_file_type_char(ent.type), ent.size.apparent, ent.size.actual, name);
Expand All @@ -587,7 +590,8 @@ static void scanner_scan(struct scanner *scanner_dir)
}

report->topn_array[0]->size = st_ent.st_size;
strncpy(report->topn_array[0]->name,path_full,sizeof(path_full));
strncpy(report->topn_array[0]->name, path_full, DUC_PATH_MAX - 1);
report->topn_array[0]->name[DUC_PATH_MAX - 1] = '\0';
qsort(report->topn_array, req->topn_cnt, sizeof(struct duc_topn_file *), topn_comp);
}
}
Expand Down Expand Up @@ -761,7 +765,7 @@ struct duc_index_report *duc_index(duc_index_req *req, const char *path, duc_ind
db_write_report(duc, report);
}

free(path_canon);
duc_free(path_canon);

return report;
}
Expand All @@ -770,7 +774,7 @@ struct duc_index_report *duc_index(duc_index_req *req, const char *path, duc_ind

int duc_index_report_free(struct duc_index_report *rep)
{
free(rep);
duc_free(rep);
return 0;
}

Expand Down