Fix crashes on indexing

src/libduc/buffer.c

@@ -47,8 +47,8 @@ void buffer_free(struct buffer *b)
 // Add item to buffer, but grow by doubling if needed
 static int buffer_put(struct buffer *b, const void *data, size_t len)
 {
-	if(b->ptr + len <= b->len) {
-		while(b->len + len > b->max) {
+	if(b->ptr + len > b->max) {
+		while(b->ptr + len > b->max) {
 			b->max *= 2;
 		}
 		b->data = duc_realloc(b->data, b->max);
@@ -271,7 +271,8 @@ void buffer_get_index_report(struct buffer *b, struct duc_index_report *report)
 	    buffer_get_varint(b, &length);
 	    buffer_get_string(b, &vs);
 	    report->topn_array[i] = duc_malloc0(sizeof(duc_topn_file));
-	    strncpy(report->topn_array[i]->name, vs, strlen(vs));
+	    strncpy(report->topn_array[i]->name, vs, DUC_PATH_MAX - 1);
+	    report->topn_array[i]->name[DUC_PATH_MAX - 1] = '\0';
 	    buffer_get_varint(b, &vi); report->topn_array[i]->size = vi;
 	}
 }

src/libduc/canonicalize.c

@@ -232,7 +232,7 @@ char *duc_canonicalize_path(const char *in)
 
 	if(n == 0) utstring_printf(&out, "/");
 
-	free(s.cs);
+	duc_free(s.cs);
 	return utstring_body(&out);	
 }
 

src/libduc/db-tkrzw.c

@@ -50,41 +50,45 @@ duc_errno tkrzwdb_to_errno(TkrzwDBM *hdb)
 	}
 }
 
+static int options_append(char *options, size_t *options_len, size_t options_size, const char *fragment)
+{
+	size_t flen = strlen(fragment);
+	if (*options_len + flen >= options_size)
+		return -1;
+	memcpy(options + *options_len, fragment, flen + 1);
+	*options_len += flen;
+	return 0;
+}
+
 struct db *db_open(const char *path_db, int flags, duc_errno *e)
 {
 	struct db *db;
 	int compress = 0;
 	int writeable = 0;
 	char options[256] = "dbm=HashDBM,file=StdFile,offset_width=5";
+	size_t options_len = strlen(options);
 
-	if (flags & DUC_OPEN_FORCE) { 
-	    char trunc[] = ",truncate=true";
-	    strcat(options,trunc);
-	}
+	if (flags & DUC_OPEN_FORCE)
+		options_append(options, &options_len, sizeof(options), ",truncate=true");
 
 	// Ideally we would know the filesystem here so we can scale things properly, but this is a major re-work of API, so for now just define some new DUC_FS_*" factors...
-	if (flags & DUC_FS_BIG) {
-	    char big[] = ",num_buckets=100000000";
-	    strcat(options,big);
-	}
+	if (flags & DUC_FS_BIG)
+		options_append(options, &options_len, sizeof(options), ",num_buckets=100000000");
 
-	if (flags & DUC_FS_BIGGER) {
-	    char bigger[] = ",num_buckets=1000000000";
-	    strcat(options,bigger);
-	}
+	if (flags & DUC_FS_BIGGER)
+		options_append(options, &options_len, sizeof(options), ",num_buckets=1000000000");
 
-	if (flags & DUC_FS_BIGGEST) {
-	    char biggest[] = ",num_buckets=10000000000";
-	    strcat(options,biggest);
-	}
+	if (flags & DUC_FS_BIGGEST)
+		options_append(options, &options_len, sizeof(options), ",num_buckets=10000000000");
 
 	if (flags & DUC_OPEN_RW) writeable = 1;
 	if (flags & DUC_OPEN_COMPRESS) {
-	    /* Do no compression for now, need to update configure tests first */
-	    char comp[64];
-	    sprintf(comp,",record_comp_mode=%s",DUC_TKRZW_REC_COMP);
-	    printf("opening tkzrw DB with compression: %s\n",DUC_TKRZW_REC_COMP);
-	    strcat(options,comp);
+		/* Do no compression for now, need to update configure tests first */
+		char comp[64];
+		int r = snprintf(comp, sizeof(comp), ",record_comp_mode=%s", DUC_TKRZW_REC_COMP);
+		printf("opening tkzrw DB with compression: %s\n",DUC_TKRZW_REC_COMP);
+		if (r > 0 && (size_t)r < sizeof(comp))
+			options_append(options, &options_len, sizeof(options), comp);
 	}
 
 	db = duc_malloc(sizeof *db);

src/libduc/db.c

@@ -47,18 +47,24 @@ duc_errno db_write_report(duc *duc, const struct duc_index_report *report)
 			       sizeof(report->histogram));
 		}
 
-		/* write topn array, FIXME to really work... */
-		char str[] = "duc_index_topn_info";
-		int str_len = sizeof(str);
-		tmp = db_get(duc->db, str, str_len , &tmpl);
-		if (tmp) {
-			tmp = duc_realloc(tmp, tmpl + sizeof(report->topn_array));
-			memcpy(tmp + tmpl, report->topn_array, sizeof(report->topn_array));
-			db_put(duc->db, str, str_len, tmp, 
-			       tmpl + sizeof(report->topn_array));
-		} else {
-			db_put(duc->db, str, str_len, report->topn_array, 
-			       sizeof(report->topn_array));
+		/* write topn array: flatten pointer array into contiguous struct data */
+		if (report->topn_cnt > 0) {
+			char str[] = "duc_index_topn_info";
+			int str_len = sizeof(str);
+			size_t topn_size = (size_t)report->topn_cnt * sizeof(duc_topn_file);
+			char *topn_flat = duc_malloc(topn_size);
+			for (int i = 0; i < report->topn_cnt; i++)
+				memcpy(topn_flat + (size_t)i * sizeof(duc_topn_file), report->topn_array[i], sizeof(duc_topn_file));
+			char *topn_prev = db_get(duc->db, str, str_len, &tmpl);
+			if (topn_prev) {
+				topn_prev = duc_realloc(topn_prev, tmpl + topn_size);
+				memcpy(topn_prev + tmpl, topn_flat, topn_size);
+				db_put(duc->db, str, str_len, topn_prev, tmpl + topn_size);
+				duc_free(topn_prev);
+			} else {
+				db_put(duc->db, str, str_len, topn_flat, topn_size);
+			}
+			duc_free(topn_flat);
 		}
 
 	} else {

src/libduc/index.c

@@ -102,34 +102,34 @@ int duc_index_req_free(duc_index_req *req)
 
 	HASH_ITER(hh, req->hard_link_map, h, hn) {
 		HASH_DEL(req->hard_link_map, h);
-		free(h);
+		duc_free(h);
 	}
 
 	HASH_ITER(hh, req->fstypes_mounted, f, fn) {
 		duc_free(f->type);
 		duc_free(f->path);
 		HASH_DEL(req->fstypes_mounted, f);
-		free(f);
+		duc_free(f);
 	}
 
 	HASH_ITER(hh, req->fstypes_include, f, fn) {
 		duc_free(f->type);
 		HASH_DEL(req->fstypes_include, f);
-		free(f);
+		duc_free(f);
 	}
 
 	HASH_ITER(hh, req->fstypes_exclude, f, fn) {
 		duc_free(f->type);
 		HASH_DEL(req->fstypes_exclude, f);
-		free(f);
+		duc_free(f);
 	}
 
 	LL_FOREACH_SAFE(req->exclude_list, e, en) {
-		free(e->name);
-		free(e);
+		duc_free(e->name);
+		duc_free(e);
 	}
 
-	free(req);
+	duc_free(req);
 
 	return 0;
 }
@@ -442,7 +442,7 @@ static struct scanner *scanner_new(struct duc *duc, struct scanner *scanner_pare
 
 err:
 	if(scanner->d) closedir(scanner->d);
-	if(scanner) free(scanner);
+	if(scanner) duc_free(scanner);
 	return NULL;
 }
 
@@ -564,12 +564,15 @@ static void scanner_scan(struct scanner *scanner_dir)
 			    i = (int) floor(log(st_ent.st_size) / log(2));
 			}
 
-			/* clamp size of histogram even if we run into monster sized file */
-			if (i >= report->histogram_buckets) {
-			    i = report->histogram_buckets;
-			    duc_log(duc, DUC_LOG_WRN, "File sizes large enough we ran out of histogram buckets %d, please increase the number of buckets and re-run your indexing.",report->histogram_buckets);
+			/* Only use histogram if buckets > 0 */
+			if (report->histogram_buckets > 0) {
+			    /* clamp size of histogram even if we run into monster sized file */
+			    if (i >= report->histogram_buckets) {
+				i = report->histogram_buckets - 1;
+				duc_log(duc, DUC_LOG_WRN, "File sizes large enough we ran out of histogram buckets %d, please increase the number of buckets and re-run your indexing.",report->histogram_buckets);
+			    }
+			    report->histogram[i]++;
 			}
-			report->histogram[i]++;
 
 			duc_log(duc, DUC_LOG_DMP, "  %c %jd %jd %s", 
 					duc_file_type_char(ent.type), ent.size.apparent, ent.size.actual, name);
@@ -587,7 +590,8 @@ static void scanner_scan(struct scanner *scanner_dir)
 				}
 
 				report->topn_array[0]->size = st_ent.st_size;
-				strncpy(report->topn_array[0]->name,path_full,sizeof(path_full));
+				strncpy(report->topn_array[0]->name, path_full, DUC_PATH_MAX - 1);
+				report->topn_array[0]->name[DUC_PATH_MAX - 1] = '\0';
 				qsort(report->topn_array, req->topn_cnt, sizeof(struct duc_topn_file *), topn_comp);
 			    }
 			}
@@ -761,7 +765,7 @@ struct duc_index_report *duc_index(duc_index_req *req, const char *path, duc_ind
 		db_write_report(duc, report);
 	}
 
-	free(path_canon);
+	duc_free(path_canon);
 
 	return report;
 }
@@ -770,7 +774,7 @@ struct duc_index_report *duc_index(duc_index_req *req, const char *path, duc_ind
 
 int duc_index_report_free(struct duc_index_report *rep)
 {
-	free(rep);
+	duc_free(rep);
 	return 0;
 }