diff --git a/requirements.txt b/requirements.txt index ef1b1207..094a68e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,33 +1,34 @@ -cycler==0.11.0 -Flask==2.2.5 -Flask_Cors==4.0.0 -fonttools==4.38.0 -importlib-metadata==6.7.0 -Jinja2==3.1.2 -joblib==1.3.2 -kiwisolver==1.4.5 -kmapper==2.0.1 -llvmlite==0.39.1 -MarkupSafe==2.1.3 -matplotlib==3.5.2 -networkx==2.6.3 -numba==0.56.4 -numpy==1.21.6 -packaging==23.2 -pandas==1.3.5 -Pillow==9.2.0 -pymilvus==2.1.0 -pynndescent==0.5.11 -pyparsing==3.1.1 -python-dateutil==2.8.2 -scikit-learn==1.0.2 -scipy==1.7.3 -six==1.16.0 -tensorflow==2.7.0 -threadpoolctl==3.1.0 -torch==1.13.1 -tqdm==4.66.1 -typing_extensions==4.7.1 -umap==0.1.1 -umap-learn==0.5.3 -zipp==3.15.0 \ No newline at end of file +cycler +Flask +Flask_Cors +fonttools +importlib-metadata +Jinja2 +joblib +kiwisolver +kmapper +llvmlite +MarkupSafe +matplotlib +networkx +numba +numpy +packaging +pandas +Pillow +pymilvus +pynndescent +pyparsing +python-dateutil +scikit-learn +scipy +six +tensorflow +threadpoolctl +torch +torchvision +transformers +tqdm +typing_extensions +umap-learn +zipp \ No newline at end of file diff --git a/tool/server/server.py b/tool/server/server.py index 076243e4..edd919ba 100644 --- a/tool/server/server.py +++ b/tool/server/server.py @@ -1,29 +1,33 @@ import os import sys -# from llm_agent import call_llm_agent -from run_visualization import visualize_run +import numpy as np -from flask import request, Flask, jsonify, make_response, send_file,send_from_directory +from flask import Flask, jsonify, make_response, request, send_file, send_from_directory from flask_cors import CORS, cross_origin -sys.path.append('.') -sys.path.append('..') -sys.path.append('../..') -sys.path.append('../visualize') +# from llm_agent import call_llm_agent +from run_visualization import visualize_run + +sys.path.append(".") +sys.path.append("..") +sys.path.append("../..") +sys.path.append("../visualize") -from server_utils import * +import server_utils +#from server_utils import * # flask for API server app = Flask(__name__) cors = CORS(app, supports_credentials=True) -app.config['CORS_HEADERS'] = 'Content-Type' +app.config["CORS_HEADERS"] = "Content-Type" # Check for "--dev" argument is_dev_mode = "--dev" in sys.argv + @app.route("/", methods=["GET", "POST"]) def GUI(): - return send_from_directory('../frontend', 'index.html') + return send_from_directory("../frontend", "index.html") """ @@ -35,47 +39,51 @@ def GUI(): color_list (list): list of colors label_text_list (list): list of label text """ -@app.route('/getTrainingProcessInfo', methods=["GET"]) + + +@app.route("/getTrainingProcessInfo", methods=["GET"]) @cross_origin() def get_training_process_info(): - content_path = request.args.get('content_path') - - epochs_dir = os.path.join(content_path, 'epochs') + content_path = request.args.get("content_path") + + epochs_dir = os.path.join(content_path, "epochs") available_epochs = [] if os.path.exists(epochs_dir) and os.path.isdir(epochs_dir): try: for item in os.listdir(epochs_dir): - if item.startswith('epoch_'): + if item.startswith("epoch_"): full_path = os.path.join(epochs_dir, item) if os.path.isdir(full_path): - epoch_num_str = item[len('epoch_'):] + epoch_num_str = item[len("epoch_") :] if epoch_num_str.isdigit(): available_epochs.append(int(epoch_num_str)) - + available_epochs.sort() except Exception as e: print(f"Error scanning epochs directory: {e}") available_epochs = [] - config = read_file_as_json(os.path.join(content_path, 'dataset', 'info.json')) - - if config == None or 'classes' not in config: + config = server_utils.read_file_as_json(os.path.join(content_path, "dataset", "info.json")) + + if config == None or "classes" not in config: # infer from labels.npy - label_file = os.path.join(content_path, 'dataset', 'labels.npy') + label_file = os.path.join(content_path, "dataset", "labels.npy") labels = np.load(label_file, allow_pickle=True) class_num = len(np.unique(labels)) - color_list = get_coloring_list(class_num) + color_list = server_utils.get_coloring_list(class_num) label_text_list = [str(i) for i in range(class_num)] else: - color_list = get_coloring_list(len(config['classes'])) - label_text_list = config['classes'] - - result = jsonify({ - 'color_list': color_list, - 'label_text_list': label_text_list, - 'available_epochs': available_epochs - }) + color_list = server_utils.get_coloring_list(len(config["classes"])) + label_text_list = config["classes"] + + result = jsonify( + { + "color_list": color_list, + "label_text_list": label_text_list, + "available_epochs": available_epochs, + } + ) return make_response(result, 200) @@ -91,22 +99,47 @@ def get_training_process_info(): project (list) label_list (list): label list of samples in projection """ -@app.route('/updateProjection', methods = ["POST"]) + + +@app.route("/updateProjection", methods=["POST"]) @cross_origin() def update_projection(): req = request.get_json() - content_path = req['content_path'] - vis_id = req['vis_id'] - epoch = int(req['epoch']) + content_path = req["content_path"] + vis_id = req["vis_id"] + epoch = int(req["epoch"]) - projection = load_projection(content_path, vis_id, epoch) + projection = server_utils.load_projection(content_path, vis_id, epoch) - result = jsonify({ - 'projection': projection, - }) + result = jsonify( + { + "projection": projection, + } + ) return make_response(result, 200) +@app.route("/refineProjection", methods=["POST"]) +@cross_origin() +def refine_projection(): + req = request.get_json() + content_path = req["content_path"] + vis_id = req["vis_id"] + epoch = int(req["epoch"]) + sample_index = int(req["sample_index"]) + + updated_coord = server_utils.local_refine(epoch=epoch, + content_path=content_path, + vis_id=vis_id, + sample_index=sample_index) + + result = jsonify( + { + "updated_coords": updated_coord, + } + ) + return make_response(result, 200) + """ Api: start training visualization model and get visualization result @@ -118,20 +151,23 @@ def update_projection(): Response: None """ -@app.route('/startVisualizing', methods = ["POST"]) + + +@app.route("/startVisualizing", methods=["POST"]) def start_visualizing(): req = request.get_json() - content_path = req['content_path'] - vis_method = req['vis_method'] - vis_id = req['vis_id'] - data_type = req['data_type'] - task_type = req['task_type'] - vis_config = req['vis_config'] - + content_path = req["content_path"] + vis_method = req["vis_method"] + vis_id = req["vis_id"] + data_type = req["data_type"] + task_type = req["task_type"] + vis_config = req["vis_config"] + visualize_run(content_path, vis_method, vis_id, data_type, task_type, vis_config) - + return make_response({}, 200) + """ Api: get text data of all samples @@ -140,36 +176,40 @@ def start_visualizing(): Response: text_list (lsit of str) """ -@app.route('/getAllText', methods = ["POST"]) + + +@app.route("/getAllText", methods=["POST"]) def get_all_text(): req = request.get_json() - content_path = req['content_path'] + content_path = req["content_path"] - text_list = get_all_texts(content_path) + text_list = server_utils.get_all_texts(content_path) if text_list is None: - return make_response(jsonify({'error_message': "getting all texts failed"}), 400) + return make_response( + jsonify({"error_message": "getting all texts failed"}), 400 + ) - result = jsonify({ - 'text_list': text_list - }) + result = jsonify({"text_list": text_list}) return make_response(result, 200) -@app.route('/getAlignment', methods = ["POST"]) + +@app.route("/getAlignment", methods=["POST"]) def get_alignment(): req = request.get_json() - content_path = req['content_path'] + content_path = req["content_path"] - alignment = get_alignment_data(content_path) + alignment = server_utils.get_alignment_data(content_path) if alignment is None: - return make_response(jsonify({'error_message': "getting alignment failed"}), 400) + return make_response( + jsonify({"error_message": "getting alignment failed"}), 400 + ) - result = jsonify({ - 'alignment': alignment - }) + result = jsonify({"alignment": alignment}) return make_response(result, 200) + """ Api: get selected attributes of the dataset @@ -182,17 +222,19 @@ def get_alignment(): attribute2 (object) ... """ -@app.route('/getAttributes', methods = ["POST"]) + + +@app.route("/getAttributes", methods=["POST"]) @cross_origin() def get_attributes(): req = request.get_json() - content_path = req['content_path'] - epoch = req['epoch'] - attributes = req['attributes'] + content_path = req["content_path"] + epoch = req["epoch"] + attributes = req["attributes"] result = {} for attribute in attributes: - result[attribute] = load_single_attribute(content_path, epoch, attribute) + result[attribute] = server_utils.load_single_attribute(content_path, epoch, attribute) result = jsonify(result) return make_response(result, 200) @@ -209,23 +251,23 @@ def get_attributes(): Response: indices (list of int): indeices of samples that satisfy the filter """ -@app.route('/getSimpleFilterResult', methods = ["POST"]) + + +@app.route("/getSimpleFilterResult", methods=["POST"]) @cross_origin() def get_simple_filter_result(): req = request.get_json() - content_path = req['content_path'] - epoch = int(req['epoch']) - filters = req['filters'] + content_path = req["content_path"] + epoch = int(req["epoch"]) + filters = req["filters"] - config = read_file_as_json(os.path.join(content_path, 'config.json')) - indices, error_message = get_filter_result(config, content_path, epoch, filters) + config = server_utils.read_file_as_json(os.path.join(content_path, "config.json")) + indices, error_message = server_utils.get_filter_result(config, content_path, epoch, filters) if indices is None: - return make_response(jsonify({'error_message': error_message}), 400) + return make_response(jsonify({"error_message": error_message}), 400) - result = jsonify({ - 'indices': indices - }) + result = jsonify({"indices": indices}) return make_response(result, 200) @@ -240,23 +282,26 @@ def get_simple_filter_result(): scale (list of float) Response: background_image_base64 (str): base64 encoded im -""" -@app.route('/getBackground', methods = ["POST"]) +""" + + +@app.route("/getBackground", methods=["POST"]) @cross_origin() def get_background(): req = request.get_json() - content_path = req['content_path'] - vis_id = req['vis_id'] - epoch = int(req['epoch']) - + content_path = req["content_path"] + vis_id = req["vis_id"] + epoch = int(req["epoch"]) + try: - base64_image = load_background(content_path, vis_id, epoch) - result = jsonify({ - 'background_image_base64': base64_image - }) + base64_image = server_utils.load_background(content_path, vis_id, epoch) + result = jsonify({"background_image_base64": base64_image}) return make_response(result, 200) except Exception as e: - return make_response(jsonify({'error_message': 'Error in loading background'}), 400) + return make_response( + jsonify({"error_message": "Error in loading background"}), 400 + ) + """ Api: get image data of one sample @@ -267,26 +312,24 @@ def get_background(): Response: image_base64 (str): base64 encoded image """ -@app.route('/getImageData', methods = ["POST"]) + + +@app.route("/getImageData", methods=["POST"]) @cross_origin() def get_image_data(): req = request.get_json() - content_path = req['content_path'] - if('index' not in req): - return make_response(jsonify({'image_base64': ''}), 200) - - index = req['index'] + content_path = req["content_path"] + if "index" not in req: + return make_response(jsonify({"image_base64": ""}), 200) + + index = req["index"] try: - base64_image = load_one_image(content_path, index) - result = jsonify({ - 'image_base64': base64_image - }) + base64_image = server_utils.load_one_image(content_path, index) + result = jsonify({"image_base64": base64_image}) return make_response(result, 200) except Exception as e: - result = jsonify({ - 'image_base64': '' - }) + result = jsonify({"image_base64": ""}) return make_response(result, 200) @@ -299,26 +342,24 @@ def get_image_data(): Response: text (str): text data """ -@app.route('/getTextData', methods = ["POST"]) + + +@app.route("/getTextData", methods=["POST"]) @cross_origin() def get_text_data(): req = request.get_json() - content_path = req['content_path'] - if('index' not in req): - return make_response(jsonify({'text': ''}), 200) - - index = req['index'] + content_path = req["content_path"] + if "index" not in req: + return make_response(jsonify({"text": ""}), 200) + + index = req["index"] try: - text = load_one_text(content_path, index) - result = jsonify({ - 'text': text - }) + text = server_utils.load_one_text(content_path, index) + result = jsonify({"text": text}) return make_response(result, 200) except Exception as e: - result = jsonify({ - 'text': '' - }) + result = jsonify({"text": ""}) return make_response(result, 200) @@ -331,22 +372,29 @@ def get_text_data(): Response: neighbors (array[][]) """ -@app.route('/getOriginalNeighbors', methods = ["POST"]) + + +@app.route("/getOriginalNeighbors", methods=["POST"]) @cross_origin() def get_original_neighbors(): req = request.get_json() - content_path = req['content_path'] - epoch = int(req['epoch']) - + content_path = req["content_path"] + epoch = int(req["epoch"]) + try: - neighbors = calculate_high_dimensional_neighbors(content_path, epoch) - result = jsonify({ - 'neighbors': neighbors, - }) + neighbors = server_utils.calculate_high_dimensional_neighbors(content_path, epoch) + result = jsonify( + { + "neighbors": neighbors, + } + ) return make_response(result, 200) except Exception as e: print(e) - return make_response(jsonify({'error_message': 'Error in calculating neighbors'}), 400) + return make_response( + jsonify({"error_message": "Error in calculating neighbors"}), 400 + ) + """ Api: get projection neighbors of one sample @@ -358,87 +406,136 @@ def get_original_neighbors(): Response: neighbors (array[][]) """ -@app.route('/getProjectionNeighbors', methods = ["POST"]) + + +@app.route("/getProjectionNeighbors", methods=["POST"]) @cross_origin() def get_projection_neighbors(): req = request.get_json() - content_path = req['content_path'] - vis_id = req['vis_id'] - epoch = int(req['epoch']) - + content_path = req["content_path"] + vis_id = req["vis_id"] + epoch = int(req["epoch"]) + try: - neighbors = calculate_projection_neighbors(content_path, vis_id, epoch) - result = jsonify({ - 'neighbors': neighbors, - }) + neighbors = server_utils.calculate_projection_neighbors(content_path, vis_id, epoch) + result = jsonify( + { + "neighbors": neighbors, + } + ) return make_response(result, 200) except Exception as e: print(e) - return make_response(jsonify({'error_message': 'Error in calculating neighbors'}), 400) + return make_response( + jsonify({"error_message": "Error in calculating neighbors"}), 400 + ) - -@app.route('/getVisualizeMetrics', methods = ["POST"]) + + +# get neighbors and projection for only one point +@app.route("/getNeighborsForSample", methods=["POST"]) +@cross_origin() +def get_neighbors_for_sample(): + req = request.get_json() + content_path = req["content_path"] + vis_id = req["vis_id"] + epoch = int(req["epoch"]) + sample_index = int(req["sample_index"]) + + try: + original_neighbors = server_utils.calculate_neighbors_for_point( + content_path, vis_id, epoch, sample_index + ) + projection_neighbors = server_utils.calculate_projection_neighbors_for_point( + content_path, vis_id, epoch, sample_index + ) + result = jsonify( + {"originalNeighbors": original_neighbors, "projectionNeighbors": projection_neighbors} + ) + return make_response(result, 200) + except Exception as e: + print(e) + return make_response( + jsonify({"error_message": "Error in calculating neighbors for sample"}), 400 + ) + + +@app.route("/getVisualizeMetrics", methods=["POST"]) @cross_origin() def get_visualize_metrics(): req = request.get_json() - content_path = req['content_path'] - vis_id = req['vis_id'] - epoch = int(req['epoch']) - + content_path = req["content_path"] + vis_id = req["vis_id"] + epoch = int(req["epoch"]) + try: - metrics = calculate_visualize_metrics(content_path, vis_id, epoch) + metrics = server_utils.calculate_visualize_metrics(content_path, vis_id, epoch) result = jsonify(metrics) return make_response(result, 200) except Exception as e: print(e) - return make_response(jsonify({'error_message': 'Error in calculating metrics'}), 400) + return make_response( + jsonify({"error_message": "Error in calculating metrics"}), 400 + ) -@app.route('/getInfluenceSamples', methods=["POST"]) +@app.route("/getInfluenceSamples", methods=["POST"]) @cross_origin() def get_influence_samples(): req = request.get_json() - content_path = req['content_path'] - epoch = int(req['epoch']) - training_event = req['training_event'] - num_samples = int(req['num_samples']) + content_path = req["content_path"] + epoch = int(req["epoch"]) + training_event = req["training_event"] + num_samples = int(req["num_samples"]) try: - if training_event['type'] == 'InconsistentMovement': + if training_event["type"] == "InconsistentMovement": # attribution of closeness or separation between a pair of samples print("Tracing InconsistentMovement") - influence_samples = movement_attribution(content_path, epoch, training_event, num_samples) - else: + influence_samples = server_utils.movement_attribution( + content_path, epoch, training_event, num_samples + ) + else: # atribution of a particular prediction print("Tracing PredictionError") - influence_samples = prediction_attribution(content_path, epoch, training_event, num_samples) - - result = jsonify({ - "influence_samples": influence_samples, - }) + influence_samples = server_utils.prediction_attribution( + content_path, epoch, training_event, num_samples + ) + + result = jsonify( + { + "influence_samples": influence_samples, + } + ) return make_response(result, 200) except Exception as e: print(e) - return make_response(jsonify({'error_message': 'Error in calculating influence samples'}), 400) + return make_response( + jsonify({"error_message": "Error in calculating influence samples"}), 400 + ) -@app.route('/calculateTrainingEvents', methods=["POST"]) +@app.route("/calculateTrainingEvents", methods=["POST"]) @cross_origin() def calculate_training_events(): req = request.get_json() - content_path = req['content_path'] - epoch = int(req['epoch']) - event_types = req['event_types'] + content_path = req["content_path"] + epoch = int(req["epoch"]) + event_types = req["event_types"] try: - training_events = compute_training_events(content_path, epoch, event_types) - result = jsonify({ - "training_events": training_events, - }) + training_events = server_utils.compute_training_events(content_path, epoch, event_types) + result = jsonify( + { + "training_events": training_events, + } + ) return make_response(result, 200) except Exception as e: print(e) - return make_response(jsonify({'error_message': 'Error in calculating training events'}), 400) + return make_response( + jsonify({"error_message": "Error in calculating training events"}), 400 + ) def check_port_inuse(port, host): @@ -455,26 +552,28 @@ def check_port_inuse(port, host): if s: s.close() + # for contrast if __name__ == "__main__": - host = '0.0.0.0' + host = "0.0.0.0" port = 5050 while check_port_inuse(port, host): port = port + 1 if not is_dev_mode: - app.run(host=host, port=port) + # added threaded=True to handle multiple requests to the backend (for the batches) + app.run(host=host, port=port, threaded=True) else: - from livereload import Server from flask_debugtoolbar import DebugToolbarExtension + from livereload import Server app.debug = True - app.config['SECRET_KEY'] = 'a-random-secret-key' + app.config["SECRET_KEY"] = "a-random-secret-key" toolbar = DebugToolbarExtension(app) server = Server(app.wsgi_app) - server.watch('../frontend/**/*.css') - server.watch('../frontend/**/*.html') - server.watch('../frontend/**/*.js') + server.watch("../frontend/**/*.css") + server.watch("../frontend/**/*.html") + server.watch("../frontend/**/*.js") server.serve(host=host, port=port) diff --git a/tool/server/server_utils.py b/tool/server/server_utils.py index 59cd329c..c8b1d196 100644 --- a/tool/server/server_utils.py +++ b/tool/server/server_utils.py @@ -18,16 +18,19 @@ from torch.utils.data import Dataset, DataLoader from transformers import RobertaTokenizer -sys.path.append('..') -sys.path.append('../visualize') +sys.path.append("..") +sys.path.append("../visualize") from visualize.data_provider import DataProvider +from visualize.strategy.losses import SingleVisLoss, UmapLoss, ReconstructionLoss from visualize.training_event import TrainingEventDetector from influence_function.IF import EmpiricalIF, PairWiseEmpiricalIF from influence_function.CustomEncoderModel import CustomEncoderModel +from umap.umap_ import find_ab_params +from visualize.visualize_model import VisModel # Func: infer available epochs files, return a list of available epochs def infer_epoch_structure(content_path): - epochs_dir = os.path.join(content_path, 'epochs') + epochs_dir = os.path.join(content_path, "epochs") available_epochs = [] if os.path.exists(epochs_dir) and os.path.isdir(epochs_dir): for folder_name in os.listdir(epochs_dir): @@ -44,7 +47,7 @@ def infer_epoch_structure(content_path): # Func: get coloring list def get_coloring_list(class_num): # color = get_standard_classes_color(class_num) * 255 - color_map = plt.get_cmap('tab10') + color_map = plt.get_cmap("tab10") color = color_map(range(class_num)) color_255 = (color[:, :3] * 255).astype(np.uint8) return color_255.tolist() @@ -56,40 +59,40 @@ def load_projection(content_path, vis_id, epoch): projection_list = projection.tolist() index_dict = load_or_create_index(content_path) - all_indices = index_dict['train'] + index_dict['test'] + all_indices = index_dict["train"] + index_dict["test"] projection_list = [projection_list[i] for i in all_indices] return projection_list # Func: load one sample from content_path def load_one_sample(config, content_path, index): - attributes = config['dataset']['attributes'] - if 'sample' not in attributes: + attributes = config["dataset"]["attributes"] + if "sample" not in attributes: raise NotImplementedError("sample is not in attributes") - file_path_pattern = attributes['sample']['source']['pattern'] - file_path = file_path_pattern.replace('${index}', str(index)) + file_path_pattern = attributes["sample"]["source"]["pattern"] + file_path = file_path_pattern.replace("${index}", str(index)) file_path = os.path.join(content_path, file_path) _, file_extension = os.path.splitext(file_path) - if file_extension == '.txt': + if file_extension == ".txt": sample = "" - single_file = attributes['sample']['source']['type']=='folder' + single_file = attributes["sample"]["source"]["type"]=="folder" if single_file: # this indicates that all the text samples are saved in one file - with open(file_path, 'r') as f: + with open(file_path, "r") as f: all_sample = f.readlines() sample = all_sample[index] else: - with open(file_path, 'r') as f: + with open(file_path, "r") as f: sample = f.readline() - return 'text',sample + return "text",sample - elif file_extension == '.png' or file_extension == '.jpg': + elif file_extension == ".png" or file_extension == ".jpg": img_stream = "" - with open(file_path, 'rb') as img_f: + with open(file_path, "rb") as img_f: img_stream = img_f.read() img_stream = base64.b64encode(img_stream).decode() - return 'image','data:image/png;base64,' + img_stream + return "image","data:image/png;base64," + img_stream else: raise NotImplementedError("Unsupported file extension: {}".format(file_extension)) @@ -99,21 +102,21 @@ def get_all_texts(content_path, from_file=True): text_list = [] if from_file: - file_path = os.path.join(content_path, 'dataset', 'text.txt') - with open(file_path, 'r') as f: + file_path = os.path.join(content_path, "dataset", "text.txt") + with open(file_path, "r") as f: content = f.read() lines = content.splitlines() text_list = lines else: - parent_directory = os.path.join(content_path, 'dataset', 'text') + parent_directory = os.path.join(content_path, "dataset", "text") files_and_folders = os.listdir(parent_directory) - numbered_files = [f for f in files_and_folders if f.endswith('.txt') and f[-5].isdigit()] - numbered_files.sort(key=lambda f: int(re.search(r'[0-9]+', f)[0])) + numbered_files = [f for f in files_and_folders if f.endswith(".txt") and f[-5].isdigit()] + numbered_files.sort(key=lambda f: int(re.search(r"[0-9]+", f)[0])) for file_name in numbered_files: file_path = os.path.join(parent_directory, file_name) - with open(file_path, 'r') as file: + with open(file_path, "r") as file: content = file.read() text_list.append(content) @@ -165,10 +168,10 @@ def union(u, v): def read_label_file(file_path): _, file_extension = os.path.splitext(file_path) - if file_extension == '.npy': + if file_extension == ".npy": data = np.load(file_path) label_list = data.tolist() - elif file_extension == '.pth': + elif file_extension == ".pth": data = torch.load(file_path) if isinstance(data, torch.Tensor): label_list = data.tolist() @@ -182,74 +185,74 @@ def read_label_file(file_path): # Func: get simple filtered indices def get_filter_result(config, content_path, epoch, filters): - index_file_path = os.path.join(content_path, 'index.json') + index_file_path = os.path.join(content_path, "index.json") if not os.path.exists(index_file_path): - return None, 'index.json not found' + return None, "index.json not found" indice_obj = read_file_as_json(index_file_path) - all_indices = indice_obj['train'] + indice_obj['test'] + all_indices = indice_obj["train"] + indice_obj["test"] result = all_indices for filter in filters: - filter_type = filter['filter_type'] - label_text_list = config['dataset']['classes'] + filter_type = filter["filter_type"] + label_text_list = config["dataset"]["classes"] - if filter_type == 'label': - filter_data = filter['filter_data'] + if filter_type == "label": + filter_data = filter["filter_data"] - attributes = config['dataset']['attributes'] - file_path_pattern = attributes['label']['source']['pattern'] + attributes = config["dataset"]["attributes"] + file_path_pattern = attributes["label"]["source"]["pattern"] file_path = os.path.join(content_path, file_path_pattern) if not os.path.exists(file_path): - return None, 'label file not found' + return None, "label file not found" label_list = read_label_file(file_path) filtered_indices = [index for index, label in zip(all_indices, label_list) if label_text_list[label] == filter_data] result = list(set(result) & set(filtered_indices)) - elif filter_type == 'prediction': - filter_data = filter['filter_data'] + elif filter_type == "prediction": + filter_data = filter["filter_data"] - attributes = config['dataset']['attributes'] - file_path_pattern = attributes['prediction']['source']['pattern'] - file_path_pattern = file_path_pattern.replace('${epoch}', str(epoch)) + attributes = config["dataset"]["attributes"] + file_path_pattern = attributes["prediction"]["source"]["pattern"] + file_path_pattern = file_path_pattern.replace("${epoch}", str(epoch)) file_path = os.path.join(content_path, file_path_pattern) if not os.path.exists(file_path): - return None, 'prediction file not found' + return None, "prediction file not found" prediction_list = read_label_file(file_path) filtered_indices = [index for index, label in zip(all_indices, prediction_list) if label_text_list[label] == filter_data] result = list(set(result) & set(filtered_indices)) - elif filter_type == 'train': - result = list(set(result) & set(indice_obj['train'])) + elif filter_type == "train": + result = list(set(result) & set(indice_obj["train"])) - elif filter_type == 'test': - result = list(set(result) & set(indice_obj['test'])) + elif filter_type == "test": + result = list(set(result) & set(indice_obj["test"])) - return result,'' + return result,"" def load_background(content_path, vis_id, epoch): - file_path = os.path.join(content_path, 'visualize',vis_id,'epochs',f'epoch_{epoch}', 'background.png') + file_path = os.path.join(content_path, "visualize",vis_id,"epochs",f"epoch_{epoch}", "background.png") if os.path.exists(file_path): return convert_to_base64(file_path) return "" def convert_to_base64(image_path): with open(image_path, "rb") as image_file: - base64_image = base64.b64encode(image_file.read()).decode('utf-8') + base64_image = base64.b64encode(image_file.read()).decode("utf-8") return base64_image def load_one_image(content_path, index): - file_path = os.path.join(content_path, 'dataset', 'image', f'{index}.png') + file_path = os.path.join(content_path, "dataset", "image", f"{index}.png") return convert_to_base64(file_path) def load_one_text(content_path, index): - file_path = os.path.join(content_path, 'dataset', 'text.txt') - with open(file_path, 'r') as f: + file_path = os.path.join(content_path, "dataset", "text.txt") + with open(file_path, "r") as f: content = f.read() lines = content.splitlines() if index < len(lines): @@ -258,11 +261,11 @@ def load_one_text(content_path, index): return "" def calculate_high_dimensional_neighbors(content_path, epoch, max_neighbors=10): - featrue_list = load_single_attribute(content_path, epoch, 'representation') + featrue_list = load_single_attribute(content_path, epoch, "representation") features = np.array(featrue_list) num_samples = len(features) - nbrs = NearestNeighbors(n_neighbors=max_neighbors + 1, algorithm='auto').fit(features) + nbrs = NearestNeighbors(n_neighbors=max_neighbors + 1, algorithm="auto").fit(features) distances, indices = nbrs.kneighbors(features) neighbors = [[] for _ in range(num_samples)] @@ -278,7 +281,7 @@ def calculate_projection_neighbors(content_path, vis_id, epoch, max_neighbors=10 projection = np.array(projection_list) num_samples = len(projection) - nbrs = NearestNeighbors(n_neighbors=max_neighbors + 1, algorithm='auto').fit(projection) + nbrs = NearestNeighbors(n_neighbors=max_neighbors + 1, algorithm="auto").fit(projection) distances, indices = nbrs.kneighbors(projection) neighbors = [[] for _ in range(num_samples)] @@ -289,32 +292,65 @@ def calculate_projection_neighbors(content_path, vis_id, epoch, max_neighbors=10 return neighbors +def calculate_neighbors_for_point(content_path, vis_id, epoch, point_index, max_neighbors=10): + feature_ls = load_single_attribute(content_path, epoch, "representation") + features = np.array(feature_ls) + neighbors = NearestNeighbors(n_neighbors=max_neighbors + 1, algorithm="auto").fit(features) + + sample_feature = features[point_index].reshape(1, -1) + distances, indices = neighbors.kneighbors(sample_feature) + + neighbors_ls = list() + + for nbr in range(1, max_neighbors + 1): + neighbor_idx = indices[0][nbr] + neighbors_ls.append(int(neighbor_idx)) + + return neighbors_ls + +def calculate_projection_neighbors_for_point(content_path, vis_id, epoch, point_index, max_neighbors=10): + + # TODO: maybe use PyNNDescent so its faster (?) + projection_ls = load_projection(content_path, vis_id, epoch) + projection = np.array(projection_ls) + neighbors = NearestNeighbors(n_neighbors=max_neighbors + 1, algorithm="auto").fit(projection) + + sample_projection = projection[point_index].reshape(1, -1) + distances, indices = neighbors.kneighbors(sample_projection) + + neighbors_ls = list() + + for nbr in range(1, max_neighbors + 1): + neighbor_idx = indices[0][nbr] + neighbors_ls.append(int(neighbor_idx)) + + return neighbors_ls # Func: Load a single attribute from a file based on the configuration and epoch def load_single_attribute(content_path, epoch, attribute): - if attribute == 'label': - file_path = os.path.join(content_path, 'dataset', 'labels.npy') + if attribute == "label": + file_path = os.path.join(content_path, "dataset", "labels.npy") attr_data = read_label_file(file_path) - elif attribute == 'intra_similarity': - file_path = os.path.join(content_path, 'epochs', f'epoch_{epoch}', 'intra_similarity.npy') + elif attribute == "intra_similarity": + file_path = os.path.join(content_path, "epochs", f"epoch_{epoch}", "intra_similarity.npy") attr_data = read_from_file(file_path) - elif attribute == 'inter_similarity': - file_path = os.path.join(content_path, 'epochs', f'epoch_{epoch}', 'inter_similarity.npy') + elif attribute == "inter_similarity": + file_path = os.path.join(content_path, "epochs", f"epoch_{epoch}", "inter_similarity.npy") attr_data = read_from_file(file_path) - elif attribute == 'representation': - file_path = os.path.join(content_path, 'epochs', f'epoch_{epoch}', 'embeddings.npy') + elif attribute == "representation": + file_path = os.path.join(content_path, "epochs", f"epoch_{epoch}", "embeddings.npy") attr_data = read_from_file(file_path) - elif attribute == 'prediction': - file_path = os.path.join(content_path, 'epochs', f'epoch_{epoch}', 'predictions.npy') + elif attribute == "prediction": + file_path = os.path.join(content_path, "epochs", f"epoch_{epoch}", "predictions.npy") attr_data = read_from_file(file_path) - elif attribute == 'index': + elif attribute == "index": attr_data = load_or_create_index(content_path) else: raise NotImplementedError(f"Unknown attribute: {attribute}") index_dict = load_or_create_index(content_path) - all_indices = index_dict['train'] + index_dict['test'] - if attribute != 'index': + all_indices = index_dict["train"] + index_dict["test"] + if attribute != "index": attr_data = [attr_data[i] for i in all_indices] return attr_data @@ -322,10 +358,10 @@ def load_single_attribute(content_path, epoch, attribute): def read_from_file(file_path): _, file_extension = os.path.splitext(file_path) - if file_extension == '.npy': + if file_extension == ".npy": data = np.load(file_path) result = data.tolist() - elif file_extension == '.pth': + elif file_extension == ".pth": data = torch.load(file_path) if isinstance(data, torch.Tensor): result = data.tolist() @@ -333,9 +369,9 @@ def read_from_file(file_path): result = data else: raise ValueError(f"Unsupported data type in .pth file: {type(data)}") - elif file_extension == '.json': + elif file_extension == ".json": try: - with open(file_path, 'r') as f: + with open(file_path, "r") as f: result = json.load(f) except Exception as e: raise ValueError(f"Error in reading json file from {file_path}: {e}") @@ -352,24 +388,24 @@ def read_file_as_json(file_path: str): return json.load(f) def load_or_create_index(content_path): - index_file_path = os.path.join(content_path, 'dataset', 'index.json') + index_file_path = os.path.join(content_path, "dataset", "index.json") if os.path.exists(index_file_path): - with open(index_file_path, 'r') as f: + with open(index_file_path, "r") as f: index_data = json.load(f) return index_data # If index.json does not exist, create it - file_path = os.path.join(content_path, 'dataset', 'labels.npy') + file_path = os.path.join(content_path, "dataset", "labels.npy") labels = read_label_file(file_path) num_samples = len(labels) index_data = { - 'train': list(range(num_samples)), - 'test': [] + "train": list(range(num_samples)), + "test": [] } # Save the index data to a file - with open(index_file_path, 'w') as f: + with open(index_file_path, "w") as f: json.dump(index_data, f) return index_data @@ -456,8 +492,8 @@ def prediction_attribution(content_path, epoch, training_event, num_samples=10): import model as subject_model info = read_file_as_json(os.path.join(content_path, "dataset", "info.json")) - model = eval("subject_model.{}()".format(info['model'])) - classes = info['classes'] + model = eval("subject_model.{}()".format(info["model"])) + classes = info["classes"] subject_model_location = os.path.join(content_path, "epochs", f"epoch_{epoch}", "model.pth") device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu") model.load_state_dict(torch.load(subject_model_location, map_location=torch.device("cpu"))) @@ -466,7 +502,7 @@ def prediction_attribution(content_path, epoch, training_event, num_samples=10): # construct dataloader dataset_path = os.path.join(content_path, "dataset") - cifar_path = os.path.join(dataset_path, 'cifar-10-batches-py') + cifar_path = os.path.join(dataset_path, "cifar-10-batches-py") download = not os.path.exists(cifar_path) or not os.listdir(cifar_path) transform_train = transforms.Compose([ @@ -487,19 +523,19 @@ def prediction_attribution(content_path, epoch, training_event, num_samples=10): IF = EmpiricalIF(dl_train=trainloader, model=model, - param_filter_fn=lambda name, param: 'classifier' in name, + param_filter_fn=lambda name, param: "classifier" in name, criterion=torch.nn.CrossEntropyLoss(reduction="none")) - test_sample = trainloader.dataset[training_event['index']] + test_sample = trainloader.dataset[training_event["index"]] test_input, _ = test_sample test_input = test_input.unsqueeze(0) # Add batch dimension - test_target = torch.tensor([classes.index(training_event['influenceTarget'])]).to(device) # Add batch dimension + test_target = torch.tensor([classes.index(training_event["influenceTarget"])]).to(device) # Add batch dimension IF_scores = IF.query_influence(test_input, test_target) # Get the indices of the top num_samples maximum and minimum scores max_indices = np.argsort(IF_scores)[-num_samples:][::-1] - labels = load_single_attribute(content_path, epoch, 'label') + labels = load_single_attribute(content_path, epoch, "label") influence_samples = [] for index in max_indices.tolist(): @@ -516,11 +552,11 @@ class CodeSearchNetDataset(Dataset): def __init__(self, file_path, tokenizer, sample_limit=None): self.samples = [] count = 0 - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, "r", encoding="utf-8") as f: for line in tqdm(f, desc="读取数据集"): line_data = json.loads(line) - docstring_tensor = tokenizer(line_data['docstring'], padding='max_length', truncation=True, max_length=256, return_tensors='pt')['input_ids'].squeeze(0) - code_tensor = tokenizer(line_data['code'], padding='max_length', truncation=True, max_length=256, return_tensors='pt')['input_ids'].squeeze(0) + docstring_tensor = tokenizer(line_data["docstring"], padding="max_length", truncation=True, max_length=256, return_tensors="pt")["input_ids"].squeeze(0) + code_tensor = tokenizer(line_data["code"], padding="max_length", truncation=True, max_length=256, return_tensors="pt")["input_ids"].squeeze(0) self.samples.append((docstring_tensor, code_tensor)) count += 1 if sample_limit and count > sample_limit: @@ -536,7 +572,7 @@ def __getitem__(self, idx): def movement_attribution(content_path, epoch, training_event, num_samples=10): # define and load subject model device = torch.device("cuda:3" if torch.cuda.is_available() else "cpu") - tokenizer = RobertaTokenizer.from_pretrained('/home/kwy/models/codebert-base') + tokenizer = RobertaTokenizer.from_pretrained("/home/kwy/models/codebert-base") subject_model_location = os.path.join(content_path, "epochs", f"epoch_{epoch}", "model.pth") model = CustomEncoderModel( @@ -553,8 +589,8 @@ def movement_attribution(content_path, epoch, training_event, num_samples=10): trainloader = DataLoader(train_dataset, batch_size=128, shuffle=False) # sub-sample (code or doc) index - index = training_event['index'] - index1 = training_event['index1'] + index = training_event["index"] + index1 = training_event["index1"] # convert to original sampel index ori_index = int(index / 2) @@ -571,7 +607,7 @@ def movement_attribution(content_path, epoch, training_event, num_samples=10): query_input_part2 = ori_sample1[tp1] # code_tensor # init IF - pairwise_if = PairWiseEmpiricalIF(dl_train=trainloader,model=model,param_filter_fn=lambda name, param: 'transformer_encoder' in name) + pairwise_if = PairWiseEmpiricalIF(dl_train=trainloader,model=model,param_filter_fn=lambda name, param: "transformer_encoder" in name) influences_case = pairwise_if.query_influence(query_input_part1, query_input_part2, query_is_positive=(ori_index == ori_index1)) print("Influence case:", influences_case[:3]) @@ -593,4 +629,93 @@ def compute_training_events(content_path, epoch, event_types): data_provider = DataProvider(config) detector = TrainingEventDetector(content_path, epoch, data_provider) events = detector.detect_events(event_types) - return events \ No newline at end of file + return events + + +def local_refine(content_path, epoch, sample_index, vis_id, k=10): + learning_rate = 0.0001 + steps = 10 + # load embeddings + epoch_path = os.path.join(content_path, "epochs", f"epoch_{epoch}", "embeddings.npy") + embeddings = np.load(epoch_path) + + # find local neigbors + nbrs = NearestNeighbors(n_neighbors=k + 1).fit(embeddings) + _, indices = nbrs.kneighbors([embeddings[sample_index]]) + local_indices = indices[0] + local_embeddings = embeddings[local_indices] + + + #initialize timevis model + device = torch.device("cpu") + checkpoint_path = os.path.join(content_path, "visualize", vis_id, "vis_model.pth") + checkpoint = torch.load(checkpoint_path, map_location=device) + state_dict = checkpoint["state_dict"] + + # tried to pass it from the frontend but couldnt make it work so i get it from the state + encoder_keys_ls = sorted([k for k in state_dict if k.startswith("encoder") and "weight" in k]) + decoder_keys_ls = sorted([k for k in state_dict if k.startswith("decoder") and "weight" in k]) + + encoder_dims_ls = [state_dict[encoder_keys_ls[0]].shape[1]] + [state_dict[k].shape[0] for k in encoder_keys_ls] + decoder_dims_ls = [state_dict[decoder_keys_ls[0]].shape[1]] + [state_dict[k].shape[0] for k in decoder_keys_ls] + + model = VisModel(encoder_dims_ls, decoder_dims_ls).to(device) + model.load_state_dict(state_dict) + + + # build knn again but for the selected point + n_neighbors = min(5, len(local_embeddings) - 1) + local_neighbors = NearestNeighbors(n_neighbors=n_neighbors + 1).fit(local_embeddings) + _, local_knn = local_neighbors.kneighbors(local_embeddings) + + # connectios to re train UMAP + edge_to_ls, edge_from_ls = [], [] + for i, neighbors in enumerate(local_knn): + for j in neighbors[1:]: + edge_to_ls.append(i) + edge_from_ls.append(j) + + edge_to_index_ls = np.array(edge_to_ls) + edge_from_index_ls = np.array(edge_from_ls) + + # define losses, copypaste from timevis_strategy.py + negative_sample_rate = 5 + min_dist = 0.1 + _a, _b = find_ab_params(1.0, min_dist) + umap_fn = UmapLoss(negative_sample_rate, device, _a, _b, repulsion_strength=1.0) + recon_fn = ReconstructionLoss(beta=1.0) + criterion = SingleVisLoss(umap_fn, recon_fn, lambd=1) + + local_tensor = torch.tensor(local_embeddings, dtype=torch.float32).to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + model.train() + + # loop for fine tuning + for i in range(steps): + optimizer.zero_grad() + edge_to_feat = local_tensor[edge_to_index_ls] + edge_from_feat = local_tensor[edge_from_index_ls] + a_to = torch.zeros(len(edge_to_index_ls), 1, dtype=torch.float32).to(device) + a_from = torch.zeros(len(edge_from_index_ls), 1, dtype=torch.float32).to(device) + outputs = model(edge_to_feat, edge_from_feat) + umao_loss, recon_loss, loss = criterion(edge_to_feat, edge_from_feat, a_to, a_from, outputs) + loss.backward() + optimizer.step() + + model.eval() + with torch.no_grad(): + outputs = model(local_tensor, local_tensor) + refined_2d = outputs["umap"][0].cpu().numpy() + + + updated_coords_dd = {} + for local_pos, global_idx in enumerate(local_indices): + updated_coords_dd[str(int(global_idx))] = refined_2d[local_pos].tolist() + + return updated_coords_dd + + + + + + \ No newline at end of file diff --git a/web/src/communication/backend.ts b/web/src/communication/backend.ts index 5fbad593..ad3a25f0 100644 --- a/web/src/communication/backend.ts +++ b/web/src/communication/backend.ts @@ -88,6 +88,25 @@ export async function fetchEpochProjection( return basicPostWithJsonResponse('/updateProjection', data, options); } + +export async function refineProjection( + contentPath: string, + visId: string, + epoch: number, + sampleIndex: number, + visConfig: any, + options?: NetworkOptions +) { + const data = { + "content_path": contentPath, + "vis_id": visId, + "sample_index": sampleIndex, + "vis_config": visConfig, + "epoch": `${epoch}`, + }; + return basicPostWithJsonResponse('/refineProjection', data, options); +} + export function getText(contentPath: string, options?: NetworkOptions) { const data = { "content_path": contentPath @@ -224,4 +243,23 @@ export function calculateTrainingEvents( export function testConnection(message: string, options?: NetworkOptions) { const data = { message }; return basicPostWithJsonResponse('/testConnection', data, options); +} + +// new function to fetch all necessary data for a specific epoch in one request +export function getNeighborsForSample( + contentPath: string, + visId: string, + epoch: number, + sampleIndex: number, + options?: NetworkOptions +) { + const data = { + "content_path": contentPath, + "vis_id": visId, + "epoch": epoch, + "sample_index": sampleIndex + }; + + // request to the backend to fetch both original and projection neighbors for the hovered sample + return basicPostWithJsonResponse('/getNeighborsForSample', data, options); } \ No newline at end of file diff --git a/web/src/component/chart.tsx b/web/src/component/chart.tsx index 0aaabf23..4edcc2cb 100644 --- a/web/src/component/chart.tsx +++ b/web/src/component/chart.tsx @@ -1,8 +1,9 @@ // ChartComponent.tsx -import { memo, useEffect, useMemo, useRef, useState } from 'react'; +import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { EmbeddingView, type EmbeddingViewProps, type DataPoint, type ViewportState } from 'embedding-atlas/react'; import { useDefaultStore } from "../state/state.unified"; import { transferArray2Color } from './utils'; +import * as BackendAPI from '../communication/backend'; type EmbeddingData = NonNullable; @@ -29,6 +30,10 @@ export const ChartComponent = memo(() => { const { availableEpochs } = useDefaultStore(["availableEpochs"]); const { showTrail } = useDefaultStore(["showTrail"]); const { setSelectedIndices } = useDefaultStore(["setSelectedIndices"]); + // added for on demand calls and cache + const { contentPath, visId, neighborCache, setValue } = useDefaultStore(["contentPath", "visId", "neighborCache", "setValue"]); + + const [isFetchingNeighbors, setIsFetchingNeighbors] = useState(false); const epochData = allEpochData[epoch]; @@ -37,6 +42,37 @@ export const ChartComponent = memo(() => { // selection can be added later when needed let [viewportState, setViewportState] = useState(null); + // define hoveredIndex so it doesnt trigger the call to get neighbors when i hover on a point + useEffect(() => { + setHoveredIndex(undefined); + }, [epochData, contentPath]); + + // fetch neighbors when user clicks on a hovered point and not when i just hoiver over it + const handleChartClick = useCallback(() => { + if (!tooltip) return + if (!contentPath || !visId || epoch == null) return + if (!revealOriginalNeighbors && !revealProjectionNeighbors) return + + const clickedIndex = tooltip.identifier as number + setSelectedIndices([clickedIndex]) + const cacheKey = `${epoch}-${clickedIndex}` + if (neighborCache[cacheKey]) return + + setIsFetchingNeighbors(true) + BackendAPI.getNeighborsForSample(contentPath, visId, epoch, clickedIndex) + .then((result: any) => { + setValue('neighborCache', { + ...neighborCache, + [cacheKey]: { + originalNeighbors: result.originalNeighbors || result.neighbors || [], + projectionNeighbors: result.projectionNeighbors || result.projection_neighbors || [], + } + }); + }) + .catch((err: any) => console.warn('Failed to fetch neighbors:', err)) + .finally(() => setIsFetchingNeighbors(false)) + }, [tooltip, epoch, contentPath, visId, revealOriginalNeighbors, revealProjectionNeighbors, neighborCache, setValue]); + // observe container size change useEffect(() => { const node = atlasRef.current; @@ -193,9 +229,12 @@ export const ChartComponent = memo(() => { if (!prepared || !epochData) return { center: null, original: [], projection: [], dataX: new Float32Array(0), dataY: new Float32Array(0), pointSize, revealOriginalNeighbors, revealProjectionNeighbors } as any; const idsByPos = prepared.dataPoints.map((p) => p.identifier as number); if (!tooltip) return { center: null, original: [], projection: [], dataX: prepared.simpleData.x as Float32Array, dataY: prepared.simpleData.y as Float32Array, pointSize, revealOriginalNeighbors, revealProjectionNeighbors, idsByPos, showLabel, showIndex, labelDict, textData, inherentLabelData, viewportState, showTrail, availableEpochs, allEpochData, currentEpoch: epoch, setSelectedIndices, selectedIndices } as any; + // now read from cache const hoverId = tooltip.identifier as number; - const orig = (epochData.originalNeighbors?.[hoverId] ?? []).filter((nid) => posMap.has(nid)); - const proj = (epochData.projectionNeighbors?.[hoverId] ?? []).filter((nid) => posMap.has(nid)); + const cacheKey = `${epoch}-${hoverId}`; + const cached = neighborCache[cacheKey]; + const orig = (cached?.originalNeighbors ?? []).filter((nid: number) => posMap.has(nid)); + const proj = (cached?.projectionNeighbors ?? []).filter((nid: number) => posMap.has(nid)); return { center: tooltip, original: orig, @@ -219,7 +258,7 @@ export const ChartComponent = memo(() => { setSelectedIndices, selectedIndices, }; - }, [prepared, epochData, tooltip, posMap, pointSize, revealOriginalNeighbors, revealProjectionNeighbors, showLabel, showIndex, labelDict, textData, inherentLabelData, viewportState, showTrail, availableEpochs, allEpochData, epoch, trailRefresh, selectedIndices]); + }, [prepared, epochData, tooltip, posMap, pointSize, revealOriginalNeighbors, revealProjectionNeighbors, showLabel, showIndex, labelDict, textData, inherentLabelData, viewportState, showTrail, availableEpochs, allEpochData, epoch, trailRefresh, selectedIndices, neighborCache]); class NeighborOverlay { private el: HTMLDivElement | null = null; @@ -293,7 +332,6 @@ export const ChartComponent = memo(() => { if (minIdx >= 0 && minD2 <= threshold) { const id = this.props.idsByPos[minIdx]; if (this.props.setSelectedIndices) { - console.log(`Click on id ${id}`); this.props.setSelectedIndices([id]); } } @@ -543,9 +581,20 @@ export const ChartComponent = memo(() => { width: '100%', height: '100%', }} + onClick={handleChartClick} >
{content ??
} + + {/* show loading bar when click point and fetch neighbors */} + {isFetchingNeighbors && ( +
+ Loading neighbors... +
+
+
+
+ )}
); diff --git a/web/src/component/custom/basic-components.tsx b/web/src/component/custom/basic-components.tsx index 50d98f94..69795e7f 100644 --- a/web/src/component/custom/basic-components.tsx +++ b/web/src/component/custom/basic-components.tsx @@ -1,14 +1,46 @@ +import { Collapse } from "antd"; +import { HolderOutlined } from "@ant-design/icons"; + // TODO put these blocks to a universal file // TODO add resize/drag/dock-to mouse interaction -export function FunctionalBlock(props: { label?: string; children?: null | React.ReactNode | React.ReactNode[]; }) { + +type FunctionalBlockProps = { + label?: string; + children?: React.ReactNode; + defaultCollapsed?: boolean; + dragHandleProps?: React.HTMLAttributes; +}; + +export function FunctionalBlock(props: FunctionalBlockProps) { + if (!props.label) return
{props.children}
; + + const header = ( +
+ {props.dragHandleProps && ( + e.stopPropagation()} + > + + + )} + {props.label} +
+ ); + return ( -
- {props.label &&
{props.label}
} - {props.children} +
+
); } -export function ComponentBlock(props: { label?: string; children?: null | React.ReactNode | React.ReactNode[]; }) { + +export function ComponentBlock(props: { label?: string; children?: React.ReactNode }) { return (
{props.label &&
{props.label}
} diff --git a/web/src/component/function-panel.tsx b/web/src/component/function-panel.tsx index dcb8517d..cabf7772 100644 --- a/web/src/component/function-panel.tsx +++ b/web/src/component/function-panel.tsx @@ -1,20 +1,14 @@ -import { AutoComplete, Input, List, Tag, RefSelectProps, Checkbox, Switch, Select, Slider } from 'antd'; -import { useDefaultStore } from '../state/state.unified'; -import { useEffect, useRef, useState } from 'react'; -import { ComponentBlock, FunctionalBlock } from './custom/basic-components'; -import { styled } from 'styled-components'; - +import { AutoComplete, Input, List, Tag, RefSelectProps, Checkbox, Switch, Select, Slider, Collapse, Button } from "antd"; +import { useDefaultStore } from "../state/state.unified"; +import { useEffect, useRef, useState } from "react"; +import { ComponentBlock, FunctionalBlock } from "./custom/basic-components"; +import { styled } from "styled-components"; +import { refineProjection } from '../communication/backend'; type SampleTag = { num: number; title: string; } -interface LabelProps { - label: string; - colorArray: number[]; - onColorChange: (newColor: [number, number, number]) => void; -} - const CompactCheckboxGroup = styled(Checkbox.Group)` display: flex; flex-wrap: wrap; @@ -52,61 +46,23 @@ const CompactCheckboxGroup = styled(Checkbox.Group)` } `; -const ColoredClassLabel: React.FC = ({ label, colorArray, onColorChange }) => { - const inputRef = useRef(null); - +function rgbArrToHex(rgbArray: number[]) { + return "#" + rgbArray.map(c => c.toString(16).padStart(2, "0")).join(""); +} - function setColorPickerOpacity(value: number) { - const colorPickerItem = inputRef.current; - if (!colorPickerItem) return; - if (value) { - colorPickerItem.style.opacity = '1'; - colorPickerItem.style.pointerEvents = 'auto'; - } else { - colorPickerItem.style.opacity = '0'; - colorPickerItem.style.pointerEvents = 'none'; - } - } +export function FunctionPanel() { - return ( -
setColorPickerOpacity(1)} - onMouseLeave={() => setColorPickerOpacity(0)} - > - onColorChange(hexToRgbArray((e.target as HTMLInputElement).value))} - /> - - {label} - -
- ) -} -function rgbArrToHex(rgbArray: number[]) { - return '#' + rgbArray.map(c => c.toString(16).padStart(2, '0')).join(''); -} -function hexToRgbArray(hex: string): [number, number, number] { - hex = hex.replace(/^#/, ''); - const bigint = parseInt(hex, 16); - const r = (bigint >> 16) & 255; - const g = (bigint >> 8) & 255; - const b = bigint & 255; - return [r, g, b]; -} -export function FunctionPanel() { - const { tokenList, labelDict, colorDict, setColorDict, selectedIndices, setSelectedIndices, setShownData, pointSize, setPointSize, mode, setMode } = - useDefaultStore(["tokenList","labelDict", "colorDict", "setColorDict", "selectedIndices", "setSelectedIndices", "setShownData", "pointSize", "setPointSize", "mode", "setMode"]); + const { tokenList, labelDict, colorDict, selectedIndices, setSelectedIndices, setShownData, pointSize, setPointSize, mode, setMode } = + useDefaultStore(["tokenList", "labelDict", "colorDict", "selectedIndices", "setSelectedIndices", "setShownData", "pointSize", "setPointSize", "mode", "setMode"]); const { revealOriginalNeighbors, revealProjectionNeighbors, setRevealOriginalNeighbors, setRevealProjectionNeighbors } = useDefaultStore(["revealOriginalNeighbors", "revealProjectionNeighbors", "setRevealOriginalNeighbors", "setRevealProjectionNeighbors"]); const { showIndex, showLabel, showBackground, showTrail, setShowIndex, setShowLabel, setShowBackground, setShowTrail } = - useDefaultStore(["showIndex","showLabel","showBackground","showTrail","setShowIndex","setShowLabel","setShowBackground","setShowTrail"]); + useDefaultStore(["showIndex", "showLabel", "showBackground", "showTrail", "setShowIndex", "setShowLabel", "setShowBackground", "setShowTrail"]); + const { inherentLabelData } = useDefaultStore(["inherentLabelData"]); + useEffect(() => { if (pointSize < 1) { @@ -116,42 +72,45 @@ export function FunctionPanel() { } }, [pointSize, setPointSize]); - const pointSizeMarks: Record = { 1: '1', 2: '2', 3: '3', 4: '4', 5: '5' }; - const pointSizeLabel = pointSizeMarks[pointSize] ?? pointSize.toString(); - - function changeLabelColor(i: number, newColor: [number, number, number]) { - setColorDict(new Map([...colorDict, [i, newColor]])); - } - - // NOTE always add state as middle dependency - const [searchValue, setSearchValue] = useState(''); - const { tokenList: searchFromOptions } = useDefaultStore(['tokenList']); + const pointSizeMarks: Record = { 1: "1", 2: "2", 3: "3", 4: "4", 5: "5" }; - const limitOfHistory = 5; + const [searchValue, setSearchValue] = useState(""); + const { tokenList: searchFromOptions } = useDefaultStore(["tokenList"]); const [searchHistory, setSearchHistory] = useState([]); const searchHistoryFiltered = searchHistory.filter((item) => item.includes(searchValue)); const [searchHistoryOpen, setSearchHistoryOpen] = useState(false); const searchElementRef = useRef(null); - const [allSearchResult, setAllSearchResult] = useState([]); - const searchFrom = (text: string, items: SampleTag[], limit: number | null = 3) => { - const res: SampleTag[] = []; + const { visConfig, contentPath, visId, epoch, patchEpochProjection } = useDefaultStore(["visConfig", "contentPath", "visId", "epoch", "patchEpochProjection"]); + const [isRefining, setIsRefining] = useState(false); - let cnt = 0; + + const handleRefine = async () => { + if (selectedIndices.length === 0) return; + const sampleIndex = selectedIndices[0]; + try { + setIsRefining(true); + const { updated_coords } = await refineProjection( + contentPath, visId, epoch, sampleIndex, visConfig + ); + patchEpochProjection(epoch, updated_coords); + } finally { + setIsRefining(false); + } + }; + const searchFrom = (text: string, items: SampleTag[], limit: number | null = 3) => { + const lower = text.toLowerCase(); + const results: SampleTag[] = []; for (const item of items) { - if (item.title.toLowerCase().includes(text.toLowerCase())) { - if (limit !== null && cnt >= limit) { - return res; - } - res.push(item); - cnt++; + if (item.title.toLowerCase().includes(lower)) { + results.push(item); + if (limit !== null && results.length >= limit) break; } } - - return res; - } + return results; + }; const handleSearch = (text: string, byEnter: boolean = false) => { if (text === searchValue) return; @@ -180,25 +139,15 @@ export function FunctionPanel() { }; const addHistory = (text: string) => { if (!text) return; - - const nonDuplicateHistory = searchHistory.filter((item) => item !== text); - setSearchHistory([text, ...nonDuplicateHistory].slice(0, limitOfHistory)); - } - const renderSearchHistoryOption = (text: string) => { - return { - value: text, - label: text - } - } - const searchHistoryRender = (history: string[]) => { - return history.map(renderSearchHistoryOption); - } + setSearchHistory([text, ...searchHistory.filter((h) => h !== text)].slice(0, 5)); + }; + const searchHistoryRender = (history: string[]) => history.map((text) => ({ value: text, label: text })); const searchResultRender = (item: SampleTag) => { return ( { const newSelectedIndices = selectedIndices.includes(item.num) ? selectedIndices.filter(i => i !== item.num) @@ -229,223 +178,207 @@ export function FunctionPanel() { useEffect(() => { setSelectedItems(Array.from(selectedIndices).map((num) => ({ num, - title: tokenList ? tokenList[num] ?? '' : '' + title: tokenList ? tokenList[num] ?? "" : "" }))); }, [selectedIndices, tokenList]); return (
- - { handleSearch(value) }} - onBlur={() => { - addHistory(searchValue); // TODO only add successful history - setSearchHistoryOpen(false); - }} - onFocus={() => handleSearch(searchValue)} - onKeyDown={(e: { key: string; }) => { - if (e.key === 'Enter') { - handleSearch(searchValue, true); - setSearchHistoryOpen(false); - } else if (e.key === 'Escape') { - searchElementRef.current?.blur(); - } - }} - onSelect={() => { - searchElementRef.current?.blur(); - }} - onClear={() => { - setSearchHistoryOpen(false); - }} - defaultActiveFirstOption={false} - notFoundContent={
No item found
} - allowClear - > - { - setSearchHistoryOpen(true); - }} /> - - { - (allSearchResult.length > 0 || searchValue !== '') - && - - { - allSearchResult.length > 0 - ? - ( - - ) - : - (searchValue &&
No item found
) + + { handleSearch(value) }} + onBlur={() => { + addHistory(searchValue); // TODO only add successful history + setSearchHistoryOpen(false); + }} + onFocus={() => handleSearch(searchValue)} + onKeyDown={(e: { key: string; }) => { + if (e.key === 'Enter') { + handleSearch(searchValue, true); + setSearchHistoryOpen(false); + } else if (e.key === 'Escape') { + searchElementRef.current?.blur(); + } + }} + onSelect={() => { + searchElementRef.current?.blur(); + }} + onClear={() => { + setSearchHistoryOpen(false); + }} + defaultActiveFirstOption={false} + notFoundContent={
No item found
} + allowClear + > + { + setSearchHistoryOpen(true); + }} /> + + {(allSearchResult.length > 0 || searchValue !== "") + && + + { + allSearchResult.length > 0 + ? + + : + ( + searchValue &&
No item found
) + } + } - + } - - - -
- { - Array.from(labelDict.keys()).length - ? - Array.from(labelDict.keys()).map((labelNum) => - changeLabelColor(labelNum, newColor)} - /> - ) - : -
No class is determined
- } -
- - - - -
- { - selectedItems.length - ? - selectedItems.map((item) => ( - void; }) => { - e.preventDefault(); - handleClose(item); - }} - onClose={(e: { preventDefault: () => void; }) => { - e.preventDefault(); - handleClose(item); - }} - key={item.num} - > - {item.num}. {item.title} - - )) - : -
No selected item
- } -
- - - - -
-
- Point Size - setPointSize(v as number)} - style={{ minWidth: 80, flex: 1 }} - /> + selectedBlock={ + +
+ { + selectedItems.length + ? + selectedItems.map((item) => ( + void; }) => { + e.preventDefault(); + handleClose(item); + }} + onClose={(e: { preventDefault: () => void; }) => { + e.preventDefault(); + handleClose(item); + }} + key={item.num} + > + {item.num}. {item.title} + + )) + : +
No selected item
+ }
-
- Mode - setMode(v)} + options={[{ label: "Points", value: "points" }, { label: "Density", value: "density" }]} /> +
+
+ Neighbors + { - if (v === 'none') { - setRevealOriginalNeighbors(false); - setRevealProjectionNeighbors(false); - } else if (v === 'original') { - setRevealOriginalNeighbors(true); - setRevealProjectionNeighbors(false); - } else if (v === 'projection') { - setRevealOriginalNeighbors(false); - setRevealProjectionNeighbors(true); - } else if (v === 'both') { - setRevealOriginalNeighbors(true); - setRevealProjectionNeighbors(true); - } - }} + + } + filterBlock={ + +
+ { setShownData(checkedValues as string[]); }} />
-
- Display -
-
-
- Show Label - setShowLabel(v)} /> -
-
- Show Index - setShowIndex(v)} /> -
-
- Show Trail - setShowTrail(v)} /> -
-
- Show Background - setShowBackground(v)} /> -
-
-
-
-
- - - - -
- { - setShownData(checkedValues as string[]); - }} - /> -
- - - - - + + } + highlightBlock={} + />
) } @@ -472,7 +405,7 @@ function HighlightOptionBlock() { setHighlightData(enabledTypes); }; - const renderHighlightTypeItem = (highlight: { type: string, label: string, enabled: boolean, icon: string, description: string}) => { + const renderHighlightTypeItem = (highlight: { type: string, label: string, enabled: boolean, icon: string, description: string }) => { return (
); +} +function ColorLegendPanel({ colorDict, labelDict, inherentLabelData }: { + colorDict: Map, + labelDict: Map, + inherentLabelData: number[] +}) { + const classCounts: Record = {} + inherentLabelData.forEach((label) => { + classCounts[label] = (classCounts[label] || 0) + 1 + }); + + if (labelDict.size === 0) { + return null + } + + return ( +
+ Color Legend, + children: ( +
+ {Array.from(labelDict.entries()).map(([labelNum, labelName]) => { + const color = colorDict.get(labelNum); + const colorHex = color ? rgbArrToHex(color) : "#888888"; + return ( +
+
+ {labelName} + ({classCounts[labelNum] || 0}) +
+ ); + })} +
+ ) + }]} /> +
+ ); +} + +type BlockId = "search" | "selected" | "legend" | "settings" | "filter" | "highlight" + +function DraggableBlockList(props: { + searchBlock: React.ReactNode + selectedBlock: React.ReactNode + legendBlock: React.ReactNode + settingsBlock: React.ReactNode + filterBlock: React.ReactNode + highlightBlock: React.ReactNode +}) { + const [order, setOrder] = useState(["search", "selected", "legend", "settings", "filter", "highlight"]); + const dragItem = useRef(null) + const dragOverItem = useRef(null) + + const blocks: Record = { + search: { label: "Search", content: props.searchBlock }, + selected: { label: "Selected", content: props.selectedBlock }, + legend: { label: "Color Legend", content: props.legendBlock }, + settings: { label: "Settings", content: props.settingsBlock }, + filter: { label: "Filter", content: props.filterBlock }, + highlight: { label: "Highlight", content: props.highlightBlock }, + }; + + const onDragEnd = () => { + const from = dragItem.current + const to = dragOverItem.current + if (!from || !to || from === to) return + const next = [...order] + next.splice(next.indexOf(from), 1) + next.splice(next.indexOf(to), 0, from) + setOrder(next) + dragItem.current = null + dragOverItem.current = null + }; + + return ( + <> + {order.map((id) => ( +
{ dragItem.current = id }} + onDragEnter={() => { dragOverItem.current = id }} + onDragEnd={onDragEnd} + onDragOver={(e) => e.preventDefault()} + > + {id === "legend" + ? props.legendBlock + : ( + e.stopPropagation() }}> + {blocks[id].content} + + ) + } +
+ ))} + + ); } \ No newline at end of file diff --git a/web/src/component/main-block.tsx b/web/src/component/main-block.tsx index 13dcfbc9..a36d742b 100644 --- a/web/src/component/main-block.tsx +++ b/web/src/component/main-block.tsx @@ -128,7 +128,7 @@ function Timeline({ epoch, epochs, progress, onSwitchEpoch }: { epoch: number, e // Render nodes and links (simple lines between nodes) return ( -
+
= 100) return null; + + const loadedEpochs = Math.round((progress / 100) * totalEpochs); + + return ( +
+
+ Loading epoch {loadedEpochs} / {totalEpochs} +
+
+
+
+
+ {Math.round(progress)}% +
+
+ ); +} + export function MainBlock() { const { epoch, setEpoch } = useDefaultStore(['epoch', 'setEpoch']); const { availableEpochs } = useDefaultStore(['availableEpochs']); @@ -237,9 +268,12 @@ export function MainBlock() { // only consider single container for now return (
- +
+ + +