Add lazy loading for epoch data and fix state updates

requirements.txt

@@ -1,33 +1,7 @@
-cycler==0.11.0
-Flask==2.2.5
-Flask_Cors==4.0.0
-fonttools==4.38.0
-importlib-metadata==6.7.0
-Jinja2==3.1.2
-joblib==1.3.2
-kiwisolver==1.4.5
-kmapper==2.0.1
-llvmlite==0.39.1
-MarkupSafe==2.1.3
-matplotlib==3.5.2
-networkx==2.6.3
-numba==0.56.4
-numpy==1.21.6
-packaging==23.2
-pandas==1.3.5
-Pillow==9.2.0
-pymilvus==2.1.0
-pynndescent==0.5.11
-pyparsing==3.1.1
-python-dateutil==2.8.2
-scikit-learn==1.0.2
-scipy==1.7.3
-six==1.16.0
-tensorflow==2.7.0
-threadpoolctl==3.1.0
-torch==1.13.1
-tqdm==4.66.1
-typing_extensions==4.7.1
-umap==0.1.1
-umap-learn==0.5.3
-zipp==3.15.0
+scikit-learn
+tqdm
+numpy
+Pillow
+matplotlib
+flask
+flask-cors

tool/server/server.py

@@ -1,7 +1,5 @@
 import os
 import sys
-# from llm_agent import call_llm_agent
-from run_visualization import visualize_run
 
 from flask import request, Flask, jsonify, make_response, send_file,send_from_directory
 from flask_cors import CORS, cross_origin

tool/server/server_utils.py

@@ -12,18 +12,13 @@
 from PIL import Image
 
 import matplotlib.pyplot as plt
-import torch
-import torchvision
-import torchvision.transforms as transforms
-from torch.utils.data import Dataset, DataLoader
-from transformers import RobertaTokenizer
 
 sys.path.append('..')
 sys.path.append('../visualize')
-from visualize.data_provider import DataProvider
-from visualize.training_event import TrainingEventDetector
-from influence_function.IF import EmpiricalIF, PairWiseEmpiricalIF
-from influence_function.CustomEncoderModel import CustomEncoderModel
+# from visualize.data_provider import DataProvider
+# from visualize.training_event import TrainingEventDetector
+# from influence_function.IF import EmpiricalIF, PairWiseEmpiricalIF
+# from influence_function.CustomEncoderModel import CustomEncoderModel
 
 # Func: infer available epochs files, return a list of available epochs
 def infer_epoch_structure(content_path):
@@ -511,26 +506,6 @@ def prediction_attribution(content_path, epoch, training_event, num_samples=10):
         })
 
     return influence_samples
-
-class CodeSearchNetDataset(Dataset):
-    def __init__(self, file_path, tokenizer, sample_limit=None):
-        self.samples = []
-        count = 0
-        with open(file_path, 'r', encoding='utf-8') as f:
-            for line in tqdm(f, desc="读取数据集"):
-                line_data = json.loads(line)
-                docstring_tensor = tokenizer(line_data['docstring'], padding='max_length', truncation=True, max_length=256, return_tensors='pt')['input_ids'].squeeze(0)
-                code_tensor = tokenizer(line_data['code'], padding='max_length', truncation=True, max_length=256, return_tensors='pt')['input_ids'].squeeze(0)
-                self.samples.append((docstring_tensor, code_tensor))
-                count += 1
-                if sample_limit and count > sample_limit:
-                    break
-
-    def __len__(self):
-        return len(self.samples)
-
-    def __getitem__(self, idx):
-        return self.samples[idx]
 
 
 def movement_attribution(content_path, epoch, training_event, num_samples=10):