|
25 | 25 | { |
26 | 26 | "metadata": { |
27 | 27 | "ExecuteTime": { |
28 | | - "end_time": "2025-07-23T07:19:33.714851Z", |
29 | | - "start_time": "2025-07-23T07:19:30.010353Z" |
| 28 | + "end_time": "2025-07-25T08:06:06.794006Z", |
| 29 | + "start_time": "2025-07-25T08:06:03.385595Z" |
30 | 30 | } |
31 | 31 | }, |
32 | 32 | "cell_type": "code", |
|
54 | 54 | { |
55 | 55 | "metadata": { |
56 | 56 | "ExecuteTime": { |
57 | | - "end_time": "2025-07-23T07:19:36.118443Z", |
58 | | - "start_time": "2025-07-23T07:19:34.063665Z" |
| 57 | + "end_time": "2025-07-25T08:06:08.457844Z", |
| 58 | + "start_time": "2025-07-25T08:06:07.389495Z" |
59 | 59 | } |
60 | 60 | }, |
61 | 61 | "cell_type": "code", |
|
87 | 87 | { |
88 | 88 | "metadata": { |
89 | 89 | "ExecuteTime": { |
90 | | - "end_time": "2025-07-23T07:19:36.129148Z", |
91 | | - "start_time": "2025-07-23T07:19:36.126869Z" |
| 90 | + "end_time": "2025-07-25T08:06:09.098346Z", |
| 91 | + "start_time": "2025-07-25T08:06:09.095374Z" |
92 | 92 | } |
93 | 93 | }, |
94 | 94 | "cell_type": "code", |
|
118 | 118 | { |
119 | 119 | "metadata": { |
120 | 120 | "ExecuteTime": { |
121 | | - "end_time": "2025-07-23T07:19:37.318293Z", |
122 | | - "start_time": "2025-07-23T07:19:37.220430Z" |
| 121 | + "end_time": "2025-07-25T08:06:09.680771Z", |
| 122 | + "start_time": "2025-07-25T08:06:09.529678Z" |
123 | 123 | } |
124 | 124 | }, |
125 | 125 | "cell_type": "code", |
|
135 | 135 | "output_type": "stream", |
136 | 136 | "text": [ |
137 | 137 | "Loading conversations from data/sample_conversations.csv...\n", |
138 | | - "✓ Loaded 100 conversations\n", |
139 | | - "✓ Sample conversation has 6 messages\n", |
| 138 | + "✓ Loaded 47 conversations\n", |
| 139 | + "✓ Sample conversation has 16 messages\n", |
140 | 140 | "✓ Extracted 2 unique outcomes\n", |
141 | | - " - resolved: Issue was successfully resolved\n", |
142 | | - " - unresolved: Issue was not resolved\n" |
| 141 | + " - unresolved: Issue was not resolved\n", |
| 142 | + " - resolved: Issue was successfully resolved\n" |
143 | 143 | ] |
144 | 144 | } |
145 | 145 | ], |
|
154 | 154 | { |
155 | 155 | "metadata": { |
156 | 156 | "ExecuteTime": { |
157 | | - "end_time": "2025-07-23T07:19:41.743896Z", |
158 | | - "start_time": "2025-07-23T07:19:38.432667Z" |
| 157 | + "end_time": "2025-07-25T08:06:15.868901Z", |
| 158 | + "start_time": "2025-07-25T08:06:10.106998Z" |
159 | 159 | } |
160 | 160 | }, |
161 | 161 | "cell_type": "code", |
|
176 | 176 | "name": "stdout", |
177 | 177 | "output_type": "stream", |
178 | 178 | "text": [ |
179 | | - "✓ Created vector store with 409 documents\n" |
| 179 | + "✓ Created vector store with 840 documents\n" |
180 | 180 | ] |
181 | 181 | } |
182 | 182 | ], |
|
191 | 191 | { |
192 | 192 | "metadata": { |
193 | 193 | "ExecuteTime": { |
194 | | - "end_time": "2025-07-23T07:20:30.227359Z", |
195 | | - "start_time": "2025-07-23T07:19:41.753690Z" |
| 194 | + "end_time": "2025-07-25T08:08:13.858232Z", |
| 195 | + "start_time": "2025-07-25T08:06:15.875031Z" |
196 | 196 | } |
197 | 197 | }, |
198 | 198 | "cell_type": "code", |
|
202 | 202 | " default_chat_model=chat_model,\n", |
203 | 203 | " outcomes=outcomes,\n", |
204 | 204 | " vector_store=vector_store,\n", |
205 | | - " max_messages=10\n", |
| 205 | + " max_messages=25\n", |
206 | 206 | ").build()\n", |
207 | 207 | "\n", |
208 | 208 | "# Run simulation with first 5 conversations as starting points\n", |
|
217 | 217 | "name": "stderr", |
218 | 218 | "output_type": "stream", |
219 | 219 | "text": [ |
220 | | - "2025-07-23 10:19:41,755 - Starting intent extraction on 5 conversations\n", |
221 | | - "2025-07-23 10:19:46,194 - Finished extracting original intents; generated 5 scenarios\n", |
222 | | - "2025-07-23 10:19:46,195 - Starting conversation simulations (self.max_concurrent_conversations=20)\n", |
223 | | - "2025-07-23 10:19:51,197 - Progress: 0/5 scenarios completed\n", |
224 | | - "2025-07-23 10:20:01,199 - Progress: 1/5 scenarios completed\n", |
225 | | - "2025-07-23 10:20:16,202 - Progress: 2/5 scenarios completed\n", |
226 | | - "2025-07-23 10:20:21,204 - Progress: 5/5 scenarios completed\n", |
227 | | - "2025-07-23 10:20:21,206 - Finished simulating conversations; simulated 5 conversations, with 0 failures\n", |
228 | | - "2025-07-23 10:20:21,208 - Starting analysis of simulation results\n", |
229 | | - "2025-07-23 10:20:30,225 - Finished analyzing results\n" |
| 220 | + "2025-07-25 11:06:15,876 - Starting intent extraction on 5 conversations\n", |
| 221 | + "2025-07-25 11:06:18,934 - Finished extracting original intents; generated 5 scenarios\n", |
| 222 | + "2025-07-25 11:06:18,934 - Starting conversation simulations (self.max_concurrent_conversations=20)\n", |
| 223 | + "2025-07-25 11:06:23,935 - Progress: 0/5 scenarios completed\n", |
| 224 | + "2025-07-25 11:06:48,942 - Progress: 1/5 scenarios completed\n", |
| 225 | + "2025-07-25 11:07:23,950 - Progress: 3/5 scenarios completed\n", |
| 226 | + "2025-07-25 11:07:38,955 - Progress: 4/5 scenarios completed\n", |
| 227 | + "2025-07-25 11:08:03,962 - Progress: 5/5 scenarios completed\n", |
| 228 | + "2025-07-25 11:08:03,964 - Finished simulating conversations; simulated 5 conversations, with 0 failures\n", |
| 229 | + "2025-07-25 11:08:03,965 - Starting analysis of simulation results\n", |
| 230 | + "2025-07-25 11:08:13,856 - Finished analyzing results\n" |
230 | 231 | ] |
231 | 232 | }, |
232 | 233 | { |
|
248 | 249 | { |
249 | 250 | "metadata": { |
250 | 251 | "ExecuteTime": { |
251 | | - "end_time": "2025-07-23T07:20:30.241663Z", |
252 | | - "start_time": "2025-07-23T07:20:30.238988Z" |
| 252 | + "end_time": "2025-07-25T08:08:13.871712Z", |
| 253 | + "start_time": "2025-07-25T08:08:13.869520Z" |
253 | 254 | } |
254 | 255 | }, |
255 | 256 | "cell_type": "code", |
|
268 | 269 | "Simulated conversations: 5\n", |
269 | 270 | "\n", |
270 | 271 | "Average messages per conversation:\n", |
271 | | - " Original: 4.4\n", |
272 | | - " Simulated: 5.0\n", |
| 272 | + " Original: 16.8\n", |
| 273 | + " Simulated: 13.0\n", |
273 | 274 | "\n", |
274 | 275 | "Outcome distribution comparison:\n", |
275 | 276 | "Outcome Original Simulated \n", |
276 | 277 | "--------------------------------------------------\n", |
277 | | - "resolved 4 (80.0%) 4 (80.0%)\n", |
278 | | - "unresolved 1 (20.0%) 1 (20.0%)\n", |
| 278 | + "resolved 1 (20.0%) 2 (40.0%)\n", |
| 279 | + "unresolved 4 (80.0%) 3 (60.0%)\n", |
279 | 280 | "\n", |
280 | | - "Sample conversation (6 messages):\n", |
281 | | - " 1. customer: Last night, I waited in line for 2 hours in the business office, but because I only had a copy of my...\n", |
282 | | - " 2. agent: We're very sorry for the inconvenience and frustration you've experienced. I am from the Guangdong C...\n", |
283 | | - " 3. customer: I've already been through enough hassle with this process. Why can't you resolve the issue directly ...\n", |
284 | | - " 4. agent: I apologize for the inconvenience and understand your frustration. I will escalate your issue intern...\n", |
285 | | - " ... and 2 more messages\n", |
| 281 | + "Sample conversation (13 messages):\n", |
| 282 | + " 1. agent: Thank you for calling PC Support, how can I help you today?\n", |
| 283 | + " 2. customer: My laptop screen is flickering and sometimes goes completely black. I need assistance to get this fi...\n", |
| 284 | + " 3. agent: I'm sorry to hear about your screen issues. Let's start by checking the basics. Can you tell me if t...\n", |
| 285 | + " 4. customer: The screen flickers both when the laptop is still and when I move it. I haven't made any recent chan...\n", |
| 286 | + " ... and 9 more messages\n", |
286 | 287 | "========================================\n" |
287 | 288 | ] |
288 | 289 | } |
|
312 | 313 | " - [Streamlit web interface](https://github.com/SparkBeyond/agentune/blob/main/agentune_simulate/streamlit/README.md)" |
313 | 314 | ], |
314 | 315 | "id": "6adea4e078c54658" |
315 | | - }, |
316 | | - { |
317 | | - "metadata": {}, |
318 | | - "cell_type": "code", |
319 | | - "outputs": [], |
320 | | - "execution_count": null, |
321 | | - "source": "", |
322 | | - "id": "76af0307231433b8" |
323 | 316 | } |
324 | 317 | ], |
325 | 318 | "metadata": { |
|
0 commit comments