Update app.py

app.py

@@ -68,8 +68,6 @@ def generate_response_with_visualization(model, tokenizer, device, messages, gen
     Returns:
         List of visualization states showing the progression and final text
     """
-    # Set random seed for reproducibility
-    torch.manual_seed(42)
     
     # Process constraints
     if constraints is None:
@@ -212,29 +210,12 @@ def generate_response_with_visualization(model, tokenizer, device, messages, gen
     response_text = tokenizer.decode(response_tokens, skip_special_tokens=True)
     
     # Clean the response text
-    final_text = clean_output_text(response_text)
+    final_text = tokenizer.decode(response_tokens, 
+                                 skip_special_tokens=True,
+                                 clean_up_tokenization_spaces=True)
     
     return visualization_states, final_text
 
-def clean_output_text(text):
-    """Clean the output text to remove special tokens and fix spacing"""
-    # Remove any remaining [MASK] tokens
-    text = text.replace(MASK_TOKEN, "")
-    
-    # Fix common spacing issues with tokenization
-    text = re.sub(r'\s+', ' ', text)  # Remove multiple spaces
-    text = re.sub(r' \.', '.', text)  # Fix spacing before periods
-    text = re.sub(r' ,', ',', text)   # Fix spacing before commas
-    text = re.sub(r' !', '!', text)   # Fix spacing before exclamation marks
-    text = re.sub(r' \?', '?', text)  # Fix spacing before question marks
-    text = re.sub(r' ;', ';', text)   # Fix spacing before semicolons
-    text = re.sub(r' :', ':', text)   # Fix spacing before colons
-    
-    # Fix beginning and end spacing
-    text = text.strip()
-    
-    return text
-
 css = '''
 .category-legend{display:none}
 '''