Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
@@ -96,7 +96,7 @@ SENTIMENT_LABELS = ["undermining", "supportive"]
|
|
96 |
|
97 |
THRESHOLDS = {
|
98 |
"recovery phase": 0.324,
|
99 |
-
"control": 0.
|
100 |
"gaslighting": 0.285,
|
101 |
"guilt tripping": 0.267,
|
102 |
"dismissiveness": 0.123,
|
@@ -110,7 +110,7 @@ THRESHOLDS = {
|
|
110 |
|
111 |
|
112 |
PATTERN_WEIGHTS = {
|
113 |
-
"recovery": 0.7,
|
114 |
"control": 1.4,
|
115 |
"gaslighting": 1.3,
|
116 |
"guilt tripping": 1.2,
|
@@ -328,6 +328,86 @@ def get_risk_stage(patterns, sentiment):
|
|
328 |
except Exception as e:
|
329 |
logger.error(f"Error determining risk stage: {e}")
|
330 |
return 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
331 |
|
332 |
@spaces.GPU
|
333 |
def compute_abuse_score(matched_scores, sentiment):
|
@@ -453,6 +533,16 @@ def analyze_single_message(text, thresholds):
|
|
453 |
if explicit_abuse:
|
454 |
abuse_score = max(abuse_score, 70.0)
|
455 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
456 |
# Get DARVO score
|
457 |
darvo_score = predict_darvo_score(text)
|
458 |
|
@@ -810,6 +900,15 @@ def analyze_composite(msg1, msg2, msg3, *answers_and_none):
|
|
810 |
logger.debug(" ✓ No escalation factors")
|
811 |
|
812 |
logger.debug(f"\n📊 Total Escalation Bump: +{escalation_bump}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
813 |
|
814 |
# Combined Risk Calculation
|
815 |
logger.debug("\n🎯 FINAL RISK CALCULATION")
|
|
|
96 |
|
97 |
THRESHOLDS = {
|
98 |
"recovery phase": 0.324,
|
99 |
+
"control": 0.33,
|
100 |
"gaslighting": 0.285,
|
101 |
"guilt tripping": 0.267,
|
102 |
"dismissiveness": 0.123,
|
|
|
110 |
|
111 |
|
112 |
PATTERN_WEIGHTS = {
|
113 |
+
"recovery phase": 0.7,
|
114 |
"control": 1.4,
|
115 |
"gaslighting": 1.3,
|
116 |
"guilt tripping": 1.2,
|
|
|
328 |
except Exception as e:
|
329 |
logger.error(f"Error determining risk stage: {e}")
|
330 |
return 1
|
331 |
+
def detect_threat_pattern(text, patterns):
|
332 |
+
"""Detect if a message contains threat patterns"""
|
333 |
+
# Threat indicators in text
|
334 |
+
threat_words = [
|
335 |
+
"regret", "sorry", "pay", "hurt", "suffer", "destroy", "ruin",
|
336 |
+
"expose", "tell everyone", "never see", "take away", "lose",
|
337 |
+
"control", "make sure", "won't let", "force", "warn", "never",
|
338 |
+
"punish", "teach you", "learn", "show you", "remember",
|
339 |
+
"if you", "don't forget", "i control", "i'll make sure", # Added these specific phrases
|
340 |
+
"bank account", "phone", "money", "access" # Added financial control indicators
|
341 |
+
]
|
342 |
+
|
343 |
+
# Check for conditional threats (if/then structures)
|
344 |
+
text_lower = text.lower()
|
345 |
+
conditional_threat = (
|
346 |
+
"if" in text_lower and
|
347 |
+
any(word in text_lower for word in ["regret", "make sure", "control"])
|
348 |
+
)
|
349 |
+
|
350 |
+
has_threat_words = any(word in text_lower for word in threat_words)
|
351 |
+
|
352 |
+
# Check for threat patterns
|
353 |
+
threat_patterns = {"control", "gaslighting", "blame shifting", "insults"}
|
354 |
+
has_threat_patterns = any(p in threat_patterns for p in patterns)
|
355 |
+
|
356 |
+
return has_threat_words or has_threat_patterns or conditional_threat
|
357 |
+
|
358 |
+
|
359 |
+
|
360 |
+
def detect_compound_threat(text, patterns):
|
361 |
+
"""Detect compound threats in a single message"""
|
362 |
+
try:
|
363 |
+
# Rule A: Single Message Multiple Patterns
|
364 |
+
high_risk_patterns = {"control", "gaslighting", "blame shifting", "insults"}
|
365 |
+
high_risk_count = sum(1 for p in patterns if p in high_risk_patterns)
|
366 |
+
|
367 |
+
has_threat = detect_threat_pattern(text, patterns)
|
368 |
+
|
369 |
+
# Special case for control + threats
|
370 |
+
has_control = "control" in patterns
|
371 |
+
has_conditional_threat = "if" in text.lower() and any(word in text.lower()
|
372 |
+
for word in ["regret", "make sure", "control"])
|
373 |
+
|
374 |
+
# Single message compound threat
|
375 |
+
if (has_threat and high_risk_count >= 2) or (has_control and has_conditional_threat):
|
376 |
+
return True, "single_message"
|
377 |
+
|
378 |
+
return False, None
|
379 |
+
except Exception as e:
|
380 |
+
logger.error(f"Error in compound threat detection: {e}")
|
381 |
+
return False, None
|
382 |
+
|
383 |
+
|
384 |
+
def analyze_message_batch_threats(messages, results):
|
385 |
+
"""Analyze multiple messages for compound threats"""
|
386 |
+
threat_messages = []
|
387 |
+
support_messages = []
|
388 |
+
|
389 |
+
for i, (msg, (result, _)) in enumerate(zip(messages, results)):
|
390 |
+
if not msg.strip(): # Skip empty messages
|
391 |
+
continue
|
392 |
+
|
393 |
+
patterns = result[1] # Get detected patterns
|
394 |
+
|
395 |
+
# Check for threat in this message
|
396 |
+
if detect_threat_pattern(msg, patterns):
|
397 |
+
threat_messages.append(i)
|
398 |
+
|
399 |
+
# Check for supporting patterns
|
400 |
+
if any(p in {"control", "gaslighting", "blame shifting"} for p in patterns):
|
401 |
+
support_messages.append(i)
|
402 |
+
|
403 |
+
# Rule B: Multi-Message Accumulation
|
404 |
+
if len(threat_messages) >= 2:
|
405 |
+
return True, "multiple_threats"
|
406 |
+
elif len(threat_messages) == 1 and len(support_messages) >= 2:
|
407 |
+
return True, "threat_with_support"
|
408 |
+
|
409 |
+
return False, None
|
410 |
+
|
411 |
|
412 |
@spaces.GPU
|
413 |
def compute_abuse_score(matched_scores, sentiment):
|
|
|
533 |
if explicit_abuse:
|
534 |
abuse_score = max(abuse_score, 70.0)
|
535 |
|
536 |
+
# Check for compound threats
|
537 |
+
compound_threat_flag, threat_type = detect_compound_threat(
|
538 |
+
text, threshold_labels
|
539 |
+
)
|
540 |
+
|
541 |
+
if compound_threat_flag:
|
542 |
+
logger.debug(f"⚠️ Compound threat detected in message: {threat_type}")
|
543 |
+
abuse_score = max(abuse_score, 85.0) # Force high score for compound threats
|
544 |
+
|
545 |
+
|
546 |
# Get DARVO score
|
547 |
darvo_score = predict_darvo_score(text)
|
548 |
|
|
|
900 |
logger.debug(" ✓ No escalation factors")
|
901 |
|
902 |
logger.debug(f"\n📊 Total Escalation Bump: +{escalation_bump}")
|
903 |
+
# Check for compound threats across messages
|
904 |
+
compound_threat_flag, threat_type = analyze_message_batch_threats(
|
905 |
+
[msg1, msg2, msg3], results
|
906 |
+
)
|
907 |
+
|
908 |
+
if compound_threat_flag:
|
909 |
+
logger.debug(f"⚠️ Compound threat detected across messages: {threat_type}")
|
910 |
+
pattern_escalation_risk = "Critical" # Override risk level
|
911 |
+
logger.debug("Risk level elevated to CRITICAL due to compound threats")
|
912 |
|
913 |
# Combined Risk Calculation
|
914 |
logger.debug("\n🎯 FINAL RISK CALCULATION")
|