SamanthaStorm commited on
Commit
9c4d0f5
·
verified ·
1 Parent(s): 025be57

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +101 -2
app.py CHANGED
@@ -96,7 +96,7 @@ SENTIMENT_LABELS = ["undermining", "supportive"]
96
 
97
  THRESHOLDS = {
98
  "recovery phase": 0.324,
99
- "control": 0.433,
100
  "gaslighting": 0.285,
101
  "guilt tripping": 0.267,
102
  "dismissiveness": 0.123,
@@ -110,7 +110,7 @@ THRESHOLDS = {
110
 
111
 
112
  PATTERN_WEIGHTS = {
113
- "recovery": 0.7,
114
  "control": 1.4,
115
  "gaslighting": 1.3,
116
  "guilt tripping": 1.2,
@@ -328,6 +328,86 @@ def get_risk_stage(patterns, sentiment):
328
  except Exception as e:
329
  logger.error(f"Error determining risk stage: {e}")
330
  return 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
331
 
332
  @spaces.GPU
333
  def compute_abuse_score(matched_scores, sentiment):
@@ -453,6 +533,16 @@ def analyze_single_message(text, thresholds):
453
  if explicit_abuse:
454
  abuse_score = max(abuse_score, 70.0)
455
 
 
 
 
 
 
 
 
 
 
 
456
  # Get DARVO score
457
  darvo_score = predict_darvo_score(text)
458
 
@@ -810,6 +900,15 @@ def analyze_composite(msg1, msg2, msg3, *answers_and_none):
810
  logger.debug(" ✓ No escalation factors")
811
 
812
  logger.debug(f"\n📊 Total Escalation Bump: +{escalation_bump}")
 
 
 
 
 
 
 
 
 
813
 
814
  # Combined Risk Calculation
815
  logger.debug("\n🎯 FINAL RISK CALCULATION")
 
96
 
97
  THRESHOLDS = {
98
  "recovery phase": 0.324,
99
+ "control": 0.33,
100
  "gaslighting": 0.285,
101
  "guilt tripping": 0.267,
102
  "dismissiveness": 0.123,
 
110
 
111
 
112
  PATTERN_WEIGHTS = {
113
+ "recovery phase": 0.7,
114
  "control": 1.4,
115
  "gaslighting": 1.3,
116
  "guilt tripping": 1.2,
 
328
  except Exception as e:
329
  logger.error(f"Error determining risk stage: {e}")
330
  return 1
331
+ def detect_threat_pattern(text, patterns):
332
+ """Detect if a message contains threat patterns"""
333
+ # Threat indicators in text
334
+ threat_words = [
335
+ "regret", "sorry", "pay", "hurt", "suffer", "destroy", "ruin",
336
+ "expose", "tell everyone", "never see", "take away", "lose",
337
+ "control", "make sure", "won't let", "force", "warn", "never",
338
+ "punish", "teach you", "learn", "show you", "remember",
339
+ "if you", "don't forget", "i control", "i'll make sure", # Added these specific phrases
340
+ "bank account", "phone", "money", "access" # Added financial control indicators
341
+ ]
342
+
343
+ # Check for conditional threats (if/then structures)
344
+ text_lower = text.lower()
345
+ conditional_threat = (
346
+ "if" in text_lower and
347
+ any(word in text_lower for word in ["regret", "make sure", "control"])
348
+ )
349
+
350
+ has_threat_words = any(word in text_lower for word in threat_words)
351
+
352
+ # Check for threat patterns
353
+ threat_patterns = {"control", "gaslighting", "blame shifting", "insults"}
354
+ has_threat_patterns = any(p in threat_patterns for p in patterns)
355
+
356
+ return has_threat_words or has_threat_patterns or conditional_threat
357
+
358
+
359
+
360
+ def detect_compound_threat(text, patterns):
361
+ """Detect compound threats in a single message"""
362
+ try:
363
+ # Rule A: Single Message Multiple Patterns
364
+ high_risk_patterns = {"control", "gaslighting", "blame shifting", "insults"}
365
+ high_risk_count = sum(1 for p in patterns if p in high_risk_patterns)
366
+
367
+ has_threat = detect_threat_pattern(text, patterns)
368
+
369
+ # Special case for control + threats
370
+ has_control = "control" in patterns
371
+ has_conditional_threat = "if" in text.lower() and any(word in text.lower()
372
+ for word in ["regret", "make sure", "control"])
373
+
374
+ # Single message compound threat
375
+ if (has_threat and high_risk_count >= 2) or (has_control and has_conditional_threat):
376
+ return True, "single_message"
377
+
378
+ return False, None
379
+ except Exception as e:
380
+ logger.error(f"Error in compound threat detection: {e}")
381
+ return False, None
382
+
383
+
384
+ def analyze_message_batch_threats(messages, results):
385
+ """Analyze multiple messages for compound threats"""
386
+ threat_messages = []
387
+ support_messages = []
388
+
389
+ for i, (msg, (result, _)) in enumerate(zip(messages, results)):
390
+ if not msg.strip(): # Skip empty messages
391
+ continue
392
+
393
+ patterns = result[1] # Get detected patterns
394
+
395
+ # Check for threat in this message
396
+ if detect_threat_pattern(msg, patterns):
397
+ threat_messages.append(i)
398
+
399
+ # Check for supporting patterns
400
+ if any(p in {"control", "gaslighting", "blame shifting"} for p in patterns):
401
+ support_messages.append(i)
402
+
403
+ # Rule B: Multi-Message Accumulation
404
+ if len(threat_messages) >= 2:
405
+ return True, "multiple_threats"
406
+ elif len(threat_messages) == 1 and len(support_messages) >= 2:
407
+ return True, "threat_with_support"
408
+
409
+ return False, None
410
+
411
 
412
  @spaces.GPU
413
  def compute_abuse_score(matched_scores, sentiment):
 
533
  if explicit_abuse:
534
  abuse_score = max(abuse_score, 70.0)
535
 
536
+ # Check for compound threats
537
+ compound_threat_flag, threat_type = detect_compound_threat(
538
+ text, threshold_labels
539
+ )
540
+
541
+ if compound_threat_flag:
542
+ logger.debug(f"⚠️ Compound threat detected in message: {threat_type}")
543
+ abuse_score = max(abuse_score, 85.0) # Force high score for compound threats
544
+
545
+
546
  # Get DARVO score
547
  darvo_score = predict_darvo_score(text)
548
 
 
900
  logger.debug(" ✓ No escalation factors")
901
 
902
  logger.debug(f"\n📊 Total Escalation Bump: +{escalation_bump}")
903
+ # Check for compound threats across messages
904
+ compound_threat_flag, threat_type = analyze_message_batch_threats(
905
+ [msg1, msg2, msg3], results
906
+ )
907
+
908
+ if compound_threat_flag:
909
+ logger.debug(f"⚠️ Compound threat detected across messages: {threat_type}")
910
+ pattern_escalation_risk = "Critical" # Override risk level
911
+ logger.debug("Risk level elevated to CRITICAL due to compound threats")
912
 
913
  # Combined Risk Calculation
914
  logger.debug("\n🎯 FINAL RISK CALCULATION")