diff --git a/skills/moark-truth-verify/SKILL.md b/skills/moark-truth-verify/SKILL.md new file mode 100755 index 0000000000000000000000000000000000000000..efcd83ed6c69df58f2c5f990d7fa66275307c55a --- /dev/null +++ b/skills/moark-truth-verify/SKILL.md @@ -0,0 +1,69 @@ +--- +name: moark-truth-verify +description: AI hallucination detector & fact-checker. Verify any claim in 3 depth modes — quick, standard, thorough. Searches evidence, generates counter-arguments, warns on echo-chamber bias. +metadata: + openclaw: + emoji: "🔍" + requires: + env: ["GITEEAI_API_KEY"] + primaryEnv: "GITEEAI_API_KEY" +--- + +# Truth Verify 🔍 + +**Catch AI hallucinations and unverified claims before they ship.** This skill performs a 3-step evidence-based verification: searches for supporting evidence, generates counter-arguments, and synthesizes a structured verdict — with built-in echo-chamber detection. + +## Why Use It? + +- 🛡️ **Anti-hallucination** — verify any claim with real evidence search +- ⚖️ **Adversarial design** — deliberately generates counter-arguments to challenge the claim +- 🚨 **Echo-chamber warning** — flags when evidence and counter are too aligned (likely one-sided) +- 🎚️ **3 depth modes** — quick sanity check, standard report, or thorough investigation +- 📊 **Confidence score** — numeric verdict (0-100) with evidence summary +- 🔄 **Structured output** — JSON or Markdown for downstream pipelines + +## Usage + +Set your API key once: `export GITEEAI_API_KEY=your_key` + +**Standard verification (recommended)** +```bash +python {baseDir}/scripts/perform_truth_verify.py \ + --text "Vitamin C cures the common cold" --depth standard +``` + +**Quick verification (single-round)** +```bash +python {baseDir}/scripts/perform_truth_verify.py \ + --text "Bitcoin was created in 2009" --depth quick +``` + +**Thorough verification (full investigation)** +```bash +python {baseDir}/scripts/perform_truth_verify.py \ + --text "Climate change is accelerating" --depth thorough --output json +``` + +## Options +- `--text` / `-t` (required): Claim or statement to verify +- `--depth` / `-d`: `quick` (brief verdict), `standard` (structured report, default), `thorough` (comprehensive) +- `--output` / `-o`: `markdown` (default) or `json` (for pipelines) +- `--model` / `-m`: Verification model (default: DeepSeek-R1-0528) +- `--timeout`: Per-call timeout in seconds (default: 60) +- `--api-key` / `-k`: Gitee AI API key (prefer `GITEEAI_API_KEY` env var) + +## Workflow + +1. Run `perform_truth_verify.py` with the user's parameters. +2. Find the line starting with `VERIFY_RESULT:` in the output. +3. Extract everything from that line onwards. +4. Present to the user as: `🔍 [Verification Result]` + +## Notes +- **3-step process**: (1) search evidence → (2) generate counter-arguments → (3) synthesize verdict +- `quick` mode still generates a simplified counter-argument for basic cross-validation +- `confidence` field (0-100) reflects strength of evidence +- **Echo-chamber warning** triggers if evidence and counter-argument are too aligned — re-investigate before publishing +- Counter-arguments are grounded in search evidence, not speculation +- JSON output is type-validated: `confidence` accepts float strings (e.g. "85.5") and converts safely +- Output: `VERIFY_RESULT: ` with `verdict`, `confidence`, `evidence_summary`, `counter_argument`, `notes` diff --git a/skills/moark-truth-verify/scripts/perform_truth_verify.py b/skills/moark-truth-verify/scripts/perform_truth_verify.py new file mode 100755 index 0000000000000000000000000000000000000000..5d70dd532218ac605d1fb9955b69152b0b753dad --- /dev/null +++ b/skills/moark-truth-verify/scripts/perform_truth_verify.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "openai" +# ] +# /// + +""" +Verify claims and statements using Gitee AI Chat API with web search. + +Usage: + python perform_truth_verify.py --text "claim to verify" [--depth standard] [--output markdown] [--timeout 60] [--api-key KEY] +""" + +import argparse +import json +import os +import random +import re +import sys +import time +from typing import Any +from openai import OpenAI +import openai + +DEFAULT_MODEL = "DeepSeek-R1-0528" +API_BASE_URL = "https://ai.gitee.com/v1" +DEFAULT_TIMEOUT = 60 + + +def handle_error_exit(message: str, exit_code: int = 1) -> None: + """Print error message to stderr and exit with code.""" + print(f"Error: {message}", file=sys.stderr) + sys.exit(exit_code) + + +def _extract_content(response: Any) -> str: + """Safely extract text content from an OpenAI chat completion response.""" + if not response or not getattr(response, 'choices', None): + return "" + message = response.choices[0].message + content = message.content if message else None + return content.strip() if content else "" + + +def chat_with_retry(client: OpenAI, model: str, messages: list, timeout: int, extra_body: dict | None = None, max_retries: int = 3) -> Any: + """Call OpenAI API with exponential backoff retry and jitter. + + Only catches network/rate-limit exceptions so that programming errors + (bad arguments, auth failures) propagate immediately. + """ + for attempt in range(max_retries): + try: + kwargs = { + "model": model, + "messages": messages, + "stream": False, + "timeout": timeout, + } + if extra_body: + kwargs["extra_body"] = extra_body + + return client.chat.completions.create(**kwargs) + + except (openai.APIConnectionError, openai.APITimeoutError, openai.RateLimitError) as e: + if attempt == max_retries - 1: + raise + + backoff = (2 ** attempt) + random.uniform(0, 1) + print(f"API call failed (attempt {attempt + 1}/{max_retries}) in {backoff:.1f}s: {e}", file=sys.stderr) + time.sleep(backoff) + + return None # Should never reach here + + +def get_api_key(provided_key: str | None) -> str | None: + """Get API key from argument or environment.""" + if provided_key: + return provided_key + return os.environ.get("GITEEAI_API_KEY") + + +def verify_with_search(client: OpenAI, claim: str, model: str, timeout: int) -> str: + """Search the web for evidence related to a claim.""" + response = chat_with_retry( + client=client, + model=model, + messages=[ + { + "role": "system", + "content": ( + "You are a fact-checking assistant. Search the web and provide " + "evidence that either supports or refutes the given claim. " + "Be objective and cite sources when possible." + ), + }, + {"role": "user", "content": f"Search for evidence about this claim: {claim}"}, + ], + timeout=timeout, + extra_body={"enable_search": True}, + ) + return _extract_content(response) + + +def generate_counter_argument(client: OpenAI, claim: str, evidence: str, model: str, timeout: int) -> str: + """Generate a counter-argument to stress-test the claim based on evidence.""" + # Guard: if no evidence available, return a meaningful fallback + if not evidence or not evidence.strip(): + return "Unable to generate counter-argument: no evidence found from search." + + prompt = ( + f"Claim: {claim}\n\n" + f"Evidence found:\n{evidence}\n\n" + "Provide the strongest counter-argument or alternative interpretation " + "based on the evidence above. If the evidence strongly supports the claim, " + "identify potential blind spots or limitations in the evidence." + ) + response = chat_with_retry( + client=client, + model=model, + messages=[ + { + "role": "system", + "content": ( + "You are a critical thinking assistant. Given a claim and supporting " + "evidence, generate the strongest possible counter-argument or " + "alternative interpretation. Be thorough but fair. " + "You MUST base your counter-argument on the evidence provided, " + "not on speculation alone." + ), + }, + {"role": "user", "content": prompt}, + ], + timeout=timeout, + ) + return _extract_content(response) + + +def generate_quick_counter(client: OpenAI, claim: str, evidence: str, model: str, timeout: int) -> str: + """Generate a brief counter-argument for quick mode.""" + # Guard: if no evidence available + if not evidence or not evidence.strip(): + return "No counter-argument: insufficient evidence." + + prompt = ( + f"Claim: {claim}\n\n" + f"Evidence found:\n{evidence}\n\n" + "In one sentence, state the strongest counter-argument or limitation." + ) + response = chat_with_retry( + client=client, + model=model, + messages=[ + { + "role": "system", + "content": "You are a concise critical thinking assistant. Respond in one sentence only.", + }, + {"role": "user", "content": prompt}, + ], + timeout=timeout, + ) + return _extract_content(response) + + +def generate_report( + client: OpenAI, + claim: str, + evidence: str, + counter: str, + depth: str, + model: str, + output_format: str, + timeout: int, +) -> str: + """Generate the final verification report.""" + depth_instructions = { + "quick": "Provide a brief verdict (1-2 sentences) with confidence level.", + "standard": "Provide a structured report with verdict, key evidence, and confidence level.", + "thorough": ( + "Provide a comprehensive report including: verdict, detailed evidence analysis, " + "counter-argument assessment, nuance and context, and confidence level with reasoning." + ), + } + + if output_format == "json": + format_instruction = ( + "Output ONLY a JSON object with these keys: " + "verdict (string: supported/refuted/mixed/unverifiable), " + "confidence (number 0-100), evidence_summary (string), " + "counter_argument (string), notes (string). " + "Do NOT wrap in markdown code fences." + ) + else: + format_instruction = "Output in Markdown format." + + response = chat_with_retry( + client=client, + model=model, + messages=[ + { + "role": "system", + "content": ( + "You are a truth verification analyst. Synthesize the evidence and " + "counter-arguments to produce a verification report. " + f"Task Requirements:\n\n{depth_instructions[depth]}\n\nFormat Requirements:\n{format_instruction}" + ), + }, + { + "role": "user", + "content": ( + f"Claim: {claim}\n\n" + f"Evidence found:\n{evidence}\n\n" + f"Counter-argument:\n{counter}\n\n" + ), + }, + ], + timeout=timeout, + ) + return _extract_content(response) + + +def check_comfort_zone(evidence: str, counter: str) -> str | None: + """Check whether the counter-argument is too short (< 30 chars), indicating a potential information blind spot. + + Returns a warning string if the counter-argument is absent or too brief, + which may signal that search found no dissenting views (echo-chamber risk). + Returns None if the counter-argument has sufficient length. + """ + if not evidence or not counter: + return None + if len(counter.strip()) < 30: + return ( + "⚠️ Comfort-zone alert: The counter-argument is too weak or absent, " + "which may indicate an information echo chamber. " + "Consider seeking additional independent sources." + ) + return None + + +def _enrich_json_output(data: dict, claim: str, evidence: str, counter: str, depth: str) -> dict: + """Add claim, depth, and comfort zone warning to parsed JSON output.""" + comfort = check_comfort_zone(evidence, counter) + if comfort: + data["comfort_zone_warning"] = comfort + data["claim"] = claim + data["depth"] = depth + # Deep validation of all fields + data = validate_json_output(data) + return data + + +def validate_json_output(data: dict) -> dict: + """Deep validation of JSON output fields. + + Validates and corrects field types and values to ensure consistency. + Also handles field name migration (e.g., counter_str -> counter_argument). + + Args: + data: Parsed JSON data from LLM response + + Returns: + Validated and corrected data dictionary + """ + # Define valid values for verdict + valid_verdicts = {"supported", "refuted", "mixed", "unverifiable"} + + # Validate verdict + if "verdict" not in data or not isinstance(data["verdict"], str): + data["verdict"] = "unverifiable" + else: + verdict_lower = data["verdict"].lower() + if verdict_lower not in valid_verdicts: + # Try to map common variations + if "support" in verdict_lower: + data["verdict"] = "supported" + elif "refut" in verdict_lower or "false" in verdict_lower: + data["verdict"] = "refuted" + elif "mix" in verdict_lower or "partial" in verdict_lower: + data["verdict"] = "mixed" + else: + data["verdict"] = "unverifiable" + else: + data["verdict"] = verdict_lower + + # Validate confidence + if "confidence" not in data: + data["confidence"] = 0 + else: + try: + data["confidence"] = int(float(data["confidence"])) + # Clamp to 0-100 range + data["confidence"] = max(0, min(100, data["confidence"])) + except (ValueError, TypeError): + data["confidence"] = 0 + + # Validate evidence_summary + if "evidence_summary" not in data or not isinstance(data["evidence_summary"], str): + data["evidence_summary"] = "" + + # Validate counter_argument (check for both possible field names) + # Handle field name migration: counter_str -> counter_argument + if "counter_str" in data and "counter_argument" not in data: + data["counter_argument"] = data.pop("counter_str") + + if "counter_argument" not in data or not isinstance(data["counter_argument"], str): + data["counter_argument"] = "" + + # Validate notes + if "notes" not in data or not isinstance(data["notes"], str): + data["notes"] = "" + + return data + +def build_json_output(report_text: str, claim: str, evidence: str, counter: str, depth: str) -> str: + """Build a structured JSON output, extracting from LLM if possible, else wrapping.""" + json_match = re.search(r"```(?:json)?\s*(.*?)\s*```", report_text, re.DOTALL | re.IGNORECASE) + if json_match: + try: + parsed = json.loads(json_match.group(1)) + required_keys = {"verdict", "confidence", "evidence_summary", "counter_argument", "notes"} + if required_keys.issubset(parsed.keys()): + if isinstance(parsed.get("confidence"), str): + try: + parsed["confidence"] = int(float(parsed["confidence"])) + except (ValueError, TypeError): + parsed["confidence"] = 0 + return json.dumps(_enrich_json_output(parsed, claim, evidence, counter, depth), ensure_ascii=False, indent=2) + except (json.JSONDecodeError, AttributeError): + pass + + try: + parsed = json.loads(report_text) + if isinstance(parsed, dict): + return json.dumps(_enrich_json_output(parsed, claim, evidence, counter, depth), ensure_ascii=False, indent=2) + except (json.JSONDecodeError, AttributeError): + pass + + result = _enrich_json_output({ + "verdict": "unverifiable", + "confidence": 0, + "evidence_summary": evidence[:500] if evidence else "", + "counter_argument": counter[:300] if counter else "", + "notes": report_text[:1000], + }, claim, evidence, counter, depth) + return json.dumps(result, ensure_ascii=False, indent=2) + + +def main(): + parser = argparse.ArgumentParser( + description="Verify claims and statements using Gitee AI with web search" + ) + parser.add_argument( + "--text", "-t", + required=True, + help="The claim or statement to verify", + ) + parser.add_argument( + "--depth", "-d", + choices=["quick", "standard", "thorough"], + default="standard", + help="Verification depth (default: standard)", + ) + parser.add_argument( + "--model", "-m", + default=DEFAULT_MODEL, + help=f"Model to use for verification (default: {DEFAULT_MODEL})", + ) + parser.add_argument( + "--output", "-o", + choices=["json", "markdown"], + default="markdown", + help="Output format (default: markdown)", + ) + parser.add_argument( + "--timeout", + type=int, + default=DEFAULT_TIMEOUT, + help=f"Timeout per API call in seconds (default: {DEFAULT_TIMEOUT})", + ) + parser.add_argument( + "--api-key", "-k", + help="Gitee AI API key (overrides GITEEAI_API_KEY env var)", + ) + + args = parser.parse_args() + + api_key = get_api_key(args.api_key) + if not api_key: + handle_error_exit( + "No API key provided.\n" + "Please either:\n" + " 1. Provide --api-key argument\n" + " 2. Set GITEEAI_API_KEY environment variable" + ) + + client = OpenAI( + base_url=API_BASE_URL, + api_key=api_key, + ) + + print(f"Verifying claim (depth={args.depth})...") + print(f"Claim: {args.text}") + + try: + # Step 1: Search for evidence (must complete first, counter depends on it) + print("\n[1/3] Searching for evidence...") + evidence = verify_with_search(client, args.text, args.model, args.timeout) + if not evidence: + print("Warning: No evidence found from search.", file=sys.stderr) + + # Step 2: Generate counter-argument + # Quick mode also gets a counter-argument (simplified single-sentence) + counter = "" + if args.depth == "quick": + print("[2/3] Generating brief counter-argument (quick mode)...") + counter = generate_quick_counter(client, args.text, evidence, args.model, args.timeout) + else: + print("[2/3] Generating counter-argument...") + counter = generate_counter_argument(client, args.text, evidence, args.model, args.timeout) + + # Step 3: Generate report + print("[3/3] Generating verification report...") + report = generate_report( + client, args.text, evidence, counter, args.depth, args.model, args.output, args.timeout + ) + + # Build final output + if args.output == "json": + final_output = build_json_output(report, args.text, evidence, counter, args.depth) + else: + comfort = check_comfort_zone(evidence, counter) + final_output = report + if comfort: + final_output += f"\n\n{comfort}" + + print("\nVERIFY_RESULT:") + print(final_output) + + except KeyboardInterrupt: + sys.exit(130) + except (ConnectionError, TimeoutError) as e: + handle_error_exit(f"Network error during verification: {e}") + except Exception as e: + handle_error_exit(f"Error during verification: {e}") + + +if __name__ == "__main__": + main()