diff --git a/.gitignore b/.gitignore index e6aff516..96adadbc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ .DS_Store - +.kiro/ diff --git a/ai-agents.mdx b/ai-agents.mdx index 73c00745..686eb805 100644 --- a/ai-agents.mdx +++ b/ai-agents.mdx @@ -131,7 +131,7 @@ Because CometChat AI Agents are model-agnostic, you can swap providers or upgrad {/* Footer */} -
+
2025 © CometChat diff --git a/calls.mdx b/calls.mdx index a7d5afa1..6ade1a19 100644 --- a/calls.mdx +++ b/calls.mdx @@ -283,7 +283,7 @@ import { CardGroup, Card, Icon, Badge, Steps, Columns, AccordionGroup, Accordion
{/* Footer */} -
+
2025 © CometChat diff --git a/chat-call.mdx b/chat-call.mdx index a1386dc9..3c59d0af 100644 --- a/chat-call.mdx +++ b/chat-call.mdx @@ -383,7 +383,7 @@ import { CardGroup, Card, Icon, Badge, Steps, Columns, AccordionGroup, Accordion {/* Footer */} -
+
2025 © CometChat diff --git a/chat.mdx b/chat.mdx index e9e88e94..1836a0df 100644 --- a/chat.mdx +++ b/chat.mdx @@ -444,7 +444,7 @@ import { CardGroup, Card, Icon, Badge, Steps, Columns, AccordionGroup, Accordion
{/* Footer */} -
+
2025 © CometChat diff --git a/docs.json b/docs.json index 2ccdf8f0..3c9181a7 100644 --- a/docs.json +++ b/docs.json @@ -32,13 +32,8 @@ "products": [ { "product": "Home", - "tabs": [ - { - "tab": "Home", - "pages": [ - "index" - ] - } + "pages": [ + "index" ] }, { @@ -2882,6 +2877,7 @@ "sdk/react-native/transfer-group-ownership" ] }, + "sdk/react-native/ai-moderation", "sdk/react-native/ai-agents", { "group": "Resources", @@ -3174,7 +3170,7 @@ "group": "Calling", "pages": [ "sdk/ios/calling-overview", - "sdk/ios/calling-integration", + "sdk/ios/calling-setup", "sdk/ios/default-calling", "sdk/ios/direct-calling", "sdk/ios/standalone-calling", @@ -3522,7 +3518,7 @@ "group": "Calling", "pages": [ "sdk/android/calling-overview", - "sdk/android/setup-calling", + "sdk/android/calling-setup", "sdk/android/default-calling", "sdk/android/direct-calling", "sdk/android/standalone-calling", @@ -5010,21 +5006,11 @@ "moderation/custom/custom-api" ] }, - "moderation/webhooks-overview", - "moderation/api-explorer", { - "group": "Legacy Moderation (Extensions)", + "group": "Resources", "pages": [ - "moderation/legacy/slow-mode", - "moderation/legacy/report-user", - "moderation/legacy/report-message", - "moderation/legacy/data-masking-filter", - "moderation/legacy/profanity-filter", - "moderation/legacy/image-moderation", - "moderation/legacy/sentiment-analysis", - "moderation/legacy/in-flight-message-moderation", - "moderation/legacy/virus-malware-scanner", - "moderation/legacy/xss-filter" + "moderation/api-explorer", + "moderation/legacy-extensions" ] } ] @@ -5383,43 +5369,83 @@ }, { "source": "/extensions/slow-mode", - "destination": "/moderation/slow-mode" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/xss-filter", - "destination": "/moderation/xss-filter" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/report-user", - "destination": "/moderation/report-user" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/report-message", - "destination": "/moderation/report-message" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/data-masking-filter", - "destination": "/moderation/data-masking-filter" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/profanity-filter", - "destination": "/moderation/profanity-filter" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/image-moderation", - "destination": "/moderation/image-moderation" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/sentiment-analysis", - "destination": "/moderation/sentiment-analysis" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/in-flight-message-moderation", - "destination": "/moderation/in-flight-message-moderation" + "destination": "/moderation/legacy-extensions" }, { "source": "/extensions/virus-malware-scanner", - "destination": "/moderation/virus-malware-scanner" + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/slow-mode", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/xss-filter", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/report-user", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/report-message", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/data-masking-filter", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/profanity-filter", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/image-moderation", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/sentiment-analysis", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/in-flight-message-moderation", + "destination": "/moderation/legacy-extensions" + }, + { + "source": "/moderation/virus-malware-scanner", + "destination": "/moderation/legacy-extensions" }, { "source": "/ui-kit/react/v6", @@ -5688,6 +5714,14 @@ { "source": "/ai-chatbots/bots", "destination": "/ai-chatbots/ai-bots/bots" + }, + { + "source": "/sdk/android/setup-calling", + "destination": "/sdk/android/calling-setup" + }, + { + "source": "/sdk/ios/calling-integration", + "destination": "/sdk/ios/calling-integration" } ], "integrations": { diff --git a/fundamentals/extensions-overview.mdx b/fundamentals/extensions-overview.mdx index 00914458..d6e58d1d 100644 --- a/fundamentals/extensions-overview.mdx +++ b/fundamentals/extensions-overview.mdx @@ -65,16 +65,7 @@ Extensions that help alert users of new messages. *Recommended for all apps.* *Extensions that help you to build a safe messaging environment.* *Recommended for live streaming and event apps.* -[Slow mode](/moderation/slow-mode)\ -[Report user](/moderation/report-user)\ -[Report message](/moderation/report-message)\ -[In-flight Message Moderation](/moderation/in-flight-message-moderation)\ -[Image Moderation](/moderation/image-moderation)\ -[Virus and Malware Scanner](/moderation/virus-malware-scanner)\ -[Data Masking Filter](/moderation/data-masking-filter)\ -[Profanity Filter](/moderation/profanity-filter)\ -[Sentiment Analysis](/moderation/sentiment-analysis)\ -[XSS Filter](/moderation/xss-filter) +[Legacy Moderation Extensions](/moderation/legacy-extensions) ### Security diff --git a/fundamentals/moderation-extensions.mdx b/fundamentals/moderation-extensions.mdx index f02017ea..4314fc67 100644 --- a/fundamentals/moderation-extensions.mdx +++ b/fundamentals/moderation-extensions.mdx @@ -4,18 +4,8 @@ title: "Moderation" CometChat Moderation features come in two variants: -1. The mordern rule based [Moderation](/moderation/overview) platform. +1. The modern rule based [Moderation](/moderation/overview) platform. -2. The [Legacy Moderation Extensions](/moderation/slow-mode) based on CometChat Extensions. +2. The [Legacy Moderation Extensions](/moderation/legacy-extensions) based on CometChat Extensions. - * [Slow Mode (Deprecated)](/moderation/slow-mode) - * [Report User (Legacy)](/moderation/report-user) - * [Report Message (Legacy)](/moderation/report-message) - * [Data Masking Filter (Legacy)](/moderation/data-masking-filter) - * [Profanity Filter (Legacy)](/moderation/profanity-filter) - * [Image Moderation (Legacy)](/moderation/image-moderation) - * [In-flight Message Moderation (Legacy)](/moderation/in-flight-message-moderation) - * [Virus and Malware Scanner (Legacy)](/moderation/virus-malware-scanner) - * [XSS Filter (Deprecated)](/moderation/xss-filter) - -For the best experience, we recommend to use the rule based [Moderation](/moderation/overview) platform. Please visit the above mentioned links for more details. +For the best experience, we recommend using the rule based [Moderation](/moderation/overview) platform. diff --git a/home.mdx b/home.mdx index 05eae3ba..0293bf62 100644 --- a/home.mdx +++ b/home.mdx @@ -5,57 +5,6 @@ description: "Technical documentation & Implementation guides to add In-app Mess canonical: "https://cometchat.com/docs" --- -export function openSearch() { - document.getElementById('search-bar-entry').click(); -} - -
- - -
-
-
- Get Started -
-

Seamlessly integrate real-time chat, voice, and video functionalities.

-
- -
-
-
-
-
+
diff --git a/images/mobile-uikit-moderation-overview.png b/images/mobile-uikit-moderation-overview.png new file mode 100644 index 00000000..6830e58f Binary files /dev/null and b/images/mobile-uikit-moderation-overview.png differ diff --git a/images/react-uikit-moderation-overview.png b/images/react-uikit-moderation-overview.png new file mode 100644 index 00000000..3d3c9530 Binary files /dev/null and b/images/react-uikit-moderation-overview.png differ diff --git a/index.mdx b/index.mdx index 3d3e0c41..8565bbf9 100644 --- a/index.mdx +++ b/index.mdx @@ -5,66 +5,31 @@ description: "Technical documentation & Implementation guides to add In-app Mess canonical: "https://cometchat.com/docs" --- -export function openSearch() { - document.getElementById('search-bar-entry').click(); -} +import { Columns } from 'mintlify'; -
+{/* Hero Section */} +
+
-
-
-
- Get Started +
+ +
+

CometChat

+

+ Seamlessly integrate real-time chat, voice, and video functionalities. +

-

Seamlessly integrate real-time chat, voice, and video functionalities.

-
-
-
+
-
- -Using Cursor, VS Code, Claude, Lovable, or any MCP-enabled tool? Add our CometChat Docs MCP so your AI always pulls the latest CometChat documentation. -

[Add CometChat Docs MCP](/mcp-server) -
{/** Products Section */} @@ -109,7 +74,7 @@ export function openSearch() { } + icon={Chat and Calling} iconType="solid" href="/chat-call" > @@ -118,7 +83,7 @@ export function openSearch() { } + icon={AI Agents} iconType="solid" href="/ai-agents" > @@ -127,7 +92,7 @@ export function openSearch() { } + icon={AI Moderation} iconType="solid" href="/moderation/overview" > @@ -136,7 +101,7 @@ export function openSearch() { } + icon={Notifications} iconType="solid" href="/notifications/overview" > @@ -145,7 +110,7 @@ export function openSearch() { } + icon={Insights} iconType="solid" href="/insights" > @@ -212,13 +177,16 @@ export function openSearch() { > Stay informed of any service interruptions. - - -
- - + Add CometChat Docs MCP to your AI tools for instant documentation access. + + + +
-
+
diff --git a/moderation/api-explorer.mdx b/moderation/api-explorer.mdx index e71a39fc..351d82b8 100644 --- a/moderation/api-explorer.mdx +++ b/moderation/api-explorer.mdx @@ -1,4 +1,4 @@ --- -title: "API Explorer" +title: "AI Moderation APIs" url: "/rest-api/moderation" --- \ No newline at end of file diff --git a/moderation/blocked-messages.mdx b/moderation/blocked-messages.mdx index f6c39d1c..fa8bb8a2 100644 --- a/moderation/blocked-messages.mdx +++ b/moderation/blocked-messages.mdx @@ -2,38 +2,152 @@ title: "Blocked Messages" --- -## Overview +Blocked Messages provides a centralized view of all messages that have been automatically blocked by your moderation rules. Review blocked content to identify false positives, refine your rules, and optionally approve messages that were incorrectly blocked. -The Blocked Messages endpoint in the Moderation Service API provides app owners and collaborators with the capability to retrieve details about messages that have been blocked due to violations of message moderation rules. This endpoint plays a crucial role in enabling platform app owners and collaborators to review and manage content that has been deemed inappropriate, harmful, or non-compliant with platform guidelines. + +**Blocked messages are hidden immediately.** Unlike flagged messages which wait for review, blocked messages are never delivered to recipients. Use this dashboard to monitor what's being blocked and catch false positives. + -### List Blocked Messages +--- + +## Quick Start + +Review blocked messages in under 2 minutes: -Retrieves the list of blocked messages, with the option to search messages within a specified date range. + + + Login to [CometChat Dashboard](https://app.cometchat.com) → Select your app → **Moderation** → **Blocked Messages** + + + Click on a blocked message to see the content, sender, rule that triggered the block, and timestamp + + + Click **Approve** to unblock a false positive, or **Mark as Reviewed** to acknowledge without approving + + -You can also set this up from your end using the [List Moderation Blocked Message List REST API](/rest-api/moderation/list-moderation-blocked-messages). +--- + +## How It Works + +```mermaid +flowchart LR + A[Message Sent] --> B{Rule Check} + B -->|Matches Rule| C[Message Blocked] + B -->|No Match| D[Message Delivered] + C --> E[Blocked Messages Dashboard] + E --> F{Moderator Review} + F --> G[Approve - Unblock] + F --> H[Mark as Reviewed] + F --> I[No Action] +``` + +| Status | Description | +|--------|-------------| +| **Blocked** | Message matched a rule and was not delivered | +| **Approved** | Moderator reviewed and unblocked the message | +| **Reviewed** | Moderator acknowledged but kept message blocked | + +--- + +## Managing Blocked Messages + +### List Blocked Messages + +View all blocked messages with filtering options: +- **Date range**: Filter by when messages were blocked +- **Rule**: Filter by which rule triggered the block +- **Sender**: Search by sender UID + + + + ### Approve Blocked Message -Allows the approval of messages previously blocked due to moderation violations. -Approving a blocked message automatically marks it as **reviewed**, showing that it has been checked and is okay. +Approving a blocked message: +1. Unblocks the message (makes it visible to recipients) +2. Marks it as reviewed +3. Helps identify false positives in your rules -You can also set this up from your end using the [Approve Blocked Message REST API](/rest-api/moderation/approve-blocked-messages). - + +Approving a message will deliver it to the recipient. Only approve if you're certain the message doesn't violate your guidelines. + -### Review Blocked Message +### Mark as Reviewed -Allows to manually mark a blocked message as reviewed without approving it. +Mark a message as reviewed without approving it: +- Message stays blocked +- Indicates a moderator has seen it +- Useful for tracking review progress -You can also set this up from your end using the [Mark Blocked Message as Reviewed REST API](/rest-api/moderation/review-blocked-message). +--- + +## Best Practices + + + + Check blocked messages weekly to catch false positives and refine rules. + + + Look for patterns in blocked content to identify if rules need adjustment. + + + Filter by date range to focus on recent blocks or investigate specific incidents. + + + If you see many false positives, adjust confidence levels in [Rules Management](/moderation/rules-management). + + + +--- + +## FAQ + + + + Yes. When you approve a blocked message, it becomes visible to the recipient. However, depending on how much time has passed, the message may appear out of context in the conversation. + + + Blocked messages are immediately hidden and never delivered. Flagged messages are delivered but marked for moderator review. Use "Block" for clear violations and "Flag" for borderline content. + + + 1. Increase confidence thresholds in your rules + 2. Add exceptions to your keyword lists + 3. Use more specific patterns instead of broad keywords + + + Yes, select multiple messages and use the bulk approve action. + + + +--- + +## Related Resources + + + + Configure rules that block messages + + + Review messages pending moderation + + + Manage keyword lists used by rules + + + Learn about the moderation system + + diff --git a/moderation/constraints-and-limits.mdx b/moderation/constraints-and-limits.mdx index 6f5948aa..1e3b00b5 100644 --- a/moderation/constraints-and-limits.mdx +++ b/moderation/constraints-and-limits.mdx @@ -1,42 +1,105 @@ --- -title: "Constraints And Limits" +title: "Constraints and Limits" +--- + +This page outlines the system constraints and limitations for CometChat's Moderation Service. Understanding these limits helps you design your moderation strategy effectively. + + +**Need higher limits?** Contact [CometChat Support](https://www.cometchat.com/contact) to discuss enterprise options for increased limits. + + --- ## Constraints -### Rule Management + + + | Parameter | Limit | + |-----------|-------| + | **Rule ID** | Max 100 characters, no spaces or special characters | + | **Name** | Max 100 characters | + | **Description** | Max 255 characters | + | **Filters per rule** | Max 10 filters | + | **Conditions per rule** | Max 10 conditions | + | **Rules per app** | Max 25 custom rules (excludes default rules) | + + + | Parameter | Limit | + |-----------|-------| + | **List ID** | Max 100 characters, no spaces or special characters | + | **Name** | Max 100 characters | + | **Description** | Max 255 characters | + | **CSV file size** | Max 1 MB | + | **Lists per app** | Max 25 custom lists (excludes default lists) | + + + | Parameter | Limit | + |-----------|-------| + | **AI Context** | Max 50 messages sent to OpenAI/Custom API for analysis | + | **Custom Flag Reasons** | Max 5 per app | + + + +--- + +## Supported Formats + +| Media Type | Supported Formats | +|------------|-------------------| +| **Images** | JPEG, PNG | +| **Videos** | MP4, MOV, AVI | + + +Other image and video formats are not currently supported for AI moderation. + -Here are the constraints for managing rules within the system: +--- -| Parameter | Constraint | -| ------------------- | ------------------------------------------------------------------------- | -| **Rule ID** | No spaces or special characters allowed, Maximum length of 100 characters | -| **Name** | Maximum length of 100 characters | -| **Description** | Maximum length of 255 characters | -| **Rule Filters** | Each rule can have a maximum of 10 filters | -| **Rule Conditions** | Each rule can have a maximum of 10 conditions | -| **Rules per app** | An app can have up to 25 rules (exluding default rules) | +## Known Limitations -### Lists Management + + + Moderation features require SDK version **3.0 or higher**. Ensure your app is updated before enabling moderation. + + + If a message is marked as delivered/read and previous messages include pending or disapproved ones, those will also be automatically marked as delivered/read—even though they weren't actually delivered to the receiver. + + -Here are the constraints for managing lists within the system: +--- -| Parameter | Constraint | -| ----------------- | ------------------------------------------------------------------------- | -| **ID** | No spaces or special characters allowed, Maximum length of 100 characters | -| **Name** | Maximum length of 100 characters | -| **Description** | Maximum length of 255 characters | -| **CSV File** | Accepted file size: up to 1 MB | -| **Lists per app** | An app can have up to 25 lists (excluding the default lists) | +## Quick Reference -### Advanced Settings + + + Maximum custom rules per app + + + Maximum custom lists per app + + + Maximum filters per rule + + + Maximum conditions per rule + + -* **AI Context Limit**: Up to 50 messages can be sent to OpenAI/Custom API for contextual analysis. -* **Flag Reasons Limit**: A maximum of 5 custom flag reasons can be configured per app. +--- -## Limitations +## Related Resources -* Make sure the SDK version you are using is 3.0 or higher. -* AI Image moderation currently supports & moderates JPEG and PNG image formats. -* AI video moderation currently supports & moderates MP4, MOV, and AVI formats. -* If a message is marked as delivered/read and previous messages include pending/disapproved ones, those will also be automatically marked as delivered/read, even though they haven't actually been delivered to the receiver. + + + Configure moderation rules + + + Manage keyword lists + + + Learn about the moderation system + + + Request higher limits + + diff --git a/moderation/custom/custom-api-overview.mdx b/moderation/custom/custom-api-overview.mdx index b3660af9..c65397ab 100644 --- a/moderation/custom/custom-api-overview.mdx +++ b/moderation/custom/custom-api-overview.mdx @@ -2,14 +2,53 @@ title: "Overview" --- -CometChat offers AI-powered message moderation to help maintain a safe and respectful chat environment. You can choose between two moderation options: +CometChat allows you to integrate your own moderation logic using a **Custom API**. This "bring your own moderation" approach lets you use any third-party service (OpenAI Moderation, Perspective API, etc.) or your own AI model while CometChat handles message interception and action enforcement. -### **Custom API Moderation** +## How It Works -If you prefer to use a third-party moderation service or your own AI model, CometChat enables integration via a **Custom Moderation API**. With this option, you can: -- **Set Up a Webhook** – Configure an endpoint where messages will be sent for moderation. -- **Customize Authentication** – Add security layers like basic authentication. -- **Contextual Moderation** – Define how many previous messages from the conversation should be included in the webhook request for better analysis. -- **Process Moderation Decisions** – CometChat processes the webhook response and applies moderation actions accordingly. +```mermaid +sequenceDiagram + participant User + participant CometChat + participant Your Webhook + + User->>CometChat: Sends message + CometChat->>Your Webhook: POST message + context + Your Webhook->>Your Webhook: Apply moderation logic + Your Webhook->>CometChat: Return decision + CometChat->>CometChat: Apply configured action + CometChat->>User: Message delivered or blocked +``` -This moderation provide flexibility to enhance user safety and compliance within your chat platform. +1. **User sends a message** in your chat application +2. **CometChat intercepts** the message and calls your webhook endpoint +3. **Your webhook processes** the message using your custom moderation logic +4. **Your webhook responds** with a decision (violation detected or not) +5. **CometChat applies the action** you configured (block, flag, allow, etc.) + +## Getting Started + + + + Create a webhook that receives messages and returns moderation decisions. You can use any third-party moderation service or your own AI model. + + + Set up your webhook URL, authentication, and moderation rules in the CometChat Dashboard. + + Step-by-step guide to configure your custom moderation API + + + + Set up webhooks to receive notifications when messages are approved or blocked by your moderation logic. + + Learn about moderation webhook events + + + + +## Key Features + +- **Contextual Moderation** – Include previous messages from the conversation for better analysis +- **Error Handling** – Configure fallback behavior when your API is unavailable +- **Flexible Rules** – Apply custom moderation to text or image content +- **Real-time Processing** – Moderation decisions are applied before message delivery diff --git a/moderation/custom/custom-api.mdx b/moderation/custom/custom-api.mdx index a6456d2e..291d49fe 100644 --- a/moderation/custom/custom-api.mdx +++ b/moderation/custom/custom-api.mdx @@ -2,63 +2,67 @@ title: "Custom API Moderation" --- -CometChat allows you to integrate your own moderation logic using a **Custom API**. With this feature, you can define a webhook URL in the **List Configuration**, where CometChat will send messages for moderation along with relevant context from the conversation (if provided in settings). - -## **How It Works** - -1. When a user sends a message, CometChat retrieves the webhook URL from the configured **List**. -2. The message, along with previous conversation messages (if a context window is set in settings), is sent to the webhook. -3. The webhook (your external API) processes the data using your custom moderation logic. -4. The webhook responds with a structured decision containing details about the moderation outcome. -5. CometChat processes the response and applies the moderation decision in real-time. - -This approach gives you complete control over moderation, allowing you to implement **custom filtering, AI-based analysis, or any other logic** on your own servers. +CometChat allows you to integrate your own moderation logic using a **Custom API**. With this feature, you can define a webhook URL where CometChat will send messages for moderation along with relevant context from the conversation. ## Integration ### Step 1: Configure Custom API Settings 1. **Login to the CometChat Dashboard** - - * Navigate to [CometChat Dashboard](https://app.cometchat.com) and select your app. + - Navigate to [CometChat Dashboard](https://app.cometchat.com) and select your app. 2. **Navigate to Moderation Settings** - - * Go to **Moderation → Settings** in the left-hand menu. + - Go to **Moderation → Settings** in the left-hand menu. 3. **Open Custom API Settings Tab** - - * Click on the **Custom API** tab within the Moderation Settings. + - Click on the **Custom API** tab within the Moderation Settings. 4. **Fill in the Custom API Configuration** - * **Set Action on API Error** + - **Webhook URL** + - Enter the endpoint URL where CometChat will send messages for moderation. - * Define how the system should respond if the Custom API is unavailable (e.g., "Allow message" or "Block message"). + - **Authentication (Optional)** + - Enable Basic Authentication to secure your webhook endpoint. + - Provide a username and password that CometChat will include in the `Authorization` header. - * **Set Context Window** + - **Set Action on API Error** + - Define how the system should respond if the Custom API is unavailable: + - **Allow message** – Messages are delivered even if moderation fails. + - **Block message** – Messages are blocked when moderation is unavailable. - * Specify the number of previous messages in a conversation that will be used for context. + - **Set Context Window** + - Specify the number of previous messages in a conversation to include for context (0-10). 5. **Click Save Settings** -### Step 2: Enable Custom API Moderation +### Step 2: Create a Moderation Rule 1. Navigate to **Moderation → Rules**. 2. Click **"Create New Rule"**. 3. Select **Custom API** as the moderation type. -4. The rule you create should be of type **"Text Contains"** or **"Image Contains"**. -5. Save the rule. +4. Choose the rule type: + - **Text Contains** – For text message moderation + - **Image Contains** – For image message moderation +5. Configure the action to take when content is flagged (block, flag for review, etc.). +6. Save the rule. -## Payload Sent to Webhook +## Webhook Request -When a message is sent, CometChat invokes your webhook with a payload that includes: +### Headers -* he latest message (the one just sent) — provided in full detail (entire message object) +When CometChat calls your webhook, it includes the following headers: -* The previous messages — provided as plain text only, for context (based on the context window setting) +| Header | Description | +|--------|-------------| +| `Content-Type` | `application/json` | +| `Authorization` | Basic auth credentials (if configured) | -This structure allows you to apply moderation logic to the current message while considering its surrounding context. +### Payload + +The payload includes: +- The latest message (the one just sent) — provided in full detail (entire message object) +- The previous messages — provided as plain text only, for context (based on the context window setting) ```json { @@ -115,21 +119,83 @@ This structure allows you to apply moderation logic to the current message while } }, "sentAt": 1747717214, - "updatedAt": 1747717214, + "updatedAt": 1747717214 } } ] } ``` -## Webhook Response Format +## Webhook Response + +Your webhook must return a JSON response indicating the moderation decision. + +### When content violates rules + +```json +{ + "isMatchingCondition": true, + "confidence": 0.95, + "reason": "Contains hate speech" +} +``` + +### When content is safe + +```json +{ + "isMatchingCondition": false, + "confidence": 0.98, + "reason": "" +} +``` + +### Response Fields -The webhook should return a response in the following format: +| Field | Type | Description | +|-------|------|-------------| +| `isMatchingCondition` | boolean | `true` if the message violates the rule, `false` if safe | +| `confidence` | number | Confidence score of the decision (0.0 - 1.0) | +| `reason` | string | Reason for flagging (can be empty for safe content) | + +## Example Webhook Implementation + +Here's a simple Node.js/Express example: ```javascript - { - isMatchingCondition: true, // True if the message violates the rule - confidence: 0.95, // Confidence score of the decision - reason: "Contains hate speech" // Reason for flagging +const express = require('express'); +const app = express(); + +app.use(express.json()); + +app.post('/moderate', (req, res) => { + const { contextMessages } = req.body; + + // Get the latest message (last item in array) + const latestEntry = contextMessages[contextMessages.length - 1]; + const senderId = Object.keys(latestEntry)[0]; + const messageData = latestEntry[senderId]; + + // Extract text content + const text = typeof messageData === 'string' + ? messageData + : messageData.data?.text || ''; + + // Your moderation logic here + const isViolation = containsBadContent(text); + + res.json({ + isMatchingCondition: isViolation, + confidence: 0.95, + reason: isViolation ? 'Content policy violation' : '' + }); +}); + +function containsBadContent(text) { + // Implement your moderation logic + // Could call OpenAI Moderation API, Perspective API, etc. + return false; } + +app.listen(3000); ``` diff --git a/moderation/flagged-messages.mdx b/moderation/flagged-messages.mdx index fad279ed..5aed3e4e 100644 --- a/moderation/flagged-messages.mdx +++ b/moderation/flagged-messages.mdx @@ -2,70 +2,182 @@ title: "Flagged Messages" --- -## Overview +Flagged Messages allows moderators to review messages that have been flagged for potentially violating moderation rules. Messages can be flagged automatically by the rule engine or manually by end users who find content inappropriate. -The Flagged Messages endpoint in the Moderation Service API enables app owners and collaborators to review messages that have been flagged for potentially violating moderation rules. Messages can be flagged automatically by the rule engine based on predefined rules, or manually by end users who find certain content inappropriate or concerning. + +**Two-step moderation.** Unlike blocked messages which are hidden immediately, flagged messages remain visible until a moderator reviews and takes action (approve or block). + -When a message is flagged, it becomes visible in the CometChat Dashboard under the flagged messages section. Moderators can then review these flagged messages and take one of two actions: +--- -1. **Approve** - The message is deemed acceptable and the moderation status remains approved -2. **Block** - The message violates platform policies and the moderation status is changed to disapproved +## Quick Start -Once either action is taken, the message is automatically moved to the Reviewed list, which contains all messages that have been reviewed by moderators. +Review flagged messages in under 2 minutes: -#### Manual Flagging by End Users + + + Login to [CometChat Dashboard](https://app.cometchat.com) → Select your app → **Moderation** → **Flagged Messages** + + + Click on a flagged message to see details, context, and the reason it was flagged + + + Click **Approve** (message stays visible) or **Block** (message is hidden) + + -End users can manually flag messages they find inappropriate or concerning by selecting from a predefined list of reasons. These flag reasons are configurable through the CometChat Dashboard, where you can either use the default options or create custom flagging reasons tailored to your platform's needs. Once configured, these reasons will appear in the user interface, allowing users to select the most appropriate reason when reporting a message. - - - -Messages can also be flagged via [Flag Message API](/rest-api/moderation/flag-a-message) +--- -#### Automatic Flagging by Rule Engine +## How It Works + +```mermaid +flowchart LR + A[Message Sent] --> B{Flagging} + B --> C[Manual Flag by User] + B --> D[Auto Flag by Rule Engine] + C --> E[Review Queue] + D --> E + E --> F{Moderator Action} + F --> G[Approve] + F --> H[Block] + G --> I[Reviewed List] + H --> I +``` + +| Step | Description | +|------|-------------| +| 1. Message Sent | A user sends a message in a conversation | +| 2. Flagging | Message is flagged either manually by users or automatically by the rule engine | +| 3. Review Queue | Flagged message appears in the Dashboard for moderator review | +| 4. Moderator Action | Moderator reviews and either approves or blocks the message | +| 5. Reviewed List | Message moves to the reviewed list with its final status | -Messages can be automatically flagged when they violate predefined moderation rules. When setting up moderation rules in the Dashboard, you can configure the action to "flag" messages that match specific criteria. This ensures that potentially problematic content is automatically identified and queued for review without requiring manual intervention from users. -For e.g. if the message contains any profane words, then it would be automatically flagged. - - - +--- -#### Configuring Flag Reasons +## Flagging Methods + + + + End users can manually flag messages they find inappropriate by selecting from a predefined list of reasons. + + + + + + **Configure flag reasons:** Dashboard → **Moderation** → **Advanced Settings** + + + + + + + Messages are automatically flagged when they match moderation rules configured with the "Flag" action. + + + + + + **Set up auto-flagging:** Create a rule in [Rules Management](/moderation/rules-management) with Action set to "Flag" + + -You can customize the flagging reasons available to end users through the CometChat Dashboard. The platform provides a set of default flagging reasons that cover common moderation scenarios, or you can create custom reasons that align with your platform's specific community guidelines and policies. These configuration options are accessible through the Advanced Settings within the Moderation section of your dashboard. - - - -Flag reasons can be also be configured via REST APIs [Reason APIs](/rest-api/moderation/create-reasons). +--- + +## Enable Flagging in Your App + + + + The Report Message feature is available in CometChat UI Kits. Users can report messages directly from the message actions menu: + + + } href="/ui-kit/react/core-features#report-message" horizontal /> + } href="/ui-kit/react-native/core-features" horizontal /> + } href="/ui-kit/android/core-features" horizontal /> + } href="/ui-kit/ios/core-features" horizontal /> + } href="/ui-kit/flutter/core-features" horizontal /> + } href="/ui-kit/angular/core-features" horizontal /> + } href="/ui-kit/vue/overview" horizontal /> + + + + Implement message flagging directly using CometChat Chat SDKs: + + + } href="/sdk/javascript/flag-message" horizontal /> + } href="/sdk/react-native/flag-message" horizontal /> + } href="/sdk/android/flag-message" horizontal /> + } href="/sdk/ios/flag-message" horizontal /> + } href="/sdk/flutter/flag-message" horizontal /> + + + + +--- + +## Managing Flagged Messages ### List Flagged Messages -Retrieves the list of messages that have been flagged by the moderation system or manually by moderators. +View all messages pending review in the Flagged Messages tab. + -You can also set this up from your end using the [List Flagged Messages REST API](/rest-api/moderation/list-flagged-messages). - ---- - -### Approve Flagged Message +### Approve Message -Allows the approval of messages that were previously flagged. Approving a flagged message indicates that it has been reviewed and is considered acceptable according to platform guidelines. +Approving indicates the message is acceptable. It remains visible to all users and moves to the Reviewed list. -You can also set this up from your end using the [Approve Flagged Message REST API](/rest-api/moderation/blockreview-flagged-message). - ---- - -### Block Flagged Message +### Block Message -Allows the blocking of messages that were flagged. Blocking a flagged message will automatically mark it as reviewed. +Blocking marks the message as disapproved. It's hidden from the conversation and moves to the Reviewed list. -You can also set this up from your end using the [Block Flagged Message REST API](/rest-api/moderation/blockreview-flagged-message). + +**Bulk actions supported.** Select multiple flagged messages and approve or block them in a single action. + + +--- + +## FAQ + + + + Blocked messages are hidden from the conversation and no longer visible to users. The message is marked as disapproved and moved to the Reviewed list. + + + By default, users are not notified when their message is blocked. The message simply becomes hidden from the conversation. + + + Yes, the Dashboard supports bulk actions. Select multiple flagged messages and approve or block them in a single action. + + + Flagged messages are pending review and remain visible until a moderator takes action. Blocked messages are immediately hidden and don't require review. + + + +--- + +## Related Resources + + + + Configure rules to auto-flag messages + + + View immediately blocked content + + + Create keyword lists for flagging + + + Learn about the moderation system + + diff --git a/moderation/getting-started.mdx b/moderation/getting-started.mdx index b88f4245..7631e6ed 100644 --- a/moderation/getting-started.mdx +++ b/moderation/getting-started.mdx @@ -9,90 +9,85 @@ To maintain a safe, respectful, and engaging environment for your users, our pla With Moderation Integration, you can define flexible rules, receive real-time updates, and ensure your app meets community guidelines, legal standards, and brand values without manual intervention. - -If you’re using the CometChat UI Kit or SDK, you can skip steps 2 and 3. - +--- + +## Choose Your Integration Method + + + + **Zero code required** — Moderation is built into CometChat UI Kits. Simply configure rules in the Dashboard and the UI Kit handles everything automatically. + + + **Full control** — Use the CometChat SDK to handle moderation programmatically in your custom UI implementation. + + --- ## Integrate Moderation -Note: If you're using CometChat UI Kits or SDKs, you can skip steps 2 & 3. +**Using UI Kit?** You only need Step 1 - the UI Kit handles moderation automatically! - -

Define content moderation rules for your app's messaging system.

- - - -

Set up a webhook to receive real-time moderation events.

- - - -

Use APIs to send and review flagged messages programmatically.

- - + + Define content moderation rules for your app's messaging system. + + + + If building a custom UI, implement moderation handling in your code. See [SDK Integration](#integrating-moderation-with-sdk) below. + + + Set up webhooks to receive real-time moderation events on your server. + + --- -## Key Components of Moderation Integration - - -**Step 1:** [Set up moderation rules](#setting-up-moderation-rules) - -**Step 2:** [Configure a moderation webhook](#configuring-a-moderation-webhook) - -**Step 3:** [Integrate and test the Moderation API](#integrating-and-testing-the-moderation-api) - ---- - ## Setting Up Moderation Rules Moderation rules act as filters to ensure that the messages exchanged within your app meet your safety and content guidelines. ### How It Works -- When a message, image, or video is submitted, it is automatically checked against the moderation rules you’ve configured. +- When a message, image, or video is submitted, it is automatically checked against the moderation rules you've configured. - These rules can detect offensive language, sensitive content, spam, scams, and more. - Based on your settings, content can be: - **Approved:** Delivered to the recipient. - **Disapproved:** Blocked and not delivered. - - **Flagged:** Delivered to the recipient, but flagged. + - **Flagged:** Delivered to the recipient, but flagged for review. ### Benefits - **Safety first:** Protect your users and brand from harmful or unwanted content. -- **Customizable:** Fine-tune moderation rules to suit your app’s unique needs. +- **Customizable:** Fine-tune moderation rules to suit your app's unique needs. - **Seamless experience:** Moderation happens in real time, keeping communication flowing smoothly. ### Creating Moderation Rules CometChat provides a set of **default moderation rules** designed to cover common use cases such as offensive language, spam, and inappropriate content. You can enable these rules to start moderating messages immediately, without any additional setup. -{/* ![Default moderation rules](/images/existing-rules.png) */} -If you have specific requirements, you can create **custom moderation rules** tailored to your app’s needs. Rules can be created in two ways: +If you have specific requirements, you can create **custom moderation rules** tailored to your app's needs: 1. **Using the CometChat Dashboard** — A simple, no-code interface for visually creating and managing moderation rules. - {/* ![Create rule screen](/images/create-rule.png) */} + -2. **Using the CometChat API** — Programmatically create and manage moderation rules for advanced or automated workflows. See the [Create Rule API documentation]({apiEndpoints.createRule.url}). + +2. **Using the CometChat API** — Programmatically create and manage moderation rules for advanced or automated workflows. See the [Rules Management](/moderation/rules-management) documentation. --- @@ -100,30 +95,18 @@ If you have specific requirements, you can create **custom moderation rules** ta To automate your moderation flow and receive updates in real time, configure a **moderation webhook**. This allows your system to react instantly when a message or media is moderated. + +Webhooks are optional if you're using the SDK's `onMessageModerated` listener for real-time updates. + + ### How It Works - Every time content is moderated, a webhook event is triggered and sent to the URL you specify. - Your application can then take action based on the moderation result. -### Prerequisites - -- Your webhook URL must be accessible over **HTTPS** to ensure secure data transmission. -- The URL should be publicly accessible from the internet. -- Ensure your endpoint supports the `HTTP POST` method. Event payloads will be delivered via `POST` requests in JSON format. -- Configure your endpoint to respond immediately to the CometChat server with a `200 OK` response. -- For security, set up **Basic Authentication** (username and password) for server-to-server communication. - -When your webhook URL is triggered, the HTTP header includes: - -```http -Authorization: Basic -``` - ### Handle Moderation Events -Ensure your webhook endpoint is ready to process moderation events. Refer to the [Moderation Events](/docs-beta/fundamentals/webhooks-overview#moderation-events) section for the full list. - -The key events to handle include: +The key webhook events to handle include: - `moderation_engine_approved` — Triggered when the engine automatically approves content. - `moderation_engine_blocked` — Triggered when the engine automatically blocks content. @@ -133,104 +116,362 @@ To receive these events, enable the relevant webhooks in the CometChat Dashboard > **Application → Webhooks → Create Webhook → Triggers → Moderation** -Select all three moderation triggers to ensure your app receives the necessary notifications. - -{/* ![Enable webhooks](/images/webhook-events.png) */} --- -## Integrating and Testing the Moderation API - -Once your moderation rules and webhook are configured, integrate the **Moderation API** into your application to programmatically submit content and receive moderation results. - -### Steps to Integrate and Test - -1. **Submit content:** Use the API to send messages, images, or videos for moderation. -2. **Check responses:** Verify moderation status in real time. -3. **Handle outcomes:** Apply business logic based on approved, flagged, or disapproved responses. -4. **End-to-end testing:** Test both the API response and webhook delivery to ensure complete coverage. - -When you’re ready, you can render all moderation endpoints dynamically: - -### Send message - -This endpoint is used to submit a message for moderation before it is delivered to the recipient. The message is scanned against the moderation rules configured for the app. - -**URL:** -``` -https://api-explorer.cometchat.com/reference/chat-api-send-message-moderation -``` - -**Request Body:** -```json -{ - "category": "message", - "type": "text", - "data": { - "text": "Hi new user" - }, - "sender": "cometchat-uid-2", - "receiver": "cometchat-uid-1", - "receiverType": "user", - "sentAt": 1750335220 -} -``` - -### List messages -Retrieves a list of messages submitted for moderation, along with the current moderation status of each message as determined by your configured rules. - -**URL:** -``` -https://api-explorer.cometchat.com/reference/chat-api-list-message-moderation/ -``` - -### Get message -Retrieves the details of a specific message submitted for moderation, including its current moderation status as determined by your configured rules. - -**URL:** -``` -https://api-explorer.cometchat.com/reference/chat-api-get-message-moderation/ -``` - -### Update message -Edits an existing message. The moderation status is re-evaluated based on your configured rules. - -**URL:** -``` -https://api-explorer.cometchat.com/reference/chat-api-update-message-moderation/ -``` - -### Delete message -Deletes a previously submitted message along with its associated moderation data, in accordance with your configured rules. - -**URL:** -``` -https://api-explorer.cometchat.com/reference/chat-api-delete-message-moderation/ -``` - -### Approve message -Approves a previously blocked message, allowing it to be delivered to the recipient. - -**URL:** -``` -https://api-explorer.cometchat.com/reference/chat-api-approve-moderation-blocked-messages/ -``` - ---- - -| Endpoint | Purpose | -| ----------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | -| **Send message for moderation**
`POST /moderation/messages` | Submits a new message for moderation. Triggers the engine, emits a webhook, and makes the message available via the get-message endpoint. | -| **Edit message for moderation**
`PUT /moderation/messages/:id` | Re-evaluates an edited message. If approved, the updates are accepted; if blocked, the message is withheld. | -| **Get message moderation status**
`GET /moderation/messages/:id`| Retrieves the moderation status, rule metadata, and action history of a message. | -| **List messages for moderation**
`GET /moderation/messages` | Lists all moderated messages with optional filters such as category, type, status, and receiver UID/GUID. | -| **Delete message from moderation**
`DELETE /moderation/messages/:id` | Deletes a moderated message. | +## Integrating Moderation with SDK + +Once your moderation rules are configured in the Dashboard, integrate moderation into your application using the CometChat SDK. The SDK automatically handles moderation for all messages sent through CometChat. + + +**Using UI Kit?** Skip this section - the UI Kit handles everything automatically! + + +### Send a Message and Check Moderation Status + +When you send a message, it's automatically checked against your moderation rules. The message object contains a `moderationStatus` property: + + + + ```javascript + const textMessage = new CometChat.TextMessage( + receiverUID, + "Hello, how are you?", + CometChat.RECEIVER_TYPE.USER + ); + + CometChat.sendMessage(textMessage).then( + (message) => { + const status = message.getModerationStatus(); + + switch (status) { + case CometChat.ModerationStatus.PENDING: + console.log("Message is under moderation review"); + break; + case CometChat.ModerationStatus.APPROVED: + console.log("Message approved and delivered"); + break; + case CometChat.ModerationStatus.DISAPPROVED: + console.log("Message blocked by moderation"); + break; + } + }, + (error) => { + console.log("Message sending failed:", error); + } + ); + ``` + + + ```kotlin + val textMessage = TextMessage( + receiverUID, + "Hello, how are you?", + CometChatConstants.RECEIVER_TYPE_USER + ) + + CometChat.sendMessage(textMessage, object : CometChat.CallbackListener() { + override fun onSuccess(message: TextMessage) { + when (message.moderationStatus) { + ModerationStatus.PENDING -> Log.d(TAG, "Message under review") + ModerationStatus.APPROVED -> Log.d(TAG, "Message approved") + ModerationStatus.DISAPPROVED -> Log.d(TAG, "Message blocked") + } + } + + override fun onError(e: CometChatException) { + Log.e(TAG, "Message sending failed: ${e.message}") + } + }) + ``` + + + ```swift + let textMessage = TextMessage( + receiverUid: receiverUID, + text: "Hello, how are you?", + receiverType: .user + ) + + CometChat.sendTextMessage(message: textMessage) { sentMessage in + if let message = sentMessage as? TextMessage { + switch message.getModerationStatus() { + case "pending": + print("Message under review") + case "approved": + print("Message approved") + case "disapproved": + print("Message blocked") + default: + break + } + } + } onError: { error in + print("Message sending failed: \(error?.errorDescription ?? "")") + } + ``` + + + ```javascript + const textMessage = new CometChat.TextMessage( + receiverUID, + "Hello, how are you?", + CometChat.RECEIVER_TYPE.USER + ); + + CometChat.sendMessage(textMessage).then( + (message) => { + const status = message.getModerationStatus(); + + if (status === CometChat.ModerationStatus.PENDING) { + console.log("Message under review"); + } else if (status === CometChat.ModerationStatus.DISAPPROVED) { + console.log("Message blocked"); + } + }, + (error) => console.log("Failed:", error) + ); + ``` + + + ```dart + TextMessage textMessage = TextMessage( + text: "Hello, how are you?", + receiverUid: receiverUID, + receiverType: ReceiverTypeConstants.user, + ); + + CometChat.sendMessage( + textMessage, + onSuccess: (TextMessage message) { + switch (message.moderationStatus?.value) { + case ModerationStatusEnum.PENDING: + print("Message under review"); + break; + case ModerationStatusEnum.APPROVED: + print("Message approved"); + break; + case ModerationStatusEnum.DISAPPROVED: + print("Message blocked"); + break; + } + }, + onError: (CometChatException e) { + print("Message sending failed: ${e.message}"); + }, + ); + ``` + + + +### Listen for Moderation Results + +Register a message listener to receive real-time moderation updates when the moderation engine finishes processing: + + + + ```javascript + const listenerID = "MODERATION_LISTENER"; + + CometChat.addMessageListener( + listenerID, + new CometChat.MessageListener({ + onMessageModerated: (message) => { + const status = message.getModerationStatus(); + const messageId = message.getId(); + + if (status === CometChat.ModerationStatus.APPROVED) { + console.log(`Message ${messageId} approved - show in UI`); + // Update UI to display the message + } else if (status === CometChat.ModerationStatus.DISAPPROVED) { + console.log(`Message ${messageId} blocked`); + // Hide message or show blocked placeholder + } + } + }) + ); + + // Remove listener when done + // CometChat.removeMessageListener(listenerID); + ``` + + + ```kotlin + val listenerID = "MODERATION_LISTENER" + + CometChat.addMessageListener(listenerID, object : CometChat.MessageListener() { + override fun onMessageModerated(message: BaseMessage) { + when (message.moderationStatus) { + ModerationStatus.APPROVED -> { + Log.d(TAG, "Message ${message.id} approved") + // Update UI to display the message + } + ModerationStatus.DISAPPROVED -> { + Log.d(TAG, "Message ${message.id} blocked") + // Hide message or show blocked placeholder + } + } + } + }) + + // Remove listener when done + // CometChat.removeMessageListener(listenerID) + ``` + + + ```swift + let listenerID = "MODERATION_LISTENER" + + CometChat.addMessageListener(listenerID, self) + + // Implement CometChatMessageDelegate + extension YourViewController: CometChatMessageDelegate { + func onMessageModerated(moderatedMessage: BaseMessage) { + if let message = moderatedMessage as? TextMessage { + switch message.getModerationStatus() { + case "approved": + print("Message \(message.id) approved") + // Update UI to display the message + case "disapproved": + print("Message \(message.id) blocked") + // Hide message or show blocked placeholder + default: + break + } + } + } + } + + // Remove listener when done + // CometChat.removeMessageListener(listenerID) + ``` + + + ```javascript + const listenerID = "MODERATION_LISTENER"; + + CometChat.addMessageListener( + listenerID, + new CometChat.MessageListener({ + onMessageModerated: (message) => { + const status = message.getModerationStatus(); + + if (status === CometChat.ModerationStatus.APPROVED) { + // Update UI to display the message + } else if (status === CometChat.ModerationStatus.DISAPPROVED) { + // Hide message or show blocked placeholder + } + } + }) + ); + ``` + + + ```dart + const listenerID = "MODERATION_LISTENER"; + + CometChat.addMessageListener( + listenerID, + MessageListener( + onMessageModerated: (BaseMessage message) { + if (message.moderationStatus?.value == ModerationStatusEnum.APPROVED.value) { + print("Message ${message.id} approved"); + // Update UI to display the message + } else if (message.moderationStatus?.value == ModerationStatusEnum.DISAPPROVED.value) { + print("Message ${message.id} blocked"); + // Hide message or show blocked placeholder + } + }, + ), + ); + + // Remove listener when done + // CometChat.removeMessageListener(listenerID); + ``` + + + + +### Handle Blocked Messages in UI + +When a message is blocked (disapproved), handle it appropriately in your UI: + + + + ```javascript + function handleBlockedMessage(message) { + const messageId = message.getId(); + const senderUid = message.getSender().getUid(); + + // Option 1: Hide the message completely + removeMessageFromUI(messageId); + + // Option 2: Show a placeholder + showPlaceholder(messageId, "This message was blocked by moderation"); + + // Option 3: Notify the sender + if (senderUid === currentUserUID) { + showToast("Your message was blocked due to policy violation"); + } + } + ``` + + + ```kotlin + fun handleBlockedMessage(message: BaseMessage) { + val messageId = message.id + val senderUid = message.sender.uid + + // Option 1: Hide the message completely + removeMessageFromUI(messageId) + + // Option 2: Show a placeholder + showPlaceholder(messageId, "This message was blocked by moderation") + + // Option 3: Notify the sender + if (senderUid == currentUserUID) { + Toast.makeText(context, "Your message was blocked", Toast.LENGTH_SHORT).show() + } + } + ``` + + + ```swift + func handleBlockedMessage(_ message: BaseMessage) { + let messageId = message.id + let senderUid = message.sender?.uid + + // Option 1: Hide the message completely + removeMessageFromUI(messageId) + + // Option 2: Show a placeholder + showPlaceholder(messageId, text: "This message was blocked by moderation") + + // Option 3: Notify the sender + if senderUid == currentUserUID { + showAlert("Your message was blocked due to policy violation") + } + } + ``` + + + +### Moderation Status Reference + +| Status | Value | Description | +|--------|-------|-------------| +| Pending | `PENDING` | Message is being processed by moderation engine | +| Approved | `APPROVED` | Message passed moderation and is visible to recipients | +| Disapproved | `DISAPPROVED` | Message violated rules and was blocked | --- ## Summary -By combining well-defined moderation rules, automated webhooks, and robust API integration, you can build a safe, scalable, and user-friendly content moderation system tailored to your app’s values and audience expectations. +By combining well-defined moderation rules with SDK integration, you can build a safe and user-friendly content moderation system: + +- **UI Kit users**: Just configure rules in the Dashboard - everything else is automatic +- **SDK users**: Implement `onMessageModerated` listener to handle moderation results diff --git a/moderation/legacy-extensions.mdx b/moderation/legacy-extensions.mdx new file mode 100644 index 00000000..b60a3692 --- /dev/null +++ b/moderation/legacy-extensions.mdx @@ -0,0 +1,348 @@ +--- +title: "Legacy Moderation Extensions" +sidebarTitle: "Legacy Extensions" +--- + + +These extensions are considered legacy and are scheduled for deprecation. They are no longer recommended for new integrations and will not receive feature updates or enhancements. + +For new projects, use [Custom API Moderation](/moderation/custom/custom-api-overview) or [OpenAI Moderation](/moderation/open-ai/openai-overview). + + +## Slow Mode + +Slow down messages in groups to make them legible during high-traffic events. + +When enabled in a group, participants can only send messages after configured intervals. Admins and moderators are not restricted. + +**Enable Slow Mode** + +```js +CometChat.callExtension('slow-mode', 'POST', 'v1/configure', { + "guid": "cometchat-guid-1", + "slowDownTimeInMS": 660000 +}).then(response => { + // Success +}).catch(error => { + // Error +}); +``` + +**Disable Slow Mode** + +```js +CometChat.callExtension('slow-mode', 'DELETE', 'v1/configure', { + "guid": "cometchat-guid-1" +}).then(response => { + // Success +}).catch(error => { + // Error +}); +``` + +**Fetch Slow Mode Details** + +```js +const guid = "cometchat-guid-1"; +CometChat.callExtension('slow-mode', 'GET', `v1/fetch-configuration?guid=${guid}`, null) + .then(response => { + // { isSlowed, slowDownTimeInMS, lastMessageSentAtTimestamp } + }); +``` + +--- + +## Report User + +Enables users to report other users for offensive or suspicious behavior. + +**Settings** + +1. Go to **Extensions** → Enable **Report User** +2. Configure moderation criteria (max reports before notification) +3. Set up webhook URL for reports + +**Report a User** + +```js +CometChat.callExtension('report-user', 'POST', 'v1/report', { + "uid": "cometchat-uid-3", + "reason": "Misbehaving", + "guid": "cometchat-guid-1" // Optional: only for group reports +}).then(response => { + // { success: true } +}); +``` + +**View Reports** + +Open the Extension's settings page and click "View Reports" to take action (Kick, Ban, Block, or Ignore). + +--- + +## Report Message + +Enable users to report messages in conversations. + +**Report a Message** + +```js +CometChat.callExtension('report-message', 'POST', 'v1/report', { + "msgId": 123, + "reason": "Contains profanity" +}).then(response => { + // { success: true } +}); +``` + +**View Reports** + +Open the Extension's settings page and click "View Reports" to Delete or Ignore reported messages. + +--- + +## Data Masking Filter + +Hide phone numbers, email addresses, and other sensitive information in messages. + +**Settings** + +1. Enable the extension +2. Configure default masks (Emails, SSN, US phone numbers) +3. Add custom regex patterns for additional masking + +**Response Format** + +```json +{ + "@injected": { + "extensions": { + "data-masking": { + "data": { + "sensitive_data": "yes", + "message_masked": "My number is ***** & my email id is ****" + } + } + } + } +} +``` + +**Implementation** + +```js +var metadata = message.getMetadata(); +if (metadata != null) { + var injectedObject = metadata["@injected"]; + if (injectedObject?.extensions?.["data-masking"]) { + var dataMaskingObject = injectedObject.extensions["data-masking"]["data"]; + var message_masked = dataMaskingObject["message_masked"]; + } +} +``` + +--- + +## Profanity Filter + +Mask or hide profanity in messages using a customizable blacklist. + +**Settings** + +1. Enable the extension +2. Optionally enable "Drop messages with Profanity" +3. Add comma-separated list of words to filter (supports emojis) + +**Response Format** + +```json +{ + "@injected": { + "extensions": { + "profanity-filter": { + "profanity": "yes", + "message_clean": "This is ****" + } + } + } +} +``` + +**Implementation** + +```js +var metadata = message.getMetadata(); +if (metadata != null) { + var injectedObject = metadata["@injected"]; + if (injectedObject?.extensions?.["profanity-filter"]) { + var profanityFilterObject = injectedObject.extensions["profanity-filter"]; + var cleanMessage = profanityFilterObject["message_clean"]; + } +} +``` + +--- + +## Image Moderation + +AI-powered image moderation to detect unsafe content. + +Images are classified into: +- Explicit Nudity +- Suggestive Nudity +- Violence +- Visually Disturbing + +**Response Format** + +```json +{ + "@injected": { + "extensions": { + "image-moderation": { + "unsafe": "yes", + "confidence": "99", + "category": "explicit_nudity" + } + } + } +} +``` + +A confidence value less than 50 is likely a false-positive. + +**Implementation** + +```js +const metadata = message.getMetadata(); +if (metadata != null) { + const injectedObject = metadata["@injected"]; + if (injectedObject?.extensions?.["image-moderation"]) { + const { attachments } = injectedObject.extensions["image-moderation"]; + for (const attachment of attachments) { + if (!attachment.error) { + const { unsafe } = attachment.data.verdict; + } + } + } +} +``` + +--- + +## Sentiment Analysis + +Understand the tone or sentiment of messages. + +Messages are classified as: Positive, Neutral, Negative, or Mixed. + +Supported languages: German, English, Spanish, Italian, Portuguese, French, Japanese, Korean, Hindi, Arabic, Chinese (simplified & traditional) + +**Response Format** + +```json +{ + "@injected": { + "extensions": { + "sentiment-analysis": { + "sentiment": "positive", + "sentiment_score": { + "positive": 95, + "neutral": 4, + "negative": 0, + "mixed": 0 + } + } + } + } +} +``` + +--- + +## In-Flight Message Moderation + +Manually moderate messages before delivery. + +**Settings** + +Configure moderation criteria: +- Moderate all messages, one-on-one only, or group only +- Moderate messages from specific users (UIDs) +- Moderate messages to specific users or groups + +**Actions** + +From the Dashboard, you can: +- **Approve**: Send the message to the receiver +- **Reject**: Delete the message +- **Kick**: Remove user from group (can rejoin) +- **Ban**: Ban user from group (cannot rejoin) + +--- + +## Virus & Malware Scanner + +Scan uploaded files for malicious content using [Scanii](https://scanii.com). + +**Prerequisites** + +Create a Scanii account and get your API Key and Secret. + +**Response Format** + +```json +{ + "@injected": { + "extensions": { + "virus-malware-scanner": { + "attachments": [{ + "data": { + "url": "https://media.com/file.mp3", + "verdict": { + "scan_results": [] + } + }, + "error": null + }] + } + } + } +} +``` + +An empty `scan_results` array means the file is safe. + +--- + +## XSS Filter + +Sanitize messages to prevent cross-site scripting attacks. Applicable only for Web SDK. + +**Response Format** + +```json +{ + "@injected": { + "extensions": { + "xss-filter": { + "hasXSS": "yes", + "sanitized_message": "" + } + } + } +} +``` + +**Implementation** + +```js +var metadata = message.getMetadata(); +if (metadata != null) { + var injectedObject = metadata["@injected"]; + if (injectedObject?.extensions?.["xss-filter"]) { + var xssFilterObject = injectedObject.extensions["xss-filter"]; + var sanitized_message = xssFilterObject["sanitized_message"]; + } +} +``` diff --git a/moderation/legacy/data-masking-filter.mdx b/moderation/legacy/data-masking-filter.mdx deleted file mode 100644 index 8bae55ac..00000000 --- a/moderation/legacy/data-masking-filter.mdx +++ /dev/null @@ -1,151 +0,0 @@ ---- -title: "Data Masking Filter (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -The Data Masking Extension allows you to hide phone numbers, email address and other sensitive information in messages. You as a developer, can add regular expressions for matching & masking. - -## Extension settings - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. - -2. Go to the Extensions section and enable the Data Masking Filter extension. - -3. Open the Settings for this extension and configure the following: - - 1. Drop Message: If enabled, any message with sensitive information will be dropped. - 2. Default Masks: Masks for Emails, Social Security Numbers (SSN), US phone numbers are built in. - 3. Custom Masks: Add more regex that will act as masks for some form of sensitive information. - -4. Save the extension settings. - - - -Refer [this](https://www.w3schools.com/jsref/jsref_obj_regexp.asp) for more details on Regular Expressions. - - - -## How does it work? - -Once the Extension is enabled for your App and the Extension Settings are done, the recipients will receive metadata with the masked message. Here is a sample response: - - - -```json -"@injected": { - "extensions": { - "data-masking": { - "data": { - "sensitive_data": "yes", - "message_masked": "My number is ***** & my email id is ****" - } - } - } -} -``` - - - - - -If the data-masking key is missing, it means that the extension is either not enabled or has timed out. - -## Implementation - -At the recipients' end, from the message object, you can fetch the metadata by calling the getMetadata() method. Using this metadata, you can fetch the masked message. - - - -```js -var metadata = message.getMetadata(); -if (metadata != null) { - var injectedObject = metadata["@injected"]; - if (injectedObject != null && injectedObject.hasOwnProperty("extensions")) { - var extensionsObject = injectedObject["extensions"]; - if ( - extensionsObject != null && - extensionsObject.hasOwnProperty("data-masking") - ) { - var dataMaskingFilterObject = extensionsObject["data-masking"]["data"]; - var sensitive_data = dataMaskingFilterObject["sensitive_data"]; - var message_masked = dataMaskingFilterObject["message_masked"]; - } - } -} -``` - - - - -```java -JSONObject metadata = message.getMetadata(); -if (metadata != null) { - JSONObject injectedObject = metadata.getJSONObject("@injected"); - if (injectedObject != null && injectedObject.has("extensions")) { - JSONObject extensionsObject = injectedObject.getJSONObject("extensions"); - if (extensionsObject != null && extensionsObject.has("data-masking")) - { - JSONObject dataMaskingObject = extensionsObject.getJSONObject("data-masking"); - JSONObject data = dataMaskingObject.getJSONObject("data"); - String sensitive_data = data.getString("sensitive_data"); - String message_masked = data.getString("message_masked"); - } - } -} -``` - - - - -```kotlin -if (metadata != null) { - if (metadata.has("@injected")) { - val injectedJSONObject = metadata.getJSONObject("@injected") - if (injectedJSONObject != null && injectedJSONObject.has("extensions")) { - val extensionsObject = injectedJSONObject.getJSONObject("extensions") - - if (extensionsObject != null && extensionsObject.has("data-masking")) { - val dataMaskingDetails = extensionsObject.getJSONObject("data") - val dataMaskingObject = dataMaskingDetails.getJSONObject("data") - val sensitive_data = dataMaskingObject.getString("sensitive_data") - val message_masked = dataMaskingObject.getString("message_masked")) - } - } - } -} -``` - - - - -```swift -let textMessage = message as? TextMessage -var metadata : [String : Any]? = textMessage.metaData -if metadata != nil { - - var injectedObject : [String : Any]? = (metadata?["@injected"] as? [String : Any])! - - if injectedObject != nil && (injectedObject!["extensions"] != nil){ - - var extensionsObject : [String : Any]? = injectedObject?["extensions"] as? [String : Any] - - if extensionsObject != nil && extensionsObject?["data-masking"] != nil { - var dataMasking = extensionsObject?["data-masking"] as! [String : Any] - var dataMaskingDetails = dataMasking?["data"] as! [String : Any] - let sensitive_data = dataMaskingDetails["sensitive_data"] as! String - let message_masked = dataMaskingDetails["message_masked"] as! String - } - } -} -``` - - - - diff --git a/moderation/legacy/image-moderation.mdx b/moderation/legacy/image-moderation.mdx deleted file mode 100644 index aceabb0f..00000000 --- a/moderation/legacy/image-moderation.mdx +++ /dev/null @@ -1,207 +0,0 @@ ---- -title: "Image Moderation (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -AI-powered image moderation to detect unsafe content. - -## Extension settings - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. -2. Go to the Extensions section and enable the Image Moderation extension. -3. Open up the Settings and choose to Drop messages with NSFW images. - -## How does it work? - -After analyzing, it classifies the image into four categories: - -1. Explicit Nudity -2. Suggestive Nudity -3. Violence -4. Visually Disturbing. - -Along with that, you will receive the confidence, on a scale of 0 to 100. - - - -```json -"@injected": { - "extensions": { - "image-moderation": { - "unsafe": "yes/no", - "confidence": "99", - "category": "explicit_nudity/suggestive/violence/visually_disturbing", - "attachments": [ - { - "data": { - "name": "1584307225_38928710_1d3e5acc1b009e1c4ce239bedc2851f9.jpeg", - "extension": "jpeg", - "size": 402852, - "mimeType": "image/png", - "url": "https://media.com/1594986359_2067554844_9.png", - "verdict": { - "unsafe": "yes/no", - "confidence": "99", - "category": "explicit_nudity/suggestive/violence/visually_disturbing", - } - }, - "error": null, - }, - { - "data": { - "name": "1584307225_38928710_1d3e5acc1b009e1c4ce239bedc2851f9.jpeg", - "extension": "jpeg", - "size": 402852, - "mimeType": "image/png", - "url": "https://media.com/1594986359_2067554844_9.png", - "verdict": null, - }, - "error": { - "code": "ERROR_CODE", - "message": "Error Message", - "devMessage": "Error message", - "source": "ext-api" - } - } - ] - } - } -} -``` - - - - - -A value for `confidence` that is less than 50 is likely to be a false-positive. So we recommend moderating only if `confidence` is higher than 50. - -If the image-moderation key is missing, it means that either the extension is not enabled or has timed out. - - - -The `unsafe`, confidence & category keys to the outside of `attachments` are the result for the first attachment from the `attachments` array. These have been retained for backward compatibility only.\ -You can iterate over `attachments` array for better implementation. - - - -## Implementation - -You can then either show a warning or drop the image message. - - - - - -At the recipients' end, from the `message` object, you can fetch the metadata by calling the `getMetadata()` method. Using this metadata, you can fetch information whether the image is safe or unsafe. - - - -```js -const metadata = message.getMetadata(); -if (metadata != null) { - const injectedObject = metadata["@injected"]; - if (injectedObject != null && injectedObject.hasOwnProperty("extensions")) { - const extensionsObject = injectedObject["extensions"]; - if ( - extensionsObject != null && - extensionsObject.hasOwnProperty("image-moderation") - ) { - const { attachments } = extensionsObject["image-moderation"]; - for (const attachment of attachments) { - if (!attachment.error) { - const { unsafe } = attachment.data.verdict; - // Check the other parameters as required. - } - } - } - } -} -``` - - - - -```java -JSONObject metadata = message.getMetadata(); -if (metadata != null) { - JSONObject injectedObject = metadata.getJSONObject("@injected"); - if (injectedObject != null && injectedObject.has("extensions")) { - JSONObject extensionsObject = injectedObject.getJSONObject("extensions"); - if (extensionsObject != null && extensionsObject.has("image-moderation")) - { - JSONObject tg = extensionsObject.getJSONObject("image-moderation"); - JSONArray attachments = tg.getJSONArray("attachments"); - - for (int i = 0; i < attachments.length(); i++) { - JSONObject attachment = attachments.getJSONObject(i); - JSONObject error = attachment.getJSONObject("error"); - if (error == null) { - JSONObject data = attachment.getJSONObject("data"); - JSONObject verdict = data.getJSONObject("verdict"); - String unsafe = thumbnails.getString("unsafe"); - // Check the other parameters as required. - } - } - } - } -} -``` - - - - -```swift -if let metaData = message?.metaData , let injected = metaData["@injected"] as? [String : Any], let extensions =injected["extensions"] as? [String : Any], let attachments = extensions["image-moderation"] as? [[String : Any]] { - - for attachment in attachments { - - if let data = attachment["data"] as? [String:Any] , let verdict = data["verdict"] as? [String:any] { - // Check for the parameters as required. - if let unsafe = URL(string: verdict["unsafe"] as? String) { -// Use the url accordingly. - } - - // check for attachment.error if "verdict" is null - } -} -} -``` - - - - -```kotlin -if (metadata != null) { - if (metadata.has("@injected")) { - val injected = metadata.getJSONObject("@injected") - if (injected != null && injected.has("extensions")) { - val extensions = injectedJSONObject.getJSONObject("extensions") - if (extensions != null && extensions.has("image-moderation")) { - val tg = extensions.getJSONObject("image-moderation") - val attachments = tg.getJSONArray("attachments") - for (i in 0 until attachments.length()) { - val attachment = attachments.getJSONObject(i) - val error = attachment.getJSONObject("error") - if (error == null) { - val data = attachment.getJSONObject("data") - val verdict = data.getJSONObject("verdict") - val unsafe = verdict.getString("unsafe") - // use other parameters are required. - } - } - } - } - } -} -``` - - - - diff --git a/moderation/legacy/in-flight-message-moderation.mdx b/moderation/legacy/in-flight-message-moderation.mdx deleted file mode 100644 index 6dd1a27b..00000000 --- a/moderation/legacy/in-flight-message-moderation.mdx +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: "In Flight Message Moderation (Legacy)" -sidebarTitle: "In-flight Message Moderation (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -In-flight message Moderation extension allows you to moderate messages manually. - -## Settings - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. - -2. Go to the Extensions section and enable the In-flight moderation extension. - -3. Open the Settings for this extension and set the following: - - 1. Moderation Criteria - - 1. Moderate all the messages - 2. Moderate only one-on-one messages - 3. Moderate only Group messages - - 2. Moderate messages sent BY certain users (Comma-separated UIDs). - - 3. Moderate messages sent TO ceratin users (Comma-separated UIDs). - - 4. Moderate messages sent TO a certain groups (Comma-separated GUIDs). - -## How does it work? - -Once the extension is enabled and the settings are saved, navigate to the In-flight Message Moderation section of the Dashboard to manually moderate messages: - - - - - -All the messages that match the moderation criteria will get listed from oldest to newest. You can perform the following actions for a particular message: - -1. **Approve:** The message gets sent to the Receiver and disappears from the list. -2. **Reject:** The message gets deleted and is not sent to the intended receiver. -3. **Kick:** Available for Group conversations. A notorious user gets kicked out of the group. They can rejoin the group. -4. **Ban:** Available for Group conversations. A notorious user gets banned from the group. They cannot rejoin later unless they are unbanned. - -Apart from the above actions, you can also quickly access the Settings for the Extension. You can toggle Auto-refresh Messages and also toggle the Switch to Enable/Disable the Moderation. - - - -If you disable the Extension, only the new messages will flow without being routed through the In-flight Message Moderation Section. However, the existing messages that you see in the section will have to be Approved in order to be received by the intended recipient. - - diff --git a/moderation/legacy/profanity-filter.mdx b/moderation/legacy/profanity-filter.mdx deleted file mode 100644 index 4c62c768..00000000 --- a/moderation/legacy/profanity-filter.mdx +++ /dev/null @@ -1,155 +0,0 @@ ---- -title: "Profanity Filter (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -The Profanity Filter Extension helps you to mask or hide profanity in a message. We check for words from a blacklist (which you can customize) and then mask them. - -## Extension settings - -1. Login to the [CometChat](https://app.cometchat.com/login) and select your app. -2. Go to the Extensions section and enable the Profanity Filter extension. -3. Open Settings and choose to Drop messages with Profanity. -4. Also, you can provide the comma separated list of words that you would like to filter. - -### Support for filtering emojis - -Apart from words, we have also added support for filtering Profane emojis along with words. The only thing that you need to keep in mind is, certain emojis have skin tones or other variations associated with them. - -As part of the comma separated values, all the variations need to be covered. - -**For eg:** In case of finger pointing left, you need to add: 👈🏿,👈🏾,👈🏽,👈🏼,👈🏻,👈. Please note that the **yellow variant** has to be entered at the end of the list. - -Refer [https://emojipedia.org/](https://emojipedia.org/) for details. - -## How does it work? - -The recipients will receive the normal message in the text field of the message object. Also, there will be a clean/filtered version of the message in the metadata section as shown below: - - - -```json -"@injected": { - "extensions": { - "profanity-filter": { - "profanity": "yes", - "message_clean": "This is ****" - } - } -} -``` - - - - - -If the profanity-filter key is missing, it means that the extension is either not enabled or has timed out. - -If there is some error inside the profanity-filter object, the list of bad words is probably empty. Save the settings with bad words to get the desired results. - -## Implementation - -By checking if `profanity` is set to `yes`, you can also consider showing a warning before reading the message. - -From the extension settings, you can also choose to drop the message altogether. - -At the recipients' end, from the `message` object, you can fetch the metadata by calling the `getMetadata()` method. Using this metadata, you can fetch the masked message. - - - -```js -var metadata = message.getMetadata(); -if (metadata != null) { - var injectedObject = metadata["@injected"]; - if (injectedObject != null && injectedObject.hasOwnProperty("extensions")) { - var extensionsObject = injectedObject["extensions"]; - if ( - extensionsObject != null && - extensionsObject.hasOwnProperty("profanity-filter") - ) { - var profanityFilterObject = extensionsObject["profanity-filter"]; - var profanity = profanityFilterObject["profanity"]; - var cleanMessage = profanityFilterObject["message_clean"]; - } - } -} -``` - - - - -```java -JSONObject metadata = message.getMetadata(); -if (metadata != null) { - JSONObject injectedObject = metadata.getJSONObject("@injected"); - if (injectedObject != null && injectedObject.has("extensions")) { - JSONObject extensionsObject = injectedObject.getJSONObject("extensions"); - if (extensionsObject != null && extensionsObject.has("profanity-filter")) - { - JSONObject profanityFilterObject = extensionsObject.getJSONObject("profanity-filter"); - String profanity = profanityFilterObject.getString("profanity"); - String cleanMessage = profanityFilterObject.getString("message_clean"); - - } - } -} -``` - - - - -```kotlin -if (metadata != null) { - if (metadata.has("@injected")) { - val injectedJSONObject = metadata.getJSONObject("@injected") - if (injectedJSONObject != null && injectedJSONObject.has("extensions")) { - val extensionsObject = injectedJSONObject.getJSONObject("extensions") - - if (extensionsObject != null && extensionsObject.has("profanity-filter")) { - val profanityFilterObject = extensionsObject.getJSONObject("profanity-filter") - val dataObject = profanityFilterObject.getJSONObject("data") - if (dataObject.has("profanity")) - val profanity = dataObject.getString("profanity") - if (dataObject.has("message_clean")) - val message_clean= dataObject.getString("message_clean")) - } - } - } - } -``` - - - - -```swift -let textMessage = message as? TextMessage -var metadata : [String : Any]? = textMessage.metaData -if metadata != nil { - - var injectedObject : [String : Any]? = (metadata?["@injected"] as? [String : Any])! - - if injectedObject != nil && (injectedObject!["extensions"] != nil){ - - var extensionsObject : [String : Any]? = injectedObject?["extensions"] as? [String : Any] - - if extensionsObject != nil && extensionsObject?["profanity-filter"] != nil { - - var profanityFilterObject = extensionsObject?["profanity-filter"] as! [String : Any] - - let profanity = profanityFilterObject["profanity"] as! String - let cleanMessage = profanityFilterObject["language_translated"] as! String - } - } -} -``` - - - - diff --git a/moderation/legacy/report-message.mdx b/moderation/legacy/report-message.mdx deleted file mode 100644 index 293a92f5..00000000 --- a/moderation/legacy/report-message.mdx +++ /dev/null @@ -1,131 +0,0 @@ ---- -title: "Report Message (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -Enable your users to report messages in a group. - - - - - -**Extension settings** - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. - -2. Go to the Extensions section and enable the Report messages extension. - -3. Open the settings for this extension. - -4. The settings page has the following: - - * **Moderation criteria:** The max number of reports after which you want to be notified. - * **Moderation actions:** Get the list of reports on the configured Webhook URL. - - - - - -## How does it work? - -The extension has the following functionalities: - -1. Allowing end-users to report messages. -2. Allowing admins to login to the dashboard to take action on the reports. - -### 1. Reporting a message - -Messages can be reported in either group conversations or one-on-one conversations. - -In the context menu of a message, you can have a "Report" button. Clicking it should open up a modal asking for the reason. - -Here's the description of the parameters that need to be passed to the extension: - -| Parameters | Value | Description | -| ---------- | ------- | --------------------------------------------- | -| msgId | Integer | The ID of the message that has to be reported | -| reason | String | The reason for reporting the message. | - -Once you have the message to be reported along with the reason, make use of the `callExtension` method provided by the SDK to submit the report: - - - -```js -CometChat.callExtension('report-message', 'POST', 'v1/report', { - "msgId": 123, - "reason": "Contains profanity" -}).then(response => { - // { success: true } -}) -.catch(error => { - // Error occurred -}); -``` - - - - -```java -import org.json.simple.JSONObject; - -JSONObject body=new JSONObject(); - -body.put("msgId", 123); -body.put("reason", "Contains profanity"); - -CometChat.callExtension("report-message", "POST", "/v1/report", body, - new CometChat.CallbackListener < JSONObject > () { - @Override - public void onSuccess(JSONObject jsonObject) { - //On Success - } - @Override - public void onError(CometChatException e) { - //On Failure - } -}); -``` - - - - -```swift -CometChat.callExtension(slug: "report-message", type: .post, endPoint: "v1/report", body: [ - "msgId": 123, - "reason":"Contains profanity" -] as [String : Any], onSuccess: { (response) in - // Success - }) { (error) in - // Error occured - } -``` - - - - - -### 2. View reports and take action - - - - - -In order to list and take action on the reported users: - -1. Open up the Extension's settings page - -2. Click "View Reports" link. This will load all the reports. - -3. The following actions can be taken for users reported in Group: - - 1. Delete => Reported message will be deleted. - 2. Ignore => The report is ignored. - -4. To load new reports, click on the Refresh button. diff --git a/moderation/legacy/report-user.mdx b/moderation/legacy/report-user.mdx deleted file mode 100644 index 9bef8170..00000000 --- a/moderation/legacy/report-user.mdx +++ /dev/null @@ -1,147 +0,0 @@ ---- -title: "Report User (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -Enables your users to report users who use offensive or suspicious messages in the chat. - - - - - -## Extension settings - -1. Login to CometChat and select your app. - -2. Go to the Extensions section and enable the Report user extension. - -3. Open the settings for this extension. - -4. The settings page has the following: - - * **Moderation criteria:** The max number of reports after which you want to be notified. - * **Moderation actions**: Get the list of reports on the configured Webhook URL. - - - - - -## How does it work? - -The extension has the following functionalities: - -1. Allowing end-users to report other users. -2. Allowing admins to login to the Dashboard to take action on the reports. - -### 1. Reporting a user - -Users can be reported in either group conversations or one-on-one conversations. - -By clicking on the user's avatar, you can show an item in the context menu called "Report". Clicking on the "Report" button should open up a modal asking for the reason. - -Here's the description of the parameters that need to be passed to the extension: - -| Parameters | Value | Description | -| ---------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | -| uid | String | The UID of the user that needs to be reported | -| reason | String | Reason for reporting. This should be max 150 characters. | -| guid | String | The GUID of the group in which the user is being reported.If the user is being reported in a one-on-one conversation, this can be skipped. | - -Once you have the user to be reported along with the reason, make use of the callExtension method provided by the SDK to submit the report: - - - -```js -CometChat.callExtension('report-user', 'POST', 'v1/report', { - "uid": "cometchat-uid-3", - "reason": "Misbehaving", - // "guid": "cometchat-guid-1" // Used only when reporting the user in a group -}).then(response => { - // { success: true } -}) -.catch(error => { - // Error occurred -}); -``` - - - - -```java -import org.json.simple.JSONObject; - -JSONObject body=new JSONObject(); - -body.put("uid", "cometchat-uid-3"); -body.put("reason", "Misbehaving"); -// body.put("guid", "cometchat-guid-1"); // Used only when reporting the user in a group - -CometChat.callExtension("report-user", "POST", "/v1/report", body, - new CometChat.CallbackListener < JSONObject > () { - @Override - public void onSuccess(JSONObject jsonObject) { - //On Success - } - @Override - public void onError(CometChatException e) { - //On Failure - } -}); -``` - - - - -```swift -CometChat.callExtension(slug: "report-user", type: .post, endPoint: "v1/report", body: [ - "uid": "cometchat-uid-3", - "reason":"Misbehaving", - "guid":"cometchat-guid-1" // Used only when reporting the user in a group -] as [String : Any], onSuccess: { (response) in - // Success - }) { (error) in - // Error occured - } -``` - - - - - -### 2. View reports and take action on a reported user - - - - - -In order to list and take an action on the reported users: - -1. Open up the Extension's settings page - -2. Click "View Reports" link. This will load all the reports. - -3. Select the criteria from the dropdown: - - 1. One-on-one conversations => Lists the users who have been reported in One-on-one conversations. - 2. Group conversations => List the users who have been reported in a Group. - 3. All reports => Lists all the reports. - -4. The following actions can be taken for users reported in Group: - - 1. Kick => Reported user is kicked out of the group. - 2. Ban => Reported user is banned from the group. - 3. Ignore => The report is ignored. - -5. The following actions can be take for users reported in one-on-one conversations: - - 1. Block => The reported user is blocked on behalf of the reporter. - 2. Ignore => The report is ignored. - -6. To load new reports, click on the Refresh button. diff --git a/moderation/legacy/sentiment-analysis.mdx b/moderation/legacy/sentiment-analysis.mdx deleted file mode 100644 index 2959e66d..00000000 --- a/moderation/legacy/sentiment-analysis.mdx +++ /dev/null @@ -1,198 +0,0 @@ ---- -title: "Sentiment Analysis (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -The Sentiment Analysis extension helps you to understand the tone or sentiment of a message. - -## Extension settings - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. -2. Go to the Extensions section and enable the Sentiment Analysis extension. -3. Open up Settings and choose to Drop messages with Negative sentiments. - -## How does it work? - -A message can be classified into 4 categories: - -1. Positive -2. Neutral -3. Negative -4. Mixed - -Along with these categories, we specify the confidence for that category, on a scale of 0 to 100. - -The sentiment about the message can be found in the metadata object as shown below: - - - -```json -"@injected": { - "extensions": { - "sentiment-analysis": { - "sentiment": "positive", - "sentiment_score": { - "positive": 95, - "neutral": 4, - "negative": 0, - "mixed": 0 - } - } - } -} -``` - - - - - -If the sentiment-analysis key is missing, then either the extension is not enabled or has timed out. - -Sentiment analysis extension is compatible with the languages listed below - -| Supported languages | -| ----------------------------- | -| German (de) | -| English (en) | -| Spanish (es) | -| Italian (it) | -| Portuguese (pt) | -| French (fr) | -| Japanese (ja) | -| Korean (ko) | -| Hindi (hi) | -| Arabic (ar) | -| Chinese (simplified) (zh) | -| Chinese (traditional) (zh-TW) | - -## Implementation - -Using this information, you can show either a warning or drop the message completely. Here is how Twitter shows a message: - - - - - -At the recipients' end, from the `message` object, you can fetch the metadata by calling the `getMetadata()` method. Using this metadata, you can fetch the sentiment of the message. - - - -```js -var metadata = message.getMetadata(); -if (metadata != null) { - var injectedObject = metadata["@injected"]; - if (injectedObject != null && injectedObject.hasOwnProperty("extensions")) { - var extensionsObject = injectedObject["extensions"]; - if ( - extensionsObject != null && - extensionsObject.hasOwnProperty("sentiment-analysis") - ) { - var sentimentAnalysisObject = extensionsObject["sentiment-analysis"]; - var sentiment = sentimentAnalysisObject["sentiment"]; - if (sentimentAnalysisObject.hasOwnProperty("sentiment_score")) { - var positive = sentimentAnalysisObject["positive"]; - var neutral = sentimentAnalysisObject["neutral"]; - var negative = sentimentAnalysisObject["negative"]; - var mixed = sentimentAnalysisObject["mixed"]; - } - } - } -} -``` - - - - -```java -JSONObject metadata = message.getMetadata(); -if (metadata != null) { - JSONObject injectedObject = metadata.getJSONObject("@injected"); - if (injectedObject != null && injectedObject.has("extensions")) { - JSONObject extensionsObject = injectedObject.getJSONObject("extensions"); - if (extensionsObject != null && extensionsObject.has("sentiment-analysis")) - { - JSONObject sentimentAnalysisObject = extensionsObject.getJSONObject("sentiment-analysis"); - String sentiment = sentimentAnalysisObject.getString("sentiment"); - if(sentimentAnalysisObject.has("sentiment_score")){ - int positive = sentimentAnalysisObject.getInt("positive"); - int neutral = sentimentAnalysisObject.getInt("neutral"); - int negative = sentimentAnalysisObject.getInt("negative"); - int mixed = sentimentAnalysisObject.getInt("mixed"); - } - - } - } -} -``` - - - - -```kotlin -if (metadata != null) { - if (metadata.has("@injected")) { - val injectedJSONObject = metadata.getJSONObject("@injected") - if (injectedJSONObject != null && injectedJSONObject.has("extensions")) { - val extensionsObject = injectedJSONObject.getJSONObject("extensions") - if (extensionsObject.has("sentiment-analysis")) { - - val sentimentObject = extensionsObject.getJSONObject("sentiment-analysis") - val sentiment= sentimentObject.getString("sentiment") - if (sentimentObject.has("sentiment_score")) { - if (sentimentObject.has("positive")) - val positive=sentimentObject.getInt("positive") - if (sentimentObject.has("neutral")) - val neutral= sentimentObject.getInt("neutral") - if (sentimentObject.has("negative")) - val negative= sentimentObject.getInt("negative") - if (sentimentObject.has("mixed")) - val mixed= sentimentObject.getInt("mixed") - } - } - } - } - } -``` - - - - -```swift -let textMessage = message as? TextMessage -var metadata : [String : Any]? = message.metaData -if metadata != nil { - - var injectedObject : [String : Any]? = (metadata?["@injected"] as? [String : Any])! - - if injectedObject != nil && (injectedObject!["extensions"] != nil){ - - var extensionsObject : [String : Any]? = injectedObject?["extensions"] as? [String : Any] - - if extensionsObject != nil && extensionsObject?["sentiment-analysis"] != nil { - - var sentimentAnalysisObject = extensionsObject?["sentiment-analysis"] as! [String : Any] - - let sentiment = sentimentAnalysisObject["sentiment"] as! String - - if sentimentAnalysisObject["sentiment_score"] { - - let positive = imageModerationObject["positive"] as! Int - let neutral = imageModerationObject["neutral"] as! Int - let negative = imageModerationObject["negative"] as! Int - let mixed = imageModerationObject["mixed"] as! Int - } - } - } -} -``` - - - - diff --git a/moderation/legacy/slow-mode.mdx b/moderation/legacy/slow-mode.mdx deleted file mode 100644 index f282f1e5..00000000 --- a/moderation/legacy/slow-mode.mdx +++ /dev/null @@ -1,271 +0,0 @@ ---- -title: "Slow Mode (Deprecated)" ---- - - - -Deprecated: This extension is no longer maintained and will not receive further updates. - - - -Slow down messages to make them legible! - - - - - -## Extension settings - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. -2. Go to the Extensions section and enable the Slow mode extension. - -## How does Slow mode work? - -Slow mode extension works great in groups with a large number of participants. Especially, during a live event with a potential for a flood of messages being sent every second. When the extension is enabled and enforced in a group, it allows the participants to send messages only after a certain intervals. This helps to keep chats readable for everyone during large events. - -The extension has the following 4 parts: - -1. Enabling the slow mode for a group -2. Disabling the slow mode for a group -3. Enforcing slow mode for participants -4. Fetching the slow mode details - -### Enabling slow mode - -Slow mode can be enabled only by the group admins or moderators. Participants cannot enable the slow mode. - -Once slow mode is enabled in a group, the information is shared with its members in real-time as a custom message. With this, it can be enforced immediately. - -You need to implement the `onCustomMessageReceived` listener in order to receive the Slow mode related messages. The message sent has the category of `custom` and type `extension_slow-mode`. - -Following are the inputs required to enable slow mode in a particular group: - -| Parameter | Type | Description | -| ------------------ | ------ | ------------------------------------------------------------------------------------------------------------------- | -| `guid` | string | The group's ID in which the slow mode needs to be enabled. | -| `slowDownTimeInMS` | int | The time in milliseconds for which the participants have to wait before being able to send the consecutive message. | - -You can make use of the `callExtension` method exposed by CometChat SDKs to enable slow mode as an admin/moderator. - - - -```js -CometChat.callExtension('slow-mode', 'POST', 'v1/configure', { - "guid": "cometchat-guid-1", - "slowDownTimeInMS": 660000, -}).then(response => { - // Success true -}) -.catch(error => { - // Error occured -}); -``` - - - - -```java -import org.json.simple.JSONObject; - -JSONObject body=new JSONObject(); - -body.put("guid", "cometchat-guid-1"); -body.put("slowDownTimeInMS", 660000); - -CometChat.callExtension("slow-mode", "POST", "/v1/configure", body, - new CometChat.CallbackListener < JSONObject > () { - @Override - public void onSuccess(JSONObject jsonObject) { - //On Success - } - @Override - public void onError(CometChatException e) { - //On Failure - } -}); -``` - - - - -```swift -CometChat.callExtension(slug: "slow-mode", type: .post, endPoint: "v1/configure", body: ["guid": "cometchat-guid-1" ,"slowDownTimeInMS": 660000] as [String : Any], onSuccess: { (response) in - // Success - }) { (error) in - // Error occured - } -``` - - - - - -### Disabling slow mode - -Slow mode can be disabled only by the group admins or moderators. Participants cannot disable the slow mode. - -Once slow mode is disabled in a group, the information is shared with its members in real-time as a custom message. With this, it can be turned off for that group immediately. - -Following are the inputs required to enable slow mode in a particular group: - -| Parameter | Type | Description | -| --------- | ------ | ----------------------------------------------------------- | -| `guid` | string | The group's ID in which the slow mode needs to be disabled. | - -You have to implement the `onCustomMessageReceived` listener in order to receive the Slow mode related messages. The message sent has the category of `custom` and type `extension_slow-mode`. - -You can make use of the `callExtension` method exposed by CometChat SDKs to disable slow mode as an admin/moderator. - - - -```js -CometChat.callExtension('slow-mode', 'DELETE', 'v1/configure', { - "guid": "cometchat-guid-1" -}).then(response => { - // Success true -}) -.catch(error => { - // Error occured -}); -``` - - - - -```java -import org.json.simple.JSONObject; - -JSONObject body=new JSONObject(); - -body.put("guid", "cometchat-guid-1"); - -CometChat.callExtension("slow-mode", "DELETE", "/v1/configure", body, - new CometChat.CallbackListener < JSONObject > () { - @Override - public void onSuccess(JSONObject jsonObject) { - //On Success - } - @Override - public void onError(CometChatException e) { - //On Failure - } -}); -``` - - - - -```swift -CometChat.callExtension(slug: "slow-mode", type: .delete, endPoint: "v1/configure", body: nil, onSuccess: { (response) in - // Success - }) { (error) in - // Error occured - } -``` - - - - - -### Enforcing slow mode - -This is handled by the extension for groups that have slow mode enabled as mentioned below: - -#### 1. For participants of a group - -If the moderator or admin of a group has enabled slow mode with the time of 1 min, then the participants will have to wait for 1 minute after sending a message. Once the participant has waited for 1 min, he/she then becomes eligible to send out the next message. - -If there's an attempt by the participants to send a message before the interval has expired for them, the message gets blocked by the extension. - -#### 2. For admins and moderators - -The extension does not restrict moderators and admins of a group. Slow mode is enforced only for the participants. - - -Groups without admins/moderators - -Groups that are created from the dashboard do not have an admin or moderator. Hence, care needs to be taken to add a member to such groups and change their scope to either moderator or admin. - - - -#### 3. Change in member scope - -If a participant is made an admin or moderator of a group, the slow mode is no longer applicable for him/her. - -Similarly, if an admin or moderator of a group is demoted to a participant, the slow mode will be applicable immediately to him/her as mentioned above. - -### Fetching slow mode details - -As mentioned above, the details about the enabling or disabling the slow mode are shared in real-time with the members as a custom message. But, it might happen that a few members are offline or are added later on to the group. Hence, it is important to make the same details available to them for a consistent experience. - -You can make use of the `callExtension` method exposed by the CometChat SDKs to fetch the details about slow mode as a member of a group. - - - -```js -const guid = "cometchat-guid-1"; -CometChat.callExtension('slow-mode', 'GET', `v1/fetch-configuration?guid=${guid}`, null).then(response => { - // Configuration for the mentioned group -}) -.catch(error => { - // Error occured -}); -``` - - - - -```java -CometChat.callExtension("slow-mode", "GET", "/v1/fetch-configuration?guid=cometchat-guid-1", null, - new CometChat.CallbackListener < JSONObject > () { - @Override - public void onSuccess(JSONObject jsonObject) { - //On Success - } - @Override - public void onError(CometChatException e) { - //On Failure - } -}); -``` - - - - -```swift -CometChat.callExtension(slug: "slow-mode", type: .get, endPoint: "v1/fetch-configuration?guid=cometchat-guid-1", body: nil, onSuccess: { (response) in - // Success - }) { (error) in - // Error occured - } -``` - - - - - -The response has the following format: - -| Parameters | Type | Description | -| ---------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `isSlowed` | boolean | Whether the slow mode is enabled in the mentioned group.If false, the following fields are not present in the response. | -| `slowDownTimeInMS` | int | The time interval for which a participant has to wait for sending messages. | -| `lastMessageSentAtTimestamp` | timestamp | The timestamp at which the last message was sent by the logged in user in the mentioned group.If the scope of the logged in user is Admin or Moderator in the mentioned group, this field is not included in the response. | - -## Implementation - -#### 1. For admins and moderators - -When a group chat is opened, and the scope of the logged-in user is either an admin or a moderator of that group, he/she should be able to toggle slow mode for that group. - -When a group has multiple admins/moderators, and one of the admins enables (or disables) the slow mode for a group, the UI for other admins/moderators should update and show the control to disable (or enable) the slow mode. This can be achieved in real-time due to the custom message that is sent. - -Whenever, an admin or a moderator in a group switches conversations to a different group, use the fetch-configuration call to check for the above mentioned details. It may happen, that the logged in user is admin/moderator for one group but a participant in another. - -#### 2. For participants of a group - -When a group chat is opened, and the scope of the logged-in user is participant, he/she should be able to send messages only after the the configured intervals. - -You can enforce the blocking behaviour by disabling the message composer or the send button on the UI. It will get enabled only once the participant has waited for the defined amount of slow-down time. - -While the participant is waiting, a timer can be displayed to indicate the amount of time left after which they can send the next message. diff --git a/moderation/legacy/virus-malware-scanner.mdx b/moderation/legacy/virus-malware-scanner.mdx deleted file mode 100644 index aa67257b..00000000 --- a/moderation/legacy/virus-malware-scanner.mdx +++ /dev/null @@ -1,201 +0,0 @@ ---- -title: "Virus And Malware Scanner (Legacy)" ---- - - - -**Legacy Notice**: This extension is considered legacy and is scheduled for deprecation in the near future. It is no longer recommended for new integrations. - -Please note: Legacy extensions are no longer actively maintained and will not receive feature updates or enhancements. - - - -The Virus & Malware Scanner Extension allows the developer to scan files uploaded by users so that a warning can be added for malicious content. - -## Before you begin - -This Extension uses a third-party API service - [Scanii](https://scanii.com) - to scan media messages. Create an account with Scanii API Service and get your pair of **API Key** and **Secret**. - -## Extension settings - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. -2. Go to the Extensions section and enable the Virus and Malware Scanner extension. -3. Open the Settings for this extension. -4. Enter Scanii API Key and Scanii Secret and click on save. - -## How does it work? - -Once the Extension is enabled for your App and the Extension Settings are saved, the recipients will receive metadata with an array of results. - -The Virus Scan results will be updated later for the message and hence you need to implement the `onMessageEdited` listener. Please check the **Edit a Message** page under the Messaging section of each SDK for more details. - -This can be used to show warning messages: - - - -```json -{ - "@injected": { - "extensions": { - "virus-malware-scanner": { - "attachments": [ - { - "data": { - "url": "https://media.com/1646056756_400568974.mp3", - "name": "a2.mp3", - "size": 1519658, - "verdict": { - "scan_results": [], - }, - "mimeType": "audio/mpeg", - "extension": "mp3", - }, - "error": null, - }, - { - "data": { - "url": "https://media.com/1646056756_400568933.mp3", - "name": "a1.mp3", - "size": 1519658, - "mimeType": "audio/mpeg", - "extension": "mp3", - "verdict": null - }, - "error": { - "code": "ERROR_CODE", - "message": "Error Message", - "devMessage": "Error message", - "source": "ext-api" - } - } - ], - "scan_results": [], - }, - }, - }, -} -``` - - - - - -If the scan\_results is an empty array, it means the message is safe. - -If the virus-malware-scanner key is missing, then either the extension is not enabled or your Scanii credits are over. - - - -The `scan_results`, to the outside of `attachments` are the result for the first attachment from the `attachments` array. This has been retained for backward compatibility only.\ -You can iterate over `attachments` array for better implementation. - - - -## Implementation - -At the recipients' end, from the message object, you can fetch the metadata by calling the getMetadata() method. Using this metadata, you can fetch the Rich Media Embed. - - - -```js -const metadata = message.getMetadata(); -if (metadata != null) { - const injectedObject = metadata["@injected"]; - if (injectedObject != null && injectedObject.hasOwnProperty("extensions")) { - const extensionsObject = injectedObject["extensions"]; - if ( - extensionsObject != null && - extensionsObject.hasOwnProperty("virus-malware-scanner") - ) { - const { attachments } = extensionsObject["virus-malware-scanner"]; - for (const attachment of attachments) { - if (!attachment.error) { - const { scan_results } = attachment.data.verdict; - // Check the other parameters as required. - } - } - } - } -} -``` - - - - -```java -JSONObject metadata = message.getMetadata(); -if (metadata != null) { - JSONObject injectedObject = metadata.getJSONObject("@injected"); - if (injectedObject != null && injectedObject.has("extensions")) { - JSONObject extensionsObject = injectedObject.getJSONObject("extensions"); - if (extensionsObject != null && extensionsObject.has("virus-malware-scanner")) - { - JSONObject tg = extensionsObject.getJSONObject("virus-malware-scanner"); - JSONArray attachments = tg.getJSONArray("attachments"); - - for (int i = 0; i < attachments.length(); i++) { - JSONObject attachment = attachments.getJSONObject(i); - JSONObject error = attachment.getJSONObject("error"); - if (error == null) { - JSONObject data = attachment.getJSONObject("data"); - JSONObject verdict = data.getJSONObject("verdict"); - String unsafe = thumbnails.getString("scan_results"); - // Check the other parameters as required. - } - } - } - } -} -``` - - - - -```kotlin -if (metadata != null) { - if (metadata.has("@injected")) { - val injected = metadata.getJSONObject("@injected") - if (injected != null && injected.has("extensions")) { - val extensions = injectedJSONObject.getJSONObject("extensions") - if (extensions != null && extensions.has("virus-malware-scanner")) { - val tg = extensions.getJSONObject("virus-malware-scanner") - val attachments = tg.getJSONArray("attachments") - for (i in 0 until attachments.length()) { - val attachment = attachments.getJSONObject(i) - val error = attachment.getJSONObject("error") - if (error == null) { - val data = attachment.getJSONObject("data") - val verdict = data.getJSONObject("verdict") - val unsafe = verdict.getString("scan_results") - // use other parameters are required. - } - } - } - } - } -} -``` - - - - -```swift -if let metaData = message?.metaData , let injected = metaData["@injected"] as? [String : Any], let extensions =injected["extensions"] as? [String : Any], let attachments = extensions["virus-malware-scanner"] as? [[String : Any]] { - - for attachment in attachments { - - if let data = attachment["data"] as? [String:Any] , let verdict = data["verdict"] as? [String:any] { - // Check for the parameters as required. - if let unsafe = URL(string: verdict["scan_results"] as? String) { -// Use the url accordingly. - } - - // check for attachment.error if "verdict" is null - } -} -} -``` - - - - diff --git a/moderation/legacy/xss-filter.mdx b/moderation/legacy/xss-filter.mdx deleted file mode 100644 index 1d074e10..00000000 --- a/moderation/legacy/xss-filter.mdx +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: "XSS Filter (Deprecated)" ---- - - - -Deprecated: This extension is no longer maintained and will not receive further updates. - - - -A client-side script can be injected into the message which can lead to an attack. The XSS Filter Extension helps you to sanitize the messages. - -This is applicable only for the Web SDK. - -## Extension settings - -1. Login to [CometChat](https://app.cometchat.com/login) and select your app. -2. Go to the Extensions section and enable the XSS Filter extension. -3. Open up the Settings and choose to Drop messages with XSS Scripts. - -## How does it work? - -Once the extension has been enabled from the Dashboard, recipients will receive metadata with the sanitized text. - -Here is a sample response: - - - -```json -"@injected": { - "extensions": { - "xss-filter": { - "hasXSS": "yes"|"no", - "sanitized_message": "" - } - } -} -``` - - - - - -If the xss-filter key is missing, it means that the extension is either not enabled or has timed out. - -## Implementation - -At the recipients' end, from the message object, you can fetch the metadata by calling the getMetadata() method. Using this metadata, you can fetch the sanitized message. - - - -```js -var metadata = message.getMetadata(); -if (metadata != null) { - var injectedObject = metadata["@injected"]; - if (injectedObject != null && injectedObject.hasOwnProperty("extensions")) { - var extensionsObject = injectedObject["extensions"]; - if (extensionsObject != null && - extensionsObject.hasOwnProperty("xss-filter")){ - var xssFilterObject = extensionsObject["xss-filter"]; - var hasXSS = xssFilterObject["hasXSS"]; - var sanitized_message = xssFilterObject["sanitized_message"]; - } - } -} -``` - - - - - -As mentioned earlier, the XSS is only possible for the web. So, the mobile platforms do not require you to fetch the sanitized message. diff --git a/moderation/lists-management.mdx b/moderation/lists-management.mdx index 992193ef..3d9b78ed 100644 --- a/moderation/lists-management.mdx +++ b/moderation/lists-management.mdx @@ -2,173 +2,219 @@ title: "Lists Management" --- -## Overview +Lists Management provides tools for creating and managing lists of keywords, regex patterns, or sentences that power your moderation rules. When detected in user-generated content, these lists trigger moderation actions like blocking or flagging messages. -The Lists Management endpoints in the Moderation Service API provide essential tools for creating and managing lists of keywords or regex patterns that are crucial for effective message moderation. These endpoints enable app owners and collaborators to define specific terms, phrases, or patterns that, when detected in user-generated content, trigger moderation actions. The next section provides a detailed elaboration of the capabilities offered. + +**Lists are reusable.** Create a list once and use it across multiple rules. Update the list in one place and all rules using it are automatically updated. + - - - - -To begin managing lists: - -* Login to your [CometChat dashboard](https://app.cometchat.com/login) and choose your app. -* Navigate to **Moderation** > **Settings** in the left-hand menu. -* Select the **Lists** tab. - -## Default Lists - -Default lists are predefined lists of words, patterns and sentences that are readily available for use on your platform. Here are the standard default lists available: - -### Profane Words - -Our default list is a comprehensive compilation of predefined profane words and phrases. This list is designed to enhance message moderation efforts by automatically identifying and flagging inappropriate language. - -### Platform Circumvention - -The Platform Cicurvention list contains a curated set of sentences and words designed to identify attempts to circumvent platform rules and policies. These phrases are used by the AI Platform Circumvention Rule to detect and prevent efforts aimed at bypassing restrictions, ensuring compliance and maintaining platform integrity. - -### Spam Detection - -The Default Spam Detection List identifies repetitive or irrelevant messages promoting products, services, or schemes without user consent. It helps filter out unwanted content, including bulk messages, phishing attempts, and fraudulent offers, ensuring a cleaner and more secure communication experience. - -### Scam Detection - -The Default Scam Detection List includes messages crafted to deceive users by creating a sense of urgency, promising false rewards, or impersonating trusted entities. These messages often aim to manipulate users into sharing personal information, making payments, or clicking on malicious links. The list helps identify and block scams, protecting users from fraud, phishing attempts, and other deceptive practices. - -### Fraud or Scam Indicators Prompt - -The Fraud or Scam Indicators list is designed to detect manipulated images used for fraudulent or deceptive activities. It helps flag fake documents, counterfeit products, and misleading visuals that could be used to scam users or spread misinformation. - -### Terrorism or Extremist Promotion Prompt - -The Terrorism or Extremist Promotion list identifies imagery that endorses terrorism, violent extremism, or radical ideologies. It helps prevent the spread of extremist propaganda, recruitment materials, and content that incites violence. - -### Minor Safety and Exploitation Prompt - -The Minor Safety and Exploitation list is used to detect sexualized or exploitative imagery of minors. It helps prevent child abuse, grooming, and the sharing of harmful content, ensuring compliance with child protection policies. - -### Privacy or Personal Data Prompt - -The Privacy or Personal Data list flags images that expose sensitive or private information, such as identification documents, financial details, or personal records. This helps prevent identity theft, unauthorized data leaks, and privacy violations. - -### Graphic Violence or Gore Prompt - -The Graphic Violence or Gore list identifies violent or gory imagery, including depictions of severe injuries, crime scenes, or graphic deaths. It helps limit exposure to disturbing content and ensures a safer viewing experience. - -### Explicit or Sexual Content Prompt - -The Explicit or Sexual Content list is designed to detect nudity, sexually explicit imagery, or highly suggestive content. It helps enforce platform guidelines by filtering out inappropriate material. - -### Hate or Harassment Prompt - -The Hate or Harassment list flags imagery containing hateful symbols, offensive gestures, or harassment. It helps identify and prevent content that promotes discrimination, hate speech, or targeted abuse. - -### Hate and Harassment Prompt - -The Hate and Harassment list detects messages that contain hate speech, threats, slurs, or harassment directed at individuals or groups. It helps create a respectful and safe online environment by preventing abusive behavior. - -### Explicit or Inappropriate Content Prompt +--- -The Explicit or Inappropriate Content list identifies text that includes explicit sexual descriptions, extreme violence, or other unsuitable material. It helps ensure compliance with content policies and maintains platform integrity. +## Quick Start -### Impersonation or Fraud Prompt +Create a custom keyword list in under 2 minutes: -The Impersonation or Fraud list detects deceptive attempts to impersonate individuals, businesses, or organizations. It helps prevent identity theft, scam attempts, and fraudulent activities. + + + Login to [CometChat Dashboard](https://app.cometchat.com) → Select your app → **Moderation** → **Settings** → **Lists** + + + Click **Add** → Enter a name, select category (word/pattern/sentence), and add your terms + + + Go to **Rules** → Create or edit a rule → Select your list in the Conditions section + + -### Non-Consensual Sexual Content or Exploitation Prompt + + + -The Non-Consensual Sexual Content or Exploitation list flags messages that depict or encourage non-consensual sexual acts, grooming, or coercion. It helps protect users from exploitation and ensures adherence to safety policies. +--- -### Privacy and Sensitive Info Prompt +## List Types + + + + Simple word matching. Use for: + - Profanity and offensive terms + - Brand names or competitors + - Industry-specific blocked terms + + **Example:** `badword, offensive, blocked` + + + Regular expression matching for complex patterns. Use for: + - Phone number formats + - Email patterns + - Custom data formats + + **Example:** `\b\d{3}[-.]?\d{3}[-.]?\d{4}\b` (US phone numbers) + + + AI-powered semantic matching. Use for: + - Scam message variations + - Platform circumvention attempts + - Context-aware blocking + + **Example:** "Send me your credit card details" will match similar phishing attempts + + -The Privacy and Sensitive Info list identifies messages that share personal or sensitive information without consent. It helps protect user privacy by preventing unauthorized data exposure. +--- -### Self-Harm or Suicidal Content Prompt +## Default Lists Summary + +Pre-configured lists ready to use with your rules: + + + + | List | Description | Use Case | + |------|-------------|----------| + | **Profane Words** | Common profanity and inappropriate language | General chat moderation | + | **Platform Circumvention** | Phrases attempting to bypass rules | Marketplace, dating apps | + | **Spam Detection** | Repetitive/promotional message patterns | High-traffic apps | + | **Scam Detection** | Deceptive messages and phishing attempts | Finance, marketplace | + | **Hate and Harassment Prompt** | Hate speech, threats, and slurs | Community platforms | + | **Explicit Content Prompt** | Sexual descriptions and violence | Family-friendly apps | + | **Privacy and Sensitive Info Prompt** | Personal data sharing | Healthcare, finance | + | **Self-Harm Content Prompt** | Self-harm and suicidal content | Mental health safety | + | **Spam and Scam Prompt** | Phishing and fraudulent schemes | All apps | + | **Violent Threats Prompt** | Violence and terrorism promotion | All apps | + | **Impersonation Prompt** | Identity fraud attempts | Business apps | + | **Non-Consensual Content Prompt** | Exploitation and grooming | Dating, social apps | + + + | List | Description | Use Case | + |------|-------------|----------| + | **Hate or Harassment Prompt** | Hateful symbols and offensive gestures | Community platforms | + | **Explicit or Sexual Content Prompt** | Nudity and sexually explicit imagery | Social apps | + | **Graphic Violence Prompt** | Violent imagery and gore | Family-friendly apps | + | **Privacy or Personal Data Prompt** | IDs, financial docs, personal records | Finance, healthcare | + | **Minor Safety Prompt** | Child exploitation content | All apps | + | **Fraud or Scam Indicators Prompt** | Fake documents and counterfeit images | Marketplace | + | **Terrorism Prompt** | Extremist propaganda and recruitment | All apps | + + -The Self-Harm or Suicidal Content list detects messages indicating self-harm, suicidal thoughts, or encouragement of self-injury. It helps enable early intervention and support mental health safety. +--- -### Spam and Scam Prompt +## When to Create Custom Lists + + + + Add terms specific to your industry that default lists don't cover: + - **Gaming:** Game-specific slurs, exploit terms + - **Finance:** Competitor names, prohibited investment terms + - **Healthcare:** Medical misinformation terms + + + Block mentions of: + - Competitor names + - Prohibited external links + - Trademark violations + + + Add localized profanity or slang that default English lists miss. + + + Create lists for your unique community guidelines: + - Prohibited topics + - Off-platform contact attempts + - Community-specific terms + + -The Spam and Scam list identifies spam messages, phishing attempts, and fraudulent schemes. It helps filter out unwanted content, including bulk messages and misleading offers, ensuring a cleaner and more secure communication environment. +--- -### Violent or Terroristic Threats Prompt +## Best Practices + + + + Use default lists first, then create custom lists only for gaps in coverage. + + + Scammers constantly change wording. Sentence similarity catches variations automatically. + + + Test regex patterns before deploying. A bad pattern can block legitimate content. + + + Check blocked messages monthly and update lists based on new patterns. + + + +### Tips for Effective Lists + +1. **Keep lists focused** - Create separate lists for different purposes (profanity vs. spam vs. competitors) +2. **Use CSV for bulk imports** - Upload large keyword lists via CSV instead of manual entry +3. **Combine list types** - Use keywords for exact matches + sentence similarity for variations +4. **Document your lists** - Add clear descriptions so team members understand each list's purpose -The Violent or Terroristic Threats list detects content that promotes violence, terrorism, or extremist actions. It helps prevent harmful speech, glorification of violence, and threats against individuals or groups. +--- ## Managing Lists ### Create List -Allows you to define new moderation lists specifying the words or patterns under which text or custom messages should be blocked. - -Creating a new list from the dashboard: - -1. Click the Add button within the Lists tab. - -2. Create the list by saving the following details: - - * Name: Descriptive name for the moderation list. - * ID: The unique identifier of the list. - * Category: Choose the type for list, either 'word', 'pattern' or 'sentence similarity'. - * Description: Detailed explanation of the list. - * Your Source type for lists could be either words, patterns or sentences, separated by a comma or a CSV file. +1. Click **Add** in the Lists tab +2. Configure: + - **Name**: Descriptive name for the list + - **ID**: Unique identifier + - **Category**: Word, Pattern, or Sentence Similarity + - **Description**: Purpose of the list + - **Source**: Enter terms manually (comma-separated) or upload CSV +3. Click **Save** -3. Save +### List All Lists -You can also set this up from your end using the [Create Moderation List REST API](https://api-explorer.cometchat.com/reference/create-rule-keyword). - -### Fetch All Lists - -Fetches the details of existing list lists. +All configured lists are displayed in the Lists tab with their name, category, and item count. -You can also set this up from your end using the [List Moderation Lists REST API](https://api-explorer.cometchat.com/reference/list-rule-keywords). - -### Get List - -Fetches the details of an existing list. You can set this up from your end using the [Get Moderation List REST API](https://api-explorer.cometchat.com/reference/get-rule-keyword). - ### Update List -Allows you to update existing lists, which includes modifying the list name, category, and individual words or patterns within the list. - -Updating a list from the dashboard: - -1. Click on "Edit" in the action menu of the List you want to update. - -2. Update the list by saving the following details: - - * Name: Descriptive name for the moderation list. - * Category: Choose the type for List, either 'word', 'pattern' or 'sentence similarity'. - * Description: Detailed explanation of the list. - * Your Source type for list could be either words, patterns or sentences separated by a comma or a CSV file. - -3. Save - -You can also set this up from your end using the [Update Moderation List REST API](https://api-explorer.cometchat.com/reference/update-rule-keyword). +1. Click **Edit** in the action menu +2. Modify the list settings or add/remove terms +3. Click **Save** ### Delete List -Allows for the removal of lists from the system that are no longer needed. - -Deleting a list from the dashboard: +Click **Delete** in the action menu and confirm. + + +Deleting a list will affect any rules using it. Check rule dependencies before deleting. + -* Click "Delete" in the action menu of the list you want to remove, then confirm. +--- -You can also set this up from your end using the [Delete Moderation List REST API](https://api-explorer.cometchat.com/reference/delete-rule-keyword). +## Related Resources + + + + Create rules that use your lists + + + View content blocked by your lists + + + Review flagged content + + + Learn about the moderation system + + diff --git a/moderation/open-ai/openai-custom.mdx b/moderation/open-ai/openai-custom.mdx index cf8e0b07..9cbeeabf 100644 --- a/moderation/open-ai/openai-custom.mdx +++ b/moderation/open-ai/openai-custom.mdx @@ -36,7 +36,7 @@ CometChat allows you to integrate OpenAI for real-time message moderation, enabl ### **Step 2: Enable OpenAI Moderation** -1. Navigate to **Moderation → List**. +1. Navigate to **Moderation → Rules**. 2. Click **"Create New Rule"**. 3. Select **OpenAI** as the moderation type. 4. Select a predefined **prompt** from the List Section or create your own. diff --git a/moderation/open-ai/openai-overview.mdx b/moderation/open-ai/openai-overview.mdx index 3163b81e..492290b9 100644 --- a/moderation/open-ai/openai-overview.mdx +++ b/moderation/open-ai/openai-overview.mdx @@ -2,8 +2,6 @@ title: "Overview" --- -# Overview - CometChat offers AI-powered message moderation to help maintain a safe and respectful chat environment. You can choose between two moderation options: ### **OpenAI Moderation** diff --git a/moderation/overview.mdx b/moderation/overview.mdx index 382e9d27..dae288ba 100644 --- a/moderation/overview.mdx +++ b/moderation/overview.mdx @@ -2,82 +2,167 @@ title: "Overview" --- -The Moderation feature provides a comprehensive suite of capabilities designed to manage and enforce message moderation rules across various types of messages, ensuring your platform remains safe and compliant for all users. These capabilities include rule management for creating, updating, and deleting moderation rules, as well as keyword lists for detecting inappropriate content. Additionally, automated actions promptly address potential violations, and detailed reports on blocked messages help to continuously improve safety and compliance measures. By leveraging these robust functionalities, you can effectively maintain a secure and welcoming environment on your platform. +The Moderation feature provides a comprehensive suite of capabilities designed to manage and enforce message moderation rules across various types of messages, ensuring your platform remains safe and compliant for all users. + +## Key Benefits + +- **Automated Content Filtering** - Automatically detect and handle inappropriate content before it reaches users +- **Customizable Rules** - Create rules tailored to your platform's specific community guidelines +- **Multi-format Support** - Moderate text messages, images, and videos +- **Real-time Protection** - Messages are moderated instantly as they are sent +- **Manual Review** - Users can report content, and moderators can review flagged messages + +## How It Works + +```mermaid +flowchart LR + A[Message Sent] --> B{Moderation Rules} + B -->|Pass| C[Message Delivered] + B -->|Violation| D{Action} + D --> E[Block] + D --> F[Flag for Review] + E --> G[Blocked Messages] + F --> H[Flagged Messages] + H --> I{Moderator Review} + I --> J[Approve] + I --> K[Block] +``` + +## Quick Navigation + + + + Create and manage moderation rules to detect inappropriate content + + + Manage keyword lists and regex patterns for content filtering + + + Review messages flagged by users or the rule engine + + + View messages that were blocked due to rule violations + + + +## Getting Started + + + + Navigate to the [CometChat Dashboard](https://app.cometchat.com) and go to **Moderation** section + + + Set up rules to automatically detect and handle inappropriate content. See [Rules Management](/moderation/rules-management) + + + Customize the reasons users can select when reporting messages. Go to **Moderation > Advanced Settings** + + + Enable moderation features in your UI Kit or implement using the SDK/REST API + + + +## Platform Integration + +Moderation is supported across all CometChat platforms. Choose your integration method: + +### UI Kits + +UI Kits provide built-in support for message moderation and the Report Message feature: + + + } href="/ui-kit/react/core-features#moderation" /> + } href="/ui-kit/react-native/core-features#moderation" /> + } href="/ui-kit/android/core-features#moderation" /> + } href="/ui-kit/ios/core-features#moderation" /> + } href="/ui-kit/flutter/core-features#moderation" /> + } href="/ui-kit/angular/core-features#moderation" /> + + +### Chat SDKs + +Implement message flagging directly using CometChat Chat SDKs: + + + } href="/sdk/javascript/ai-moderation" /> + } href="/sdk/react-native/ai-moderation" /> + } href="/sdk/android/ai-moderation" /> + } href="/sdk/ios/ai-moderation" /> + } href="/sdk/flutter/ai-moderation" /> + -import { Steps } from 'mintlify'; - -Here’s an in-depth look at the key functionalities provided by the Moderation Service: +--- ## Rules Management This feature enables you to define and manage a set of moderation rules tailored to address inappropriate messages under various conditions. You can establish specific criteria that determine what constitutes unacceptable behavior or content, such as the use of offensive language, unsafe content, or sharing sensitive information. -By customizing these rules, you ensure that the moderation system effectively identifies and manages messages that violate your platform's standards, thereby maintaining a safe and respectful environment for all users. The ability to manage these rules includes adding new rules, updating existing ones, and removing obsolete rules, providing a flexible and dynamic approach to message moderation. +By customizing these rules, you ensure that the moderation system effectively identifies and manages messages that violate your platform's standards, thereby maintaining a safe and respectful environment for all users. -For more detailed management, refer to the [rules management](/moderation/rules-management) section. +For more detailed management, refer to the [Rules Management](/moderation/rules-management) section. ## Lists Management -This feature allows you to create and manage comprehensive lists of keywords or regex patterns that are used for message moderation. These lists serve as a vital component in identifying and handling inappropriate content. You can customize these lists to include specific terms and patterns that are relevant to your platform's moderation needs. +This feature allows you to create and manage comprehensive lists of keywords or regex patterns that are used for message moderation. These lists serve as a vital component in identifying and handling inappropriate content. -Once created, these keyword lists can be linked to various moderation rules when creating or updating rules, ensuring that the moderation system effectively detects and manages content that violates your standards. The ability to manage these lists includes adding new keywords or patterns or sentences, updating existing ones, and removing those that are no longer relevant, providing a flexible and responsive approach to message moderation. +Once created, these keyword lists can be linked to various moderation rules when creating or updating rules, ensuring that the moderation system effectively detects and manages content that violates your standards. -For more detailed management, refer to the [lists management](/moderation/lists-management) section. +For more detailed management, refer to the [Lists Management](/moderation/lists-management) section. ## Flagged Messages -This feature enables you to access and manage all messages that require moderation review. You can view a complete list of messages that have been automatically flagged by the rule engine for policy violations or manually reported by users for inappropriate content. The system provides filtering to narrow down results using date ranges. +This feature enables you to access and manage all messages that require moderation review. You can view a complete list of messages that have been automatically flagged by the rule engine for policy violations or manually reported by users for inappropriate content. For more details, refer to the [Flagged Messages](/moderation/flagged-messages) section. ## Blocked Messages -This feature allows you to retrieve all the violated messages. You can retrieve a comprehensive list of messages that have been blocked due to violations of moderation rules. Additionally, you can perform searches within this list to find specific messages or filter results based on date ranges and find details for the violation. This functionality helps you effectively monitor and manage inappropriate content, ensuring a safe and compliant environment on your platform. +This feature allows you to retrieve all the violated messages. You can retrieve a comprehensive list of messages that have been blocked due to violations of moderation rules. Additionally, you can perform searches within this list to find specific messages or filter results based on date ranges. For more details, refer to the [Blocked Messages](/moderation/blocked-messages) section. -*** - -## Overview of Moderation Rules - -Our platform offers a wide range of moderation rules to help you detect and manage various types of risky, sensitive, or inappropriate content. Below is an overview of the available rules categorized by content type: - -### 🚩 Message Moderation Rules - -| Name | Description | -|------|-------------| -| **Word Pattern Match** | Identifies profane or offensive words using word matching. | -| **Contact Details Removal** | Detects and removes phone numbers from text. | -| **Email Detection** | Detects and removes email addresses from messages. | -| **Spam Detection (English)** | Detects spam messages in English. | -| **Scam Detection (English)** | Detects scam or fraudulent text in English. | -| **Platform Circumvention (English)** | Identifies attempts to bypass platform rules. | -| **Toxicity Detection (English)** | Detects toxic or harmful language in text. | -| **Explicit or Inappropriate Content Prompt** | Detects explicit sexual descriptions, graphic violence, or other unsuitable text. | -| **Privacy and Sensitive Info Prompt** | Identifies sensitive personal information shared without consent. | -| **Hate and Harassment Prompt** | Detects hateful or harassing language toward individuals or groups. | -| **Self-Harm or Suicidal Content Prompt** | Detects content suggesting self-harm or suicidal thoughts. | -| **Impersonation or Fraud Prompt** | Detects deceptive attempts to impersonate individuals or organizations. | -| **Violent or Terroristic Threats Prompt** | Detects content promoting violence or extremism. | -| **Non-Consensual Sexual Content Prompt** | Detects sexual exploitation, grooming, or non-consensual content. | -| **Spam and Scam Prompt** | Identifies spam, phishing attempts, and fraudulent schemes. | - -### 🖼️ Image Moderation Rules - -| Name | Description | -|------|-------------| -| **Unsafe & Prohibited Content** | Detects unsafe or prohibited content in images. | -| **Terrorism or Extremist Promotion Prompt** | Detects extremist propaganda, terrorist symbols, or images promoting violent ideologies. | -| **Minor Safety and Exploitation Prompt** | Detects child sexual content or exploitative imagery of minors. | -| **Self-Harm or Suicidal Content Prompt** | Detects imagery suggesting self-harm or suicidal ideation. | -| **Privacy or Personal Data Prompt** | Identifies images containing personal or sensitive data. | -| **Graphic Violence or Gore Prompt** | Detects images of extreme violence or gore. | -| **Explicit or Sexual Content Prompt** | Identifies nudity, explicit sexual content, or suggestive imagery. | -| **Hate or Harassment Prompt** | Detects hate symbols, harassment, or extremist imagery. | -| **Fraud or Scam Indicators Prompt** | Flags manipulated or fraudulent images, such as fake IDs. | - -### 🎥 Video Moderation Rules - -| Name | Description | -|------|-------------| -| **Unsafe & Prohibited Content** | Detects unsafe or prohibited content in video files. | +--- + +## Available Moderation Rules + +Our platform offers a wide range of moderation rules to help you detect and manage various types of risky, sensitive, or inappropriate content. + + + + | Name | Description | + |------|-------------| + | **Word Pattern Match** | Identifies profane or offensive words using word matching | + | **Contact Details Removal** | Detects and removes phone numbers from text | + | **Email Detection** | Detects and removes email addresses from messages | + | **Spam Detection (English)** | Detects spam messages in English | + | **Scam Detection (English)** | Detects scam or fraudulent text in English | + | **Platform Circumvention (English)** | Identifies attempts to bypass platform rules | + | **Toxicity Detection (English)** | Detects toxic or harmful language in text | + | **Explicit or Inappropriate Content** | Detects explicit sexual descriptions, graphic violence, or unsuitable text | + | **Privacy and Sensitive Info** | Identifies sensitive personal information shared without consent | + | **Hate and Harassment** | Detects hateful or harassing language toward individuals or groups | + | **Self-Harm or Suicidal Content** | Detects content suggesting self-harm or suicidal thoughts | + | **Impersonation or Fraud** | Detects deceptive attempts to impersonate individuals or organizations | + | **Violent or Terroristic Threats** | Detects content promoting violence or extremism | + | **Non-Consensual Sexual Content** | Detects sexual exploitation, grooming, or non-consensual content | + | **Spam and Scam** | Identifies spam, phishing attempts, and fraudulent schemes | + + + | Name | Description | + |------|-------------| + | **Unsafe & Prohibited Content** | Detects unsafe or prohibited content in images | + | **Terrorism or Extremist Promotion** | Detects extremist propaganda, terrorist symbols, or violent ideologies | + | **Minor Safety and Exploitation** | Detects child sexual content or exploitative imagery of minors | + | **Self-Harm or Suicidal Content** | Detects imagery suggesting self-harm or suicidal ideation | + | **Privacy or Personal Data** | Identifies images containing personal or sensitive data | + | **Graphic Violence or Gore** | Detects images of extreme violence or gore | + | **Explicit or Sexual Content** | Identifies nudity, explicit sexual content, or suggestive imagery | + | **Hate or Harassment** | Detects hate symbols, harassment, or extremist imagery | + | **Fraud or Scam Indicators** | Flags manipulated or fraudulent images, such as fake IDs | + + + | Name | Description | + |------|-------------| + | **Unsafe & Prohibited Content** | Detects unsafe or prohibited content in video files | + + diff --git a/moderation/reviewed-messages.mdx b/moderation/reviewed-messages.mdx index ee4a1e93..e9383893 100644 --- a/moderation/reviewed-messages.mdx +++ b/moderation/reviewed-messages.mdx @@ -2,14 +2,118 @@ title: "Reviewed Messages" --- -## Overview +Reviewed Messages is your audit trail for all moderation decisions. When a moderator takes action on a flagged or blocked message (approve, block, or mark as reviewed), it automatically moves here for record-keeping. -The Reviewed Messages endpoint in the Moderation Service API provides app owners and collaborators with access to messages that have been processed by moderators after being flagged/blocked. This endpoint contains the complete audit trail of moderation decisions made on previously flagged/blocked content. + +**Complete moderation history.** Use this dashboard to track moderator activity, audit decisions, and maintain compliance records. + -Once a moderator makes either decision, the message is automatically transferred from the Flagged/Blocked Messages list to the Reviewed Messages list. This reviewed list serves as a comprehensive record of all moderation decisions, providing transparency and accountability in your content moderation workflow. +--- + +## Quick Start + +Access your moderation history: + + + + Login to [CometChat Dashboard](https://app.cometchat.com) → Select your app → **Moderation** → **Reviewed Messages** + + + Use date range, status, or moderator filters to find specific decisions + + + Click on any message to see the original content, rule triggered, and moderator action taken + + -Reviewed messages list can also be fetched via API [List Reviewed Messages](/rest-api/moderation/list-reviewed-messages). +--- + +## How Messages Get Here + +```mermaid +flowchart LR + A[Flagged Message] -->|Approve| C[Reviewed Messages] + A -->|Block| C + B[Blocked Message] -->|Approve| C + B -->|Mark as Reviewed| C +``` + +| Source | Action Taken | Result | +|--------|--------------|--------| +| Flagged Message | Approved | Message visible, marked reviewed | +| Flagged Message | Blocked | Message hidden, marked reviewed | +| Blocked Message | Approved | Message unblocked, marked reviewed | +| Blocked Message | Mark as Reviewed | Message stays blocked, marked reviewed | + +--- + +## What You Can See + +Each reviewed message includes: + +| Field | Description | +|-------|-------------| +| **Message Content** | The original text, image, or media that was moderated | +| **Sender** | User who sent the message | +| **Rule Triggered** | Which moderation rule flagged/blocked the message | +| **Action Taken** | Approved, Blocked, or Marked as Reviewed | +| **Moderator** | Who made the decision | +| **Timestamp** | When the decision was made | + +--- + +## Use Cases + + + + Maintain records of all moderation decisions for regulatory compliance. + + + Track moderator activity and decision patterns. + + + Analyze which rules generate the most reviews to optimize your setup. + + + Reference historical decisions when users dispute moderation actions. + + + +--- + +## FAQ + + + + You cannot directly undo from the Reviewed Messages list. However, if a message was blocked, you can find it and approve it to make it visible again. + + + Reviewed messages are stored according to your data retention settings. Check your app settings for specific retention periods. + + + Yes, you can export reviewed messages for compliance or analysis purposes from the Dashboard. + + + +--- + +## Related Resources + + + + Messages pending review + + + Automatically blocked content + + + Configure moderation rules + + + Learn about the moderation system + + diff --git a/moderation/rules-management.mdx b/moderation/rules-management.mdx index 73706ca1..10829fd8 100644 --- a/moderation/rules-management.mdx +++ b/moderation/rules-management.mdx @@ -4,614 +4,479 @@ title: "Rules Management" ## Overview -The Rules Management endpoints in the Moderation Service API provide the functionality to define and manage moderation rules that help in identifying and handling inappropriate content based on a variety of conditions. These endpoints empower app owners and collaborators to create a customized message moderation strategy tailored to the specific needs of their platform. The next section provides a detailed elaboration of the capabilities offered. +Rules Management allows you to define and manage moderation rules that automatically detect and handle inappropriate content. Configure rules once in the Dashboard, and they're automatically applied to all messages sent through CometChat. - - - - -To begin managing rules: - -* Login to your [CometChat dashboard](https://app.cometchat.com/login) and choose your app. -* Navigate to **Moderation** > **Settings** in the left-hand menu. -* Select the **Rules** tab. - -## Default Rules - -Default rules are predefined sets of message moderation conditions that are readily available for use on your platform, and automatically applied to moderate messages when enabled. These default rules form the foundation of an effective message moderation strategy, combining automation with customizable options to ensure a safe, respectful, and compliant environment for platform users. Here are the standard default rules available: - -### Profanity Filter - -This feature automatically detects and manages text and custom messages containing offensive language, profanity, or derogatory remarks using a predefined list of offensive keywords to block inappropriate content. Ensuring user interactions maintain a respectful tone and comply with community standards, enhances overall platform decency. - -**Example** - -Before enabling the profanity filter, messages containing profane words are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### Contact Details Filter - -This feature detects and manages messages containing phone numbers by applying rules to prevent the sharing of private information that could compromise user privacy or security. It protects users from potential misuse of personal data and ensures compliance with data protection regulations. - -**Example** - -Before enabling the contact details filter, messages containing phone numbers are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### Email Filter - -This feature detects and manages messages containing email addresses by applying rules to prevent the sharing of private information that could compromise user privacy or security. It protects users from potential misuse of personal data and ensures compliance with data protection regulations. - -**Example** - -Before enabling the email filter, a message containing an email address is delivered to the receiver and can be seen on the receiver's chat screen. After enabling the filter, such messages are not delivered to the receiver, like in the example where the personal email isn't delivered to the receiver. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### AI-based Image Moderation - -This feature identifies and manages image-type messages containing sensitive, explicit, or prohibited content using advanced artificial intelligence algorithms for image recognition. Once detected, the system automatically blocks the images that violate platform guidelines, ensuring that such content is not displayed to users. This proactive approach safeguards users from exposure to harmful visual material, maintaining a safe and compliant environment on the platform. - -\**Example* - -Non-violating images are being delivered as seen in the example. Enabling this filter blocks violating images that are not delivered to the receiver, like in the example where the second image is indicated by a single tick in the message status on the sender's screen and isn't delivered to the receiver. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### AI-based Video Moderation - -This feature identifies and manages video-type messages containing sensitive, explicit, or prohibited content using advanced artificial intelligence algorithms for image recognition. Once detected, the system automatically blocks the images that violate platform guidelines, ensuring that such content is not displayed to users. This proactive approach safeguards users from exposure to harmful visual material, maintaining a safe and compliant environment on the platform. - -**Example** - -Before enabling the AI-based Video Moderation filter, a message containing violating videos is delivered to the receiver, like in the example where the first video can be seen on the receiver's chat screen. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status on the sender's chat screen. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### AI Message Toxicity - -The AI Message Toxicity Detection rule is a powerful, AI-driven tool designed to identify and flag toxic, harmful, or inappropriate language within user-generated messages. This feature analyzes text in real-time, detecting patterns of abusive speech, such as threats, harassment, hate speech, and other forms of offensive communication. By automatically blocking these messages based on predefined moderation rules, the tool helps prevent the spread of toxic content, fostering a safer and more respectful communication environment. This system empowers platform administrators to maintain community standards, allowing them to intervene or moderate flagged messages promptly. It also supports various languages and contexts, ensuring that the platform remains compliant with safety guidelines and user conduct policies. - -**Example** - -Before enabling the AI message toxicity rule, a message containing a sentence which violates AI message toxicity is delivered to the receiver and can be seen on the receiver's chat screen. After enabling the filter, such messages are not delivered to the receiver. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### AI Platform Circumvention - -The AI Platform Circumvention Rule employs a list of categories related to sentence similarity to identify and manage attempts by users to circumvent platform rules. This filter analyzes user-generated content for patterns and phrases that may indicate efforts to bypass established guidelines. By leveraging AI technology, it compares new submissions against a predefined set of sentence structures and categories to detect similarities that suggest rule violations. - -**Example** - -Before enabling the platform circumvention filter, a message containing a sentence which violates platform circumvention is delivered to the receiver and can be seen on the receiver's chat screen. After enabling the filter, such messages are not delivered to the receiver. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### AI Scam Detection - -The AI Scam Detection rule leverages advanced AI-powered text moderation techniques to identify and prevent scam-related messages in real-time. By analyzing message patterns and identifying specific language markers and behaviors commonly associated with scams, this rule ensures that fraudulent schemes are swiftly intercepted before reaching users. This proactive detection system scans for misleading content, phishing attempts, fake offers, and other tactics typically employed by scammers, thereby safeguarding users and maintaining the trust and security of the platform. It also continuously adapts to evolving scam strategies through machine learning, making it more effective over time. - -**Example** - -Before enabling the AI Scam Detection rule, a message containing a sentence which violates AI Scam Detection rule is delivered to the receiver and can be seen on the receiver's chat screen. After enabling the filter, such messages are not delivered to the receiver. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### AI Spam Detection - -AI Spam Detection uses sophisticated AI algorithms to automatically detect and filter out spam messages in real-time. By analyzing message content and patterns, it effectively identifies unwanted or irrelevant communications, reducing the risk of spam flooding your platform. This feature helps ensure a cleaner, more efficient messaging experience, allowing users to focus on genuine, meaningful interactions. - -**Example** - -Before enabling the AI Spam Detection rule, a message containing a sentence which violates AI Spam Detection rule is delivered to the receiver and can be seen on the receiver's chat screen. After enabling the filter, such messages are not delivered to the receiver. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### OpenAI (Message): Hate and Harassment Prompt (All Languages) - -This feature uses a predefined OpenAI moderation prompt to detect hateful or harassing language toward individuals or groups. By automatically identifying and blocking such content, it ensures a respectful and inclusive environment, fostering positive interactions among users. - -**Example** - -Before enabling the hate and harassment detection, messages containing hateful or harassing language are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Message): Privacy and Sensitive Info Prompt (All Languages) - -This feature leverages OpenAI to detect messages that share personal or sensitive information without consent. It helps prevent unauthorized disclosure of private data, safeguarding user privacy and maintaining compliance with data protection standards. - -**Example** - -Before enabling the privacy and sensitive information detection, messages containing personal or sensitive information are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Message): Explicit or Inappropriate Content Prompt (All Languages) - -This feature identifies and manages messages containing explicit sexual descriptions, graphic violence, or other unsuitable text using OpenAI moderation. It ensures that such content is automatically blocked, maintaining a safe and appropriate environment for all users. - -**Example** - -Before enabling the explicit or inappropriate content detection, messages containing explicit or inappropriate content are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Message): Spam and Scam Prompt (All Languages) - -This feature uses OpenAI to detect and block spam messages, phishing attempts, and fraudulent schemes. By filtering out malicious or unwanted content, it enhances user trust and protects them from potential scams or harmful activities. - -**Example** - -Before enabling the spam and scam detection, messages containing spam or scam content are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Message): Violent or Terroristic Threats Prompt (All Languages) - -This feature identifies content that encourages, promotes, or glorifies violence or extremism using OpenAI moderation. It ensures that such messages are automatically blocked, contributing to a safer and more secure platform for all users. + +**Using UI Kit or SDK?** Once you configure rules in the Dashboard, they are automatically applied to all messages. No additional code is required - the UI Kit and SDK handle moderation seamlessly. + -**Example** - -Before enabling the violent or terroristic threats detection, messages containing violent or terroristic content are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Message): Non-Consensual Sexual Content or Exploitation Prompt (All Languages) - -This feature detects messages related to sexual exploitation, grooming, or non-consensual content using OpenAI moderation. It proactively blocks such content, protecting users from harmful interactions and maintaining a safe environment. - -**Example** - -Before enabling the non-consensual sexual content or exploitation detection, messages containing such content are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Message): Impersonation or Fraud Prompt (All Languages) - -This feature identifies deceptive attempts to impersonate individuals or organizations using OpenAI moderation. By detecting and blocking such content, it prevents fraudulent activities and ensures the authenticity of user interactions. - -**Example** - -Before enabling the impersonation or fraud detection, messages containing impersonation or fraudulent content are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Message): Self-Harm or Suicidal Content Prompt (All Languages) - -This feature uses OpenAI to detect messages suggesting self-harm, suicidal thoughts, or related instructions. It helps identify and address potentially harmful content, providing a supportive environment and connecting users with appropriate resources when needed. - -**Example** - -Before enabling the self-harm or suicidal content detection, messages containing such content are delivered to the receiver, as indicated by double ticks in the message status. After enabling the filter, such messages are not delivered to the receiver, which is indicated by a single tick in the message status. - - - - - -The blocked messages are then visible on the dashboard for monitoring purposes. - - - - - -### OpenAI (Image): Hate or Harassment Prompt - -This feature uses a predefined OpenAI moderation prompt to detect hate symbols, extremist insignia, and harassing imagery in images. By automatically identifying and blocking such content, it ensures a respectful and safe environment for all users. - -**Example** - -Before enabling the hate or harassment detection for images, images containing hate symbols or harassing content are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. - - - - - -The blocked images are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Image): Explicit or Sexual Content Prompt - -This feature leverages OpenAI to identify nudity, explicit sexual content, or suggestive imagery unsuitable for general audiences. It ensures that such images are automatically blocked, maintaining a safe and appropriate environment. - -**Example** - -Before enabling the explicit or sexual content detection, images containing explicit or suggestive content are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. - - - - - -The blocked images are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Image): Graphic Violence or Gore Prompt - -This feature uses OpenAI to detect images of extreme violence, gore, or other disturbing content. It ensures that such images are automatically blocked, contributing to a safer and more secure platform. - -**Example** - -Before enabling the graphic violence or gore detection, images containing violent or disturbing content are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. - - - - - -The blocked images are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Image): Privacy or Personal Data Prompt - -This feature identifies images containing personal or sensitive data, such as IDs, addresses, or financial documents, using OpenAI moderation. It helps prevent unauthorized sharing of private information, safeguarding user privacy. - -**Example** - -Before enabling the privacy or personal data detection, images containing sensitive information are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. - - - - - -The blocked images are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Image): Self-Harm or Suicidal Content Prompt - -This feature uses OpenAI to detect imagery suggesting self-harm, suicidal ideation, or content that promotes self-injury. It helps identify and address potentially harmful content, providing a supportive environment. - -**Example** - -Before enabling the self-harm or suicidal content detection, images containing such content are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. - - - - - -The blocked images are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Image): Minor Safety and Exploitation Prompt - -This feature detects child sexual content, exploitative imagery of minors, or unsafe depictions of children using OpenAI moderation. It proactively blocks such content, protecting minors and maintaining a safe environment. - -**Example** - -Before enabling the minor safety and exploitation detection, images containing exploitative or unsafe content are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. - - - - - -The blocked images are then visible on the dashboard for monitoring purposes. - - - - - -*** - -### OpenAI (Image): Fraud or Scam Indicators Prompt +--- -This feature flags manipulated or fraudulent images, such as fake IDs or doctored screenshots, using OpenAI moderation. It helps prevent fraudulent activities and ensures the authenticity of user interactions. +## Quick Start -**Example** +Enable moderation in under 2 minutes: -Before enabling the fraud or scam indicators detection, images containing fraudulent or manipulated content are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. + + + Login to [CometChat Dashboard](https://app.cometchat.com) → Select your app → **Moderation** → **Settings** → **Rules** + + + Toggle on the rules you need (e.g., Profanity Filter, AI Image Moderation) + + + Send a test message that violates the rule - it should be blocked automatically + + - + -The blocked images are then visible on the dashboard for monitoring purposes. +--- - - - +## Available Rules Summary + + + + | Rule | Description | Use Case | + |------|-------------|----------| + | **Profanity Filter** | Blocks offensive language using keyword matching | General chat apps | + | **Contact Details Filter** | Blocks phone numbers | Marketplaces, dating apps | + | **Email Filter** | Blocks email addresses | Prevent off-platform contact | + | **AI Toxicity Detection** | AI-powered detection of toxic language | Community platforms | + | **AI Spam Detection** | Detects spam messages | High-traffic apps | + | **AI Scam Detection** | Identifies fraudulent messages | Marketplaces, finance apps | + | **AI Platform Circumvention** | Detects attempts to bypass rules | All apps | + + + | Rule | Description | Use Case | + |------|-------------|----------| + | **AI Image Moderation** | Blocks explicit/unsafe images | All apps with image sharing | + | **OpenAI Explicit Content** | Detects nudity and sexual content | Social apps, dating | + | **OpenAI Violence/Gore** | Blocks graphic violence | Family-friendly apps | + | **OpenAI Hate Symbols** | Detects hate imagery | Community platforms | + | **OpenAI Privacy Data** | Blocks images with personal data | Finance, healthcare | + + + | Rule | Description | Use Case | + |------|-------------|----------| + | **AI Video Moderation** | Blocks explicit/unsafe video content | Apps with video sharing | + + + | Rule | Type | Description | + |------|------|-------------| + | **Hate & Harassment** | Text/Image | Detects hateful or harassing content | + | **Explicit Content** | Text/Image | Blocks sexual or graphic content | + | **Privacy & Sensitive Info** | Text/Image | Protects personal data | + | **Spam & Scam** | Text | Identifies fraudulent schemes | + | **Violence & Terrorism** | Text/Image | Blocks violent extremism | + | **Self-Harm Content** | Text/Image | Detects self-harm references | + | **Impersonation & Fraud** | Text | Prevents identity fraud | + | **Minor Safety** | Image | Protects minors from exploitation | + + -*** +--- -### OpenAI (Image): Terrorism or Extremist Promotion Prompt +## Recommended Rules by Use Case + + + + **Essential:** + - Profanity Filter + - AI Toxicity Detection + - AI Image Moderation + - OpenAI Hate & Harassment + + **Recommended:** + - AI Spam Detection + - OpenAI Explicit Content + + + **Essential:** + - Contact Details Filter + - Email Filter + - AI Scam Detection + - AI Platform Circumvention + + **Recommended:** + - Profanity Filter + - AI Image Moderation + + + **Essential:** + - Contact Details Filter + - Email Filter + - AI Image Moderation + - OpenAI Explicit Content + + **Recommended:** + - AI Scam Detection + - OpenAI Privacy Data (Image) + + + **Essential:** + - Profanity Filter + - AI Toxicity Detection + - AI Spam Detection + + **Recommended:** + - OpenAI Hate & Harassment + - AI Image Moderation + + + **Essential:** + - OpenAI Privacy & Sensitive Info + - OpenAI Privacy Data (Image) + - AI Scam Detection + + **Recommended:** + - Profanity Filter + - Contact Details Filter + + -This feature detects extremist propaganda, terrorist symbols, or images promoting violent ideologies using OpenAI moderation. It ensures that such images are automatically blocked, contributing to a safer platform. +--- -**Example** +## Best Practices + + + + Enable default rules first, then customize based on your needs. Don't try to configure everything at once. + + + Test rules in a staging environment. Send test messages to verify rules work as expected. + + + Check the [Blocked Messages](/moderation/blocked-messages) dashboard regularly to catch false positives. + + + Review [Rule Revisions](#rule-revisions) to track changes and refine rules over time. + + + +### Tips for Effective Moderation + +1. **Layer your rules** - Use multiple rules together (e.g., Profanity Filter + AI Toxicity) for better coverage +2. **Adjust confidence levels** - Lower confidence = more aggressive blocking, higher = fewer false positives +3. **Use Flag action for borderline content** - Instead of blocking, flag messages for manual review +4. **Create custom keyword lists** - Add industry-specific terms to the [Lists Management](/moderation/lists-management) -Before enabling the terrorism or extremist promotion detection, images containing extremist or violent content are delivered to the receiver. After enabling the filter, such images are not delivered to the receiver. +--- - - - +## Default Rules -The blocked images are then visible on the dashboard for monitoring purposes. +Default rules are pre-configured and ready to use. Simply toggle them on in the Dashboard. + + + + + + Automatically detects and blocks messages containing offensive language, profanity, or derogatory remarks using a predefined list of offensive keywords. + + **Example:** Before enabling, profane messages are delivered (double ticks). After enabling, they're blocked (single tick). + + + + + + + Detects and blocks messages containing phone numbers to prevent sharing of private contact information. + + + + + + + Detects and blocks messages containing email addresses to prevent off-platform communication. + + + + + + + AI-powered detection of toxic, harmful, or inappropriate language including threats, harassment, and hate speech. Supports multiple languages. + + + + + + + Detects attempts by users to bypass platform rules using sentence similarity analysis. + + + + + + + Identifies scam-related messages including phishing attempts, fake offers, and fraudulent schemes. + + + + + + + Detects and filters spam messages by analyzing content patterns and identifying unwanted communications. + + + + + + + + + + + Uses AI to identify and block images containing sensitive, explicit, or prohibited content. + + + + + + + Detects hate symbols, extremist insignia, and harassing imagery. + + + + + + + Identifies nudity, explicit sexual content, or suggestive imagery. + + + + + + + Detects images of extreme violence, gore, or disturbing content. + + + + + + + Identifies images containing personal data like IDs, addresses, or financial documents. + + + + + + + Detects imagery suggesting self-harm or suicidal ideation. + + + + + + + Detects exploitative imagery of minors or unsafe depictions of children. + + + + + + + Flags manipulated or fraudulent images like fake IDs or doctored screenshots. + + + + + + + Detects extremist propaganda, terrorist symbols, or violent ideologies. + + + + + + + + + + + Uses AI to identify and block videos containing sensitive, explicit, or prohibited content. + + + + + + + + + + + + + + + Detects hateful or harassing language toward individuals or groups using OpenAI moderation. + + + + + + + Detects messages sharing personal or sensitive information without consent. + + + + + + + Identifies explicit sexual descriptions, graphic violence, or unsuitable text. + + + + + + + Detects spam messages, phishing attempts, and fraudulent schemes. + + + + + + + Identifies content promoting violence or extremism. + + + + + + + Detects sexual exploitation, grooming, or non-consensual content. + + + + + + + Identifies deceptive attempts to impersonate individuals or organizations. + + + + + + + Detects messages suggesting self-harm or suicidal thoughts. + + + + + + + + - - - +--- -## Rule Filters, Conditions and Actions +## Rule Configuration ### Filters -Filters allow you to narrow down messages based on the Sender or Receiver of a message. +Filters narrow down which messages are checked by a rule based on sender or receiver attributes: -For Senders, you can filter by specific properties like UID, Role, Name, and Tags, or see when the sender was created. Similarly, for Receivers, you can filter by properties such as Name, GUID, Tags, Group type or see when the receiver was created, and the Type of receiver (for example, a user or group). This enables targeted filtering based on user or group attributes within the conversation. +| Filter Type | Properties | +|-------------|------------| +| **Sender** | UID, Role, Name, Tags, Created date | +| **Receiver** | Name, GUID, Tags, Group type, Created date, Type (user/group) | ### Conditions -Conditions allow you to define criteria for blocking messages based on their type—text, image, video, or custom. +Conditions define what content triggers the rule: -You can select a keyword list, define a list of words or patterns, for text and custom messages. +| Content Type | Available Conditions | +|--------------|---------------------| +| **Text/Custom** | Keyword list, Word patterns, Toxicity, Sentiment, Sentence similarity | +| **Image/Video** | Violence, Gambling, Alcohol, Drugs, Nudity, Hate symbols, Unsafe content | -In addition to selecting specific words, patterns, or lists for text and custom messages, you can also choose filters based on Toxicity, Sentiment, or Sentence Similarity for more advanced moderation and content analysis. You can refine Toxicity filtering by selecting categories such as Identity Attack, Insult, Obscene, Mild Toxicity, or Severe Toxicity. For Sentiment, you can choose to filter messages based on positive or negative sentiment. In Sentence Similarity, you have the option to apply a default or custom list. Additionally, you can set a confidence percentage for each criterion to determine the threshold for blocking messages. - -For media messages you can select among categories like Violence, Gambling, Alcohol, Drugs and Tobacco, Rude gestures, Explicity nudity, Non-explicit nudity, Swimwear or underwear, Visually disturbing, Hate symbols or Any unsafe content. Additionally, you can set a confidence percentage for each criterion to determine the threshold for blocking messages. +You can set a **confidence percentage** for AI-based conditions to control sensitivity. ### Actions -Actions specify what happens when content matches the conditions. In addition to blocking the message by default, actions include options such as banning or kicking a user from a group and blocking a user. +Actions determine what happens when content matches the conditions: -## Configuring rules +| Action | Description | +|--------|-------------| +| **Block** | Message is not delivered (default) | +| **Flag** | Message is delivered but flagged for review | +| **Ban User** | User is banned from the group | +| **Kick User** | User is removed from the group | +| **Block User** | User is blocked platform-wide | -### Create Rule +--- + +## Managing Rules -Allows you to define new moderation rules specifying the conditions under which messages should be blocked. +### Create Rule -Creating a new rule from the dashboard: - -1. Click the Add button within the Rules tab. - -2. Configure the Rule by saving the following details: - - * Name: Name for the moderation rule. - * Rule ID: The unique identifier of the rule. - * Description: Detailed explanation of the rule's purpose. - * Filter: List of filters that must be met for the rule to trigger. - * Condition: List of conditions that must be met for the rule to trigger. - * Action: Choose from a set of actions to be taken when a violation is detected. - -3. Save - -4. Enable the Rule to start moderating! - -You can also set this up from your end using the [Create Moderation Rule REST API](https://api-explorer.cometchat.com/reference/create-rule). +1. Click **Add** in the Rules tab +2. Configure: + - **Name**: Descriptive name for the rule + - **Rule ID**: Unique identifier + - **Description**: Purpose of the rule + - **Filter**: Who the rule applies to + - **Condition**: What triggers the rule + - **Action**: What happens when triggered +3. Click **Save** +4. **Enable** the rule to start moderating ### List Rules -Fetches the details of the existing list of rules. +All configured rules are displayed in the Rules tab with their name, status, and actions. -You can also set this up from your end using the [List Moderation Rules REST API](https://api-explorer.cometchat.com/reference/list-rules). - -### Get Rule - -Fetches the details of a rule. You can set this up from your end using the [Get Moderation Rule REST API](https://api-explorer.cometchat.com/reference/get-rule). - ### Update Rule -Enables modifications to existing rules. This includes changing conditions, updating actions, or refining parameters to improve accuracy. +1. Click **Edit** in the action menu +2. Modify the rule settings +3. Click **Save** -Updating a rule from the dashboard: - -1. Click on "Edit" in the action menu of the rule you want to update. - -2. Update the Rule by saving the following details: - - * Name: Descriptive name for the moderation rule. - * Description: Detailed explanation of the rule's purpose. - * Filter: List of filters that must be met for the rule to trigger. - * Condition: List of conditions that must be met for the rule to trigger. - * Action: Choose from a set of actions to be taken when a violation is detected. - -3. Save - -You can also set this up from your end using the [Update Moderation Rule REST API](https://api-explorer.cometchat.com/reference/update-rule). - ### Delete Rule -Permits the deletion of outdated or unnecessary rules from the system. This helps in maintaining an efficient and relevant set of moderation guidelines. +Click **Delete** in the action menu and confirm. -Deleting a rule from the dashboard: - -* Click "Delete" in the action menu of the rule you want to remove, then confirm. - -You can also set this up from your end using the [Delete Moderation Rule REST API](https://api-explorer.cometchat.com/reference/delete-rule). - ### Rule Revisions -The ability to fetch all revisions of a rule in a moderation system allows app owners and collaborators to retrieve a comprehensive history of updates and changes made to specific moderation rules over time. This feature provides detailed insights into how rules have been adjusted and refined to better manage and moderate content on the platform. +Track the history of changes made to a rule: + +1. Click **View** in the action menu +2. Navigate to **Rule History** -Viewing the rule revisions on the dashboard: - -1. Click "View" in the action menu of the rule for which you wish to see revisions. -2. Navigate to the Rule History section. +--- -You can also set this up from your end using the [Get Moderation Rule Revisions REST API](https://api-explorer.cometchat.com/reference/list-rule-revisions). +## Related Resources + + + + Create custom keyword lists for your rules + + + View and manage blocked content + + + Review flagged content + + + Handle moderation in custom UI + + diff --git a/moderation/webhooks-overview.mdx b/moderation/webhooks-overview.mdx deleted file mode 100644 index cb1273e9..00000000 --- a/moderation/webhooks-overview.mdx +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: "Webhook Overview" -url: "/fundamentals/webhooks-overview" ---- \ No newline at end of file diff --git a/sdk/android/2.0/extensions.mdx b/sdk/android/2.0/extensions.mdx index ed254094..68e86c55 100644 --- a/sdk/android/2.0/extensions.mdx +++ b/sdk/android/2.0/extensions.mdx @@ -49,10 +49,4 @@ Extensions that help increase user engagement. *Recommended for advanced apps.* *Extensions that help you to build a safe messaging environment.* *Recommended for live streaming and event apps.* -[Data Masking Filter](/moderation/data-masking-filter)\ -[Profanity Filter](/moderation/profanity-filter)\ -[Image Moderation](/moderation/image-moderation)\ -[Sentiment Analysis](/moderation/sentiment-analysis)\ -[In-flight Message Moderation](/moderation/in-flight-message-moderation)\ -[Virus and Malware Scanner](/moderation/virus-malware-scanner)\ -[XSS Filter](/moderation/xss-filter) +[Legacy Moderation Extensions](/moderation/legacy-extensions) diff --git a/sdk/android/ai-moderation.mdx b/sdk/android/ai-moderation.mdx index 0fb7cdd9..dfe91143 100644 --- a/sdk/android/ai-moderation.mdx +++ b/sdk/android/ai-moderation.mdx @@ -2,93 +2,233 @@ title: "AI Moderation" --- -# Overview +## Overview AI Moderation in the CometChat SDK helps ensure that your chat application remains safe and compliant by automatically reviewing messages for inappropriate content. This feature leverages AI to moderate messages in real-time, reducing manual intervention and improving user experience. -For a broader understanding of moderation features, see the [Moderation Overview](../../moderation/overview). + +For a broader understanding of moderation features, configuring rules, and managing flagged messages, see the [Moderation Overview](/moderation/overview). + -# What Triggers Moderation? +## Prerequisites + +Before using AI Moderation, ensure the following: + +1. Moderation is enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Moderation rules are configured under **Moderation > Rules** +3. You're using CometChat SDK version that supports moderation + +## How It Works + +```mermaid +sequenceDiagram + participant App + participant SDK + participant CometChat + participant Moderation + + App->>SDK: sendMessage() + SDK->>CometChat: Send Message + CometChat->>Moderation: Process Message + CometChat-->>SDK: Message (status: PENDING) + SDK-->>App: onSuccess (PENDING) + Moderation-->>CometChat: Moderation Result + CometChat-->>SDK: onMessageModerated + SDK-->>App: APPROVED or DISAPPROVED +``` + +| Step | Description | +|------|-------------| +| 1. Send Message | App sends a text, image, or video message | +| 2. Pending Status | Message is sent with `PENDING` moderation status | +| 3. AI Processing | Moderation service analyzes the content | +| 4. Result Event | `onMessageModerated` event fires with final status | + +## Supported Message Types Moderation is triggered **only** for the following message types: -- Text Messages -- Image Messages -- Video Messages +| Message Type | Moderated | Notes | +|--------------|-----------|-------| +| Text Messages | ✅ | Content analyzed for inappropriate text | +| Image Messages | ✅ | Images scanned for unsafe content | +| Video Messages | ✅ | Videos analyzed for prohibited content | +| Custom Messages | ❌ | Not subject to AI moderation | +| Action Messages | ❌ | Not subject to AI moderation | + +## Moderation Status -Other message types are not subject to AI moderation. +The `getModerationStatus()` method returns one of the following values: -# Handling Message Moderation Status +| Status | Enum Value | Description | +|--------|------------|-------------| +| Pending | `ModerationStatus.PENDING` | Message is being processed by moderation | +| Approved | `ModerationStatus.APPROVED` | Message passed moderation and is visible | +| Disapproved | `ModerationStatus.DISAPPROVED` | Message violated rules and was blocked | -When sending text, image, or video messages, the moderation process introduces the following changes to the message flow: +## Implementation -1. **Initial Status:** - - On successfully sending a text, image, or video message, the response will include the moderation status as `pending`. - - You can fetch this status using the `getModerationStatus()` method, which returns an Enum: `ModerationStatus.PENDING`. +### Step 1: Send a Message and Check Initial Status -2. **Moderation Result Event:** - - Once the moderation service processes the message, a real-time event is triggered in your `MessageListener` as `onMessageModerated`. - - This event provides the message object with the updated moderation status, which will be either `ModerationStatus.APPROVED` or `ModerationStatus.DISAPPROVED`. +When you send a text, image, or video message, check the initial moderation status: -**Example:** + + + ```kotlin + val textMessage = TextMessage(receiverUID, "Hello, how are you?", CometChatConstants.RECEIVER_TYPE_USER) + + CometChat.sendMessage(textMessage, object : CometChat.CallbackListener() { + override fun onSuccess(message: TextMessage) { + // Check moderation status + if (message.moderationStatus == ModerationStatus.PENDING) { + Log.d(TAG, "Message is under moderation review") + // Show pending indicator in UI + } + } + + override fun onError(e: CometChatException) { + Log.e(TAG, "Message sending failed: ${e.message}") + } + }) + ``` + + + ```java + TextMessage textMessage = new TextMessage(receiverUID, "Hello, how are you?", CometChatConstants.RECEIVER_TYPE_USER); + + CometChat.sendMessage(textMessage, new CometChat.CallbackListener() { + @Override + public void onSuccess(TextMessage message) { + // Check moderation status + if (message.getModerationStatus().equals(ModerationStatus.PENDING)) { + Log.d(TAG, "Message is under moderation review"); + // Show pending indicator in UI + } + } + + @Override + public void onError(CometChatException e) { + Log.e(TAG, "Message sending failed: " + e.getMessage()); + } + }); + ``` + + + +### Step 2: Listen for Moderation Results + +Register a message listener to receive moderation results in real-time: + + + + ```kotlin + val listenerID = "MODERATION_LISTENER" + + CometChat.addMessageListener(listenerID, object : CometChat.MessageListener() { + override fun onMessageModerated(message: BaseMessage) { + when (message) { + is TextMessage -> { + when (message.moderationStatus) { + ModerationStatus.APPROVED -> { + Log.d(TAG, "Message ${message.id} approved") + // Update UI to show message normally + } + ModerationStatus.DISAPPROVED -> { + Log.d(TAG, "Message ${message.id} blocked") + // Handle blocked message (hide or show warning) + } + } + } + is MediaMessage -> { + when (message.moderationStatus) { + ModerationStatus.APPROVED -> { + Log.d(TAG, "Media message ${message.id} approved") + } + ModerationStatus.DISAPPROVED -> { + Log.d(TAG, "Media message ${message.id} blocked") + } + } + } + } + } + }) + + // Don't forget to remove the listener when done + // CometChat.removeMessageListener(listenerID) + ``` + + + ```java + String listenerID = "MODERATION_LISTENER"; + + CometChat.addMessageListener(listenerID, new CometChat.MessageListener() { + @Override + public void onMessageModerated(BaseMessage message) { + if (message instanceof TextMessage) { + TextMessage textMessage = (TextMessage) message; + if (textMessage.getModerationStatus().equals(ModerationStatus.APPROVED)) { + Log.d(TAG, "Message " + message.getId() + " approved"); + // Update UI to show message normally + } else if (textMessage.getModerationStatus().equals(ModerationStatus.DISAPPROVED)) { + Log.d(TAG, "Message " + message.getId() + " blocked"); + // Handle blocked message (hide or show warning) + } + } else if (message instanceof MediaMessage) { + MediaMessage mediaMessage = (MediaMessage) message; + if (mediaMessage.getModerationStatus().equals(ModerationStatus.APPROVED)) { + Log.d(TAG, "Media message " + message.getId() + " approved"); + } else if (mediaMessage.getModerationStatus().equals(ModerationStatus.DISAPPROVED)) { + Log.d(TAG, "Media message " + message.getId() + " blocked"); + } + } + } + }); + + // Don't forget to remove the listener when done + // CometChat.removeMessageListener(listenerID); + ``` + + + +### Step 3: Handle Disapproved Messages + +When a message is disapproved, handle it appropriately in your UI: - - ```java - // Check moderation status after sending a message. This check is only applicable for TextMessage & MediaMessage class. - if(message.getModerationStatus().equals(ModerationStatus.PENDING)) { - // Message is under moderation - } - - // Listen for moderation result - CometChat.addMessageListener("UNIQUE_LISTENER_ID", new CometChat.MessageListener() { - @Override - public void onMessageModerated(BaseMessage message) { - if(message instanceof TextMessage){ - if(((TextMessage) message).getModerationStatus().equals(ModerationStatus.APPROVED)) { - // Message approved - } else if(((TextMessage) message).getModerationStatus().equals(ModerationStatus.DISAPPROVED)) { - // Message disapproved - } - } - else if(message instanceof MediaMessage){ - if(((MediaMessage) message).getModerationStatus().equals(ModerationStatus.APPROVED)) { - // Message approved - } else if(((MediaMessage) message).getModerationStatus().equals(ModerationStatus.DISAPPROVED)) { - // Message disapproved - } - } - } - }); - ``` - - - - ```kotlin - - // Check moderation status after sending message. This check is only applicable for TextMessage & MediaMessage class. - if (message.moderationStatus == ModerationStatus.PENDING) { - // Message is under moderation - } - - // Listen for moderation result - CometChat.addMessageListener("UNIQUE_LISTENER_ID", object : CometChat.MessageListener() { - override fun onMessageModerated(message: BaseMessage) { - if(baseMessage is TextMessage){ - if(message.moderationStatus == ModerationStatus.APPROVED) { - // Message approved - } else if(message.moderationStatus == ModerationStatus.DISAPPROVED) { - // Message disapproved - } - } else if(baseMessage is MediaMessage){ - if(message.moderationStatus == ModerationStatus.APPROVED) { - // Message approved - } else if(message.moderationStatus == ModerationStatus.DISAPPROVED) { - // Message disapproved - } - } - } - }); - ``` - - \ No newline at end of file + + ```kotlin + fun handleDisapprovedMessage(message: BaseMessage) { + val messageId = message.id + + // Option 1: Hide the message completely + hideMessageFromUI(messageId) + + // Option 2: Show a placeholder message + showBlockedPlaceholder(messageId, "This message was blocked by moderation") + + // Option 3: Notify the sender (if it's their message) + if (message.sender.uid == currentUserUID) { + showNotification("Your message was blocked due to policy violation") + } + } + ``` + + + ```java + private void handleDisapprovedMessage(BaseMessage message) { + int messageId = message.getId(); + + // Option 1: Hide the message completely + hideMessageFromUI(messageId); + + // Option 2: Show a placeholder message + showBlockedPlaceholder(messageId, "This message was blocked by moderation"); + + // Option 3: Notify the sender (if it's their message) + if (message.getSender().getUid().equals(currentUserUID)) { + showNotification("Your message was blocked due to policy violation"); + } + } + ``` + + diff --git a/sdk/android/calling-overview.mdx b/sdk/android/calling-overview.mdx index d09702e5..19ef1c0f 100644 --- a/sdk/android/calling-overview.mdx +++ b/sdk/android/calling-overview.mdx @@ -18,7 +18,7 @@ dependencies { } ``` -For detailed setup instructions, see the [Calls SDK Setup](/sdk/android/setup-calling) guide. +For detailed setup instructions, see the [Calls SDK Setup](/sdk/android/calling-setup) guide. ## Choose Your Implementation diff --git a/sdk/android/setup-calling.mdx b/sdk/android/calling-setup.mdx similarity index 100% rename from sdk/android/setup-calling.mdx rename to sdk/android/calling-setup.mdx diff --git a/sdk/android/direct-calling.mdx b/sdk/android/direct-calling.mdx index 897aba29..1c552333 100644 --- a/sdk/android/direct-calling.mdx +++ b/sdk/android/direct-calling.mdx @@ -6,7 +6,7 @@ title: "Call Session" This section demonstrates how to start a call session in an Android application. Previously known as **Direct Calling**. -Before you begin, we strongly recommend you read the [calling setup guide](/sdk/android/setup-calling). +Before you begin, we strongly recommend you read the [calling setup guide](/sdk/android/calling-setup). diff --git a/sdk/android/flag-message.mdx b/sdk/android/flag-message.mdx index 292e37ff..0843d290 100644 --- a/sdk/android/flag-message.mdx +++ b/sdk/android/flag-message.mdx @@ -1,108 +1,227 @@ --- -title: "Flag A Message" +title: "Flag Message" --- -Flagging messages allows users to report inappropriate content to moderators or administrators. CometChat provides methods to flag messages with specific reasons and retrieve available flag reasons configured in your dashboard. +## Overview -## Flag a Message +Flagging messages allows users to report inappropriate content to moderators or administrators. When a message is flagged, it appears in the [CometChat Dashboard](https://app.cometchat.com) under **Moderation > Flagged Messages** for review. -*In other words, as a user, how do I report a message?* + +For a complete understanding of how flagged messages are reviewed and managed, see the [Flagged Messages](/moderation/flagged-messages) documentation. + -To flag a message, you can use the `flagMessage()` method. This method takes the message ID and a payload containing an optional reason ID and remark. +## Prerequisites - - -```kotlin - val id = 0L; // Id of the message to be flagged - val flagDetail = FlagDetail() - flagDetail.reasonId = "spam" // Required: Reason ID for flagging the message - flagDetail.remark = "This message contains promotional content" // Optional: Additional remarks - CometChat.flagMessage(id, flagDetail, object : CometChat.CallbackListener() { - override fun onSuccess(s: String?) { - Log.i(TAG, "onSuccess: Message flagged successfully: $s") - } +Before using the flag message feature: - override fun onError(e: CometChatException?) { - Log.i(TAG, "onError: Message flagging failed with error: ${e?.message}") - } - }) +1. Moderation must be enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Flag reasons should be configured under **Moderation > Advanced Settings** + +## How It Works + +```mermaid +sequenceDiagram + participant User + participant App + participant SDK + participant CometChat + participant Dashboard + + User->>App: Reports message + App->>SDK: getFlagReasons() + SDK-->>App: Available reasons + App->>User: Show reason picker + User->>App: Select reason + remark + App->>SDK: flagMessage(id, flagDetail) + SDK->>CometChat: Flag message + CometChat-->>SDK: Success response + SDK-->>App: Message flagged + CometChat->>Dashboard: Message in Flagged queue ``` - - - -```java - long id = 0; // Id of the message to be flagged - FlagDetail flagDetail = new FlagDetail(); // Details of the flagging action - flagDetail.setReasonId("spam"); // Required: Reason ID for flagging the message - flagDetail.setRemark("This message contains promotional content"); // Optional: Additional remarks - CometChat.flagMessage(id, flagDetail, new CometChat.CallbackListener() { - @Override - public void onSuccess(String s) { - Log.i(TAG, "onSuccess: Message flagged successfully: " + s); - } - @Override - public void onError(CometChatException e) { - Log.i(TAG, "onError: Message flagging failed with error: " + e.getMessage()); +## Get Flag Reasons + +Before flagging a message, retrieve the list of available flag reasons configured in your Dashboard: + + + + ```kotlin + CometChat.getFlagReasons(object : CometChat.CallbackListener>() { + override fun onSuccess(reasons: MutableList?) { + Log.d(TAG, "Flag reasons fetched: $reasons") + // Use reasons to populate your report dialog UI + reasons?.forEach { reason -> + Log.d(TAG, "Reason ID: ${reason?.id}, Title: ${reason?.reason}") } - }); -``` - + } + + override fun onError(e: CometChatException) { + Log.e(TAG, "Error fetching flag reasons: ${e.message}") + } + }) + ``` + + + ```java + CometChat.getFlagReasons(new CometChat.CallbackListener>() { + @Override + public void onSuccess(List reasons) { + Log.d(TAG, "Flag reasons fetched: " + reasons); + // Use reasons to populate your report dialog UI + for (FlagReason reason : reasons) { + Log.d(TAG, "Reason ID: " + reason.getId() + ", Title: " + reason.getReason()); + } + } + + @Override + public void onError(CometChatException e) { + Log.e(TAG, "Error fetching flag reasons: " + e.getMessage()); + } + }); + ``` + + + +### Response + +The response is a list of `FlagReason` objects containing: + +| Property | Type | Description | +|----------|------|-------------| +| id | String | Unique identifier for the reason | +| reason | String | Display text for the reason | + +## Flag a Message + +To flag a message, use the `flagMessage()` method with the message ID and a `FlagDetail` object: + + + + ```kotlin + val messageId = 123L // ID of the message to flag + + val flagDetail = FlagDetail().apply { + reasonId = "spam" // Required: ID from getFlagReasons() + remark = "This message contains promotional content" // Optional + } + + CometChat.flagMessage(messageId, flagDetail, object : CometChat.CallbackListener() { + override fun onSuccess(response: String?) { + Log.d(TAG, "Message flagged successfully: $response") + } + + override fun onError(e: CometChatException?) { + Log.e(TAG, "Message flagging failed: ${e?.message}") + } + }) + ``` + + + ```java + long messageId = 123L; // ID of the message to flag + + FlagDetail flagDetail = new FlagDetail(); + flagDetail.setReasonId("spam"); // Required: ID from getFlagReasons() + flagDetail.setRemark("This message contains promotional content"); // Optional + + CometChat.flagMessage(messageId, flagDetail, new CometChat.CallbackListener() { + @Override + public void onSuccess(String response) { + Log.d(TAG, "Message flagged successfully: " + response); + } + + @Override + public void onError(CometChatException e) { + Log.e(TAG, "Message flagging failed: " + e.getMessage()); + } + }); + ``` + ### Parameters -| Parameter | Type | Required | Description | -|-------------------------|------------|----------|---------------------------------------------| -| id | long | Yes | The ID of the message to be flagged | -| flagDetail | FlagDetail | Yes | Contains flagging details | -| flagDetail.setReasonId | String | Yes | ID of the flag reason (from getFlagReasons) | -| flagDetail.setRemark | String | No | Additional context or explanation | +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| messageId | long | Yes | The ID of the message to flag | +| flagDetail | FlagDetail | Yes | Contains flagging details | +| flagDetail.reasonId | String | Yes | ID of the flag reason (from `getFlagReasons()`) | +| flagDetail.remark | String | No | Additional context or explanation from the user | ### Response -On successful flagging, you'll receive a response object: - -```String +```json { "message": "Message {id} has been flagged successfully." } ``` -## Get Flag Reasons +## Complete Example -*In other words, what are the available reasons for flagging a message?* - -Before flagging a message, you can retrieve the list of available flag reasons using the `getFlagReasons()` method. These reasons are configured in the CometChat Dashboard. +Here's a complete implementation showing how to build a report message flow: - -```java - CometChat.getFlagReasons(new CometChat.CallbackListener>() { - @Override - public void onSuccess(List reasons) { - Log.e(TAG, "Flag Reasons fetched: " + reasons); - } - - @Override - public void onError(CometChatException e) { - Log.e(TAG, "Error fetching Flag Reasons: " + e.getMessage()); - } - }); -``` - - - -```kotlin - CometChat.getFlagReasons(object : CometChat.CallbackListener?>() { - override fun onSuccess(reasons: MutableList?) { - Log.e(TAG, "Flag Reasons fetched: $reasons") + + ```kotlin + class ReportMessageHandler { + private var flagReasons: List = emptyList() + + // Load flag reasons (call this on app init or when needed) + fun loadFlagReasons(callback: (List) -> Unit) { + CometChat.getFlagReasons(object : CometChat.CallbackListener>() { + override fun onSuccess(reasons: MutableList?) { + flagReasons = reasons ?: emptyList() + callback(flagReasons) + } + + override fun onError(e: CometChatException) { + Log.e(TAG, "Failed to load flag reasons: ${e.message}") + callback(emptyList()) + } + }) + } + + // Get reasons for UI display + fun getReasons(): List = flagReasons + + // Flag a message with selected reason + fun flagMessage( + messageId: Long, + reasonId: String, + remark: String? = null, + callback: (Boolean, String?) -> Unit + ) { + val flagDetail = FlagDetail().apply { + this.reasonId = reasonId + remark?.let { this.remark = it } } - override fun onError(e: CometChatException) { - Log.e(TAG, "Error fetching Flag Reasons: " + e.message) - } - }) -``` - + CometChat.flagMessage(messageId, flagDetail, object : CometChat.CallbackListener() { + override fun onSuccess(response: String?) { + callback(true, response) + } + + override fun onError(e: CometChatException?) { + callback(false, e?.message) + } + }) + } + } + + // Usage + val reportHandler = ReportMessageHandler() + + // Load reasons when app initializes + reportHandler.loadFlagReasons { reasons -> + // Display reasons in UI for user to select + } + + // When user submits the report + reportHandler.flagMessage(123L, "spam", "User is sending promotional links") { success, message -> + if (success) { + showToast("Message reported successfully") + } + } + ``` + diff --git a/sdk/android/standalone-calling.mdx b/sdk/android/standalone-calling.mdx index 354a7ae6..77b4e628 100644 --- a/sdk/android/standalone-calling.mdx +++ b/sdk/android/standalone-calling.mdx @@ -7,7 +7,7 @@ title: "Standalone Calling" This section demonstrates how to implement calling functionality using only the CometChat Calls SDK, without requiring the Chat SDK. This is ideal for applications that need video/audio calling capabilities without the full chat infrastructure. -Before you begin, ensure you have completed the [Calls SDK setup](/sdk/android/setup-calling). +Before you begin, ensure you have completed the [Calls SDK setup](/sdk/android/calling-setup). ## User Authentication diff --git a/sdk/flutter/ai-moderation.mdx b/sdk/flutter/ai-moderation.mdx index a5370e5f..130524a4 100644 --- a/sdk/flutter/ai-moderation.mdx +++ b/sdk/flutter/ai-moderation.mdx @@ -2,64 +2,171 @@ title: "AI Moderation" --- -# Overview +## Overview AI Moderation in the CometChat SDK helps ensure that your chat application remains safe and compliant by automatically reviewing messages for inappropriate content. This feature leverages AI to moderate messages in real-time, reducing manual intervention and improving user experience. -For a broader understanding of moderation features, see the [Moderation Overview](../../moderation/overview). + +For a broader understanding of moderation features, configuring rules, and managing flagged messages, see the [Moderation Overview](/moderation/overview). + -# What Triggers Moderation? +## Prerequisites + +Before using AI Moderation, ensure the following: + +1. Moderation is enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Moderation rules are configured under **Moderation > Rules** +3. You're using CometChat SDK version that supports moderation + +## How It Works + +```mermaid +sequenceDiagram + participant App + participant SDK + participant CometChat + participant Moderation + + App->>SDK: sendMessage() + SDK->>CometChat: Send Message + CometChat->>Moderation: Process Message + CometChat-->>SDK: Message (status: PENDING) + SDK-->>App: onSuccess (PENDING) + Moderation-->>CometChat: Moderation Result + CometChat-->>SDK: onMessageModerated + SDK-->>App: APPROVED or DISAPPROVED +``` + +| Step | Description | +|------|-------------| +| 1. Send Message | App sends a text, image, or video message | +| 2. Pending Status | Message is sent with `PENDING` moderation status | +| 3. AI Processing | Moderation service analyzes the content | +| 4. Result Event | `onMessageModerated` event fires with final status | + +## Supported Message Types Moderation is triggered **only** for the following message types: -- Text Messages -- Image Messages -- Video Messages +| Message Type | Moderated | Notes | +|--------------|-----------|-------| +| Text Messages | ✅ | Content analyzed for inappropriate text | +| Image Messages | ✅ | Images scanned for unsafe content | +| Video Messages | ✅ | Videos analyzed for prohibited content | +| Custom Messages | ❌ | Not subject to AI moderation | +| Action Messages | ❌ | Not subject to AI moderation | + +## Moderation Status + +The `moderationStatus` property returns one of the following enum values: + +| Status | Enum Value | Description | +|--------|------------|-------------| +| Pending | `ModerationStatusEnum.PENDING` | Message is being processed by moderation | +| Approved | `ModerationStatusEnum.APPROVED` | Message passed moderation and is visible | +| Disapproved | `ModerationStatusEnum.DISAPPROVED` | Message violated rules and was blocked | -Other message types are not subject to AI moderation. +## Implementation -# Handling Message Moderation Status +### Step 1: Send a Message and Check Initial Status -When sending text, image, or video messages, the moderation process introduces the following changes to the message flow: +When you send a text, image, or video message, check the initial moderation status: -1. **Initial Status:** - - On successfully sending a text, image, or video message, the response will include the moderation status as `pending`. - - You can fetch this status using the `moderationStatus` property, which returns an Enum: `CometChat.ModerationStatus.PENDING`. + + + ```dart + TextMessage textMessage = TextMessage( + text: "Hello, how are you?", + receiverUid: receiverUID, + receiverType: ReceiverTypeConstants.user, + ); + + CometChat.sendMessage( + textMessage, + onSuccess: (TextMessage message) { + // Check moderation status + if (message.moderationStatus?.value == ModerationStatusEnum.PENDING.value) { + print("Message is under moderation review"); + // Show pending indicator in UI + } + }, + onError: (CometChatException e) { + print("Message sending failed: ${e.message}"); + }, + ); + ``` + + + +### Step 2: Listen for Moderation Results + +Implement the `MessageListener` to receive moderation results in real-time: + + + + ```dart + class ModerationListener with MessageListener { + + @override + void onMessageModerated(BaseMessage message) { + if (message is TextMessage) { + switch (message.moderationStatus?.value) { + case ModerationStatusEnum.APPROVED: + print("Message ${message.id} approved"); + // Update UI to show message normally + break; + + case ModerationStatusEnum.DISAPPROVED: + print("Message ${message.id} blocked"); + // Handle blocked message (hide or show warning) + handleDisapprovedMessage(message); + break; + } + } else if (message is MediaMessage) { + switch (message.moderationStatus?.value) { + case ModerationStatusEnum.APPROVED: + print("Media message ${message.id} approved"); + break; + + case ModerationStatusEnum.DISAPPROVED: + print("Media message ${message.id} blocked"); + handleDisapprovedMessage(message); + break; + } + } + } + } + + // Register the listener + CometChat.addMessageListener("MODERATION_LISTENER", ModerationListener()); + + // Don't forget to remove the listener when done + // CometChat.removeMessageListener("MODERATION_LISTENER"); + ``` + + -2. **Moderation Result Event:** - - Once the moderation service processes the message, a real-time event is triggered in your `MessageListener` as `onMessageModerated`. - - This event provides the message object with the updated moderation status, which will be either `CometChat.ModerationStatus.APPROVED` or `CometChat.ModerationStatus.DISAPPROVED`. +### Step 3: Handle Disapproved Messages -**Example:** +When a message is disapproved, handle it appropriately in your UI: - - ```dart - // Check moderation status after sending a message. This check is only applicable for TextMessage & MediaMessage class. - if(message.moderationStatus.value == ModerationStatusEnum.PENDING.value) { - // Message is under moderation - } - - // Listen for moderation result - class Class_Name with MessageListener { - // CometChat.addMessageListener("listenerId", this); - @override - void onMessageModerated(BaseMessage message) { - if(message is TextMessage) { - if(message.moderationStatus?.value == ModerationStatusEnum.APPROVED.value) { - // Message approved - } else if(message.moderationStatus?.value == ModerationStatusEnum.DISAPPROVED.value) { - // Message disapproved - } - } else if(message is MediaMessage) { - if(message.moderationStatus?.value == ModerationStatusEnum.APPROVED.value) { - // Message approved - } else if(message.moderationStatus?.value == ModerationStatusEnum.DISAPPROVED.value) { - // Message disapproved - } - } - } - } - ``` - + + ```dart + void handleDisapprovedMessage(BaseMessage message) { + int messageId = message.id; + + // Option 1: Hide the message completely + hideMessageFromUI(messageId); + + // Option 2: Show a placeholder message + showBlockedPlaceholder(messageId, "This message was blocked by moderation"); + + // Option 3: Notify the sender (if it's their message) + if (message.sender?.uid == currentUserUID) { + showNotification("Your message was blocked due to policy violation"); + } + } + ``` + diff --git a/sdk/flutter/flag-message.mdx b/sdk/flutter/flag-message.mdx index eb5b90fe..02b71550 100644 --- a/sdk/flutter/flag-message.mdx +++ b/sdk/flutter/flag-message.mdx @@ -1,81 +1,199 @@ --- -title: "Flag A Message" +title: "Flag Message" --- -Flagging messages allows users to report inappropriate content to moderators or administrators. CometChat provides methods to flag messages with specific reasons and retrieve available flag reasons configured in your dashboard. +## Overview -## Flag a Message +Flagging messages allows users to report inappropriate content to moderators or administrators. When a message is flagged, it appears in the [CometChat Dashboard](https://app.cometchat.com) under **Moderation > Flagged Messages** for review. -*In other words, as a user, how do I report a message?* + +For a complete understanding of how flagged messages are reviewed and managed, see the [Flagged Messages](/moderation/flagged-messages) documentation. + -To flag a message, you can use the `flagMessage()` method. This method takes the message ID and a payload containing an optional reason ID and remark. +## Prerequisites - +Before using the flag message feature: + +1. Moderation must be enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Flag reasons should be configured under **Moderation > Advanced Settings** + +## How It Works - -```dart - int messageId = 0; // ID of the message to be flagged - - FlagDetail flagDetail = FlagDetail() - ..reasonId = "spam" // Required - ..remark = "This message contains promotional content"; // Optional - - CometChat.flagMessage( - messageId, - flagDetail, - onSuccess: (String response) { - print("Message flagged successfully: $response"); - }, - onError: (CometChatException e) { - print("Message flagging failed with error: ${e.message}"); - }, - ); +```mermaid +sequenceDiagram + participant User + participant App + participant SDK + participant CometChat + participant Dashboard + + User->>App: Reports message + App->>SDK: getFlagReasons() + SDK-->>App: Available reasons + App->>User: Show reason picker + User->>App: Select reason + remark + App->>SDK: flagMessage(id, flagDetail) + SDK->>CometChat: Flag message + CometChat-->>SDK: Success response + SDK-->>App: Message flagged + CometChat->>Dashboard: Message in Flagged queue ``` - + +## Get Flag Reasons + +Before flagging a message, retrieve the list of available flag reasons configured in your Dashboard: + + + + ```dart + CometChat.getFlagReasons( + onSuccess: (List reasons) { + print("Flag reasons fetched: $reasons"); + // Use reasons to populate your report dialog UI + for (var reason in reasons) { + print("Reason ID: ${reason.id}, Title: ${reason.title}"); + } + }, + onError: (CometChatException e) { + print("Error fetching flag reasons: ${e.message}"); + }, + ); + ``` + + + +### Response + +The response is a list of `FlagReason` objects containing: + +| Property | Type | Description | +|----------|------|-------------| +| id | String | Unique identifier for the reason | +| title | String | Display text for the reason | + +## Flag a Message + +To flag a message, use the `flagMessage()` method with the message ID and a `FlagDetail` object: + + + + ```dart + int messageId = 123; // ID of the message to flag + + FlagDetail flagDetail = FlagDetail() + ..reasonId = "spam" // Required: ID from getFlagReasons() + ..remark = "This message contains promotional content"; // Optional + + CometChat.flagMessage( + messageId, + flagDetail, + onSuccess: (String response) { + print("Message flagged successfully: $response"); + }, + onError: (CometChatException e) { + print("Message flagging failed: ${e.message}"); + }, + ); + ``` + ### Parameters -| Parameter | Type | Required | Description | -|-------------------------|------------|----------|---------------------------------------------| -| id | long | Yes | The ID of the message to be flagged | -| flagDetail | FlagDetail | Yes | Contains flagging details | -| flagDetail.setReasonId | String | Yes | ID of the flag reason (from getFlagReasons) | -| flagDetail.setRemark | String | No | Additional context or explanation | +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| messageId | int | Yes | The ID of the message to flag | +| flagDetail | FlagDetail | Yes | Contains flagging details | +| flagDetail.reasonId | String | Yes | ID of the flag reason (from `getFlagReasons()`) | +| flagDetail.remark | String | No | Additional context or explanation from the user | ### Response -On successful flagging, you'll receive a response object: - -```String +```json { "message": "Message {id} has been flagged successfully." } ``` -## Get Flag Reasons - -*In other words, what are the available reasons for flagging a message?* +## Complete Example -Before flagging a message, you can retrieve the list of available flag reasons using the `getFlagReasons()` method. These reasons are configured in the CometChat Dashboard. +Here's a complete implementation showing how to build a report message flow: - -```dart + + ```dart + class ReportMessageHandler { + List _flagReasons = []; + + // Load flag reasons (call this on app init or when needed) + Future> loadFlagReasons() async { + final completer = Completer>(); + CometChat.getFlagReasons( - onSuccess: (List reasons) { - print("Flag Reasons fetched: $reasons"); - - // Example: iterate through reasons - for (var reason in reasons) { - print("Reason ID: ${reason.id}, Title: ${reason.title}"); + onSuccess: (List reasons) { + _flagReasons = reasons; + completer.complete(reasons); + }, + onError: (CometChatException e) { + print("Failed to load flag reasons: ${e.message}"); + completer.complete([]); + }, + ); + + return completer.future; } - }, - onError: (CometChatException e) { - print("Error fetching Flag Reasons: ${e.message}"); - }, - ); -``` - + // Get reasons for UI display + List getReasons() => _flagReasons; + + // Flag a message with selected reason + Future flagMessage( + int messageId, + String reasonId, { + String? remark, + }) async { + final completer = Completer(); + + FlagDetail flagDetail = FlagDetail() + ..reasonId = reasonId; + + if (remark != null) { + flagDetail.remark = remark; + } + + CometChat.flagMessage( + messageId, + flagDetail, + onSuccess: (String response) { + print("Message flagged successfully"); + completer.complete(true); + }, + onError: (CometChatException e) { + print("Failed to flag message: ${e.message}"); + completer.complete(false); + }, + ); + + return completer.future; + } + } + + // Usage + final reportHandler = ReportMessageHandler(); + + // Load reasons when app initializes + await reportHandler.loadFlagReasons(); + + // When user submits the report + final success = await reportHandler.flagMessage( + 123, + "spam", + remark: "User is sending promotional links", + ); + + if (success) { + showToast("Message reported successfully"); + } + ``` + diff --git a/sdk/ios/2.0/extensions.mdx b/sdk/ios/2.0/extensions.mdx index bfb9c4ee..e68c97c5 100644 --- a/sdk/ios/2.0/extensions.mdx +++ b/sdk/ios/2.0/extensions.mdx @@ -49,10 +49,4 @@ Extensions that help increase user engagement. *Recommended for advanced apps.* \_Extensions that help you to build a safe messaging environment.\_*Recommended for live streaming and event apps.* -[Data Masking Filter](/moderation/data-masking-filter)\ -[Profanity Filter](/moderation/profanity-filter)\ -[Image Moderation](/moderation/image-moderation)\ -[Sentiment Analysis](/moderation/sentiment-analysis)\ -[In-flight Message Moderation](/moderation/in-flight-message-moderation)\ -[Virus and Malware Scanner](/moderation/virus-malware-scanner)\ -[XSS Filter](/moderation/xss-filter) +[Legacy Moderation Extensions](/moderation/legacy-extensions) diff --git a/sdk/ios/ai-moderation.mdx b/sdk/ios/ai-moderation.mdx index 998b2d88..eda461d9 100644 --- a/sdk/ios/ai-moderation.mdx +++ b/sdk/ios/ai-moderation.mdx @@ -2,101 +2,232 @@ title: "AI Moderation" --- -# Overview +## Overview AI Moderation in the CometChat SDK helps ensure that your chat application remains safe and compliant by automatically reviewing messages for inappropriate content. This feature leverages AI to moderate messages in real-time, reducing manual intervention and improving user experience. -For a broader understanding of moderation features, see the [Moderation Overview](../../moderation/overview). + +For a broader understanding of moderation features, configuring rules, and managing flagged messages, see the [Moderation Overview](/moderation/overview). + -# What Triggers Moderation? +## Prerequisites + +Before using AI Moderation, ensure the following: + +1. Moderation is enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Moderation rules are configured under **Moderation > Rules** +3. You're using CometChat SDK version that supports moderation + +## How It Works + +```mermaid +sequenceDiagram + participant App + participant SDK + participant CometChat + participant Moderation + + App->>SDK: sendMessage() + SDK->>CometChat: Send Message + CometChat->>Moderation: Process Message + CometChat-->>SDK: Message (status: pending) + SDK-->>App: onSuccess (pending) + Moderation-->>CometChat: Moderation Result + CometChat-->>SDK: onMessageModerated + SDK-->>App: approved or disapproved +``` + +| Step | Description | +|------|-------------| +| 1. Send Message | App sends a text, image, or video message | +| 2. Pending Status | Message is sent with `pending` moderation status | +| 3. AI Processing | Moderation service analyzes the content | +| 4. Result Event | `onMessageModerated` event fires with final status | + +## Supported Message Types Moderation is triggered **only** for the following message types: -- Text Messages -- Image Messages -- Video Messages +| Message Type | Moderated | Notes | +|--------------|-----------|-------| +| Text Messages | ✅ | Content analyzed for inappropriate text | +| Image Messages | ✅ | Images scanned for unsafe content | +| Video Messages | ✅ | Videos analyzed for prohibited content | +| Custom Messages | ❌ | Not subject to AI moderation | +| Action Messages | ❌ | Not subject to AI moderation | + +## Moderation Status + +The `getModerationStatus()` method returns one of the following string values: + +| Status | Value | Description | +|--------|-------|-------------| +| Pending | `"pending"` | Message is being processed by moderation | +| Approved | `"approved"` | Message passed moderation and is visible | +| Disapproved | `"disapproved"` | Message violated rules and was blocked | + +## Implementation + +### Step 1: Send a Message and Check Initial Status + +When you send a text, image, or video message, check the initial moderation status: + + + + ```swift + let textMessage = TextMessage(receiverUid: receiverUID, text: "Hello, how are you?", receiverType: .user) + + CometChat.sendTextMessage(message: textMessage) { sentMessage in + // Check moderation status + if let message = sentMessage as? TextMessage { + if message.getModerationStatus() == "pending" { + print("Message is under moderation review") + // Show pending indicator in UI + } + } + } onError: { error in + print("Message sending failed: \(error?.errorDescription ?? "")") + } + ``` + + + ```objc + TextMessage *textMessage = [[TextMessage alloc] initWithReceiverUid:receiverUID text:@"Hello, how are you?" receiverType:ReceiverTypeUser]; + + [CometChat sendTextMessageWithMessage:textMessage onSuccess:^(TextMessage *sentMessage) { + // Check moderation status + if ([[sentMessage getModerationStatus] isEqualToString:@"pending"]) { + NSLog(@"Message is under moderation review"); + // Show pending indicator in UI + } + } onError:^(CometChatException *error) { + NSLog(@"Message sending failed: %@", error.errorDescription); + }]; + ``` + + + +### Step 2: Listen for Moderation Results + +Implement the `onMessageModerated` delegate method to receive moderation results in real-time: + + + + ```swift + extension YourViewController: CometChatMessageDelegate { + + func onMessageModerated(moderatedMessage: BaseMessage) { + if let message = moderatedMessage as? TextMessage { + switch message.getModerationStatus() { + case "approved": + print("Message \(message.id) approved") + // Update UI to show message normally + + case "disapproved": + print("Message \(message.id) blocked") + // Handle blocked message (hide or show warning) + handleDisapprovedMessage(message) + + default: + break + } + } else if let message = moderatedMessage as? MediaMessage { + switch message.getModerationStatus() { + case "approved": + print("Media message \(message.id) approved") + + case "disapproved": + print("Media message \(message.id) blocked") + handleDisapprovedMessage(message) + + default: + break + } + } + } + } -Other message types are not subject to AI moderation. + // Register the delegate + CometChat.addMessageListener("MODERATION_LISTENER", self) -# Handling Message Moderation Status + // Don't forget to remove the listener when done + // CometChat.removeMessageListener("MODERATION_LISTENER") + ``` + + + ```objc + - (void)onMessageModerated:(BaseMessage *)message { + if ([message isKindOfClass:[TextMessage class]]) { + TextMessage *textMessage = (TextMessage *)message; + + if ([[textMessage getModerationStatus] isEqualToString:@"approved"]) { + NSLog(@"Message %d approved", message.id); + // Update UI to show message normally + } else if ([[textMessage getModerationStatus] isEqualToString:@"disapproved"]) { + NSLog(@"Message %d blocked", message.id); + // Handle blocked message (hide or show warning) + [self handleDisapprovedMessage:message]; + } + } else if ([message isKindOfClass:[MediaMessage class]]) { + MediaMessage *mediaMessage = (MediaMessage *)message; + + if ([[mediaMessage getModerationStatus] isEqualToString:@"approved"]) { + NSLog(@"Media message %d approved", message.id); + } else if ([[mediaMessage getModerationStatus] isEqualToString:@"disapproved"]) { + NSLog(@"Media message %d blocked", message.id); + [self handleDisapprovedMessage:message]; + } + } + } -When sending text, image, or video messages, the moderation process introduces the following changes to the message flow: + // Register the delegate + [CometChat addMessageListener:@"MODERATION_LISTENER" delegate:self]; -1. **Initial Status:** - - On successfully sending a text, image, or video message, the response will include the moderation status as `pending`. - - You can fetch this status using the `getModerationStatus()` method, which returns a string `pending`. + // Don't forget to remove the listener when done + // [CometChat removeMessageListener:@"MODERATION_LISTENER"]; + ``` + + -2. **Moderation Result Event:** - - Once the moderation service processes the message, a real-time event is triggered in your `MessageListener` as `onMessageModerated`. - - This event provides the message object with the updated moderation status, which will be either `approved` or `disapproved`. +### Step 3: Handle Disapproved Messages -**Example:** +When a message is disapproved, handle it appropriately in your UI: - - ```swift - // Check moderation status after sending a message. This check is only applicable for TextMessage & MediaMessage class. - if let message = sentMessage as? TextMessage { - if message.getModerationStatus() == "pending" { - // Message is under moderation - } - } else if let message = sentMessage as? MediaMessage { - if message.getModerationStatus() == "pending" { - // Message is under moderation - } - } - - // Listen for moderation result. - func onMessageModerated(moderatedMessage: BaseMessage) { - if let message = moderatedMessage as? TextMessage { - if message.getModerationStatus() == "approved" { - // Message approved - } else if message.getModerationStatus() == "disapproved" { - // Message disapproved - } - } else if let message = moderatedMessage as? MediaMessage { - if message.getModerationStatus() == "approved" { - // Message approved - } else if message.getModerationStatus() == "disapproved" { - // Message disapproved - } - } - } - ``` - - - ```objc - // Check moderation status after sending a message. This check is only applicable for TextMessage & MediaMessage class. - if ([sentMessage isKindOfClass:[TextMessage class]]) { - TextMessage *message = (TextMessage *)sentMessage; - if ([[message getModerationStatus] isEqualToString:@"pending"]) { - // Message is under moderation - } - } else if ([sentMessage isKindOfClass:[MediaMessage class]]) { - MediaMessage *message = (MediaMessage *)sentMessage; - if ([[message getModerationStatus] isEqualToString:@"pending"]) { - // Message is under moderation - } - } - - // Listen for moderation result - - (void)onMessageModerated:(BaseMessage *)message { - if ([message isKindOfClass:[TextMessage class]]) { - TextMessage *textMessage = (TextMessage *)message; - if ([[textMessage getModerationStatus] isEqualToString:@"approved"]) { - // Message approved - } else if ([[textMessage getModerationStatus] isEqualToString:@"disapproved"]) { - // Message disapproved - } - } else if ([message isKindOfClass:[MediaMessage class]]) { - MediaMessage *mediaMessage = (MediaMessage *)message; - if ([[mediaMessage getModerationStatus] isEqualToString:@"approved"]) { - // Message approved - } else if ([[mediaMessage getModerationStatus] isEqualToString:@"disapproved"]) { - // Message disapproved - } - } - } - ``` - - \ No newline at end of file + + ```swift + func handleDisapprovedMessage(_ message: BaseMessage) { + let messageId = message.id + + // Option 1: Hide the message completely + hideMessageFromUI(messageId) + + // Option 2: Show a placeholder message + showBlockedPlaceholder(messageId, text: "This message was blocked by moderation") + + // Option 3: Notify the sender (if it's their message) + if message.sender?.uid == currentUserUID { + showNotification("Your message was blocked due to policy violation") + } + } + ``` + + + ```objc + - (void)handleDisapprovedMessage:(BaseMessage *)message { + int messageId = message.id; + + // Option 1: Hide the message completely + [self hideMessageFromUI:messageId]; + + // Option 2: Show a placeholder message + [self showBlockedPlaceholder:messageId text:@"This message was blocked by moderation"]; + + // Option 3: Notify the sender (if it's their message) + if ([message.sender.uid isEqualToString:currentUserUID]) { + [self showNotification:@"Your message was blocked due to policy violation"]; + } + } + ``` + + diff --git a/sdk/ios/calling-overview.mdx b/sdk/ios/calling-overview.mdx index 1ecf19a3..4883e77b 100644 --- a/sdk/ios/calling-overview.mdx +++ b/sdk/ios/calling-overview.mdx @@ -10,7 +10,7 @@ CometChat provides voice and video calling capabilities for your iOS application ## Prerequisites 1. CometChat SDK installed and configured. See the [Setup](/sdk/ios/setup) guide. -2. CometChat Calls SDK added to your project. For detailed setup instructions, see the [Calls SDK Setup](/sdk/ios/calling-integration) guide. +2. CometChat Calls SDK added to your project. For detailed setup instructions, see the [Calls SDK Setup](/sdk/ios/calling-setup) guide. ## Choose Your Implementation diff --git a/sdk/ios/calling-integration.mdx b/sdk/ios/calling-setup.mdx similarity index 99% rename from sdk/ios/calling-integration.mdx rename to sdk/ios/calling-setup.mdx index e21fd6ce..064816a0 100644 --- a/sdk/ios/calling-integration.mdx +++ b/sdk/ios/calling-setup.mdx @@ -1,5 +1,5 @@ --- -title: "Integration" +title: "Setup" --- diff --git a/sdk/ios/direct-calling.mdx b/sdk/ios/direct-calling.mdx index 5b9750a4..b525aa90 100644 --- a/sdk/ios/direct-calling.mdx +++ b/sdk/ios/direct-calling.mdx @@ -6,7 +6,7 @@ title: "Call Session" This section demonstrates how to start a call session in an iOS application. Previously known as **Direct Calling**. -Before you begin, we strongly recommend you read the [calling setup guide](/sdk/ios/calling-integration). +Before you begin, we strongly recommend you read the [calling setup guide](/sdk/ios/calling-setup). If you want to implement a complete calling experience with ringing functionality (incoming/outgoing call UI), follow the [Ringing](/sdk/ios/default-calling) guide first. Once the call is accepted, return here to start the call session. diff --git a/sdk/ios/flag-message.mdx b/sdk/ios/flag-message.mdx index 2382239b..37a2be33 100644 --- a/sdk/ios/flag-message.mdx +++ b/sdk/ios/flag-message.mdx @@ -1,62 +1,175 @@ --- -title: "Flag A Message" +title: "Flag Message" --- -Flagging messages allows users to report inappropriate content to moderators or administrators. CometChat provides methods to flag messages with specific reasons and retrieve available flag reasons configured in your dashboard. +## Overview -## Flag a Message +Flagging messages allows users to report inappropriate content to moderators or administrators. When a message is flagged, it appears in the [CometChat Dashboard](https://app.cometchat.com) under **Moderation > Flagged Messages** for review. -*In other words, as a user, how do I report a message?* + +For a complete understanding of how flagged messages are reviewed and managed, see the [Flagged Messages](/moderation/flagged-messages) documentation. + -To flag a message, you can use the `flagMessage()` method. This method takes the message ID and a payload containing an optional reason ID and remark. +## Prerequisites - - -```swift -let flagDetail = FlagDetail(messageId: message.id, reasonId: reason.id, remark: "i find this inappropriate") - -CometChat.flagMessage(messageId: message.id, detail: <#T##FlagDetail#>) { flagResponse in - print(flagResponse) -} onError: { error in - print(error?.errorDescription) -} +Before using the flag message feature: + +1. Moderation must be enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Flag reasons should be configured under **Moderation > Advanced Settings** + +## How It Works + +```mermaid +sequenceDiagram + participant User + participant App + participant SDK + participant CometChat + participant Dashboard + + User->>App: Reports message + App->>SDK: getFlagReasons() + SDK-->>App: Available reasons + App->>User: Show reason picker + User->>App: Select reason + remark + App->>SDK: flagMessage(messageId, detail) + SDK->>CometChat: Flag message + CometChat-->>SDK: Success response + SDK-->>App: Message flagged + CometChat->>Dashboard: Message in Flagged queue ``` - + +## Get Flag Reasons + +Before flagging a message, retrieve the list of available flag reasons configured in your Dashboard: + + + + ```swift + CometChat.getFlagReasons { reasons in + print("Flag reasons fetched: \(reasons)") + // Use reasons to populate your report dialog UI + for reason in reasons { + print("Reason ID: \(reason.id ?? ""), Title: \(reason.reason ?? "")") + } + } onError: { error in + print("Error fetching flag reasons: \(error?.errorDescription ?? "")") + } + ``` + + + +### Response + +The response is an array of `FlagReason` objects containing: + +| Property | Type | Description | +|----------|------|-------------| +| id | String | Unique identifier for the reason | +| reason | String | Display text for the reason | + +## Flag a Message + +To flag a message, use the `flagMessage()` method with the message ID and a `FlagDetail` object: + + + + ```swift + let messageId = 123 // ID of the message to flag + + let flagDetail = FlagDetail( + messageId: messageId, + reasonId: "spam", // Required: ID from getFlagReasons() + remark: "This message contains promotional content" // Optional + ) + + CometChat.flagMessage(messageId: messageId, detail: flagDetail) { response in + print("Message flagged successfully: \(response)") + } onError: { error in + print("Message flagging failed: \(error?.errorDescription ?? "")") + } + ``` + ### Parameters -| Parameter | Type | Required | Description | -|-------------------------|------------|----------|---------------------------------------------| -| id | long | Yes | The ID of the message to be flagged | -| flagDetail | FlagDetail | Yes | Contains flagging details | -| flagDetail.setReasonId | String | Yes | ID of the flag reason (from getFlagReasons) | -| flagDetail.setRemark | String | No | Additional context or explanation | +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| messageId | Int | Yes | The ID of the message to flag | +| detail | FlagDetail | Yes | Contains flagging details | +| detail.reasonId | String | Yes | ID of the flag reason (from `getFlagReasons()`) | +| detail.remark | String | No | Additional context or explanation from the user | ### Response -On successful flagging, you'll receive a response object: - -```String +```json { "message": "Message {id} has been flagged successfully." } ``` -## Get Flag Reasons - -*In other words, what are the available reasons for flagging a message?* +## Complete Example -Before flagging a message, you can retrieve the list of available flag reasons using the `getFlagReasons()` method. These reasons are configured in the CometChat Dashboard. +Here's a complete implementation showing how to build a report message flow: - -```swift -CometChat.getFlagReasons { reasons in - print("flag reasons: \(reasons)") -} onError: { error in - print(error?.errorDescription) -} -``` - + + ```swift + class ReportMessageHandler { + private var flagReasons: [FlagReason] = [] + + // Load flag reasons (call this on app init or when needed) + func loadFlagReasons(completion: @escaping ([FlagReason]) -> Void) { + CometChat.getFlagReasons { [weak self] reasons in + self?.flagReasons = reasons + completion(reasons) + } onError: { error in + print("Failed to load flag reasons: \(error?.errorDescription ?? "")") + completion([]) + } + } + + // Get reasons for UI display + func getReasons() -> [FlagReason] { + return flagReasons + } + + // Flag a message with selected reason + func flagMessage( + messageId: Int, + reasonId: String, + remark: String? = nil, + completion: @escaping (Bool, String?) -> Void + ) { + let flagDetail = FlagDetail( + messageId: messageId, + reasonId: reasonId, + remark: remark ?? "" + ) + + CometChat.flagMessage(messageId: messageId, detail: flagDetail) { response in + completion(true, response) + } onError: { error in + completion(false, error?.errorDescription) + } + } + } + + // Usage + let reportHandler = ReportMessageHandler() + + // Load reasons when app initializes + reportHandler.loadFlagReasons { reasons in + // Display reasons in UI for user to select + } + + // When user submits the report + reportHandler.flagMessage(messageId: 123, reasonId: "spam", remark: "User is sending promotional links") { success, message in + if success { + showToast("Message reported successfully") + } + } + ``` + diff --git a/sdk/ios/standalone-calling.mdx b/sdk/ios/standalone-calling.mdx index 5e8d060d..022ee552 100644 --- a/sdk/ios/standalone-calling.mdx +++ b/sdk/ios/standalone-calling.mdx @@ -7,7 +7,7 @@ title: "Standalone Calling" This section demonstrates how to implement calling functionality using only the CometChat Calls SDK, without requiring the Chat SDK. This is ideal for applications that need video/audio calling capabilities without the full chat infrastructure. -Before you begin, ensure you have completed the [Calls SDK setup](/sdk/ios/calling-integration). +Before you begin, ensure you have completed the [Calls SDK setup](/sdk/ios/calling-setup). ## User Authentication diff --git a/sdk/javascript/ai-moderation.mdx b/sdk/javascript/ai-moderation.mdx index 0818d929..17fa27e2 100644 --- a/sdk/javascript/ai-moderation.mdx +++ b/sdk/javascript/ai-moderation.mdx @@ -2,85 +2,300 @@ title: "AI Moderation" --- -# Overview +## Overview AI Moderation in the CometChat SDK helps ensure that your chat application remains safe and compliant by automatically reviewing messages for inappropriate content. This feature leverages AI to moderate messages in real-time, reducing manual intervention and improving user experience. -For a broader understanding of moderation features, see the [Moderation Overview](../../moderation/overview). + +For a broader understanding of moderation features, configuring rules, and managing flagged messages, see the [Moderation Overview](/moderation/overview). + -# What Triggers Moderation? +## Prerequisites + +Before using AI Moderation, ensure the following: + +1. Moderation is enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Moderation rules are configured under **Moderation > Rules** +3. You're using CometChat SDK version that supports moderation + +## How It Works + +```mermaid +sequenceDiagram + participant App + participant SDK + participant CometChat + participant Moderation + + App->>SDK: sendMessage() + SDK->>CometChat: Send Message + CometChat->>Moderation: Process Message + CometChat-->>SDK: Message (status: PENDING) + SDK-->>App: onMessageSent (PENDING) + Moderation-->>CometChat: Moderation Result + CometChat-->>SDK: onMessageModerated + SDK-->>App: APPROVED or DISAPPROVED +``` + +| Step | Description | +|------|-------------| +| 1. Send Message | App sends a text, image, or video message | +| 2. Pending Status | Message is sent with `PENDING` moderation status | +| 3. AI Processing | Moderation service analyzes the content | +| 4. Result Event | `onMessageModerated` event fires with final status | + +## Supported Message Types Moderation is triggered **only** for the following message types: -- Text Messages -- Image Messages -- Video Messages +| Message Type | Moderated | Notes | +|--------------|-----------|-------| +| Text Messages | ✅ | Content analyzed for inappropriate text | +| Image Messages | ✅ | Images scanned for unsafe content | +| Video Messages | ✅ | Videos analyzed for prohibited content | +| Custom Messages | ❌ | Not subject to AI moderation | +| Action Messages | ❌ | Not subject to AI moderation | + +## Moderation Status + +The `getModerationStatus()` method returns one of the following values: + +| Status | Enum Value | Description | +|--------|------------|-------------| +| Pending | `CometChat.ModerationStatus.PENDING` | Message is being processed by moderation | +| Approved | `CometChat.ModerationStatus.APPROVED` | Message passed moderation and is visible | +| Disapproved | `CometChat.ModerationStatus.DISAPPROVED` | Message violated rules and was blocked | -Other message types are not subject to AI moderation. +## Implementation -# Handling Message Moderation Status +### Step 1: Send a Message and Check Initial Status -When sending text, image, or video messages, the moderation process introduces the following changes to the message flow: +When you send a text, image, or video message, check the initial moderation status: -1. **Initial Status:** - - On successfully sending a text, image, or video message, the response will include the moderation status as `pending`. - - You can fetch this status using the `getModerationStatus()` method, which returns an Enum: `CometChat.ModerationStatus.PENDING`. + + + ```javascript + const textMessage = new CometChat.TextMessage( + receiverUID, + "Hello, how are you?", + CometChat.RECEIVER_TYPE.USER + ); + + CometChat.sendMessage(textMessage).then( + (message) => { + // Check moderation status + const status = message.getModerationStatus(); + + if (status === CometChat.ModerationStatus.PENDING) { + console.log("Message is under moderation review"); + // Show pending indicator in UI + } + }, + (error) => { + console.log("Message sending failed:", error); + } + ); + ``` + + + ```typescript + const textMessage = new CometChat.TextMessage( + receiverUID, + "Hello, how are you?", + CometChat.RECEIVER_TYPE.USER + ); + + CometChat.sendMessage(textMessage).then( + (message: CometChat.TextMessage) => { + // Check moderation status + const status: string = message.getModerationStatus(); + + if (status === CometChat.ModerationStatus.PENDING) { + console.log("Message is under moderation review"); + // Show pending indicator in UI + } + }, + (error: CometChat.CometChatException) => { + console.log("Message sending failed:", error); + } + ); + ``` + + -2. **Moderation Result Event:** - - Once the moderation service processes the message, a real-time event is triggered in your `MessageListener` as `onMessageModerated`. - - This event provides the message object with the updated moderation status, which will be either `CometChat.ModerationStatus.APPROVED` or `CometChat.ModerationStatus.DISAPPROVED`. +### Step 2: Listen for Moderation Results -**Example:** +Register a message listener to receive moderation results in real-time: - - ```js - // Check moderation status after sending a message. This check is only applicable for TextMessage & MediaMessage class. - const status = message.getModerationStatus(); - if (status === CometChat.ModerationStatus.PENDING) { - // Message is under moderation - } - - // Listen for moderation result - CometChat.addMessageListener( - 'UNIQUE_LISTENER_ID', - new CometChat.MessageListener({ - onMessageModerated: (message) => { - if (message instanceof CometChat.TextMessage || message instanceof CometChat.MediaMessage) { - if (message.getModerationStatus() === CometChat.ModerationStatus.APPROVED) { - // Message approved - } else if (message.getModerationStatus() === CometChat.ModerationStatus.DISAPPROVED) { - // Message disapproved - } - } - } - }) - ); - ``` - - - ```js - // Check moderation status after sending a message. This check is only applicable for TextMessage & MediaMessage class. - const status: string = message.getModerationStatus(); - if (status === CometChat.ModerationStatus.PENDING) { - // Message is under moderation - } - - // Listen for moderation result - CometChat.addMessageListener( - 'UNIQUE_LISTENER_ID', - new CometChat.MessageListener({ - onMessageModerated: (message: CometChat.BaseMessage) => { - if (message instanceof CometChat.TextMessage || message instanceof CometChat.MediaMessage) { - if (message.getModerationStatus() === CometChat.ModerationStatus.APPROVED) { - // Message approved - } else if (message.getModerationStatus() === CometChat.ModerationStatus.DISAPPROVED) { - // Message disapproved - } - } - } - }) - ); - ``` - - \ No newline at end of file + + ```javascript + const listenerID = "MODERATION_LISTENER"; + + CometChat.addMessageListener( + listenerID, + new CometChat.MessageListener({ + onMessageModerated: (message) => { + if ( + message instanceof CometChat.TextMessage || + message instanceof CometChat.MediaMessage + ) { + const status = message.getModerationStatus(); + const messageId = message.getId(); + + switch (status) { + case CometChat.ModerationStatus.APPROVED: + console.log(`Message ${messageId} approved`); + // Update UI to show message normally + break; + + case CometChat.ModerationStatus.DISAPPROVED: + console.log(`Message ${messageId} blocked`); + // Handle blocked message (hide or show warning) + handleDisapprovedMessage(message); + break; + } + } + } + }) + ); + + // Don't forget to remove the listener when done + // CometChat.removeMessageListener(listenerID); + ``` + + + ```typescript + const listenerID: string = "MODERATION_LISTENER"; + + CometChat.addMessageListener( + listenerID, + new CometChat.MessageListener({ + onMessageModerated: (message: CometChat.BaseMessage) => { + if ( + message instanceof CometChat.TextMessage || + message instanceof CometChat.MediaMessage + ) { + const status: string = message.getModerationStatus(); + const messageId: number = message.getId(); + + switch (status) { + case CometChat.ModerationStatus.APPROVED: + console.log(`Message ${messageId} approved`); + // Update UI to show message normally + break; + + case CometChat.ModerationStatus.DISAPPROVED: + console.log(`Message ${messageId} blocked`); + // Handle blocked message (hide or show warning) + handleDisapprovedMessage(message); + break; + } + } + } + }) + ); + + // Don't forget to remove the listener when done + // CometChat.removeMessageListener(listenerID); + ``` + + + +### Step 3: Handle Disapproved Messages + +When a message is disapproved, you should handle it appropriately in your UI: + +```javascript +function handleDisapprovedMessage(message) { + const messageId = message.getId(); + + // Option 1: Hide the message completely + hideMessageFromUI(messageId); + + // Option 2: Show a placeholder message + showBlockedPlaceholder(messageId, "This message was blocked by moderation"); + + // Option 3: Notify the sender (if it's their message) + if (message.getSender().getUid() === currentUserUID) { + showNotification("Your message was blocked due to policy violation"); + } +} +``` + +## Complete Example + +Here's a complete implementation showing the full moderation flow: + + + + ```javascript + class ModerationHandler { + constructor() { + this.pendingMessages = new Map(); + this.setupListener(); + } + + setupListener() { + CometChat.addMessageListener( + "MODERATION_LISTENER", + new CometChat.MessageListener({ + onMessageModerated: (message) => this.onModerated(message) + }) + ); + } + + async sendMessage(receiverUID, text) { + const textMessage = new CometChat.TextMessage( + receiverUID, + text, + CometChat.RECEIVER_TYPE.USER + ); + + try { + const message = await CometChat.sendMessage(textMessage); + const status = message.getModerationStatus(); + + if (status === CometChat.ModerationStatus.PENDING) { + // Track pending message + this.pendingMessages.set(message.getId(), message); + return { success: true, pending: true, message }; + } + + return { success: true, pending: false, message }; + } catch (error) { + return { success: false, error }; + } + } + + onModerated(message) { + const messageId = message.getId(); + const status = message.getModerationStatus(); + + // Remove from pending + this.pendingMessages.delete(messageId); + + // Emit event for UI update + this.emit("moderationResult", { + messageId, + status, + approved: status === CometChat.ModerationStatus.APPROVED, + message + }); + } + + cleanup() { + CometChat.removeMessageListener("MODERATION_LISTENER"); + } + } + + // Usage + const handler = new ModerationHandler(); + const result = await handler.sendMessage("user123", "Hello!"); + + if (result.pending) { + console.log("Message pending moderation..."); + } + ``` + + diff --git a/sdk/javascript/flag-message.mdx b/sdk/javascript/flag-message.mdx index a411e1fd..350149f5 100644 --- a/sdk/javascript/flag-message.mdx +++ b/sdk/javascript/flag-message.mdx @@ -1,66 +1,147 @@ --- -title: "Flag A Message" +title: "Flag Message" --- -Flagging messages allows users to report inappropriate content to moderators or administrators. CometChat provides methods to flag messages with specific reasons and retrieve available flag reasons configured in your dashboard. +## Overview -## Flag a Message +Flagging messages allows users to report inappropriate content to moderators or administrators. When a message is flagged, it appears in the [CometChat Dashboard](https://app.cometchat.com) under **Moderation > Flagged Messages** for review. + + +For a complete understanding of how flagged messages are reviewed and managed, see the [Flagged Messages](/moderation/flagged-messages) documentation. + + +## Prerequisites -*In other words, as a user, how do I report a message?* +Before using the flag message feature: -To flag a message, you can use the `flagMessage()` method. This method takes the message ID and a payload containing an optional reason ID and remark. +1. Moderation must be enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Flag reasons should be configured under **Moderation > Advanced Settings** + +## How It Works + +```mermaid +sequenceDiagram + participant User + participant App + participant SDK + participant CometChat + participant Dashboard + + User->>App: Reports message + App->>SDK: getFlagReasons() + SDK-->>App: Available reasons + App->>User: Show reason picker + User->>App: Select reason + remark + App->>SDK: flagMessage(messageId, payload) + SDK->>CometChat: Flag message + CometChat-->>SDK: Success response + SDK-->>App: Message flagged + CometChat->>Dashboard: Message in Flagged queue +``` + +## Get Flag Reasons + +Before flagging a message, retrieve the list of available flag reasons configured in your Dashboard: - + + ```javascript + CometChat.getFlagReasons().then( + (reasons) => { + console.log("Flag reasons retrieved:", reasons); + // reasons is an array of { id, reason } objects + // Use these to populate your report dialog UI + }, + (error) => { + console.log("Failed to get flag reasons:", error); + } + ); + ``` + + + ```typescript + CometChat.getFlagReasons().then( + (reasons: CometChat.FlagReason[]) => { + console.log("Flag reasons retrieved:", reasons); + // reasons is an array of { id, reason } objects + // Use these to populate your report dialog UI + }, + (error: CometChat.CometChatException) => { + console.log("Failed to get flag reasons:", error); + } + ); + ``` + + + +### Response + +The response is an array of flag reason objects: + ```javascript -let messageId = "ID_OF_THE_MESSAGE_YOU_WANT_TO_FLAG"; -let payload = { - reasonId: "spam", // Required: ID of the flag reason - remark: "This message contains promotional content" // Optional: Additional context -}; - -CometChat.flagMessage(messageId, payload).then( - response => { - console.log("Message flagged successfully", response); - }, error => { - console.log("Message flagging failed with error:", error); - } -); +[ + { "id": "spam", "reason": "Spam or misleading" }, + { "id": "harassment", "reason": "Harassment or bullying" }, + { "id": "hate_speech", "reason": "Hate speech" }, + { "id": "violence", "reason": "Violence or dangerous content" }, + { "id": "inappropriate", "reason": "Inappropriate content" }, + { "id": "other", "reason": "Other" } +] ``` - - - -```typescript -let messageId: string = "ID_OF_THE_MESSAGE_YOU_WANT_TO_FLAG"; -let payload: { reasonId: string; remark?: string } = { - reasonId: "spam", - remark: "This message contains promotional content" -}; - -CometChat.flagMessage(messageId, payload).then( - (response: CometChat.FlagMessageResponse) => { - console.log("Message flagged successfully", response); - }, (error: CometChat.CometChatException) => { - console.log("Message flagging failed with error:", error); - } -); -``` - + +## Flag a Message + +To flag a message, use the `flagMessage()` method with the message ID and a payload containing the reason: + + + + ```javascript + const messageId = "MESSAGE_ID_TO_FLAG"; + const payload = { + reasonId: "spam", // Required: ID from getFlagReasons() + remark: "This message contains promotional content" // Optional + }; + + CometChat.flagMessage(messageId, payload).then( + (response) => { + console.log("Message flagged successfully:", response); + }, + (error) => { + console.log("Message flagging failed:", error); + } + ); + ``` + + + ```typescript + const messageId: string = "MESSAGE_ID_TO_FLAG"; + const payload: { reasonId: string; remark?: string } = { + reasonId: "spam", + remark: "This message contains promotional content" + }; + + CometChat.flagMessage(messageId, payload).then( + (response: CometChat.FlagMessageResponse) => { + console.log("Message flagged successfully:", response); + }, + (error: CometChat.CometChatException) => { + console.log("Message flagging failed:", error); + } + ); + ``` + ### Parameters | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| messageId | string | Yes | The ID of the message to be flagged | -| payload | object | Yes | Contains flagging details | -| payload.reasonId | string | Yes | ID of the flag reason (from getFlagReasons) | -| payload.remark | string | No | Additional context or explanation | +| messageId | string | Yes | The ID of the message to flag | +| payload.reasonId | string | Yes | ID of the flag reason (from `getFlagReasons()`) | +| payload.remark | string | No | Additional context or explanation from the user | ### Response -On successful flagging, you'll receive a response object: - ```javascript { "success": true, @@ -68,36 +149,72 @@ On successful flagging, you'll receive a response object: } ``` -## Get Flag Reasons - -*In other words, what are the available reasons for flagging a message?* +## Complete Example -Before flagging a message, you can retrieve the list of available flag reasons using the `getFlagReasons()` method. These reasons are configured in the CometChat Dashboard. +Here's a complete implementation showing how to build a report message flow: - - ```javascript -CometChat.getFlagReasons().then( - reasons => { - console.log("Flag reasons retrieved", reasons); - // Use reasons to populate your UI - }, error => { - console.log("Failed to get flag reasons:", error); +class ReportMessageHandler { + constructor() { + this.flagReasons = []; } -); -``` - - - -```typescript -CometChat.getFlagReasons().then( - (reasons: CometChat.FlagReason[]) => { - console.log("Flag reasons retrieved", reasons); - // Use reasons to populate your UI - }, (error: CometChat.CometChatException) => { - console.log("Failed to get flag reasons:", error); + + // Load flag reasons (call this on app init or when needed) + async loadFlagReasons() { + try { + this.flagReasons = await CometChat.getFlagReasons(); + return this.flagReasons; + } catch (error) { + console.error("Failed to load flag reasons:", error); + return []; + } + } + + // Get reasons for UI display + getReasons() { + return this.flagReasons; + } + + // Flag a message with selected reason + async flagMessage(messageId, reasonId, remark = "") { + if (!reasonId) { + throw new Error("Reason ID is required"); + } + + try { + const payload = { reasonId }; + if (remark) { + payload.remark = remark; + } + + const response = await CometChat.flagMessage(messageId, payload); + console.log("Message flagged successfully"); + return { success: true, response }; + } catch (error) { + console.error("Failed to flag message:", error); + return { success: false, error }; + } } +} + +// Usage +const reportHandler = new ReportMessageHandler(); + +// Load reasons when app initializes +await reportHandler.loadFlagReasons(); + +// When user wants to report a message +const reasons = reportHandler.getReasons(); +// Display reasons in UI for user to select... + +// When user submits the report +const result = await reportHandler.flagMessage( + "message_123", + "spam", + "User is sending promotional links" ); + +if (result.success) { + showToast("Message reported successfully"); +} ``` - - diff --git a/sdk/react-native/ai-moderation.mdx b/sdk/react-native/ai-moderation.mdx new file mode 100644 index 00000000..fa0bffd7 --- /dev/null +++ b/sdk/react-native/ai-moderation.mdx @@ -0,0 +1,224 @@ +--- +title: "AI Moderation" +--- + +## Overview + +AI Moderation in the CometChat SDK helps ensure that your chat application remains safe and compliant by automatically reviewing messages for inappropriate content. This feature leverages AI to moderate messages in real-time, reducing manual intervention and improving user experience. + + +For a broader understanding of moderation features, configuring rules, and managing flagged messages, see the [Moderation Overview](/moderation/overview). + + +## Prerequisites + +Before using AI Moderation, ensure the following: + +1. Moderation is enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Moderation rules are configured under **Moderation > Rules** +3. You're using CometChat SDK version that supports moderation + +## How It Works + +```mermaid +sequenceDiagram + participant App + participant SDK + participant CometChat + participant Moderation + + App->>SDK: sendMessage() + SDK->>CometChat: Send Message + CometChat->>Moderation: Process Message + CometChat-->>SDK: Message (status: PENDING) + SDK-->>App: onMessageSent (PENDING) + Moderation-->>CometChat: Moderation Result + CometChat-->>SDK: onMessageModerated + SDK-->>App: APPROVED or DISAPPROVED +``` + +| Step | Description | +|------|-------------| +| 1. Send Message | App sends a text, image, or video message | +| 2. Pending Status | Message is sent with `PENDING` moderation status | +| 3. AI Processing | Moderation service analyzes the content | +| 4. Result Event | `onMessageModerated` event fires with final status | + +## Supported Message Types + +Moderation is triggered **only** for the following message types: + +| Message Type | Moderated | Notes | +|--------------|-----------|-------| +| Text Messages | ✅ | Content analyzed for inappropriate text | +| Image Messages | ✅ | Images scanned for unsafe content | +| Video Messages | ✅ | Videos analyzed for prohibited content | +| Custom Messages | ❌ | Not subject to AI moderation | +| Action Messages | ❌ | Not subject to AI moderation | + +## Moderation Status + +The `getModerationStatus()` method returns one of the following values: + +| Status | Enum Value | Description | +|--------|------------|-------------| +| Pending | `CometChat.ModerationStatus.PENDING` | Message is being processed by moderation | +| Approved | `CometChat.ModerationStatus.APPROVED` | Message passed moderation and is visible | +| Disapproved | `CometChat.ModerationStatus.DISAPPROVED` | Message violated rules and was blocked | + +## Implementation + +### Step 1: Send a Message and Check Initial Status + +When you send a text, image, or video message, check the initial moderation status: + + + + ```javascript + const textMessage = new CometChat.TextMessage( + receiverUID, + "Hello, how are you?", + CometChat.RECEIVER_TYPE.USER + ); + + CometChat.sendMessage(textMessage).then( + (message) => { + // Check moderation status + const status = message.getModerationStatus(); + + if (status === CometChat.ModerationStatus.PENDING) { + console.log("Message is under moderation review"); + // Show pending indicator in UI + } + }, + (error) => { + console.log("Message sending failed:", error); + } + ); + ``` + + + ```typescript + const textMessage = new CometChat.TextMessage( + receiverUID, + "Hello, how are you?", + CometChat.RECEIVER_TYPE.USER + ); + + CometChat.sendMessage(textMessage).then( + (message: CometChat.TextMessage) => { + // Check moderation status + const status: string = message.getModerationStatus(); + + if (status === CometChat.ModerationStatus.PENDING) { + console.log("Message is under moderation review"); + // Show pending indicator in UI + } + }, + (error: CometChat.CometChatException) => { + console.log("Message sending failed:", error); + } + ); + ``` + + + +### Step 2: Listen for Moderation Results + +Register a message listener to receive moderation results in real-time: + + + + ```javascript + const listenerID = "MODERATION_LISTENER"; + + CometChat.addMessageListener( + listenerID, + new CometChat.MessageListener({ + onMessageModerated: (message) => { + if ( + message instanceof CometChat.TextMessage || + message instanceof CometChat.MediaMessage + ) { + const status = message.getModerationStatus(); + const messageId = message.getId(); + + switch (status) { + case CometChat.ModerationStatus.APPROVED: + console.log(`Message ${messageId} approved`); + // Update UI to show message normally + break; + + case CometChat.ModerationStatus.DISAPPROVED: + console.log(`Message ${messageId} blocked`); + // Handle blocked message (hide or show warning) + handleDisapprovedMessage(message); + break; + } + } + } + }) + ); + + // Don't forget to remove the listener when done + // CometChat.removeMessageListener(listenerID); + ``` + + + ```typescript + const listenerID: string = "MODERATION_LISTENER"; + + CometChat.addMessageListener( + listenerID, + new CometChat.MessageListener({ + onMessageModerated: (message: CometChat.BaseMessage) => { + if ( + message instanceof CometChat.TextMessage || + message instanceof CometChat.MediaMessage + ) { + const status: string = message.getModerationStatus(); + const messageId: number = message.getId(); + + switch (status) { + case CometChat.ModerationStatus.APPROVED: + console.log(`Message ${messageId} approved`); + // Update UI to show message normally + break; + + case CometChat.ModerationStatus.DISAPPROVED: + console.log(`Message ${messageId} blocked`); + // Handle blocked message (hide or show warning) + handleDisapprovedMessage(message); + break; + } + } + } + }) + ); + + // Don't forget to remove the listener when done + // CometChat.removeMessageListener(listenerID); + ``` + + + +### Step 3: Handle Disapproved Messages + +When a message is disapproved, handle it appropriately in your UI: + +```javascript +function handleDisapprovedMessage(message) { + const messageId = message.getId(); + + // Option 1: Hide the message completely + hideMessageFromUI(messageId); + + // Option 2: Show a placeholder message + showBlockedPlaceholder(messageId, "This message was blocked by moderation"); + + // Option 3: Notify the sender (if it's their message) + if (message.getSender().getUid() === currentUserUID) { + showNotification("Your message was blocked due to policy violation"); + } +} +``` diff --git a/sdk/react-native/flag-message.mdx b/sdk/react-native/flag-message.mdx index a411e1fd..350149f5 100644 --- a/sdk/react-native/flag-message.mdx +++ b/sdk/react-native/flag-message.mdx @@ -1,66 +1,147 @@ --- -title: "Flag A Message" +title: "Flag Message" --- -Flagging messages allows users to report inappropriate content to moderators or administrators. CometChat provides methods to flag messages with specific reasons and retrieve available flag reasons configured in your dashboard. +## Overview -## Flag a Message +Flagging messages allows users to report inappropriate content to moderators or administrators. When a message is flagged, it appears in the [CometChat Dashboard](https://app.cometchat.com) under **Moderation > Flagged Messages** for review. + + +For a complete understanding of how flagged messages are reviewed and managed, see the [Flagged Messages](/moderation/flagged-messages) documentation. + + +## Prerequisites -*In other words, as a user, how do I report a message?* +Before using the flag message feature: -To flag a message, you can use the `flagMessage()` method. This method takes the message ID and a payload containing an optional reason ID and remark. +1. Moderation must be enabled for your app in the [CometChat Dashboard](https://app.cometchat.com) +2. Flag reasons should be configured under **Moderation > Advanced Settings** + +## How It Works + +```mermaid +sequenceDiagram + participant User + participant App + participant SDK + participant CometChat + participant Dashboard + + User->>App: Reports message + App->>SDK: getFlagReasons() + SDK-->>App: Available reasons + App->>User: Show reason picker + User->>App: Select reason + remark + App->>SDK: flagMessage(messageId, payload) + SDK->>CometChat: Flag message + CometChat-->>SDK: Success response + SDK-->>App: Message flagged + CometChat->>Dashboard: Message in Flagged queue +``` + +## Get Flag Reasons + +Before flagging a message, retrieve the list of available flag reasons configured in your Dashboard: - + + ```javascript + CometChat.getFlagReasons().then( + (reasons) => { + console.log("Flag reasons retrieved:", reasons); + // reasons is an array of { id, reason } objects + // Use these to populate your report dialog UI + }, + (error) => { + console.log("Failed to get flag reasons:", error); + } + ); + ``` + + + ```typescript + CometChat.getFlagReasons().then( + (reasons: CometChat.FlagReason[]) => { + console.log("Flag reasons retrieved:", reasons); + // reasons is an array of { id, reason } objects + // Use these to populate your report dialog UI + }, + (error: CometChat.CometChatException) => { + console.log("Failed to get flag reasons:", error); + } + ); + ``` + + + +### Response + +The response is an array of flag reason objects: + ```javascript -let messageId = "ID_OF_THE_MESSAGE_YOU_WANT_TO_FLAG"; -let payload = { - reasonId: "spam", // Required: ID of the flag reason - remark: "This message contains promotional content" // Optional: Additional context -}; - -CometChat.flagMessage(messageId, payload).then( - response => { - console.log("Message flagged successfully", response); - }, error => { - console.log("Message flagging failed with error:", error); - } -); +[ + { "id": "spam", "reason": "Spam or misleading" }, + { "id": "harassment", "reason": "Harassment or bullying" }, + { "id": "hate_speech", "reason": "Hate speech" }, + { "id": "violence", "reason": "Violence or dangerous content" }, + { "id": "inappropriate", "reason": "Inappropriate content" }, + { "id": "other", "reason": "Other" } +] ``` - - - -```typescript -let messageId: string = "ID_OF_THE_MESSAGE_YOU_WANT_TO_FLAG"; -let payload: { reasonId: string; remark?: string } = { - reasonId: "spam", - remark: "This message contains promotional content" -}; - -CometChat.flagMessage(messageId, payload).then( - (response: CometChat.FlagMessageResponse) => { - console.log("Message flagged successfully", response); - }, (error: CometChat.CometChatException) => { - console.log("Message flagging failed with error:", error); - } -); -``` - + +## Flag a Message + +To flag a message, use the `flagMessage()` method with the message ID and a payload containing the reason: + + + + ```javascript + const messageId = "MESSAGE_ID_TO_FLAG"; + const payload = { + reasonId: "spam", // Required: ID from getFlagReasons() + remark: "This message contains promotional content" // Optional + }; + + CometChat.flagMessage(messageId, payload).then( + (response) => { + console.log("Message flagged successfully:", response); + }, + (error) => { + console.log("Message flagging failed:", error); + } + ); + ``` + + + ```typescript + const messageId: string = "MESSAGE_ID_TO_FLAG"; + const payload: { reasonId: string; remark?: string } = { + reasonId: "spam", + remark: "This message contains promotional content" + }; + + CometChat.flagMessage(messageId, payload).then( + (response: CometChat.FlagMessageResponse) => { + console.log("Message flagged successfully:", response); + }, + (error: CometChat.CometChatException) => { + console.log("Message flagging failed:", error); + } + ); + ``` + ### Parameters | Parameter | Type | Required | Description | |-----------|------|----------|-------------| -| messageId | string | Yes | The ID of the message to be flagged | -| payload | object | Yes | Contains flagging details | -| payload.reasonId | string | Yes | ID of the flag reason (from getFlagReasons) | -| payload.remark | string | No | Additional context or explanation | +| messageId | string | Yes | The ID of the message to flag | +| payload.reasonId | string | Yes | ID of the flag reason (from `getFlagReasons()`) | +| payload.remark | string | No | Additional context or explanation from the user | ### Response -On successful flagging, you'll receive a response object: - ```javascript { "success": true, @@ -68,36 +149,72 @@ On successful flagging, you'll receive a response object: } ``` -## Get Flag Reasons - -*In other words, what are the available reasons for flagging a message?* +## Complete Example -Before flagging a message, you can retrieve the list of available flag reasons using the `getFlagReasons()` method. These reasons are configured in the CometChat Dashboard. +Here's a complete implementation showing how to build a report message flow: - - ```javascript -CometChat.getFlagReasons().then( - reasons => { - console.log("Flag reasons retrieved", reasons); - // Use reasons to populate your UI - }, error => { - console.log("Failed to get flag reasons:", error); +class ReportMessageHandler { + constructor() { + this.flagReasons = []; } -); -``` - - - -```typescript -CometChat.getFlagReasons().then( - (reasons: CometChat.FlagReason[]) => { - console.log("Flag reasons retrieved", reasons); - // Use reasons to populate your UI - }, (error: CometChat.CometChatException) => { - console.log("Failed to get flag reasons:", error); + + // Load flag reasons (call this on app init or when needed) + async loadFlagReasons() { + try { + this.flagReasons = await CometChat.getFlagReasons(); + return this.flagReasons; + } catch (error) { + console.error("Failed to load flag reasons:", error); + return []; + } + } + + // Get reasons for UI display + getReasons() { + return this.flagReasons; + } + + // Flag a message with selected reason + async flagMessage(messageId, reasonId, remark = "") { + if (!reasonId) { + throw new Error("Reason ID is required"); + } + + try { + const payload = { reasonId }; + if (remark) { + payload.remark = remark; + } + + const response = await CometChat.flagMessage(messageId, payload); + console.log("Message flagged successfully"); + return { success: true, response }; + } catch (error) { + console.error("Failed to flag message:", error); + return { success: false, error }; + } } +} + +// Usage +const reportHandler = new ReportMessageHandler(); + +// Load reasons when app initializes +await reportHandler.loadFlagReasons(); + +// When user wants to report a message +const reasons = reportHandler.getReasons(); +// Display reasons in UI for user to select... + +// When user submits the report +const result = await reportHandler.flagMessage( + "message_123", + "spam", + "User is sending promotional links" ); + +if (result.success) { + showToast("Message reported successfully"); +} ``` - - diff --git a/ui-kit/android/core-features.mdx b/ui-kit/android/core-features.mdx index 3dc1732f..a4701b2e 100644 --- a/ui-kit/android/core-features.mdx +++ b/ui-kit/android/core-features.mdx @@ -139,6 +139,10 @@ Conversation and Advanced Search is a powerful feature provided by CometChat tha The Report Message feature allows users to report inappropriate or harmful messages within the chat. Users can choose from predefined reasons and provide additional remarks for detailed context. This feature helps maintain a safe and respectful chat environment. + +Learn more about how flagged messages are handled, reviewed, and moderated in the [Flagged Messages](/moderation/flagged-messages) documentation. + + @@ -147,6 +151,22 @@ The Report Message feature allows users to report inappropriate or harmful messa | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [Message List](/ui-kit/android/message-list) | [Message List](/ui-kit/android/message-list) provides the "Report Message" option in the message actions menu, allowing users to initiate the reporting process for inappropriate messages. | +## Moderation + +CometChat's Message Moderation feature helps maintain a safe and respectful chat environment by automatically filtering and managing inappropriate content. Messages can be automatically blocked or flagged based on predefined rules, ensuring harmful content is handled before it reaches users. + + + + + + +Learn more about setting up moderation rules and managing content in the [Moderation](/moderation/overview) documentation. + + +| Components | Functionality | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Message List](/ui-kit/android/message-list) | [Message List](/ui-kit/android/message-list) automatically handles moderated messages, displaying blocked content appropriately based on your moderation settings. | + ## Threaded Conversations The Threaded Conversations feature enables users to respond directly to a specific message in a chat. This keeps conversations organized and enhances the user experience by maintaining context, especially in group chats. diff --git a/ui-kit/android/extensions.mdx b/ui-kit/android/extensions.mdx index 1ab5880b..6333a963 100644 --- a/ui-kit/android/extensions.mdx +++ b/ui-kit/android/extensions.mdx @@ -78,9 +78,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. ### Thumbnail Generation diff --git a/ui-kit/android/v4/extensions.mdx b/ui-kit/android/v4/extensions.mdx index 319cb4a6..96017abb 100644 --- a/ui-kit/android/v4/extensions.mdx +++ b/ui-kit/android/v4/extensions.mdx @@ -88,9 +88,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/v4/message-list) component of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/v4/message-list) component of UI Kits. @@ -98,9 +98,9 @@ Once you have successfully activated the [Profanity Filter Extension](/moderatio ### Data Masking -The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Data Masking Extension](/moderation/data-masking-filter). +The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/v4/message-list) component of UI Kits. +Once you have successfully activated the Data Masking Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/v4/message-list) component of UI Kits. @@ -108,9 +108,9 @@ Once you have successfully activated the [Data Masking Extension](/moderation/da ### Image Moderation -The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Image Moderation Extension](/moderation/image-moderation). +The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/v4/message-list) component of UI Kits. +Once you have successfully activated the Image Moderation Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/v4/message-list) component of UI Kits. diff --git a/ui-kit/angular/extensions.mdx b/ui-kit/angular/extensions.mdx index 0301a765..89c89485 100644 --- a/ui-kit/angular/extensions.mdx +++ b/ui-kit/angular/extensions.mdx @@ -88,9 +88,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/angular/message-list) component of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/angular/message-list) component of UI Kits. @@ -98,9 +98,9 @@ Once you have successfully activated the [Profanity Filter Extension](/moderatio ### Data Masking -The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Data Masking Extension](/moderation/data-masking-filter). +The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/angular/message-list) component of UI Kits. +Once you have successfully activated the Data Masking Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/angular/message-list) component of UI Kits. @@ -108,9 +108,9 @@ Once you have successfully activated the [Data Masking Extension](/moderation/da ### Image Moderation -The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Image Moderation Extension](/moderation/image-moderation). +The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/angular/message-list) component of UI Kits. +Once you have successfully activated the Image Moderation Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/angular/message-list) component of UI Kits. diff --git a/ui-kit/flutter/core-features.mdx b/ui-kit/flutter/core-features.mdx index cb01a210..8c1e87d7 100644 --- a/ui-kit/flutter/core-features.mdx +++ b/ui-kit/flutter/core-features.mdx @@ -110,6 +110,10 @@ Mentions is a robust feature provided by CometChat that enhances the interactivi The Report Message feature allows users to report inappropriate or harmful messages within the chat. Users can choose from predefined reasons and provide additional remarks for detailed context. This feature helps maintain a safe and respectful chat environment. + +Learn more about how flagged messages are handled, reviewed, and moderated in the [Flagged Messages](/moderation/flagged-messages) documentation. + + @@ -118,6 +122,22 @@ The Report Message feature allows users to report inappropriate or harmful messa | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [Message List](/ui-kit/flutter/message-list) | [Message List](/ui-kit/flutter/message-list) provides the "Report Message" option in the message actions menu, allowing users to initiate the reporting process for inappropriate messages. | +## Moderation + +CometChat's Message Moderation feature helps maintain a safe and respectful chat environment by automatically filtering and managing inappropriate content. Messages can be automatically blocked or flagged based on predefined rules, ensuring harmful content is handled before it reaches users. + + + + + + +Learn more about setting up moderation rules and managing content in the [Moderation](/moderation/overview) documentation. + + +| Widgets | Functionality | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Message List](/ui-kit/flutter/message-list) | [Message List](/ui-kit/flutter/message-list) automatically handles moderated messages, displaying blocked content appropriately based on your moderation settings. | + ## Threaded Conversations The Threaded Conversations feature enables users to respond directly to a specific message in a chat. This keeps conversations organized and enhances the user experience by maintaining context, especially in group chats. diff --git a/ui-kit/flutter/v4/extensions.mdx b/ui-kit/flutter/v4/extensions.mdx index acadb56d..76b43d8b 100644 --- a/ui-kit/flutter/v4/extensions.mdx +++ b/ui-kit/flutter/v4/extensions.mdx @@ -151,9 +151,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Widget](/ui-kit/flutter/v4/message-list) widget of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Widget](/ui-kit/flutter/v4/message-list) widget of UI Kits. @@ -170,9 +170,9 @@ Once you have successfully activated the [Profanity Filter Extension](/moderatio ### Data Masking -The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Data Masking Extension](/moderation/data-masking-filter). +The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Widget](/ui-kit/flutter/v4/message-list) widget of UI Kits. +Once you have successfully activated the Data Masking Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Widget](/ui-kit/flutter/v4/message-list) widget of UI Kits. @@ -189,9 +189,9 @@ Once you have successfully activated the [Data Masking Extension](/moderation/da ### Image Moderation -The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Image Moderation Extension](/moderation/image-moderation). +The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Widget](/ui-kit/flutter/v4/message-list) widget of UI Kits. +Once you have successfully activated the Image Moderation Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Widget](/ui-kit/flutter/v4/message-list) widget of UI Kits. diff --git a/ui-kit/ios/core-features.mdx b/ui-kit/ios/core-features.mdx index 2932e0a7..ce0d2f50 100644 --- a/ui-kit/ios/core-features.mdx +++ b/ui-kit/ios/core-features.mdx @@ -163,10 +163,30 @@ Conversation and Advanced Search is a powerful feature provided by CometChat tha The Report Message feature allows users to report inappropriate or harmful messages within the chat. Users can choose from predefined reasons and provide additional remarks for detailed context. This feature helps maintain a safe and respectful chat environment. + +Learn more about how flagged messages are handled, reviewed, and moderated in the [Flagged Messages](/moderation/flagged-messages) documentation. + + | Components | Functionality | | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Message List](/ui-kit/android/message-list) | [Message List](/ui-kit/android/message-list) provides the "Report Message" option in the message actions menu, allowing users to initiate the reporting process for inappropriate messages. | \ No newline at end of file +| [Message List](/ui-kit/android/message-list) | [Message List](/ui-kit/android/message-list) provides the "Report Message" option in the message actions menu, allowing users to initiate the reporting process for inappropriate messages. | + +## Moderation + +CometChat's Message Moderation feature helps maintain a safe and respectful chat environment by automatically filtering and managing inappropriate content. Messages can be automatically blocked or flagged based on predefined rules, ensuring harmful content is handled before it reaches users. + + + + + + +Learn more about setting up moderation rules and managing content in the [Moderation](/moderation/overview) documentation. + + +| Components | Functionality | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Message List](/ui-kit/ios/message-list) | [Message List](/ui-kit/ios/message-list) automatically handles moderated messages, displaying blocked content appropriately based on your moderation settings. | \ No newline at end of file diff --git a/ui-kit/ios/extensions.mdx b/ui-kit/ios/extensions.mdx index 62f7186c..fef6939e 100644 --- a/ui-kit/ios/extensions.mdx +++ b/ui-kit/ios/extensions.mdx @@ -88,9 +88,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/message-list) component of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/message-list) component of UI Kits. @@ -98,9 +98,9 @@ Once you have successfully activated the [Profanity Filter Extension](/moderatio ### Data Masking -The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Data Masking Extension](/moderation/data-masking-filter). +The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/message-list) component of UI Kits. +Once you have successfully activated the Data Masking Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/message-list) component of UI Kits. @@ -108,9 +108,9 @@ Once you have successfully activated the [Data Masking Extension](/moderation/da ### Image Moderation -The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Image Moderation Extension](/moderation/image-moderation). +The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/message-list) component of UI Kits. +Once you have successfully activated the Image Moderation Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/message-list) component of UI Kits. diff --git a/ui-kit/ios/v4/extensions.mdx b/ui-kit/ios/v4/extensions.mdx index 47df0433..d5f46e4a 100644 --- a/ui-kit/ios/v4/extensions.mdx +++ b/ui-kit/ios/v4/extensions.mdx @@ -84,9 +84,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/v4/message-list) component of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/v4/message-list) component of UI Kits. @@ -94,9 +94,9 @@ Once you have successfully activated the [Profanity Filter Extension](/moderatio ### Data Masking -The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Data Masking Extension](/moderation/data-masking-filter). +The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/v4/message-list) component of UI Kits. +Once you have successfully activated the Data Masking Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/v4/message-list) component of UI Kits. @@ -104,9 +104,9 @@ Once you have successfully activated the [Data Masking Extension](/moderation/da ### Image Moderation -The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Image Moderation Extension](/moderation/image-moderation). +The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/v4/message-list) component of UI Kits. +Once you have successfully activated the Image Moderation Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/ios/v4/message-list) component of UI Kits. diff --git a/ui-kit/react-native/core-features.mdx b/ui-kit/react-native/core-features.mdx index eff01d21..2e29a6b2 100644 --- a/ui-kit/react-native/core-features.mdx +++ b/ui-kit/react-native/core-features.mdx @@ -137,6 +137,10 @@ Conversation and Advanced Search is a powerful feature provided by CometChat tha The Report Message feature allows users to report inappropriate or harmful messages within the chat. Users can choose from predefined reasons and provide additional remarks for detailed context. This feature helps maintain a safe and respectful chat environment. + +Learn more about how flagged messages are handled, reviewed, and moderated in the [Flagged Messages](/moderation/flagged-messages) documentation. + + @@ -145,6 +149,22 @@ The Report Message feature allows users to report inappropriate or harmful messa | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [Message List](/ui-kit/react-native/message-list) | [Message List](/ui-kit/react-native/message-list) provides the "Report Message" option in the message actions menu, allowing users to initiate the reporting process for inappropriate messages. | +## Moderation + +CometChat's Message Moderation feature helps maintain a safe and respectful chat environment by automatically filtering and managing inappropriate content. Messages can be automatically blocked or flagged based on predefined rules, ensuring harmful content is handled before it reaches users. + + + + + + +Learn more about setting up moderation rules and managing content in the [Moderation](/moderation/overview) documentation. + + +| Components | Functionality | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Message List](/ui-kit/react-native/message-list) | [Message List](/ui-kit/react-native/message-list) automatically handles moderated messages, displaying blocked content appropriately based on your moderation settings. | + ## Threaded Conversations The Threaded Conversations feature enables users to respond directly to a specific message in a chat. This keeps conversations organized and enhances the user experience by maintaining context, especially in group chats. diff --git a/ui-kit/react-native/v4/extensions.mdx b/ui-kit/react-native/v4/extensions.mdx index 30eeda2d..3fb56313 100644 --- a/ui-kit/react-native/v4/extensions.mdx +++ b/ui-kit/react-native/v4/extensions.mdx @@ -151,9 +151,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. @@ -170,9 +170,9 @@ Once you have successfully activated the [Profanity Filter Extension](/moderatio ### Data Masking -The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Data Masking Extension](/moderation/data-masking-filter). +The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. +Once you have successfully activated the Data Masking Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. @@ -189,9 +189,9 @@ Once you have successfully activated the [Data Masking Extension](/moderation/da ### Image Moderation -The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Image Moderation Extension](/moderation/image-moderation). +The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. +Once you have successfully activated the Image Moderation Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/android/message-list) component of UI Kits. diff --git a/ui-kit/react/core-features.mdx b/ui-kit/react/core-features.mdx index 21a84129..e1f2051d 100644 --- a/ui-kit/react/core-features.mdx +++ b/ui-kit/react/core-features.mdx @@ -155,10 +155,30 @@ The Threaded Conversations feature enables users to respond directly to a specif | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | | [Threaded Message Preview](/ui-kit/react/threaded-message-preview) | [Threaded Message Preview](/ui-kit/react/threaded-message-preview) component displays the parent message along with the number of replies. | +## Moderation + +CometChat's Message Moderation feature helps maintain a safe and respectful chat environment by automatically filtering and managing inappropriate content. Messages can be automatically blocked or flagged based on predefined rules, ensuring harmful content is handled before it reaches users. + + + + + + +Learn more about setting up moderation rules and managing content in the [Moderation](/moderation/overview) documentation. + + +| Components | Functionality | +| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Message List](/ui-kit/react/message-list) | [Message List](/ui-kit/react/message-list) automatically handles moderated messages, displaying blocked content appropriately based on your moderation settings. | + ## Report Message The Report Message feature allows users to report inappropriate or harmful messages within the chat. Users can choose from predefined reasons and provide additional remarks for detailed context. This feature helps maintain a safe and respectful chat environment. + +Learn more about how flagged messages are handled, reviewed, and moderated in the [Flagged Messages](/moderation/flagged-messages) documentation. + + diff --git a/ui-kit/react/v4/extensions.mdx b/ui-kit/react/v4/extensions.mdx index 9f6c39aa..cfbe3971 100644 --- a/ui-kit/react/v4/extensions.mdx +++ b/ui-kit/react/v4/extensions.mdx @@ -90,9 +90,9 @@ Once you have successfully activated the [Link Preview Extension](/fundamentals/ ### Profanity Filter -The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Profanity Filter Extension](/moderation/profanity-filter). +The Profanity Filter extension helps in maintaining the chat decorum by censoring obscene and inappropriate words in the messages. For a comprehensive understanding and guide on implementing and using the Profanity Filter Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Profanity Filter Extension](/moderation/profanity-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/react/v4/message-list) component of UI Kits. +Once you have successfully activated the Profanity Filter Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/react/v4/message-list) component of UI Kits. @@ -100,9 +100,9 @@ Once you have successfully activated the [Profanity Filter Extension](/moderatio ### Data Masking -The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Data Masking Extension](/moderation/data-masking-filter). +The Data Masking extension helps secure sensitive data by masking information like credit card numbers and phone numbers in a chat message. For a comprehensive understanding and guide on implementing and using the Data Masking Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/react/v4/message-list) component of UI Kits. +Once you have successfully activated the Data Masking Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/react/v4/message-list) component of UI Kits. @@ -110,9 +110,9 @@ Once you have successfully activated the [Data Masking Extension](/moderation/da ### Image Moderation -The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Image Moderation Extension](/moderation/image-moderation). +The Image Moderation extension uses machine learning to detect and filter out inappropriate or explicit images shared in the chat. For a comprehensive understanding and guide on implementing and using the Image Moderation Extension, refer to our specific guide on the [Legacy Extensions](/moderation/legacy-extensions). -Once you have successfully activated the [Data Masking Extension](/moderation/data-masking-filter) from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/react/v4/message-list) component of UI Kits. +Once you have successfully activated the Image Moderation Extension from your CometChat Dashboard, the feature will automatically be incorporated into the Message Bubble of [MessageList Component](/ui-kit/react/v4/message-list) component of UI Kits.