feat: enhance crawler endpoint and email notification localization

- Added optional parameters `email` and `language` to the crawler endpoint for improved user notifications.
- Updated documentation in `crawler_endpoint.md` to reflect new parameters and enhance clarity.
- Improved email notification content in both English and Farsi localization files to include the application name "Hatef" for better branding.
- Enhanced email subject localization to dynamically include the number of pages indexed, improving user engagement.

These changes significantly improve the user experience by providing localized notifications and clearer documentation for the crawler API.

Author: Hatef-Rostamkhani

docs/api/crawler_endpoint.md

@@ -35,6 +35,8 @@ Add a new site to the crawl queue with optimized SPA rendering.
   "url": "https://www.digikala.com",
   "maxPages": 100,
   "maxDepth": 3,
+  "email": "[email protected]",
+  "language": "en",
   "spaRenderingEnabled": true,
   "includeFullContent": false,
   "browserlessUrl": "http://browserless:3000",
@@ -45,16 +47,18 @@ Add a new site to the crawl queue with optimized SPA rendering.
 
 #### Parameters
 
-| Parameter             | Type    | Default                   | Description                            |
-| --------------------- | ------- | ------------------------- | -------------------------------------- |
-| `url`                 | string  | **required**              | Seed URL to start crawling             |
-| `maxPages`            | integer | 1000                      | Maximum pages to crawl                 |
-| `maxDepth`            | integer | 5                         | Maximum crawl depth                    |
-| `spaRenderingEnabled` | boolean | true                      | Enable SPA rendering                   |
-| `includeFullContent`  | boolean | false                     | Store full HTML content                |
-| `browserlessUrl`      | string  | "http://browserless:3000" | Browserless service URL                |
-| `timeout`             | integer | 15000                     | Request timeout in milliseconds        |
-| `politenessDelay`     | integer | 500                       | Delay between requests in milliseconds |
+| Parameter             | Type    | Default                   | Description                                         |
+| --------------------- | ------- | ------------------------- | --------------------------------------------------- |
+| `url`                 | string  | **required**              | Seed URL to start crawling                          |
+| `maxPages`            | integer | 1000                      | Maximum pages to crawl                              |
+| `maxDepth`            | integer | 5                         | Maximum crawl depth                                 |
+| `email`               | string  | (optional)                | Email address for completion notification           |
+| `language`            | string  | "en"                      | Language for email notifications (en, fa, etc.)     |
+| `spaRenderingEnabled` | boolean | true                      | Enable SPA rendering                                |
+| `includeFullContent`  | boolean | false                     | Store full HTML content                             |
+| `browserlessUrl`      | string  | "http://browserless:3000" | Browserless service URL                             |
+| `timeout`             | integer | 15000                     | Request timeout in milliseconds                     |
+| `politenessDelay`     | integer | 500                       | Delay between requests in milliseconds              |
 
 #### Response
 

locales/en/crawling-notification.json

@@ -22,12 +22,12 @@
       "completed_at": "Completed At",
       "session_id": "Session ID"
     },
-    "description": "Your pages are now searchable in our search engine. If you'd like to crawl and index more pages from your site, please visit our crawl request page.",
+    "description": "Your pages are now searchable in Hatef search engine. If you'd like to crawl and index more pages from your site, please visit our crawl request page.",
     "cta": {
       "button_text": "Request More Crawling"
     },
     "footer": {
-      "thank_you": "Thank you for using our search engine service!",
+      "thank_you": "Thank you for using Hatef search engine service!",
       "automated_message": "This is an automated notification from Hatef Search Engine",
       "unsubscribe_text": "Unsubscribe from these notifications",
       "copyright": "© 2024 Hatef.ir - All rights reserved"

locales/fa/crawling-notification.json

@@ -22,12 +22,12 @@
       "completed_at": "تکمیل شده در",
       "session_id": "شناسه جلسه"
     },
-    "description": "صفحات شما اکنون در موتور جستجوی ما قابل جستجو هستند. اگر می‌خواهید صفحات بیشتری از سایت خود را خزش و نمایه‌سازی کنید، یا دامنه‌های اضافی برای افزودن دارید، لطفاً از صفحه درخواست خزش ما استفاده کنید.",
+    "description": "صفحات شما اکنون در موتور جستجو هاتف قابل جستجو هستند. اگر می‌خواهید صفحات بیشتری از سایت خود را خزش و نمایه‌سازی کنید، یا دامنه‌های اضافی برای افزودن دارید، لطفاً از صفحه درخواست خزش ما استفاده کنید.",
     "cta": {
       "button_text": "درخواست خزش بیشتر"
     },
     "footer": {
-      "thank_you": "از استفاده از خدمات موتور جستجوی ما متشکریم!",
+      "thank_you": "از استفاده از خدمات موتور جستجو هاتف متشکریم!",
       "automated_message": "این پیام خودکار از موتور جستجوی هاتف ارسال شده است",
       "unsubscribe_text": "لغو اشتراک از این اعلان‌ها",
       "copyright": "© ۲۰۲۴ هاتف - تمام حقوق محفوظ است"

public/js/crawl-request-template.js

@@ -33,8 +33,9 @@ function initializeTemplateData(data) {
     progressMessages = data.progressMessages || [];
 
     // Debug logging
-    console.log('Template data initialized:', templateData);
-    console.log('Base URL from template:', templateData.baseUrl || templateData.base_url || 'Not set');
+    console.log('🎯 Template data initialized:', templateData);
+    console.log('🌐 Base URL from template:', templateData.baseUrl || templateData.base_url || 'Not set');
+    console.log('🌍 Language from template:', templateData.language || 'Not set (will use API default)');
 }
 
 let currentSessionId = null;
@@ -196,10 +197,21 @@ async function startCrawl() {
         maxDepth: maxDepth
     };
 
+    // Add language from template data (for localized email notifications)
+    if (templateData.language) {
+        payload.language = templateData.language;
+        console.log('✅ Language set from template data:', templateData.language);
+    } else {
+        console.warn('⚠️ Template data language not found, email will use default language');
+        console.log('Template data:', templateData);
+    }
+    
     if (email) {
         payload.email = email;
     }
 
+    console.log('📤 Sending payload:', payload);
+    
     try {
         // Show progress section
         document.getElementById('form-section').style.display = 'none';

src/controllers/SearchController.cpp

@@ -217,6 +217,7 @@ void SearchController::addSiteToCrawl(uWS::HttpResponse<false>* res, uWS::HttpRe
 
                 // Optional parameters
                 std::string email = jsonBody.value("email", "");  // Email for completion notification
+                std::string language = jsonBody.value("language", "en");  // Language for email notification (default: English)
                 int maxPages = jsonBody.value("maxPages", 1000);
                 int maxDepth = jsonBody.value("maxDepth", 3);
                 bool restrictToSeedDomain = jsonBody.value("restrictToSeedDomain", true);
@@ -307,11 +308,11 @@ void SearchController::addSiteToCrawl(uWS::HttpResponse<false>* res, uWS::HttpRe
                     // Create completion callback for email notification if email is provided
                     CrawlCompletionCallback emailCallback = nullptr;
                     if (!email.empty()) {
-                        LOG_INFO("Setting up email notification callback for: " + email);
-                        emailCallback = [this, email, url](const std::string& sessionId, 
+                        LOG_INFO("Setting up email notification callback for: " + email + " (language: " + language + ")");
+                        emailCallback = [this, email, url, language](const std::string& sessionId, 
                                                          const std::vector<CrawlResult>& results, 
                                                          CrawlerManager* manager) {
-                            this->sendCrawlCompletionEmail(sessionId, email, url, results);
+                            this->sendCrawlCompletionEmail(sessionId, email, url, results, language);
                         };
                     }
 
@@ -1696,9 +1697,10 @@ namespace {
 }
 
 void SearchController::sendCrawlCompletionEmail(const std::string& sessionId, const std::string& email, 
-                                               const std::string& url, const std::vector<CrawlResult>& results) {
+                                               const std::string& url, const std::vector<CrawlResult>& results,
+                                               const std::string& language) {
     try {
-        LOG_INFO("Sending crawl completion email for session: " + sessionId + " to: " + email);
+        LOG_INFO("Sending crawl completion email for session: " + sessionId + " to: " + email + " (language: " + language + ")");
 
         // Get email service using lazy initialization
         auto emailService = getEmailService();
@@ -1733,8 +1735,9 @@ void SearchController::sendCrawlCompletionEmail(const std::string& sessionId, co
             }
         }
 
-        // Load localized sender name
-        std::string senderName = loadLocalizedSenderName("fa"); // Default to Persian for now
+        // Load localized sender name and subject using the provided language
+        std::string senderName = loadLocalizedSenderName(language);
+        std::string localizedSubject = loadLocalizedSubject(language, crawledPagesCount);
 
         // Prepare notification data
         search_engine::storage::EmailService::NotificationData data;
@@ -1744,7 +1747,8 @@ void SearchController::sendCrawlCompletionEmail(const std::string& sessionId, co
         data.crawledPagesCount = crawledPagesCount;
         data.crawlSessionId = sessionId;
         data.crawlCompletedAt = std::chrono::system_clock::now();
-        data.language = "fa"; // Default to Persian for now
+        data.language = language;
+        data.subject = localizedSubject; // Set localized subject
 
         // Send email asynchronously with localized sender name
         bool success = emailService->sendCrawlingNotificationAsync(data, senderName, "");
@@ -1877,4 +1881,46 @@ std::string SearchController::loadLocalizedSenderName(const std::string& languag
         LOG_ERROR("SearchController: Exception loading localized sender name for language " + language + ": " + e.what());
         return "Hatef Search Engine"; // Default fallback
     }
+}
+
+std::string SearchController::loadLocalizedSubject(const std::string& language, int pageCount) const {
+    try {
+        // Load localization file
+        std::string localesPath = "locales/" + language + "/crawling-notification.json";
+        std::string localeContent = loadFile(localesPath);
+        
+        if (localeContent.empty() && language != "en") {
+            LOG_WARNING("SearchController: Failed to load locale file: " + localesPath + ", falling back to English");
+            localesPath = "locales/en/crawling-notification.json";
+            localeContent = loadFile(localesPath);
+        }
+        
+        if (localeContent.empty()) {
+            LOG_WARNING("SearchController: Failed to load any localization file, using default subject");
+            return "Crawling Complete - " + std::to_string(pageCount) + " pages indexed"; // Default fallback
+        }
+        
+        // Parse JSON and extract subject
+        nlohmann::json localeData = nlohmann::json::parse(localeContent);
+        
+        if (localeData.contains("email") && localeData["email"].contains("subject")) {
+            std::string subject = localeData["email"]["subject"];
+            
+            // Replace {pages} placeholder with actual count
+            size_t pos = subject.find("{pages}");
+            if (pos != std::string::npos) {
+                subject.replace(pos, 7, std::to_string(pageCount));
+            }
+            
+            LOG_DEBUG("SearchController: Loaded localized subject: " + subject + " for language: " + language);
+            return subject;
+        } else {
+            LOG_WARNING("SearchController: subject not found in locale file, using default");
+            return "Crawling Complete - " + std::to_string(pageCount) + " pages indexed"; // Default fallback
+        }
+        
+    } catch (const std::exception& e) {
+        LOG_ERROR("SearchController: Exception loading localized subject for language " + language + ": " + e.what());
+        return "Crawling Complete - " + std::to_string(pageCount) + " pages indexed"; // Default fallback
+    }
 } 

src/controllers/SearchController.h

@@ -44,7 +44,8 @@ class SearchController : public routing::Controller {
 
     // Email notification for crawl completion
     void sendCrawlCompletionEmail(const std::string& sessionId, const std::string& email, 
-                                 const std::string& url, const std::vector<CrawlResult>& results);
+                                 const std::string& url, const std::vector<CrawlResult>& results,
+                                 const std::string& language);
 
     // Email service access (lazy initialization)
     search_engine::storage::EmailService* getEmailService() const;
@@ -54,6 +55,9 @@ class SearchController : public routing::Controller {
 
     // Localized sender name loading
     std::string loadLocalizedSenderName(const std::string& language) const;
+    
+    // Localized email subject loading
+    std::string loadLocalizedSubject(const std::string& language, int pageCount) const;
 
 private:
     mutable std::unique_ptr<search_engine::storage::EmailService> emailService_;

src/storage/EmailService.cpp

@@ -320,6 +320,7 @@ std::string EmailService::formatEmailHeaders(const std::string& to, const std::s
 
     headers << "To: " << to << "\r\n";
     headers << "From: " << config_.fromName << " <" << config_.fromEmail << ">\r\n";
+    headers << "Reply-To: [email protected]\r\n";
     headers << "Subject: " << subject << "\r\n";
     headers << "MIME-Version: 1.0\r\n";
 
@@ -627,7 +628,7 @@ bool EmailService::performSMTPRequest(const std::string& to, const std::string&
 }
 
 std::string EmailService::generateDefaultNotificationHTML(const NotificationData& data) {
-    LOG_INFO("EmailService: Using Inja template-based email generation");
+    LOG_INFO("EmailService: Using Inja template-based email generation for language: " + data.language);
 
     // Render the email template
     std::string templateHTML = renderEmailTemplate("email-crawling-notification.inja", data);
@@ -637,6 +638,9 @@ std::string EmailService::generateDefaultNotificationHTML(const NotificationData
         throw std::runtime_error("Failed to render email template");
     }
 
+    LOG_DEBUG("EmailService: Generated HTML content length: " + std::to_string(templateHTML.length()) + " bytes for language: " + data.language);
+    LOG_DEBUG("EmailService: HTML preview (first 200 chars): " + templateHTML.substr(0, std::min(size_t(200), templateHTML.length())));
+    
     return templateHTML;
 }
 
@@ -1103,31 +1107,72 @@ std::string EmailService::convertToPersianDate(const std::tm& gregorianDate) {
         int gMonth = gregorianDate.tm_mon + 1;
         int gDay = gregorianDate.tm_mday;
 
-        // Calculate days since March 21, 2024 (reference point: 1 Farvardin 1403)
-        int daysSinceMarch21 = 0;
+        // Determine Persian year based on Gregorian date
+        // Persian new year (Nowruz) is around March 20/21
+        int persianYear;
+        if (gMonth < 3 || (gMonth == 3 && gDay < 20)) {
+            // Before March 20: still in previous Persian year
+            persianYear = gYear - 621;
+        } else {
+            // March 20 onwards: new Persian year has started
+            persianYear = gYear - 621;
+        }
 
-        // Days in each month (from March to current month)
-        int monthDays[] = {31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 31, 28}; // March to February
+        // Calculate day of year in Persian calendar
+        int persianDayOfYear;
 
-        if (gMonth >= 3) {
-            // Current year - calculate days from March 21
-            for (int i = 3; i < gMonth; i++) {
-                daysSinceMarch21 += monthDays[i - 3];
+        if (gMonth >= 3 && (gMonth > 3 || gDay >= 20)) {
+            // From March 20 onwards in current Gregorian year
+            int daysInGregorianMonths[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+            
+            // Check for leap year
+            if ((gYear % 4 == 0 && gYear % 100 != 0) || (gYear % 400 == 0)) {
+                daysInGregorianMonths[1] = 29;
+            }
+            
+            persianDayOfYear = 0;
+            // Add days from March 20 to end of March
+            if (gMonth == 3) {
+                persianDayOfYear = gDay - 20 + 1;
+            } else {
+                persianDayOfYear = daysInGregorianMonths[2] - 20 + 1; // Days left in March (12 days)
+                // Add full months between April and current month
+                for (int m = 4; m < gMonth; m++) {
+                    persianDayOfYear += daysInGregorianMonths[m - 1];
+                }
+                // Add days in current month
+                persianDayOfYear += gDay;
             }
-            daysSinceMarch21 += gDay - 21; // March 21 is day 0
         } else {
-            // Previous year - calculate from March 21 of previous year
-            daysSinceMarch21 += 31 - 21 + 30 + 31 + 30 + 31 + 31 + 30 + 31 + 30 + 31 + 31 + 28; // March 21 to Dec 31
-            for (int i = 1; i < gMonth; i++) {
-                daysSinceMarch21 += monthDays[i - 1 + 9]; // Offset for month array
+            // Before March 20: in previous Persian year
+            persianYear--;
+            
+            int daysInGregorianMonths[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+            
+            // Check for leap year of previous Gregorian year
+            int prevGYear = gYear - 1;
+            if ((prevGYear % 4 == 0 && prevGYear % 100 != 0) || (prevGYear % 400 == 0)) {
+                daysInGregorianMonths[1] = 29;
+            }
+            
+            // Days from March 20 to Dec 31 of previous year
+            persianDayOfYear = daysInGregorianMonths[2] - 20 + 1; // Rest of March (12 days)
+            for (int m = 4; m <= 12; m++) {
+                persianDayOfYear += daysInGregorianMonths[m - 1];
+            }
+            
+            // Add days from Jan 1 to current date
+            for (int m = 1; m < gMonth; m++) {
+                // Use current year's month days for Jan-Feb
+                int currentYearMonthDays[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+                if ((gYear % 4 == 0 && gYear % 100 != 0) || (gYear % 400 == 0)) {
+                    currentYearMonthDays[1] = 29;
+                }
+                persianDayOfYear += currentYearMonthDays[m - 1];
             }
-            daysSinceMarch21 += gDay - 1;
+            persianDayOfYear += gDay;
         }
 
-        // Convert to Persian date
-        int persianYear = 1403; // Base year for 2024
-        int persianDayOfYear = daysSinceMarch21 + 1;
-        
         // Persian months: 31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, 29/30
         int persianMonthDays[] = {31, 31, 31, 31, 31, 31, 30, 30, 30, 30, 30, 29};
 
@@ -1159,6 +1204,10 @@ std::string EmailService::convertToPersianDate(const std::tm& gregorianDate) {
                                  " (" + persianMonths[persianMonth - 1] + ") " +
                                  "ساعت " + std::string(timeBuffer) + " (تهران)";
 
+        LOG_DEBUG("EmailService: Converted Gregorian " + std::to_string(gYear) + "/" + 
+                  std::to_string(gMonth) + "/" + std::to_string(gDay) + 
+                  " to Persian: " + persianDate);
+        
         return persianDate;
 
     } catch (const std::exception& e) {