new usersetting to blacklist (filter) also on description

Author: orangecoding

.gitignore

@@ -7,3 +7,4 @@ npm-debug.log
 .idea
 .vscode
 tools/release/config.json
+.agents

lib/FredyPipelineExecutioner.js

@@ -38,11 +38,15 @@ import { formatListing } from './utils/formatListing.js';
  * 3) Normalize listings to the provider schema
  * 4) Filter out incomplete/blacklisted listings
  * 5) Identify new listings (vs. previously stored hashes)
- * 6) Persist new listings
- * 7) Filter out entries similar to already seen ones
- * 8) Filter out entries that do not match the job's specFilter
- * 9) Filter out entries that do not match the job's spatialFilter
- * 10) Dispatch notifications
+ * 6) Optionally enrich new listings via provider.fetchDetails
+ * 7) Optionally re-apply the provider blacklist using the (now enriched)
+ *    description — only when the user opted in via
+ *    `blacklist_filter_on_provider_details`
+ * 8) Persist new listings
+ * 9) Filter out entries similar to already seen ones
+ * 10) Filter out entries that do not match the job's specFilter
+ * 11) Filter out entries that do not match the job's spatialFilter
+ * 12) Dispatch notifications
  */
 class FredyPipelineExecutioner {
   /**
@@ -86,6 +90,7 @@ class FredyPipelineExecutioner {
       .then(this._filter.bind(this))
       .then(this._findNew.bind(this))
       .then(this._fetchDetails.bind(this))
+      .then(this._filterAfterDetails.bind(this))
       .then(this._geocode.bind(this))
       .then(this._save.bind(this))
       .then(this._calculateDistance.bind(this))
@@ -266,6 +271,48 @@ class FredyPipelineExecutioner {
     );
   }
 
+  /**
+   * Re-apply the provider's blacklist filter after `_fetchDetails` has had a
+   * chance to enrich the listings (e.g., load the full description from the
+   * detail page). The initial `_filter` step only sees the truncated snippet
+   * exposed on the search results page, so a blacklisted term that lives
+   * deeper in the listing's full description would otherwise slip through.
+   *
+   * Opt-in: gated by the user setting `blacklist_filter_on_provider_details`.
+   * The full detail description tends to contain a lot of boilerplate (legal,
+   * exposé contact info, generic marketing copy) which can accidentally match
+   * a blacklist term and remove otherwise relevant listings. Users who want
+   * the stricter behavior must enable the setting explicitly.
+   *
+   * Throws {@link NoNewListingsWarning} when all listings are filtered out
+   * so the rest of the pipeline (save + notify) is short-circuited.
+   *
+   * @param {ParsedListing[]} listings Enriched listings to re-filter.
+   * @returns {ParsedListing[]} Listings that still pass the provider's filter.
+   * @throws {NoNewListingsWarning} When every listing is filtered out.
+   */
+  _filterAfterDetails(listings) {
+    if (typeof this._providerConfig.filter !== 'function') {
+      return listings;
+    }
+    const userId = getJob(this._jobKey)?.userId;
+    const enabled = getUserSettings(userId)?.blacklist_filter_on_provider_details === true;
+    if (!enabled) {
+      return listings;
+    }
+    const kept = listings.filter(this._providerConfig.filter);
+    const removed = listings.length - kept.length;
+    if (removed > 0) {
+      logger.debug(
+        `Re-filter after detail enrichment removed ${removed} listing(s) by blacklist (Provider: '${this._providerId}')`,
+      );
+    }
+    if (kept.length === 0) {
+      throw new NoNewListingsWarning();
+    }
+    return kept;
+  }
+
   /**
    * Determine which listings are new by comparing their IDs against stored hashes.
    *

lib/api/routes/userSettingsRoute.js

@@ -103,6 +103,28 @@ export default async function userSettingsPlugin(fastify) {
     }
   });
 
+  fastify.post('/blacklist-filter-on-details', async (request, reply) => {
+    const userId = request.session.currentUser;
+    const { blacklist_filter_on_provider_details } = request.body;
+
+    const globalSettings = await getSettings();
+    if (globalSettings.demoMode && !isAdmin(request)) {
+      return reply.code(403).send({ error: 'In demo mode, it is not allowed to change settings.' });
+    }
+
+    if (typeof blacklist_filter_on_provider_details !== 'boolean') {
+      return reply.code(400).send({ error: 'blacklist_filter_on_provider_details must be a boolean.' });
+    }
+
+    try {
+      upsertSettings({ blacklist_filter_on_provider_details }, userId);
+      return { success: true };
+    } catch (error) {
+      logger.error('Error updating blacklist-filter-on-details setting', error);
+      return reply.code(500).send({ error: error.message });
+    }
+  });
+
   fastify.post('/listings-view-mode', async (request, reply) => {
     const userId = request.session.currentUser;
     const { listings_view_mode } = request.body;

lib/provider/immoscout.js

@@ -198,7 +198,9 @@ function normalize(o) {
  * @returns {boolean}
  */
 function applyBlacklist(o) {
-  return !isOneOf(o.title, appliedBlackList);
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
 }
 /** @type {ProviderConfig} */
 const config = {

lib/provider/neubauKompass.js

@@ -42,7 +42,9 @@ function normalize(o) {
  * @returns {boolean}
  */
 function applyBlacklist(o) {
-  return !isOneOf(o.title, appliedBlackList);
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
 }
 
 /** @type {ProviderConfig} */

test/mocks/mockStore.js

@@ -17,8 +17,12 @@ export const getGeocoordinatesByAddress = (any) => {
   return null;
 };
 
+let userSettings = null;
+export function setUserSettings(settings) {
+  userSettings = settings;
+}
 export function getUserSettings(userId) {
-  return null;
+  return userSettings;
 }
 
 export async function getSettings() {

test/pipeline_filtering.test.js

@@ -3,9 +3,10 @@
  * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
  */
 
-import { expect } from 'vitest';
+import { afterEach, expect } from 'vitest';
 import { mockFredy } from './utils.js';
 import * as mockStore from './mocks/mockStore.js';
+import { get as getLastNotification } from './mocks/mockNotification.js';
 
 describe('Issue reproduction: listings filtered by similarity or area should be marked as manually deleted', () => {
   it('should call deleteListingsById when listings are filtered by similarity', async () => {
@@ -113,3 +114,223 @@ describe('Issue reproduction: listings filtered by similarity or area should be
     expect(mockStore.deletedIds).toContain('2');
   });
 });
+
+describe('Blacklist is re-applied after detail enrichment', () => {
+  afterEach(() => {
+    mockStore.setUserSettings(null);
+  });
+
+  it('filters out a listing whose blacklisted term only appears in the enriched description', async () => {
+    const Fredy = await mockFredy();
+    const providerId = 'test-provider';
+
+    mockStore.setUserSettings({
+      provider_details: [providerId],
+      blacklist_filter_on_provider_details: true,
+    });
+
+    const mockSimilarityCache = {
+      checkAndAddEntry: () => false,
+    };
+
+    const blacklist = ['allkauf'];
+
+    // The search results page returns a clean snippet (no blacklisted term).
+    // fetchDetails simulates loading the full detail page and discovers the
+    // blacklisted term hidden deep in the description.
+    const providerConfig = {
+      url: 'http://example.com',
+      getListings: () =>
+        Promise.resolve([
+          {
+            id: 'kept',
+            title: 'Nice house',
+            address: 'Some street',
+            price: '500000',
+            link: 'http://example.com/kept',
+            description: 'Cozy home with garden',
+          },
+          {
+            id: 'blacklisted',
+            title: 'Eleganz trifft Raumkomfort',
+            address: 'Other street',
+            price: '600000',
+            link: 'http://example.com/blacklisted',
+            description: 'Eleganz trifft Raumkomfort',
+          },
+        ]),
+      normalize: (l) => l,
+      filter: (l) => {
+        const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
+        return !blacklist.some((term) => text.includes(term));
+      },
+      fetchDetails: (listing) => {
+        if (listing.id === 'blacklisted') {
+          return Promise.resolve({
+            ...listing,
+            description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
+          });
+        }
+        return Promise.resolve(listing);
+      },
+      crawlFields: {
+        id: 'id',
+        title: 'title',
+        address: 'address',
+        price: 'price',
+        link: 'link',
+        description: 'description',
+      },
+      requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
+    };
+
+    const mockedJob = {
+      id: 'blacklist-test-job',
+      notificationAdapter: null,
+      specFilter: null,
+      spatialFilter: null,
+    };
+
+    const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
+
+    const result = await fredy.execute();
+
+    expect(result).toBeInstanceOf(Array);
+    const ids = result.map((l) => l.id);
+    expect(ids).toContain('kept');
+    expect(ids).not.toContain('blacklisted');
+
+    const notification = getLastNotification();
+    const notifiedIds = (notification?.payload ?? []).map((p) => p.id);
+    expect(notifiedIds).not.toContain('blacklisted');
+  });
+
+  it('short-circuits the pipeline when all listings get blacklisted after enrichment', async () => {
+    const Fredy = await mockFredy();
+    const providerId = 'all-blacklisted-provider';
+
+    mockStore.setUserSettings({
+      provider_details: [providerId],
+      blacklist_filter_on_provider_details: true,
+    });
+
+    const mockSimilarityCache = {
+      checkAndAddEntry: () => false,
+    };
+
+    const blacklist = ['allkauf'];
+
+    const providerConfig = {
+      url: 'http://example.com',
+      getListings: () =>
+        Promise.resolve([
+          {
+            id: 'only',
+            title: 'Eleganz trifft Raumkomfort',
+            address: 'Some street',
+            price: '700000',
+            link: 'http://example.com/only',
+            description: 'Eleganz trifft Raumkomfort',
+          },
+        ]),
+      normalize: (l) => l,
+      filter: (l) => {
+        const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
+        return !blacklist.some((term) => text.includes(term));
+      },
+      fetchDetails: (listing) =>
+        Promise.resolve({
+          ...listing,
+          description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
+        }),
+      crawlFields: {
+        id: 'id',
+        title: 'title',
+        address: 'address',
+        price: 'price',
+        link: 'link',
+        description: 'description',
+      },
+      requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
+    };
+
+    const mockedJob = {
+      id: 'all-blacklisted-job',
+      notificationAdapter: null,
+      specFilter: null,
+      spatialFilter: null,
+    };
+
+    const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
+
+    // Should resolve to undefined (NoNewListingsWarning is caught in _handleError).
+    const result = await fredy.execute();
+    expect(result).toBeUndefined();
+  });
+
+  it('does NOT re-filter when blacklist_filter_on_provider_details is disabled', async () => {
+    const Fredy = await mockFredy();
+    const providerId = 'opt-out-provider';
+
+    // provider_details enabled (so fetchDetails runs) but blacklist re-filter NOT enabled.
+    mockStore.setUserSettings({
+      provider_details: [providerId],
+      blacklist_filter_on_provider_details: false,
+    });
+
+    const mockSimilarityCache = {
+      checkAndAddEntry: () => false,
+    };
+
+    const blacklist = ['allkauf'];
+
+    const providerConfig = {
+      url: 'http://example.com',
+      getListings: () =>
+        Promise.resolve([
+          {
+            id: 'leaks-through',
+            title: 'Eleganz trifft Raumkomfort',
+            address: 'Other street',
+            price: '600000',
+            link: 'http://example.com/leaks-through',
+            description: 'Eleganz trifft Raumkomfort',
+          },
+        ]),
+      normalize: (l) => l,
+      filter: (l) => {
+        const text = `${l.title ?? ''} ${l.description ?? ''}`.toLowerCase();
+        return !blacklist.some((term) => text.includes(term));
+      },
+      fetchDetails: (listing) =>
+        Promise.resolve({
+          ...listing,
+          description: 'Mit allkauf Haus wird dein Traum vom Eigenheim wahr.',
+        }),
+      crawlFields: {
+        id: 'id',
+        title: 'title',
+        address: 'address',
+        price: 'price',
+        link: 'link',
+        description: 'description',
+      },
+      requiredFieldNames: ['id', 'title', 'address', 'price', 'link', 'description'],
+    };
+
+    const mockedJob = {
+      id: 'opt-out-job',
+      notificationAdapter: null,
+      specFilter: null,
+      spatialFilter: null,
+    };
+
+    const fredy = new Fredy(providerConfig, mockedJob, providerId, mockSimilarityCache, undefined);
+
+    const result = await fredy.execute();
+
+    // Listing leaks through because user has not opted in to the stricter check.
+    expect(result).toBeInstanceOf(Array);
+    expect(result.map((l) => l.id)).toContain('leaks-through');
+  });
+});

ui/src/locales/de.json

@@ -334,6 +334,11 @@
   "settings.providerDetailsPlaceholder": "Anbieter für Detail-Abruf auswählen...",
   "settings.providerDetailsUpdated": "Anbieter-Detail-Einstellung aktualisiert.",
   "settings.providerDetailsUpdateError": "Einstellung konnte nicht aktualisiert werden.",
+  "settings.blacklistFilterOnProviderDetails": "Blacklist-Filter auf Anbieter-Details anwenden",
+  "settings.blacklistFilterOnProviderDetailsHelp": "Wenn aktiv, wird die Blacklist zusätzlich gegen die vollständige Beschreibung geprüft, die durch den obigen Anbieter-Details-Schritt geladen wurde. Damit lassen sich Spam-Anbieter (z. B. 'allkauf', 'massa') herausfiltern, die nur tief in der Detail-Seite auftauchen und nicht im kurzen Vorschau-Text der Suchergebnisse stehen. Standardmäßig aus, weil die vollständige Beschreibung oft generischen Boilerplate-Text (Kontaktdaten, rechtliche Hinweise) enthält, der ein Blacklist-Wort versehentlich auslösen und passende Inserate entfernen kann. Hat keine Wirkung auf Anbieter, für die Anbieter-Details nicht aktiviert sind.",
+  "settings.blacklistFilterOnProviderDetailsEnable": "Blacklist auf die vollständige Detail-Beschreibung anwenden",
+  "settings.blacklistFilterOnProviderDetailsUpdated": "Einstellung Blacklist-auf-Details aktualisiert.",
+  "settings.blacklistFilterOnProviderDetailsUpdateError": "Einstellung konnte nicht aktualisiert werden.",
   "settings.listingDeletion": "Inserate löschen",
   "settings.listingDeletionHelp": "Wähle den Standard-Löschmodus. Soft Delete blendet Inserate aus ohne erneutes Scraping; Hard Delete entfernt sie aus der Datenbank.",
   "settings.listingDeletionSoftLabel": "Als gelöscht markieren (Soft Delete)",