Skip to content

Commit b9e4bca

Browse files
libkleinlibklein
authored
Add immoscout mobile API provider to avoid failing bot checks (#125)
* Add provider that uses the immoscout mobile API to avoid failing bot checks.
1 parent a138daf commit b9e4bca

3 files changed

Lines changed: 276 additions & 1 deletion

File tree

‎lib/FredyRuntime.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class FredyRuntime {
2626
//modify the url to make sure search order is correctly set
2727
Promise.resolve(urlModifier(this._providerConfig.url, this._providerConfig.sortByDateParam))
2828
//scraping the site and try finding new listings
29-
.then(this._getListings.bind(this))
29+
.then(this._providerConfig.getListings?.bind(this) ?? this._getListings.bind(this))
3030
//bring them in a proper form (dictated by the provider)
3131
.then(this._normalize.bind(this))
3232
//filter listings with stuff tagged by the blacklist of the provider

‎lib/provider/immoscout-mobile.js‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
/**
2+
* ImmoScout provider using the mobile API to retrieve listings.
3+
*
4+
* The mobile API provides the following endpoints:
5+
* - GET /search/total?{search parameters}: Returns the total number of listings for the given query
6+
* Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" https://api.mobile.immobilienscout24.de/search/total?searchType=region&realestatetype=apartmentrent&pricetype=calculatedtotalrent&geocodes=%2Fde%2Fberlin%2Fberlin `
7+
*
8+
* - POST /search/list?{search parameters}: Actually retrieves the listings. Body is json encoded and contains
9+
* data specifying additional results (advertisements) to return. The format is as follows:
10+
* ```
11+
* {
12+
* "supportedResultListTypes": [],
13+
* "userData": {}
14+
* }
15+
* ```
16+
* It is not necessary to provide data for the specified keys.
17+
*
18+
* Example: `curl -X POST 'https://api.mobile.immobilienscout24.de/search/list?pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region&geocodes=%2Fde%2Fberlin%2Fberlin&pagenumber=1' -H "Connection: keep-alive" -H "User-Agent: ImmoScout24_1410_30_._" -H "Accept: application/json" -H "Content-Type: application/json" -d '{"supportedResultListType": [], "userData": {}}'`
19+
20+
* - GET /expose/{id} - Returns the details of a listing. The response contains additional details not included in the
21+
* listing response.
22+
*
23+
* Example: `curl -H "User-Agent: ImmoScout24_1410_30_._" "https://api.mobile.immobilienscout24.de/expose/158382494"`
24+
*
25+
*
26+
* It is necessary to set the correct User Agent (see `getListings`) in the request header.
27+
*
28+
* Note that the mobile API is not publicly documented. I've reverse-engineered
29+
* it by intercepting traffic from an android emulator running the immoscout app.
30+
* Moreover, the search parameters differ slightly from the web API. I've mapped them
31+
* to the web API parameters by comparing a search request with all parameters set between
32+
* the web and mobile API. The mobile API actually seems to be a superset of the web API,
33+
* but I have decided not to include new parameters as I wanted to keep the existing UX (i.e.,
34+
* users only have to provide a link to an existing search).
35+
*
36+
* Limitations:
37+
* - The current implementation of this provider *does not* support non-rental properties,
38+
* although the same approach can be used to implement support. It's just a matter of
39+
* mapping the web search URL to the corresponding mobile API URL.
40+
* - Pagination support is not implemented.
41+
*/
42+
43+
import utils, {buildHash} from '../utils.js';
44+
import queryString from 'query-string';
45+
let appliedBlackList = [];
46+
47+
async function getListings(url) {
48+
const response = await fetch(url, {
49+
method: 'POST',
50+
headers: {
51+
'User-Agent': 'ImmoScout24_1410_30_._',
52+
'Content-Type': 'application/json',
53+
},
54+
body: JSON.stringify({
55+
supportedResultListTypes: [],
56+
userData: {}
57+
})
58+
});
59+
if (!response.ok) {
60+
console.error('Error fetching data from ImmoScout Mobile API:', response.statusText);
61+
return [];
62+
}
63+
64+
const responseBody = await response.json();
65+
return responseBody.resultListItems.filter((item) => item.type === 'EXPOSE_RESULT').map(expose => {
66+
const item = expose.item;
67+
const [price, size, ] = item.attributes;
68+
return {
69+
id: item.id,
70+
price: price?.value,
71+
size: size?.value,
72+
title: item.title,
73+
link: `${metaInformation.baseUrl}/expose/${item.id}`,
74+
address: item.address?.line,
75+
};
76+
});
77+
}
78+
79+
function nullOrEmpty(val) {
80+
return val == null || val.length === 0;
81+
}
82+
function normalize(o) {
83+
const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
84+
const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
85+
const id = buildHash(o.id, o.price);
86+
return Object.assign(o, { id, title, address});
87+
}
88+
function applyBlacklist(o) {
89+
return !utils.isOneOf(o.title, appliedBlackList);
90+
}
91+
const config = {
92+
url: null,
93+
sortByDateParam: 'sorting=-firstactivation',
94+
// Not actually required - used by filter to remove and listings that failed to parse
95+
crawlFields: {
96+
'id': 'id',
97+
'title': 'title',
98+
'price': 'price',
99+
'size': 'size',
100+
'link': 'link',
101+
'address': 'address'
102+
},
103+
normalize: normalize,
104+
filter: applyBlacklist,
105+
getListings: getListings
106+
};
107+
export const init = (sourceConfig, blacklist) => {
108+
config.enabled = sourceConfig.enabled;
109+
config.url = convertWebToMobile(sourceConfig.url);
110+
appliedBlackList = blacklist || [];
111+
};
112+
export const metaInformation = {
113+
name: 'Immoscout',
114+
baseUrl: 'https://www.immobilienscout24.de/',
115+
id: 'immoscout-mobile',
116+
};
117+
118+
export function convertWebToMobile(webUrl) {
119+
let url;
120+
try {
121+
url = new URL(webUrl);
122+
} catch (err) {
123+
throw new Error(`Invalid URL: ${webUrl}`);
124+
}
125+
const segments = url.pathname.split('/');
126+
if (segments.length < 6 || segments[1] !== 'Suche') {
127+
throw new Error(`Unexpected path format: ${url.pathname}`);
128+
}
129+
const geocodes = `/${segments[2]}/${segments[3]}/${segments[4]}`;
130+
131+
const paramNameMap = {
132+
heatingtypes: 'heatingtypes',
133+
haspromotion: 'haspromotion',
134+
numberofrooms: 'numberofrooms',
135+
livingspace: 'livingspace',
136+
energyefficiencyclasses: 'energyefficiencyclasses',
137+
exclusioncriteria: 'exclusioncriteria',
138+
equipment: 'equipment',
139+
petsallowedtypes: 'petsallowedtypes',
140+
price: 'price',
141+
constructionyear: 'constructionyear',
142+
apartmenttypes: 'apartmenttypes',
143+
pricetype: 'pricetype',
144+
floor: 'floor'
145+
};
146+
147+
const equipmentValueMap = {
148+
parking: 'parking',
149+
cellar: 'cellar',
150+
builtinkitchen: 'builtInKitchen',
151+
lift: 'lift',
152+
garden: 'garden',
153+
guesttoilet: 'guestToilet',
154+
balcony: 'balcony'
155+
};
156+
157+
const { query: webParams } = queryString.parseUrl(webUrl, { arrayFormat: 'comma' });
158+
delete webParams['enteredFrom'];
159+
160+
// Check for unsupported parameters
161+
Object.keys(webParams).forEach((key) => {
162+
if (!paramNameMap[key]) {
163+
throw new Error(`Unsupported Web-API parameter: "${key}"`);
164+
}
165+
});
166+
167+
// Build mobile params
168+
const mobileParams = {
169+
searchType: 'region',
170+
geocodes,
171+
realestatetype: 'apartmentrent'
172+
};
173+
174+
Object.entries(webParams).forEach(([webKey, webVal]) => {
175+
let value = webVal;
176+
177+
if (webKey === 'equipment') {
178+
// Map equipment list to camelCase values
179+
if (!Array.isArray(value)) {
180+
value = ('' + value).split(',');
181+
}
182+
value = value.map((token) => {
183+
const lower = token.toLowerCase();
184+
if (!equipmentValueMap[lower]) {
185+
throw new Error(`Unknown equipment type: "${token}"`);
186+
}
187+
return equipmentValueMap[lower];
188+
});
189+
}
190+
191+
mobileParams[paramNameMap[webKey]] = value;
192+
});
193+
194+
const mobileQuery = queryString.stringify(mobileParams, {
195+
arrayFormat: 'comma',
196+
encode: true,
197+
skipEmptyString: true
198+
});
199+
200+
return `https://api.mobile.immobilienscout24.de/search/list?${mobileQuery}`;
201+
}
202+
203+
export { config };

‎test/provider/immoscout-mobile.test.js‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import {expect} from 'chai';
2+
import {convertWebToMobile} from '../../lib/provider/immoscout-mobile.js';
3+
import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
4+
import {mockFredy, providerConfig} from '../utils.js';
5+
import {get} from '../mocks/mockNotification.js';
6+
import * as provider from '../../lib/provider/immoscout-mobile.js';
7+
8+
describe('#immoscout-mobile provider testsuite()', () => {
9+
after(() => {
10+
similarityCache.stopCacheCleanup();
11+
});
12+
13+
provider.init(providerConfig.immoscout, [], []);
14+
it('should test immoscout-mobile provider', async () => {
15+
const Fredy = await mockFredy();
16+
return await new Promise((resolve) => {
17+
const fredy = new Fredy(provider.config, null, provider.metaInformation.id, '', similarityCache);
18+
fredy.execute().then((listings) => {
19+
expect(listings).to.be.a('array');
20+
const notificationObj = get();
21+
expect(notificationObj).to.be.a('object');
22+
expect(notificationObj.serviceName).to.equal('immoscout-mobile');
23+
notificationObj.payload.forEach((notify) => {
24+
/** check the actual structure **/
25+
expect(notify.id).to.be.a('string');
26+
expect(notify.price).to.be.a('string');
27+
expect(notify.size).to.be.a('string');
28+
expect(notify.title).to.be.a('string');
29+
expect(notify.link).to.be.a('string');
30+
expect(notify.address).to.be.a('string');
31+
/** check the values if possible **/
32+
expect(notify.size).to.be.not.empty;
33+
expect(notify.title).to.be.not.empty;
34+
expect(notify.link).that.does.include('https://www.immobilienscout24.de/');
35+
});
36+
resolve();
37+
});
38+
});
39+
});
40+
});
41+
42+
describe('#immoscout-mobile URL conversion', () => {
43+
// Test URL conversion
44+
it('should convert a full web URL to mobile URL', () => {
45+
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?heatingtypes=central,selfcontainedcentral&haspromotion=false&numberofrooms=2.0-5.0&livingspace=10.0-25.0&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&exclusioncriteria=projectlisting,swapflat&equipment=parking,cellar,builtinkitchen,lift,garden,guesttoilet,balcony&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&constructionyear=1920-2026&apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&pricetype=calculatedtotalrent&floor=2-7&enteredFrom=result_list';
46+
const expectedMobileUrl = 'https://api.mobile.immobilienscout24.de/search/list?apartmenttypes=halfbasement,penthouse,other,loft,groundfloor,terracedflat,raisedgroundfloor,roofstorey,apartment,maisonette&constructionyear=1920-2026&energyefficiencyclasses=a,b,c,d,e,f,g,h,a_plus&equipment=parking,cellar,builtInKitchen,lift,garden,guestToilet,balcony&exclusioncriteria=projectlisting,swapflat&floor=2-7&geocodes=%2Fde%2Fberlin%2Fberlin&haspromotion=false&heatingtypes=central,selfcontainedcentral&livingspace=10.0-25.0&numberofrooms=2.0-5.0&petsallowedtypes=no,yes,negotiable&price=10.0-100.0&pricetype=calculatedtotalrent&realestatetype=apartmentrent&searchType=region';
47+
48+
const actualMobileUrl = convertWebToMobile(webUrl);
49+
expect(actualMobileUrl).to.equal(expectedMobileUrl);
50+
});
51+
52+
// Test URL conversion with unsupported query parameters
53+
it('should throw an error for unsupported query parameters', () => {
54+
const webUrl = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?minimuminternetspeed=100000';
55+
56+
expect(() => convertWebToMobile(webUrl)).to.throw('Unsupported Web-API parameter: "minimuminternetspeed"');
57+
});
58+
59+
// Test URL conversion with invalid URL
60+
it('should throw an error for invalid URL', () => {
61+
const invalidUrl = 'invalid-url';
62+
63+
expect(() => convertWebToMobile(invalidUrl)).to.throw('Invalid URL: invalid-url');
64+
});
65+
66+
// Test URL conversion with unexpected path format
67+
it('should throw an error for unexpected path format', () => {
68+
const webUrl = 'https://www.immobilienscout24.de/invalid/path/format';
69+
70+
expect(() => convertWebToMobile(webUrl)).to.throw('Unexpected path format: /invalid/path/format');
71+
});
72+
});

0 commit comments

Comments
 (0)