Update DataDome module (vercel#868)

### Description

Update the DataDome module to retrieve current supported signals.
Fixes a blocking issue with redirection.

### Demo URL
https://vercel-examples-ochre.vercel.app/

### Changelog

- [x] Fix redirection issue
- [x] Improve payload content
- [x] Update variable names and documentation 
- [x] Remove obsolete code

edge-middleware/bot-protection-datadome/.env.example

@@ -1,2 +1,2 @@
-NEXT_PUBLIC_DATADOME_CLIENT_KEY =
-DATADOME_SERVER_KEY =
+DATADOME_SERVER_SIDE_KEY =
+NEXT_PUBLIC_DATADOME_CLIENT_SIDE_KEY =

edge-middleware/bot-protection-datadome/README.md

@@ -5,7 +5,7 @@ description: DataDome can provide real-time bot protection and other security pr
 framework: Next.js
 useCase: Edge Middleware
 css: Tailwind
-deployUrl: https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fvercel-customer-feedback%2Fedge-middleware%2Ftree%2Fmain%2Fexamples%2Fbot-protection-datadome&env=NEXT_PUBLIC_DATADOME_CLIENT_KEY,DATADOME_SERVER_KEY&project-name=bot-protection-datadome&repository-name=bot-protection-datadome
+deployUrl: https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fvercel-customer-feedback%2Fedge-middleware%2Ftree%2Fmain%2Fexamples%2Fbot-protection-datadome&env=NEXT_PUBLIC_DATADOME_CLIENT_SIDE_KEY,DATADOME_SERVER_SIDE_KEY&project-name=bot-protection-datadome&repository-name=bot-protection-datadome
 demoUrl: https://edge-functions-bot-protection-datadome.vercel.app
 ignoreE2E: true
 relatedTemplates:
@@ -29,7 +29,7 @@ You can choose from one of the following two methods to use this repository:
 
 Deploy the example using [Vercel](https://vercel.com?utm_source=github&utm_medium=readme):
 
-[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fvercel-customer-feedback%2Fedge-middleware%2Ftree%2Fmain%2Fexamples%2Fbot-protection-datadome&env=NEXT_PUBLIC_DATADOME_CLIENT_KEY,DATADOME_SERVER_KEY&project-name=bot-protection-datadome&repository-name=bot-protection-datadome)
+[![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fvercel-customer-feedback%2Fedge-middleware%2Ftree%2Fmain%2Fexamples%2Fbot-protection-datadome&env=NEXT_PUBLIC_DATADOME_CLIENT_SIDE_KEY,DATADOME_SERVER_SIDE_KEY&project-name=bot-protection-datadome&repository-name=bot-protection-datadome)
 
 ### Clone and Deploy
 

edge-middleware/bot-protection-datadome/lib/datadome.ts

@@ -1,82 +1,100 @@
 import { NextRequest, NextResponse } from 'next/server'
 
-const DATADOME_TIMEOUT = 500
-const DATADOME_URI_REGEX_EXCLUSION =
-  /\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav|webm|webp|bmp|gif|ico|jpeg|jpg|png|svg|svgz|swf|eot|otf|ttf|woff|woff2|css|less|js)$/i
+const DATADOME_TIMEOUT = parseInt(process.env.DATADOME_TIMEOUT ?? "300")
+const DATADOME_ENDPOINT = validateEndpoint()
+const DATADOME_URI_REGEX_EXCLUSION = 
+  /\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav|webm|webp|bmp|gif|ico|jpeg|jpg|png|svg|svgz|swf|eot|otf|ttf|woff|woff2|css|less|js|map)$/i
 
 export default async function datadome(req: NextRequest) {
   const { pathname } = req.nextUrl
 
   if (DATADOME_URI_REGEX_EXCLUSION.test(pathname)) {
     return
   }
-
+  let clientId = getCookieData(req.cookies)
+  let cookiesLength = req.headers.get('cookie')?.length ?? 0
   const requestData = {
-    Key: process.env.DATADOME_SERVER_KEY,
-    RequestModuleName: 'Next.js',
-    ModuleVersion: '0.1',
-    ServerName: 'vercel',
+    Key: process.env.DATADOME_SERVER_SIDE_KEY,
     // this should be `x-real-ip` but it doesn't currently work on Edge Middleware
+    // localhost won't likely be blocked by Datadome unless you use your real IP
+    // IP: 'YOUR IP',
     IP: req.headers.get('x-forwarded-for')
       ? req.headers.get('x-forwarded-for')!.split(',')[0]
       : '127.0.0.1',
-    // localhost won't likely be blocked by Datadome unless you use your real IP
-    // IP: 'YOUR IP',
-    Port: 0,
-    TimeRequest: new Date().getTime() * 1000,
-    Protocol: req.headers.get('x-forwarded-proto'),
-    Method: req.method,
-    ServerHostname: req.headers.get('host'),
-    Request: pathname + encode(Object.fromEntries(req.nextUrl.searchParams)),
-    HeadersList: getHeadersList(req),
-    Host: req.headers.get('host'),
-    UserAgent: req.headers.get('user-agent'),
-    Referer: req.headers.get('referer'),
-    // Make sure Datadome always returns a JSON response in case of a 403
-    Accept: 'application/json',
+    RequestModuleName: 'Next.js',
+    ModuleVersion: '0.3.0',
+    AuthorizationLen: getAuthorizationLength(req),
+    Accept: req.headers.get('accept'),
     AcceptEncoding: req.headers.get('accept-encoding'),
     AcceptLanguage: req.headers.get('accept-language'),
     AcceptCharset: req.headers.get('accept-charset'),
-    Origin: req.headers.get('origin'),
-    XForwaredForIP: req.headers.get('x-forwarded-for'),
-    Connection: req.headers.get('connection'),
-    Pragma: req.headers.get('pragma'),
     CacheControl: req.headers.get('cache-control'),
+    ClientID: clientId,
+    Connection: req.headers.get('connection'),
     ContentType: req.headers.get('content-type'),
+    CookiesLen: cookiesLength,
     From: req.headers.get('from'),
-    Via: req.headers.get('via'),
-    CookiesLen: getCookiesLength(req.cookies),
-    AuthorizationLen: getAuthorizationLength(req),
+    HeadersList: getHeadersList(req),
+    Host: req.headers.get('host'),
+    Method: req.method,
+    Origin: req.headers.get('origin'),
+    Port: 0,
+    Pragma: req.headers.get('pragma'),
     PostParamLen: req.headers.get('content-length'),
-    ClientID: req.cookies.get('datadome')?.value,
+    Protocol: req.headers.get('x-forwarded-proto'),
+    Referer: req.headers.get('referer'),
+    Request: pathname + encode(Object.fromEntries(req.nextUrl.searchParams)),
+    ServerHostname: req.headers.get('host'),
+    ServerName: 'vercel',
     ServerRegion: 'sfo1',
+    TimeRequest: new Date().getTime() * 1000,
+    TrueClientIP: req.headers.get('true-client-ip'),
+    UserAgent: req.headers.get('user-agent'),
+    Via: req.headers.get('via'),
+    XForwardedForIP: req.headers.get('x-forwarded-for'),
+    SecCHDeviceMemory: req.headers.get('sec-ch-device-memory'),
+    SecCHUA: req.headers.get('sec-ch-ua'),
+    SecCHUAArch: req.headers.get('sec-ch-ua-arch'),
+    SecCHUAFullVersionList: req.headers.get('sec-ch-ua-full-version-list'),
+    SecCHUAMobile: req.headers.get('sec-ch-ua-mobile'),
+    SecCHUAModel: req.headers.get('sec-ch-ua-model'),
+    SecCHUAPlatform: req.headers.get('sec-ch-ua-platform'),
+    SecFetchDest: req.headers.get('sec-fetch-dest'),
+    SecFetchMode: req.headers.get('sec-fetch-mode'),
+    SecFetchSite: req.headers.get('sec-fetch-site'),
+    SecFetchUser: req.headers.get('sec-fetch-user'),
+    'X-Real-IP': req.headers.get('x-real-ip'),
+    'X-Requested-With': req.headers.get('x-requested-with'),
   }
-  const dataDomeReq = fetch(
-    'http://api-cloudflare.datadome.co/validate-request/',
-    {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/x-www-form-urlencoded',
-        'User-Agent': 'DataDome',
-      },
-      body: stringify(requestData),
-    }
-  )
+
+  const options = {
+    method: 'POST',
+    body: stringify(truncateRequestData(requestData)),
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded',
+      'User-Agent': 'DataDome'
+    },
+  }
+  if (req.headers.get('x-datadome-clientid')?.length) {
+    options.headers['X-DataDome-X-Set-Cookie'] = 'true'
+    requestData.ClientID = req.headers.get('x-datadome-clientid') as string
+  }
+  const dataDomeReq = fetch(DATADOME_ENDPOINT + '/validate-request/', options)
 
   const timeoutPromise = new Promise((resolve, reject) => {
     setTimeout(() => {
       reject(new Error('Datadome timeout'))
     }, DATADOME_TIMEOUT)
   })
 
-  let dataDomeRes: Response
+  let dataDomeRes: NextResponse
   const dataDomeStart = Date.now()
 
   try {
     dataDomeRes = (await Promise.race([
       dataDomeReq,
       timeoutPromise,
-    ])) as Response
+    ])) as NextResponse
   } catch (err: any) {
     console.error('Datadome failed with:', err.stack)
     return
@@ -91,7 +109,7 @@ export default async function datadome(req: NextRequest) {
   switch (dataDomeRes.status) {
     case 400:
       // Something is wrong with our authentication
-      console.log('DataDome returned 400', dataDomeRes.statusText)
+      console.log('DataDome returned 400', dataDomeRes.statusText, await dataDomeRes.text())
       return
 
     case 200:
@@ -105,18 +123,16 @@ export default async function datadome(req: NextRequest) {
 
       if (dataDomeRes.status !== 200) {
         // blocked!
+        res = new Response(dataDomeRes.body, {status: dataDomeRes.status}) as NextResponse
         const isBot = dataDomeRes.headers.get('x-datadome-isbot')
         if (isBot) {
           console.log(
             'Bot detected. Name:',
             dataDomeRes.headers.get('x-datadome-botname'),
             '– Kind:',
-            dataDomeRes.headers.get('x-datadome-botfamily')
+            dataDomeRes.headers.get('x-datadome-ruletype')
           )
         }
-
-        const data = await dataDomeRes.json()
-        res = NextResponse.rewrite(data.url)
       }
 
       // Add Datadome headers to the response
@@ -125,7 +141,11 @@ export default async function datadome(req: NextRequest) {
           res.headers.set(k, v)
         }
       )
-
+      console.log(
+        'Datadome response debug',
+        res.status,
+        JSON.stringify(Object.fromEntries(res.headers.entries()), null, 2)
+      )
       // We're sending the latency for demo purposes, this is not something you need to do
       res.headers.set('x-datadome-latency', `${Date.now() - dataDomeStart}`)
 
@@ -200,11 +220,81 @@ function stringify(obj: Record<string, string | number | null | undefined>) {
         .join('&')
     : ''
 }
+function truncateRequestData(requestData: Record<string, string | number | null | undefined>) {
+  const limits = {
+    secfetchuser: 8,
+    secchdevicememory: 8,
+    secchuamobile: 8,
+    tlsprotocol: 8,
+    secchuaarch: 16,
+    contenttype: 64,
+    secchuaplatform: 32,
+    secfetchdest: 32,
+    secfetchmode: 32,
+    secfetchsite: 64,
+    tlscipher: 64,
+    clientid: 128,
+    from: 128,
+    "x-requested-with": 128,
+    acceptcharset: 128,
+    acceptencoding: 128,
+    connection: 128,
+    pragma: 128,
+    cachecontrol: 128,
+    secchua: 128,
+    secchuamodel: 128,
+    trueclientip: 128,
+    secchuafullversionlist: 256,
+    acceptlanguage: 256,
+    via: 256,
+    headerslist: 512,
+    origin: 512,
+    serverhostname: 512,
+    servername: 512,
+    xforwardedforip: -512,
+    accept: 512,
+    host: 512,
+    useragent: 768,
+    referer: 1024,
+    request: 2048,
+  };
+
+  for (let key in requestData) {
+    const value = requestData[key];
+    const limit = limits[key.toLowerCase()];
+    if (limit && value && typeof value == 'string' && value.length > Math.abs(limit)) {
+      if (limit > 0) {
+        requestData[key] = value.substring(0, limit);
+      } else {
+        requestData[key] = value.slice(limit);
+      }
+    }
+  }
+  return requestData;
+}
 
-function getCookiesLength(cookies: NextRequest['cookies']) {
-  let cookiesLength = 0
+/**
+ * Returns the client ID from the `datadome` cookie.
+ * @param {NextRequest['cookies']} cookies - Incoming client request cookie header
+ * @returns {{ clientId: string }}
+ */
+function getCookieData(cookies: NextRequest['cookies']) {
   for (const [, cookie] of cookies) {
-    cookiesLength += cookie.value.length
+    if (cookie.name == 'datadome') {
+      return cookie.value
+    }
+  }
+  return '';
+}
+
+/**
+ *
+ * @returns the default endpoint for the API if not set, enforcing https otherwise
+ */
+function validateEndpoint() {
+  let endpoint = process.env.DATADOME_ENDPOINT ?? 'https://api.datadome.co'
+  if (!(/https?:\/\//i).test(endpoint)) {
+    endpoint = "https://" + endpoint
   }
-  return cookiesLength
+  return endpoint
 }

edge-middleware/bot-protection-datadome/pages/_app.tsx

@@ -13,14 +13,14 @@ function MyApp({ Component, pageProps }: AppProps) {
       title="Bot Protection with DataDome"
       path="edge-middleware/bot-protection-datadome"
       deployButton={{
-        env: ['NEXT_PUBLIC_DATADOME_CLIENT_KEY', 'DATADOME_SERVER_KEY'],
+        env: ['NEXT_PUBLIC_DATADOME_CLIENT_SIDE_KEY', 'DATADOME_SERVER_SIDE_KEY'],
       }}
     >
       <Component {...pageProps} />
 
       {/* datadome bot protection */}
       <Script strategy="lazyOnload" id="load-datadome">{`
-        window.ddjskey = '${process.env.NEXT_PUBLIC_DATADOME_CLIENT_KEY}'
+        window.ddjskey = '${process.env.NEXT_PUBLIC_DATADOME_CLIENT_SIDE_KEY}'
         window.ddoptions = {
           endpoint: '${DATADOME_JS}'
         }

edge-middleware/bot-protection-datadome/tsconfig.json

@@ -5,7 +5,7 @@
     "lib": ["dom", "dom.iterable", "esnext"],
     "allowJs": true,
     "skipLibCheck": true,
-    "strict": true,
+    "strict": false,
     "forceConsistentCasingInFileNames": true,
     "noEmit": true,
     "esModuleInterop": true,