Repository navigation
Expand file tree
/
Copy pathtwitter.js
More file actions
139 lines (127 loc) · 4.37 KB
/
Copy pathtwitter.js
File metadata and controls
139 lines (127 loc) · 4.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
(() => {
// Runtime settings for scrolling and exit conditions.
const config = {
intervalMs: 1200,
scrollBy: 900,
maxIdleRounds: 6,
maxRows: 0,
};
// In-memory storage and dedupe state.
const rows = [];
const seen = new Set();
let idleRounds = 0;
// Text normalization for stable CSV output.
const normalizeText = (s) => (s || "").replace(/\s+/g, " ").trim();
// Extract a stable tweet id from status links.
const getTweetId = (article) => {
const statusLink = article.querySelector('a[href*="/status/"]');
const href = statusLink ? statusLink.getAttribute("href") : "";
const m = href ? href.match(/\/status\/(\d+)/) : null;
return m ? m[1] : href || "";
};
// Extract author handle from the header block.
const getUsername = (article) => {
const nameBlock = article.querySelector('div[data-testid="User-Name"]');
if (!nameBlock) return "";
const spanList = Array.from(nameBlock.querySelectorAll("span"));
const handle = spanList.map((s) => normalizeText(s.textContent)).find((t) => t.startsWith("@"));
if (handle) return handle.slice(1);
return normalizeText(nameBlock.textContent).replace(/^@/, "");
};
// Extract post time from the <time> element.
const getTime = (article) => {
const timeEl = article.querySelector("time");
return timeEl ? timeEl.getAttribute("datetime") || "" : "";
};
// Extract main tweet text content.
const getContent = (article) => {
const textEl = article.querySelector('div[data-testid="tweetText"]');
return normalizeText(textEl ? textEl.innerText : "");
};
// Scan visible tweets and append new rows.
const collect = () => {
const articles = Array.from(document.querySelectorAll('article[data-testid="tweet"]'));
let added = 0;
for (const article of articles) {
const id = getTweetId(article);
if (!id || seen.has(id)) continue;
const username = getUsername(article);
const time = getTime(article);
const content = getContent(article);
if (!username && !time && !content) continue;
seen.add(id);
rows.push({
index: rows.length + 1,
username,
time,
content,
});
console.log("Collected:", { username, time, content });
added += 1;
if (config.maxRows > 0 && rows.length >= config.maxRows) break;
}
return added;
};
// Convert collected rows to CSV string.
const toCSV = (data) => {
const headers = Object.keys(data[0] || {});
const escapeCSV = (v) => {
const s = String(v ?? "");
return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
};
return [
headers.join(","),
...data.map((r) => headers.map((h) => escapeCSV(r[h])).join(",")),
].join("\n");
};
// Trigger a CSV file download in the browser.
const download = () => {
if (!rows.length) {
console.warn("No data collected. Scroll a bit and try again.");
return;
}
const csv = toCSV(rows);
const blob = new Blob(["\uFEFF" + csv], { type: "text/csv;charset=utf-8" });
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = `x_export_${new Date().toISOString().slice(0, 10)}.csv`;
document.body.appendChild(a);
a.click();
a.remove();
URL.revokeObjectURL(url);
console.log(`Exported ${rows.length} rows to CSV.`);
};
// Main loop: collect, decide stop, then scroll.
const tick = () => {
const added = collect();
if (added === 0) idleRounds += 1;
else idleRounds = 0;
if (added > 0) {
console.log(`Added ${added} new tweets. Total: ${rows.length}.`);
} else {
console.log("No new tweets found in this pass.");
}
if (config.maxRows > 0 && rows.length >= config.maxRows) {
stop("Reached maxRows.");
return;
}
if (idleRounds >= config.maxIdleRounds) {
stop("No new tweets found for a while.");
return;
}
window.scrollBy({ top: config.scrollBy, behavior: "smooth" });
};
// Stop the loop and export.
const stop = (reason) => {
clearInterval(timerId);
console.log(reason || "Stopped.");
download();
};
// Start the loop and expose a manual stop hook.
const timerId = setInterval(tick, config.intervalMs);
window.__x_exporter_stop = () => stop("Stopped by user.");
console.log(
"X exporter started. Use window.__x_exporter_stop() to stop and export."
);
})();