Horizon-Labs commited on
Commit
05c5a2c
·
verified ·
1 Parent(s): 139993b

Multilingual toxicity demo (transformers.js)

Browse files
Files changed (2) hide show
  1. README.md +9 -4
  2. index.html +66 -19
README.md CHANGED
@@ -1,10 +1,15 @@
1
  ---
2
  title: Multilingual Toxicity
3
- emoji: 🌍
4
- colorFrom: purple
5
  colorTo: gray
6
  sdk: static
7
- pinned: false
 
 
 
 
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
1
  ---
2
  title: Multilingual Toxicity
3
+ emoji: 🧹
4
+ colorFrom: red
5
  colorTo: gray
6
  sdk: static
7
+ pinned: true
8
+ license: apache-2.0
9
+ short_description: Detoxify-style toxicity scores in 34 languages, in-browser
10
+ models:
11
+ - Horizon-Labs/multilingual-toxicity-small
12
  ---
13
 
14
+ Runs [Horizon-Labs/multilingual-toxicity-small](https://huggingface.co/Horizon-Labs/multilingual-toxicity-small) (int8-embedding
15
+ ONNX, downloaded once and cached) in your browser with transformers.js. Nothing is uploaded.
index.html CHANGED
@@ -1,19 +1,66 @@
1
- <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width" />
6
- <title>My static Space</title>
7
- <link rel="stylesheet" href="style.css" />
8
- </head>
9
- <body>
10
- <div class="card">
11
- <h1>Welcome to your static Space!</h1>
12
- <p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
13
- <p>
14
- Also don't forget to check the
15
- <a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
16
- </p>
17
- </div>
18
- </body>
19
- </html>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html><html lang="en"><head><meta charset="utf-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/>
2
+ <title>Multilingual Toxicity</title>
3
+ <style>
4
+ :root{--bg:#fafafa;--fg:#1a1a1a;--muted:#666;--card:#fff;--border:#ddd;--accent:#5f3dc4}
5
+ @media (prefers-color-scheme:dark){:root{--bg:#111;--fg:#eee;--muted:#aaa;--card:#1b1b1b;--border:#333;--accent:#9775fa}}
6
+ body{font-family:system-ui,sans-serif;background:var(--bg);color:var(--fg);max-width:820px;margin:2rem auto;padding:0 1rem;line-height:1.5}
7
+ textarea{width:100%;font:15px/1.4 system-ui,sans-serif;padding:.6rem;box-sizing:border-box;background:var(--card);color:var(--fg);border:1px solid var(--border);border-radius:8px}
8
+ .out{background:var(--card);border:1px solid var(--border);border-radius:8px;padding:.8rem;min-height:2rem}
9
+ .item{padding:.45rem 0;border-bottom:1px solid var(--border)} .item:last-child{border-bottom:0} .txt{font-size:14px;margin-bottom:.25rem}
10
+ .stack{display:flex;height:12px;border-radius:6px;overflow:hidden;background:var(--border)} .neg{background:#e03131} .neu{background:#adb5bd} .pos{background:#2f9e44}
11
+ .lab{font-weight:600} .lab.negative{color:#e03131} .lab.positive{color:#2f9e44} .lab.neutral{color:var(--muted)}
12
+ .ex{display:inline-block;margin:.2rem .3rem .2rem 0;padding:.2rem .6rem;border:1px solid var(--border);border-radius:14px;cursor:pointer;font-size:13px;background:var(--card)}
13
+ .muted{color:var(--muted)} code{font-size:13px}
14
+ </style></head><body>
15
+ <h1>🧹 Multilingual Toxicity</h1>
16
+ <p class="muted">Scores comments for <b>toxicity, severe_toxicity, obscene, threat, insult, identity_attack, sexual_explicit</b>
17
+ (the Detoxify labels) in English and 33 other languages with
18
+ <a href="https://huggingface.co/Horizon-Labs/multilingual-toxicity-small">multilingual-toxicity-small</a> (141M, Apache-2.0),
19
+ running entirely in your browser. One comment per line; labels at or above 0.5 are bold.</p>
20
+ <div id="ex"></div>
21
+ <p><textarea id="t" rows="5"></textarea></p>
22
+ <p class="muted" id="status">Loading model…</p>
23
+ <div class="out" id="out"></div>
24
+ <p class="muted">Toxicity judgements are subjective; the model can over-flag mentions of identity groups and miss sarcasm. See
25
+ the model card for measured accuracy per language.</p>
26
+ <script type="module">
27
+ import { AutoTokenizer, AutoModelForSequenceClassification } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3";
28
+ const M = "Horizon-Labs/multilingual-toxicity-small";
29
+ const EX = ["Thanks for sharing, this was really helpful!\nYou are a complete idiot and nobody wants you here.\nI will find you and make you pay for this.",
30
+ "Halt die Klappe, du Vollidiot.\nEres un inútil, lárgate de aquí.\nMerci beaucoup pour votre aide, bonne journée !",
31
+ "Ты просто тупой, иди отсюда.\nこの記事はとても分かりやすかったです。\nتبا لك يا غبي"];
32
+ const NAMES = ["English", "German · Spanish · French", "Russian · Japanese · Arabic"];
33
+ const $ = id => document.getElementById(id);
34
+ $("ex").innerHTML = NAMES.map((e, i) => `<span class="ex" data-i="${i}">${e}</span>`).join("");
35
+ $("t").value = EX[0];
36
+ const esc = s => s.replace(/[&<>"]/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" })[c]);
37
+ let tok, model, labels, thr = {}, timer;
38
+ async function run() {
39
+ if (!model) return; const lines = $("t").value.split("\n").map(x => x.trim()).filter(Boolean).slice(0, 20);
40
+ if (!lines.length) { $("out").innerHTML = ""; return; }
41
+ const t0 = performance.now(); const html = [];
42
+ for (const t of lines) {
43
+ const { logits } = await model(await tok(t, { truncation: true, max_length: 256 }));
44
+ const p = Array.from(logits.data).map(v => 1 / (1 + Math.exp(-v)));
45
+ const top = p.map((v, i) => [labels[i], v]).sort((a, b) => b[1] - a[1]).slice(0, 7);
46
+ html.push(`<div class="item"><div class="txt">${esc(t)}</div>` + top.map(([l, v]) => {
47
+ const on = v >= (thr[l] ?? 0.5);
48
+ return `<div style="display:flex;gap:.6rem;align-items:center;margin:.1rem 0"><span style="width:9rem;${on ? "font-weight:700" : "color:var(--muted)"}">${l}</span>
49
+ <span class="stack" style="flex:1"><span class="pos" style="width:${(100 * v).toFixed(1)}%;${on ? "" : "opacity:.35"}"></span></span><span class="muted" style="width:3.5rem;font-size:12px">${(100 * v).toFixed(0)}%</span></div>`;
50
+ }).join("") + `</div>`);
51
+ }
52
+ $("out").innerHTML = html.join("");
53
+ $("status").textContent = `${lines.length} text(s) in ${(performance.now() - t0).toFixed(0)} ms (running locally)`;
54
+ }
55
+ $("t").oninput = () => { clearTimeout(timer); timer = setTimeout(run, 250); };
56
+ document.querySelectorAll(".ex").forEach(el => el.onclick = () => { $("t").value = EX[+el.dataset.i]; run(); });
57
+ (async () => {
58
+ try {
59
+ tok = await AutoTokenizer.from_pretrained(M);
60
+ model = await AutoModelForSequenceClassification.from_pretrained(M, { dtype: "q8",
61
+ progress_callback: x => { if (x.status === "progress" && x.file.endsWith(".onnx")) $("status").textContent = `Downloading model… ${x.progress.toFixed(0)}%`; } });
62
+ labels = Object.keys(model.config.id2label).sort((a, b) => a - b).map(k => model.config.id2label[k]);
63
+ run();
64
+ } catch (e) { $("status").textContent = "Failed to load model: " + e; }
65
+ })();
66
+ </script></body></html>