Open a Character.AI data export (ZIP or JSON) locally and save chats as .txt. Pure parser ported from kamiko.fun's hosted viewer, including the nested turn_key.chat_id split; minimal dependency-free ZIP reader using DecompressionStream in the browser and zlib in Node; synthetic sample fixture; MIT. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
367 lines
15 KiB
JavaScript
367 lines
15 KiB
JavaScript
/**
|
|
* parser.js — read a Character.AI data export, entirely in the caller's memory.
|
|
*
|
|
* Character.AI does not document its export. What is publicly described: a ZIP whose recent form
|
|
* holds `user.json`, `character.json` and `message.json`, with message text in `raw_content`, and
|
|
* older exports that use other layouts and field names. So this parser does not trust file names or
|
|
* one schema. It walks whatever JSON it is given and recognises two things by shape:
|
|
*
|
|
* - a CHARACTER: an object with a name and at least one of greeting, description, definition
|
|
* - a MESSAGE: an object carrying text under one of a short list of known keys
|
|
*
|
|
* Rules it holds to:
|
|
* - PURE. No network, no storage, no DOM. Input is file names plus their text, output is data.
|
|
* - It never invents a speaker. A label comes from a name the file carries, or from a flag the
|
|
* file carries that says a human wrote the line. A message with neither is shown without one.
|
|
* - It never reorders by guesswork. Messages are sorted by time only when every message in the
|
|
* chat has a readable timestamp; otherwise they stay in file order.
|
|
* - Unknown shapes produce a plain result with zero chats and zero characters. Nothing throws.
|
|
*
|
|
* This file is a plain script: it works in a browser via <script src="parser.js"> (exposes
|
|
* `window.CaiExport`) and in Node via `require('./parser.js')`. No dependencies.
|
|
*
|
|
* Shapes it returns:
|
|
* CaiSourceFile { name, text }
|
|
* CaiCharacter { id, name, description, greeting, definition, sourceFile }
|
|
* CaiMessage { speaker, text, time } speaker '' = file did not say; time '' = none
|
|
* CaiChat { key, title, characterName, sourceFile, messages }
|
|
* CaiExport { characters, chats, unreadable, messageCount }
|
|
*/
|
|
(function (root, factory) {
|
|
const api = factory();
|
|
if (typeof module === 'object' && module.exports) module.exports = api;
|
|
root.CaiExport = api;
|
|
})(typeof globalThis !== 'undefined' ? globalThis : this, function () {
|
|
'use strict';
|
|
|
|
const TEXT_KEYS = ['raw_content', 'text', 'content', 'message', 'body'];
|
|
const NAME_KEYS = ['name', 'participant__name', 'character_name', 'char_name', 'title'];
|
|
const GREETING_KEYS = ['greeting', 'first_message', 'intro'];
|
|
const DESCRIPTION_KEYS = ['description', 'short_description', 'tagline'];
|
|
const DEFINITION_KEYS = ['definition', 'long_description'];
|
|
const CHARACTER_ID_KEYS = ['external_id', 'character_id', 'character_external_id', 'character__external_id', 'char_id', 'identifier', 'id'];
|
|
const CHARACTER_REF_KEYS = ['character_id', 'character_external_id', 'character__external_id', 'char_id', 'character'];
|
|
const CHAT_ID_KEYS = ['chat_id', 'conversation_id', 'history_id', 'history_external_id', 'chat__external_id', 'chat_external_id', 'thread_id'];
|
|
const CONTAINER_ID_KEYS = [...CHAT_ID_KEYS, 'external_id', 'id'];
|
|
const SPEAKER_KEYS = ['src__name', 'author__name', 'author_name', 'sender_name', 'speaker', 'src__user__username'];
|
|
const HUMAN_FLAG_KEYS = ['is_human', 'src__is_human', 'author__is_human', 'is_user'];
|
|
const TIME_KEYS = ['create_time', 'created', 'created_at', 'timestamp', 'send_date', 'time', 'date'];
|
|
|
|
/** A ceiling on recursion so a hostile, deeply nested file cannot overflow the stack. */
|
|
const MAX_DEPTH = 40;
|
|
const HUMAN_LABEL = 'You';
|
|
|
|
function isObject(v) {
|
|
return !!v && typeof v === 'object' && !Array.isArray(v);
|
|
}
|
|
|
|
function firstString(obj, keys) {
|
|
for (const key of keys) {
|
|
const v = obj[key];
|
|
if (typeof v === 'string' && v.trim()) return v.trim();
|
|
if (typeof v === 'number' && Number.isFinite(v)) return String(v);
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function baseName(path) {
|
|
return path.split(/[\\/]/).pop() || path;
|
|
}
|
|
|
|
/** The text of a message-shaped object, or '' when it is not one. */
|
|
function messageText(obj) {
|
|
for (const key of TEXT_KEYS) {
|
|
const v = obj[key];
|
|
if (typeof v === 'string' && v.trim()) return v.replace(/\r\n/g, '\n').trim();
|
|
}
|
|
// Some layouts nest the text in a list of candidates and name the chosen one.
|
|
const candidates = obj.candidates;
|
|
if (Array.isArray(candidates)) {
|
|
const wanted = obj.primary_candidate_id;
|
|
const chosen =
|
|
candidates.find((c) => isObject(c) && wanted !== undefined && c.candidate_id === wanted) ?? candidates[0];
|
|
if (isObject(chosen)) return messageText(chosen);
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function hasCharacterFields(obj) {
|
|
return [...GREETING_KEYS, ...DESCRIPTION_KEYS, ...DEFINITION_KEYS].some(
|
|
(key) => typeof obj[key] === 'string' && obj[key].trim() !== '',
|
|
);
|
|
}
|
|
|
|
/** Keys only the account holder's own profile carries. A profile also has a name and often a
|
|
* description, so without this check the visitor's own account is listed as one of their characters. */
|
|
const ACCOUNT_KEYS = ['email', 'username'];
|
|
|
|
function isAccountProfile(obj) {
|
|
return ACCOUNT_KEYS.some((key) => typeof obj[key] === 'string' && obj[key].trim() !== '');
|
|
}
|
|
|
|
function isCharacter(obj) {
|
|
return (
|
|
firstString(obj, NAME_KEYS) !== '' &&
|
|
hasCharacterFields(obj) &&
|
|
messageText(obj) === '' &&
|
|
!isAccountProfile(obj)
|
|
);
|
|
}
|
|
|
|
function toIso(raw) {
|
|
if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) {
|
|
// Seconds or milliseconds since the epoch.
|
|
const ms = raw < 1e11 ? raw * 1000 : raw;
|
|
const d = new Date(ms);
|
|
return Number.isNaN(d.getTime()) ? '' : d.toISOString();
|
|
}
|
|
if (typeof raw === 'string' && raw.trim()) {
|
|
const d = new Date(raw);
|
|
return Number.isNaN(d.getTime()) ? '' : d.toISOString();
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function messageTime(obj) {
|
|
for (const key of TIME_KEYS) {
|
|
const iso = toIso(obj[key]);
|
|
if (iso) return iso;
|
|
}
|
|
return '';
|
|
}
|
|
|
|
/** `true`/`false` when the file says who wrote the line, `null` when it does not. */
|
|
function humanFlag(obj) {
|
|
for (const key of HUMAN_FLAG_KEYS) {
|
|
if (typeof obj[key] === 'boolean') return obj[key];
|
|
}
|
|
for (const key of ['author', 'src', 'sender']) {
|
|
const nested = obj[key];
|
|
if (isObject(nested)) {
|
|
if (typeof nested.is_human === 'boolean') return nested.is_human;
|
|
}
|
|
}
|
|
const role = typeof obj.role === 'string' ? obj.role.toLowerCase() : '';
|
|
if (role === 'user' || role === 'human') return true;
|
|
if (role === 'assistant' || role === 'char' || role === 'character' || role === 'ai' || role === 'bot') return false;
|
|
return null;
|
|
}
|
|
|
|
function explicitSpeaker(obj) {
|
|
const flat = firstString(obj, SPEAKER_KEYS);
|
|
if (flat) return flat;
|
|
for (const key of ['author', 'src', 'sender']) {
|
|
const nested = obj[key];
|
|
if (typeof nested === 'string' && nested.trim()) return nested.trim();
|
|
if (isObject(nested)) {
|
|
const name = firstString(nested, ['name', 'username', 'user__username']);
|
|
if (name) return name;
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function characterRef(obj) {
|
|
for (const key of CHARACTER_REF_KEYS) {
|
|
const v = obj[key];
|
|
if (typeof v === 'string' && v.trim()) return v.trim();
|
|
if (typeof v === 'number') return String(v);
|
|
if (isObject(v)) {
|
|
const id = firstString(v, CHARACTER_ID_KEYS);
|
|
if (id) return id;
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function isMessageArray(arr) {
|
|
if (arr.length === 0) return false;
|
|
let hits = 0;
|
|
for (const item of arr) {
|
|
if (isObject(item) && messageText(item) !== '' && !isCharacter(item)) hits += 1;
|
|
}
|
|
return hits > 0 && hits >= arr.length / 2;
|
|
}
|
|
|
|
/** A message's own chat id, checked both flat on the object and nested one level under a key such
|
|
* as `turn_key` — real Character.AI turns carry `{ turn_key: { chat_id, turn_id } }` rather than a
|
|
* flat `chat_id` (confirmed against multiple independent captures of the live chat API, which the
|
|
* export reuses the schema of), so without this nested check every turn in a flat top-level
|
|
* message.json array collapses into one chat per file, mixing unrelated characters' conversations. */
|
|
function ownChatIdOf(obj) {
|
|
const flat = firstString(obj, CHAT_ID_KEYS);
|
|
if (flat) return flat;
|
|
const turnKey = obj.turn_key;
|
|
return isObject(turnKey) ? firstString(turnKey, CHAT_ID_KEYS) : '';
|
|
}
|
|
|
|
function addMessage(out, ctx, obj, fallbackKey) {
|
|
const text = messageText(obj);
|
|
if (!text) return;
|
|
const ownChatId = ownChatIdOf(obj);
|
|
const characterId = characterRef(obj) || ctx.characterId;
|
|
const key = ownChatId
|
|
? `chat:${ownChatId}`
|
|
: ctx.chatKey || (characterId ? `character:${characterId}` : fallbackKey);
|
|
const existing = out.chats.get(key);
|
|
const chat = existing ?? {
|
|
key,
|
|
sourceFile: ctx.file,
|
|
characterId,
|
|
inlineCharacterName: ctx.inlineCharacterName,
|
|
messages: [],
|
|
};
|
|
if (!existing) out.chats.set(key, chat);
|
|
if (!chat.characterId && characterId) chat.characterId = characterId;
|
|
chat.messages.push({ speaker: explicitSpeaker(obj), human: humanFlag(obj), text, time: messageTime(obj) });
|
|
}
|
|
|
|
function walk(node, ctx, out, path, depth) {
|
|
if (depth > MAX_DEPTH) return;
|
|
|
|
if (Array.isArray(node)) {
|
|
if (isMessageArray(node)) {
|
|
const fallbackKey = `file:${ctx.file}#${path}`;
|
|
for (const item of node) {
|
|
if (isObject(item)) addMessage(out, ctx, item, fallbackKey);
|
|
}
|
|
return;
|
|
}
|
|
node.forEach((item, i) => walk(item, ctx, out, `${path}[${i}]`, depth + 1));
|
|
return;
|
|
}
|
|
|
|
if (!isObject(node)) return;
|
|
|
|
// user.json describes the account holder, never a character, whatever fields it carries.
|
|
if (isCharacter(node) && !/^user\.json$/i.test(baseName(ctx.file))) {
|
|
out.characters.push({
|
|
id: firstString(node, CHARACTER_ID_KEYS),
|
|
name: firstString(node, NAME_KEYS),
|
|
description: firstString(node, DESCRIPTION_KEYS),
|
|
greeting: firstString(node, GREETING_KEYS),
|
|
definition: firstString(node, DEFINITION_KEYS),
|
|
sourceFile: ctx.file,
|
|
});
|
|
}
|
|
|
|
// An object that owns a list is a container: its own id and character link apply to that list.
|
|
const ownsList = Object.values(node).some((v) => Array.isArray(v) && isMessageArray(v));
|
|
const containerId = ownsList ? firstString(node, CONTAINER_ID_KEYS) : '';
|
|
const nestedCharacter = isObject(node.character) ? node.character : null;
|
|
const next = {
|
|
file: ctx.file,
|
|
chatKey: containerId ? `chat:${containerId}` : ctx.chatKey,
|
|
characterId: (isCharacter(node) ? firstString(node, CHARACTER_ID_KEYS) : characterRef(node)) || ctx.characterId,
|
|
inlineCharacterName:
|
|
(isCharacter(node) ? firstString(node, NAME_KEYS) : '') ||
|
|
(nestedCharacter ? firstString(nestedCharacter, NAME_KEYS) : '') ||
|
|
firstString(node, ['character_name', 'char_name', 'participant__name']) ||
|
|
ctx.inlineCharacterName,
|
|
};
|
|
|
|
for (const [key, value] of Object.entries(node)) {
|
|
if (value && typeof value === 'object') walk(value, next, out, path ? `${path}.${key}` : key, depth + 1);
|
|
}
|
|
}
|
|
|
|
function dedupeCharacters(list) {
|
|
const seen = new Map();
|
|
for (const c of list) {
|
|
const key = c.id ? `id:${c.id}` : `name:${c.name.toLowerCase()}`;
|
|
const prior = seen.get(key);
|
|
if (!prior) {
|
|
seen.set(key, c);
|
|
continue;
|
|
}
|
|
seen.set(key, {
|
|
...prior,
|
|
description: prior.description || c.description,
|
|
greeting: prior.greeting || c.greeting,
|
|
definition: prior.definition || c.definition,
|
|
});
|
|
}
|
|
return [...seen.values()];
|
|
}
|
|
|
|
function finishChat(raw, characters, index) {
|
|
const linked = raw.characterId ? characters.find((c) => c.id !== '' && c.id === raw.characterId) : undefined;
|
|
// Real turns rarely carry a per-turn character id (the id is chat metadata the export does not
|
|
// always repeat on every message), so without this the title/characterName stays blank for most
|
|
// real chats even though the file names the character on every one of its own bot turns. This
|
|
// reads a name the file already gave a line it itself marked as not-human; it invents nothing.
|
|
const nonHumanSpeaker = raw.messages.find((m) => m.human === false && m.speaker)?.speaker ?? '';
|
|
const characterName = (linked && linked.name) || raw.inlineCharacterName || nonHumanSpeaker;
|
|
|
|
const allTimed = raw.messages.every((m) => m.time !== '');
|
|
const ordered = allTimed
|
|
? raw.messages
|
|
.map((m, i) => ({ m, i }))
|
|
.sort((a, b) => a.m.time.localeCompare(b.m.time) || a.i - b.i)
|
|
.map((x) => x.m)
|
|
: raw.messages;
|
|
|
|
const messages = ordered.map((m) => {
|
|
let speaker = m.speaker;
|
|
if (!speaker && m.human === true) speaker = HUMAN_LABEL;
|
|
// A line the file marks as not human is the character's, but only name it when the file links one.
|
|
if (!speaker && m.human === false && characterName) speaker = characterName;
|
|
return { speaker, text: m.text, time: m.time };
|
|
});
|
|
|
|
const title = characterName ? `Chat with ${characterName}` : `Chat ${index + 1} from ${baseName(raw.sourceFile)}`;
|
|
return { key: raw.key, title, characterName, sourceFile: raw.sourceFile, messages };
|
|
}
|
|
|
|
/** PURE. Parse every given file and return whatever characters and chats could be recognised. */
|
|
function parseCaiExport(files) {
|
|
const out = { characters: [], chats: new Map() };
|
|
const unreadable = [];
|
|
|
|
for (const file of files) {
|
|
let data;
|
|
try {
|
|
data = JSON.parse(file.text.replace(/^\uFEFF/, ''));
|
|
} catch {
|
|
unreadable.push(file.name);
|
|
continue;
|
|
}
|
|
walk(data, { file: file.name, chatKey: '', characterId: '', inlineCharacterName: '' }, out, '', 0);
|
|
}
|
|
|
|
const characters = dedupeCharacters(out.characters);
|
|
const chats = [...out.chats.values()]
|
|
.filter((c) => c.messages.length > 0)
|
|
.map((c, i) => finishChat(c, characters, i));
|
|
const messageCount = chats.reduce((sum, c) => sum + c.messages.length, 0);
|
|
return { characters, chats, unreadable, messageCount };
|
|
}
|
|
|
|
/** PURE. One chat as plain text: a short header, then one block per message in order. */
|
|
function chatToText(chat) {
|
|
const header = [chat.title, `Source file: ${baseName(chat.sourceFile)}`, `Messages: ${chat.messages.length}`];
|
|
const body = chat.messages.map((m) => {
|
|
const stamp = m.time ? `[${m.time.slice(0, 16).replace('T', ' ')}] ` : '';
|
|
return m.speaker ? `${stamp}${m.speaker}: ${m.text}` : `${stamp}${m.text}`;
|
|
});
|
|
return `${header.join('\n')}\n\n${body.join('\n\n')}\n`;
|
|
}
|
|
|
|
/** PURE. Every chat in one text file, separated by a rule. */
|
|
function allChatsToText(chats) {
|
|
return chats.map(chatToText).join('\n' + '='.repeat(60) + '\n\n');
|
|
}
|
|
|
|
/** PURE. A safe download name for one chat. */
|
|
function chatFileName(chat, index) {
|
|
const slug = (chat.characterName || `chat-${index + 1}`)
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/g, '-')
|
|
.replace(/^-+|-+$/g, '')
|
|
.slice(0, 40);
|
|
return `character-ai-${slug || `chat-${index + 1}`}-${index + 1}.txt`;
|
|
}
|
|
|
|
return { parseCaiExport, chatToText, allChatsToText, chatFileName, HUMAN_LABEL };
|
|
});
|