tools/ejabberdsql2prosody.lua

   1 #!/usr/bin/env lua
   2 -- Prosody IM
   3 -- Copyright (C) 2008-2010 Matthew Wild
   4 -- Copyright (C) 2008-2010 Waqas Hussain
   5 --
   6 -- This project is MIT/X11 licensed. Please see the
   7 -- COPYING file in the source package for more information.
   8 --
   9
  10 prosody = {};
  11
  12 package.path = package.path ..";../?.lua";
  13 local serialize = require "util.serialization".serialize;
  14 local st = require "util.stanza";
  15 package.loaded["util.logger"] = {init = function() return function() end; end}
  16 local dm = require "util.datamanager"
  17 dm.set_data_path("data");
  18
  19 function parseFile(filename)
  20 ------
  21
  22 local file = nil;
  23 local last = nil;
  24 local line = 1;
  25 local function read(expected)
  26         local ch;
  27         if last then
  28                 ch = last; last = nil;
  29         else
  30                 ch = file:read(1);
  31                 if ch == "\n" then line = line + 1; end
  32         end
  33         if expected and ch ~= expected then error("expected: "..expected.."; got: "..(ch or "nil").." on line "..line); end
  34         return ch;
  35 end
  36 local function pushback(ch)
  37         if last then error(); end
  38         last = ch;
  39 end
  40 local function peek()
  41         if not last then last = read(); end
  42         return last;
  43 end
  44
  45 local escapes = {
  46         ["\\0"] = "\0";
  47         ["\\'"] = "'";
  48         ["\\\""] = "\"";
  49         ["\\b"] = "\b";
  50         ["\\n"] = "\n";
  51         ["\\r"] = "\r";
  52         ["\\t"] = "\t";
  53         ["\\Z"] = "\26";
  54         ["\\\\"] = "\\";
  55         ["\\%"] = "%";
  56         ["\\_"] = "_";
  57 }
  58 local function unescape(s)
  59         return escapes[s] or error("Unknown escape sequence: "..s);
  60 end
  61 local function readString()
  62         read("'");
  63         local s = "";
  64         while true do
  65                 local ch = peek();
  66                 if ch == "\\" then
  67                         s = s..unescape(read()..read());
  68                 elseif ch == "'" then
  69                         break;
  70                 else
  71                         s = s..read();
  72                 end
  73         end
  74         read("'");
  75         return s;
  76 end
  77 local function readNonString()
  78         local s = "";
  79         while true do
  80                 if peek() == "," or peek() == ")" then
  81                         break;
  82                 else
  83                         s = s..read();
  84                 end
  85         end
  86         return tonumber(s);
  87 end
  88 local function readItem()
  89         if peek() == "'" then
  90                 return readString();
  91         else
  92                 return readNonString();
  93         end
  94 end
  95 local function readTuple()
  96         local items = {}
  97         read("(");
  98         while peek() ~= ")" do
  99                 table.insert(items, readItem());
 100                 if peek() == ")" then break; end
 101                 read(",");
 102         end
 103         read(")");
 104         return items;
 105 end
 106 local function readTuples()
 107         if peek() ~= "(" then read("("); end
 108         local tuples = {};
 109         while true do
 110                 table.insert(tuples, readTuple());
 111                 if peek() == "," then read() end
 112                 if peek() == ";" then break; end
 113         end
 114         return tuples;
 115 end
 116 local function readTableName()
 117         local tname = "";
 118         while peek() ~= "`" do tname = tname..read(); end
 119         return tname;
 120 end
 121 local function readInsert()
 122         if peek() == nil then return nil; end
 123         for ch in ("INSERT INTO `"):gmatch(".") do -- find line starting with this
 124                 if peek() == ch then
 125                         read(); -- found
 126                 else -- match failed, skip line
 127                         while peek() and read() ~= "\n" do end
 128                         return nil;
 129                 end
 130         end
 131         local tname = readTableName();
 132         for ch in ("` VALUES "):gmatch(".") do read(ch); end -- expect this
 133         local tuples = readTuples();
 134         read(";"); read("\n");
 135         return tname, tuples;
 136 end
 137
 138 local function readFile(filename)
 139         file = io.open(filename);
 140         if not file then error("File not found: "..filename); os.exit(0); end
 141         local t = {};
 142         while true do
 143                 local tname, tuples = readInsert();
 144                 if tname then
 145                         if t[tname] then
 146                                 local t_name = t[tname];
 147                                 for i=1,#tuples do
 148                                         table.insert(t_name, tuples[i]);
 149                                 end
 150                         else
 151                                 t[tname] = tuples;
 152                         end
 153                 elseif peek() == nil then
 154                         break;
 155                 end
 156         end
 157         return t;
 158 end
 159
 160 return readFile(filename);
 161
 162 ------
 163 end
 164
 165 -- XML parser
 166 local parse_xml = (function()
 167         local entity_map = setmetatable({
 168                 ["amp"] = "&";
 169                 ["gt"] = ">";
 170                 ["lt"] = "<";
 171                 ["apos"] = "'";
 172                 ["quot"] = "\"";
 173         }, {__index = function(_, s)
 174                         if s:sub(1,1) == "#" then
 175                                 if s:sub(2,2) == "x" then
 176                                         return string.char(tonumber(s:sub(3), 16));
 177                                 else
 178                                         return string.char(tonumber(s:sub(2)));
 179                                 end
 180                         end
 181                 end
 182         });
 183         local function xml_unescape(str)
 184                 return (str:gsub("&(.-);", entity_map));
 185         end
 186         local function parse_tag(s)
 187                 local name,sattr=(s):gmatch("([^%s]+)(.*)")();
 188                 local attr = {};
 189                 for a,b in (sattr):gmatch("([^=%s]+)=['\"]([^'\"]*)['\"]") do attr[a] = xml_unescape(b); end
 190                 return name, attr;
 191         end
 192         return function(xml)
 193                 local stanza = st.stanza("root");
 194                 local regexp = "<([^>]*)>([^<]*)";
 195                 for elem, text in xml:gmatch(regexp) do
 196                         if elem:sub(1,1) == "!" or elem:sub(1,1) == "?" then -- neglect comments and processing-instructions
 197                         elseif elem:sub(1,1) == "/" then -- end tag
 198                                 elem = elem:sub(2);
 199                                 stanza:up(); -- TODO check for start-end tag name match
 200                         elseif elem:sub(-1,-1) == "/" then -- empty tag
 201                                 elem = elem:sub(1,-2);
 202                                 local name,attr = parse_tag(elem);
 203                                 stanza:tag(name, attr):up();
 204                         else -- start tag
 205                                 local name,attr = parse_tag(elem);
 206                                 stanza:tag(name, attr);
 207                         end
 208                         if #text ~= 0 then -- text
 209                                 stanza:text(xml_unescape(text));
 210                         end
 211                 end
 212                 return stanza.tags[1];
 213         end
 214 end)();
 215 -- end of XML parser
 216
 217 local arg, host = ...;
 218 local help = "/? -? ? /h -h /help -help --help";
 219 if not(arg and host) or help:find(arg, 1, true) then
 220         print([[ejabberd SQL DB dump importer for Prosody
 221
 222   Usage: ejabberdsql2prosody.lua filename.txt hostname
 223
 224 The file can be generated using mysqldump:
 225   mysqldump db_name > filename.txt]]);
 226         os.exit(1);
 227 end
 228 local map = {
 229         ["last"] = {"username", "seconds", "state"};
 230         ["privacy_default_list"] = {"username", "name"};
 231         ["privacy_list"] = {"username", "name", "id"};
 232         ["privacy_list_data"] = {"id", "t", "value", "action", "ord", "match_all", "match_iq", "match_message", "match_presence_in", "match_presence_out"};
 233         ["private_storage"] = {"username", "namespace", "data"};
 234         ["rostergroups"] = {"username", "jid", "grp"};
 235         ["rosterusers"] = {"username", "jid", "nick", "subscription", "ask", "askmessage", "server", "subscribe", "type"};
 236         ["spool"] = {"username", "xml", "seq"};
 237         ["users"] = {"username", "password"};
 238         ["vcard"] = {"username", "vcard"};
 239         --["vcard_search"] = {};
 240 }
 241 local NULL = {};
 242 local t = parseFile(arg);
 243 for name, data in pairs(t) do
 244         local m = map[name];
 245         if m then
 246                 if #data > 0 and #data[1] ~= #m then
 247                         print("[warning] expected "..#m.." columns for table `"..name.."`, found "..#data[1]);
 248                 end
 249                 for i=1,#data do
 250                         local row = data[i];
 251                         for j=1,#m do
 252                                 row[m[j]] = row[j];
 253                                 row[j] = nil;
 254                         end
 255                 end
 256         end
 257 end
 258 --print(serialize(t));
 259
 260 for i, row in ipairs(t["users"] or NULL) do
 261         local node, password = row.username, row.password;
 262         local ret, err = dm.store(node, host, "accounts", {password = password});
 263         print("["..(err or "success").."] accounts: "..node.."@"..host);
 264 end
 265
 266 function roster(node, host, jid, item)
 267         local roster = dm.load(node, host, "roster") or {};
 268         roster[jid] = item;
 269         local ret, err = dm.store(node, host, "roster", roster);
 270         print("["..(err or "success").."] roster: " ..node.."@"..host.." - "..jid);
 271 end
 272 function roster_pending(node, host, jid)
 273         local roster = dm.load(node, host, "roster") or {};
 274         roster.pending = roster.pending or {};
 275         roster.pending[jid] = true;
 276         local ret, err = dm.store(node, host, "roster", roster);
 277         print("["..(err or "success").."] roster-pending: " ..node.."@"..host.." - "..jid);
 278 end
 279 function roster_group(node, host, jid, group)
 280         local roster = dm.load(node, host, "roster") or {};
 281         local item = roster[jid];
 282         if not item then print("Warning: No roster item "..jid.." for user "..node..", can't put in group "..group); return; end
 283         item.groups[group] = true;
 284         local ret, err = dm.store(node, host, "roster", roster);
 285         print("["..(err or "success").."] roster-group: " ..node.."@"..host.." - "..jid.." - "..group);
 286 end
 287 function private_storage(node, host, xmlns, stanza)
 288         local private = dm.load(node, host, "private") or {};
 289         private[stanza.name..":"..xmlns] = st.preserialize(stanza);
 290         local ret, err = dm.store(node, host, "private", private);
 291         print("["..(err or "success").."] private: " ..node.."@"..host.." - "..xmlns);
 292 end
 293 function offline_msg(node, host, t, stanza)
 294         stanza.attr.stamp = os.date("!%Y-%m-%dT%H:%M:%SZ", t);
 295         stanza.attr.stamp_legacy = os.date("!%Y%m%dT%H:%M:%S", t);
 296         local ret, err = dm.list_append(node, host, "offline", st.preserialize(stanza));
 297         print("["..(err or "success").."] offline: " ..node.."@"..host.." - "..os.date("!%Y-%m-%dT%H:%M:%SZ", t));
 298 end
 299 for i, row in ipairs(t["rosterusers"] or NULL) do
 300         local node, contact = row.username, row.jid;
 301         local name = row.nick;
 302         if name == "" then name = nil; end
 303         local subscription = row.subscription;
 304         if subscription == "N" then
 305                 subscription = "none"
 306         elseif subscription == "B" then
 307                 subscription = "both"
 308         elseif subscription == "F" then
 309                 subscription = "from"
 310         elseif subscription == "T" then
 311                 subscription = "to"
 312         else error("Unknown subscription type: "..subscription) end;
 313         local ask = row.ask;
 314         if ask == "N" then
 315                 ask = nil;
 316         elseif ask == "O" then
 317                 ask = "subscribe";
 318         elseif ask == "I" then
 319                 roster_pending(node, host, contact);
 320                 ask = nil;
 321         elseif ask == "B" then
 322                 roster_pending(node, host, contact);
 323                 ask = "subscribe";
 324         else error("Unknown ask type: "..ask); end
 325         local item = {name = name, ask = ask, subscription = subscription, groups = {}};
 326         roster(node, host, contact, item);
 327 end
 328 for i, row in ipairs(t["rostergroups"] or NULL) do
 329         roster_group(row.username, host, row.jid, row.grp);
 330 end
 331 for i, row in ipairs(t["vcard"] or NULL) do
 332         local ret, err = dm.store(row.username, host, "vcard", st.preserialize(parse_xml(row.vcard)));
 333         print("["..(err or "success").."] vCard: "..row.username.."@"..host);
 334 end
 335 for i, row in ipairs(t["private_storage"] or NULL) do
 336         private_storage(row.username, host, row.namespace, parse_xml(row.data));
 337 end
 338 table.sort(t["spool"] or NULL, function(a,b) return a.seq < b.seq; end); -- sort by sequence number, just in case
 339 local time_offset = os.difftime(os.time(os.date("!*t")), os.time(os.date("*t"))) -- to deal with timezones
 340 local date_parse = function(s)
 341         local year, month, day, hour, min, sec = s:match("(....)-?(..)-?(..)T(..):(..):(..)");
 342         return os.time({year=year, month=month, day=day, hour=hour, min=min, sec=sec-time_offset});
 343 end
 344 for i, row in ipairs(t["spool"] or NULL) do
 345         local stanza = parse_xml(row.xml);
 346         local last_child = stanza.tags[#stanza.tags];
 347         if not last_child or last_child ~= stanza[#stanza] then error("Last child of offline message is not a tag"); end
 348         if last_child.name ~= "x" and last_child.attr.xmlns ~= "jabber:x:delay" then error("Last child of offline message is not a timestamp"); end
 349         stanza[#stanza], stanza.tags[#stanza.tags] = nil, nil;
 350         local t = date_parse(last_child.attr.stamp);
 351         offline_msg(row.username, host, t, stanza);
 352 end