tools/ejabberdsql2prosody.lua

   1 #!/usr/bin/env lua
   2 -- Prosody IM
   3 -- Copyright (C) 2008-2010 Matthew Wild
   4 -- Copyright (C) 2008-2010 Waqas Hussain
   5 --
   6 -- This project is MIT/X11 licensed. Please see the
   7 -- COPYING file in the source package for more information.
   8 --
   9
  10 prosody = {};
  11
  12 package.path = package.path ..";../?.lua";
  13 local serialize = require "util.serialization".serialize;
  14 local st = require "util.stanza";
  15 package.loaded["util.logger"] = {init = function() return function() end; end}
  16 local dm = require "util.datamanager"
  17 dm.set_data_path("data");
  18
  19 function parseFile(filename)
  20 ------
  21
  22 local file = nil;
  23 local last = nil;
  24 local line = 1;
  25 local function read(expected)
  26         local ch;
  27         if last then
  28                 ch = last; last = nil;
  29         else
  30                 ch = file:read(1);
  31                 if ch == "\n" then line = line + 1; end
  32         end
  33         if expected and ch ~= expected then error("expected: "..expected.."; got: "..(ch or "nil").." on line "..line); end
  34         return ch;
  35 end
  36 local function pushback(ch)
  37         if last then error(); end
  38         last = ch;
  39 end
  40 local function peek()
  41         if not last then last = read(); end
  42         return last;
  43 end
  44
  45 local escapes = {
  46         ["\\0"] = "\0";
  47         ["\\'"] = "'";
  48         ["\\\""] = "\"";
  49         ["\\b"] = "\b";
  50         ["\\n"] = "\n";
  51         ["\\r"] = "\r";
  52         ["\\t"] = "\t";
  53         ["\\Z"] = "\26";
  54         ["\\\\"] = "\\";
  55         ["\\%"] = "%";
  56         ["\\_"] = "_";
  57 }
  58 local function unescape(s)
  59         return escapes[s] or error("Unknown escape sequence: "..s);
  60 end
  61 local function readString()
  62         read("'");
  63         local s = "";
  64         while true do
  65                 local ch = peek();
  66                 if ch == "\\" then
  67                         s = s..unescape(read()..read());
  68                 elseif ch == "'" then
  69                         break;
  70                 else
  71                         s = s..read();
  72                 end
  73         end
  74         read("'");
  75         return s;
  76 end
  77 local function readNonString()
  78         local s = "";
  79         while true do
  80                 if peek() == "," or peek() == ")" then
  81                         break;
  82                 else
  83                         s = s..read();
  84                 end
  85         end
  86         return tonumber(s);
  87 end
  88 local function readItem()
  89         if peek() == "'" then
  90                 return readString();
  91         else
  92                 return readNonString();
  93         end
  94 end
  95 local function readTuple()
  96         local items = {}
  97         read("(");
  98         while peek() ~= ")" do
  99                 table.insert(items, readItem());
 100                 if peek() == ")" then break; end
 101                 read(",");
 102         end
 103         read(")");
 104         return items;
 105 end
 106 local function readTuples()
 107         if peek() ~= "(" then read("("); end
 108         local tuples = {};
 109         while true do
 110                 table.insert(tuples, readTuple());
 111                 if peek() == "," then read() end
 112                 if peek() == ";" then break; end
 113         end
 114         return tuples;
 115 end
 116 local function readTableName()
 117         local tname = "";
 118         while peek() ~= "`" do tname = tname..read(); end
 119         return tname;
 120 end
 121 local function readInsert()
 122         if peek() == nil then return nil; end
 123         for ch in ("INSERT INTO `"):gmatch(".") do -- find line starting with this
 124                 if peek() == ch then
 125                         read(); -- found
 126                 else -- match failed, skip line
 127                         while peek() and read() ~= "\n" do end
 128                         return nil;
 129                 end
 130         end
 131         local tname = readTableName();
 132         read("`"); read(" ") -- expect this
 133         if peek() == "(" then -- skip column list
 134                 repeat until read() == ")";
 135                 read(" ");
 136         end
 137         for ch in ("VALUES "):gmatch(".") do read(ch); end -- expect this
 138         local tuples = readTuples();
 139         read(";"); read("\n");
 140         return tname, tuples;
 141 end
 142
 143 local function readFile(filename)
 144         file = io.open(filename);
 145         if not file then error("File not found: "..filename); os.exit(0); end
 146         local t = {};
 147         while true do
 148                 local tname, tuples = readInsert();
 149                 if tname then
 150                         if t[tname] then
 151                                 local t_name = t[tname];
 152                                 for i=1,#tuples do
 153                                         table.insert(t_name, tuples[i]);
 154                                 end
 155                         else
 156                                 t[tname] = tuples;
 157                         end
 158                 elseif peek() == nil then
 159                         break;
 160                 end
 161         end
 162         return t;
 163 end
 164
 165 return readFile(filename);
 166
 167 ------
 168 end
 169
 170 -- XML parser
 171 local parse_xml = (function()
 172         local entity_map = setmetatable({
 173                 ["amp"] = "&";
 174                 ["gt"] = ">";
 175                 ["lt"] = "<";
 176                 ["apos"] = "'";
 177                 ["quot"] = "\"";
 178         }, {__index = function(_, s)
 179                         if s:sub(1,1) == "#" then
 180                                 if s:sub(2,2) == "x" then
 181                                         return string.char(tonumber(s:sub(3), 16));
 182                                 else
 183                                         return string.char(tonumber(s:sub(2)));
 184                                 end
 185                         end
 186                 end
 187         });
 188         local function xml_unescape(str)
 189                 return (str:gsub("&(.-);", entity_map));
 190         end
 191         local function parse_tag(s)
 192                 local name,sattr=(s):gmatch("([^%s]+)(.*)")();
 193                 local attr = {};
 194                 for a,b in (sattr):gmatch("([^=%s]+)=['\"]([^'\"]*)['\"]") do attr[a] = xml_unescape(b); end
 195                 return name, attr;
 196         end
 197         return function(xml)
 198                 local stanza = st.stanza("root");
 199                 local regexp = "<([^>]*)>([^<]*)";
 200                 for elem, text in xml:gmatch(regexp) do
 201                         if elem:sub(1,1) == "!" or elem:sub(1,1) == "?" then -- neglect comments and processing-instructions
 202                         elseif elem:sub(1,1) == "/" then -- end tag
 203                                 elem = elem:sub(2);
 204                                 stanza:up(); -- TODO check for start-end tag name match
 205                         elseif elem:sub(-1,-1) == "/" then -- empty tag
 206                                 elem = elem:sub(1,-2);
 207                                 local name,attr = parse_tag(elem);
 208                                 stanza:tag(name, attr):up();
 209                         else -- start tag
 210                                 local name,attr = parse_tag(elem);
 211                                 stanza:tag(name, attr);
 212                         end
 213                         if #text ~= 0 then -- text
 214                                 stanza:text(xml_unescape(text));
 215                         end
 216                 end
 217                 return stanza.tags[1];
 218         end
 219 end)();
 220 -- end of XML parser
 221
 222 local arg, host = ...;
 223 local help = "/? -? ? /h -h /help -help --help";
 224 if not(arg and host) or help:find(arg, 1, true) then
 225         print([[ejabberd SQL DB dump importer for Prosody
 226
 227   Usage: ejabberdsql2prosody.lua filename.txt hostname
 228
 229 The file can be generated using mysqldump:
 230   mysqldump db_name > filename.txt]]);
 231         os.exit(1);
 232 end
 233 local map = {
 234         ["last"] = {"username", "seconds", "state"};
 235         ["privacy_default_list"] = {"username", "name"};
 236         ["privacy_list"] = {"username", "name", "id"};
 237         ["privacy_list_data"] = {"id", "t", "value", "action", "ord", "match_all", "match_iq", "match_message", "match_presence_in", "match_presence_out"};
 238         ["private_storage"] = {"username", "namespace", "data"};
 239         ["rostergroups"] = {"username", "jid", "grp"};
 240         ["rosterusers"] = {"username", "jid", "nick", "subscription", "ask", "askmessage", "server", "subscribe", "type"};
 241         ["spool"] = {"username", "xml", "seq"};
 242         ["users"] = {"username", "password"};
 243         ["vcard"] = {"username", "vcard"};
 244         --["vcard_search"] = {};
 245 }
 246 local NULL = {};
 247 local t = parseFile(arg);
 248 for name, data in pairs(t) do
 249         local m = map[name];
 250         if m then
 251                 if #data > 0 and #data[1] ~= #m then
 252                         print("[warning] expected "..#m.." columns for table `"..name.."`, found "..#data[1]);
 253                 end
 254                 for i=1,#data do
 255                         local row = data[i];
 256                         for j=1,#m do
 257                                 row[m[j]] = row[j];
 258                                 row[j] = nil;
 259                         end
 260                 end
 261         end
 262 end
 263 --print(serialize(t));
 264
 265 for i, row in ipairs(t["users"] or NULL) do
 266         local node, password = row.username, row.password;
 267         local ret, err = dm.store(node, host, "accounts", {password = password});
 268         print("["..(err or "success").."] accounts: "..node.."@"..host);
 269 end
 270
 271 function roster(node, host, jid, item)
 272         local roster = dm.load(node, host, "roster") or {};
 273         roster[jid] = item;
 274         local ret, err = dm.store(node, host, "roster", roster);
 275         print("["..(err or "success").."] roster: " ..node.."@"..host.." - "..jid);
 276 end
 277 function roster_pending(node, host, jid)
 278         local roster = dm.load(node, host, "roster") or {};
 279         roster.pending = roster.pending or {};
 280         roster.pending[jid] = true;
 281         local ret, err = dm.store(node, host, "roster", roster);
 282         print("["..(err or "success").."] roster-pending: " ..node.."@"..host.." - "..jid);
 283 end
 284 function roster_group(node, host, jid, group)
 285         local roster = dm.load(node, host, "roster") or {};
 286         local item = roster[jid];
 287         if not item then print("Warning: No roster item "..jid.." for user "..node..", can't put in group "..group); return; end
 288         item.groups[group] = true;
 289         local ret, err = dm.store(node, host, "roster", roster);
 290         print("["..(err or "success").."] roster-group: " ..node.."@"..host.." - "..jid.." - "..group);
 291 end
 292 function private_storage(node, host, xmlns, stanza)
 293         local private = dm.load(node, host, "private") or {};
 294         private[stanza.name..":"..xmlns] = st.preserialize(stanza);
 295         local ret, err = dm.store(node, host, "private", private);
 296         print("["..(err or "success").."] private: " ..node.."@"..host.." - "..xmlns);
 297 end
 298 function offline_msg(node, host, t, stanza)
 299         stanza.attr.stamp = os.date("!%Y-%m-%dT%H:%M:%SZ", t);
 300         stanza.attr.stamp_legacy = os.date("!%Y%m%dT%H:%M:%S", t);
 301         local ret, err = dm.list_append(node, host, "offline", st.preserialize(stanza));
 302         print("["..(err or "success").."] offline: " ..node.."@"..host.." - "..os.date("!%Y-%m-%dT%H:%M:%SZ", t));
 303 end
 304 for i, row in ipairs(t["rosterusers"] or NULL) do
 305         local node, contact = row.username, row.jid;
 306         local name = row.nick;
 307         if name == "" then name = nil; end
 308         local subscription = row.subscription;
 309         if subscription == "N" then
 310                 subscription = "none"
 311         elseif subscription == "B" then
 312                 subscription = "both"
 313         elseif subscription == "F" then
 314                 subscription = "from"
 315         elseif subscription == "T" then
 316                 subscription = "to"
 317         else error("Unknown subscription type: "..subscription) end;
 318         local ask = row.ask;
 319         if ask == "N" then
 320                 ask = nil;
 321         elseif ask == "O" then
 322                 ask = "subscribe";
 323         elseif ask == "I" then
 324                 roster_pending(node, host, contact);
 325                 ask = nil;
 326         elseif ask == "B" then
 327                 roster_pending(node, host, contact);
 328                 ask = "subscribe";
 329         else error("Unknown ask type: "..ask); end
 330         local item = {name = name, ask = ask, subscription = subscription, groups = {}};
 331         roster(node, host, contact, item);
 332 end
 333 for i, row in ipairs(t["rostergroups"] or NULL) do
 334         roster_group(row.username, host, row.jid, row.grp);
 335 end
 336 for i, row in ipairs(t["vcard"] or NULL) do
 337         local ret, err = dm.store(row.username, host, "vcard", st.preserialize(parse_xml(row.vcard)));
 338         print("["..(err or "success").."] vCard: "..row.username.."@"..host);
 339 end
 340 for i, row in ipairs(t["private_storage"] or NULL) do
 341         private_storage(row.username, host, row.namespace, parse_xml(row.data));
 342 end
 343 table.sort(t["spool"] or NULL, function(a,b) return a.seq < b.seq; end); -- sort by sequence number, just in case
 344 local time_offset = os.difftime(os.time(os.date("!*t")), os.time(os.date("*t"))) -- to deal with timezones
 345 local date_parse = function(s)
 346         local year, month, day, hour, min, sec = s:match("(....)-?(..)-?(..)T(..):(..):(..)");
 347         return os.time({year=year, month=month, day=day, hour=hour, min=min, sec=sec-time_offset});
 348 end
 349 for i, row in ipairs(t["spool"] or NULL) do
 350         local stanza = parse_xml(row.xml);
 351         local last_child = stanza.tags[#stanza.tags];
 352         if not last_child or last_child ~= stanza[#stanza] then error("Last child of offline message is not a tag"); end
 353         if last_child.name ~= "x" and last_child.attr.xmlns ~= "jabber:x:delay" then error("Last child of offline message is not a timestamp"); end
 354         stanza[#stanza], stanza.tags[#stanza.tags] = nil, nil;
 355         local t = date_parse(last_child.attr.stamp);
 356         offline_msg(row.username, host, t, stanza);
 357 end