tools/migration/*: Initial commit of a new migration tool. Currently supports Prosody...

diff --git a/tools/migration/config.lua b/tools/migration/config.lua
new file mode 100644 (file)
index 0000000..199b820
--- /dev/null
+++ b/tools/migration/config.lua
@@ -0,0 +1,26 @@
+input {\r
+       type = "prosody_sql";\r
+       driver = "SQLite3";\r
+       database = "out.sqlite";\r
+}\r
+output {\r
+       type = "prosody_files";\r
+       path = "out";\r
+}\r
+\r
+--[[\r
+\r
+input {\r
+       path = "../../data";\r
+       type = "prosody_files";\r
+       driver = "SQLite3";\r
+       database = "../../prosody.sqlite";\r
+}\r
+output {\r
+       type = "prosody_sql";\r
+       driver = "SQLite3";\r
+       database = "out.sqlite";\r
+       path = "out";\r
+}\r
+\r
+]]\r

diff --git a/tools/migration/main.lua b/tools/migration/main.lua
new file mode 100644 (file)
index 0000000..42e9bab
--- /dev/null
+++ b/tools/migration/main.lua
@@ -0,0 +1,27 @@
+\r
+\r
+\r
+local function loadfilein(file, env) return loadin and loadin(env, io.open(file):read("*a")) or setfenv(loadfile(file), env); end\r
+config = {};\r
+local config_env = setmetatable({}, { __index = function(t, k) return function(tbl) config[k] = tbl; end; end });\r
+loadfilein("config.lua", config_env)();\r
+\r
+package.path = "../../?.lua;"..package.path\r
+package.cpath = "../../?.dll;"..package.cpath\r
+\r
+\r
+assert(config.input, "no input specified")\r
+assert(config.output, "no output specified")\r
+local itype = assert(config.input.type, "no input.type specified");\r
+local otype = assert(config.output.type, "no output.type specified");\r
+local reader = require(itype).reader(config.input);\r
+local writer = require(otype).writer(config.output);\r
+\r
+local json = require "util.json";\r
+\r
+for x in reader do\r
+       --print(json.encode(x))\r
+       writer(x);\r
+end\r
+writer(nil); -- close\r
+\r

diff --git a/tools/migration/mtools.lua b/tools/migration/mtools.lua
new file mode 100644 (file)
index 0000000..d2bd1e7
--- /dev/null
+++ b/tools/migration/mtools.lua
@@ -0,0 +1,56 @@
+\r
+\r
+local print = print;\r
+local t_insert = table.insert;\r
+local t_sort = table.sort;\r
+\r
+module "mtools"\r
+\r
+function sorted(params)\r
+\r
+       local reader = params.reader; -- iterator to get items from\r
+       local sorter = params.sorter; -- sorting function\r
+       local filter = params.filter; -- filter function\r
+\r
+       local cache = {};\r
+       for item in reader do\r
+               if filter then item = filter(item); end\r
+               if item then t_insert(cache, item); end\r
+       end\r
+       if sorter then\r
+               t_sort(cache, sorter);\r
+       end\r
+       local i = 0;\r
+       return function()\r
+               i = i + 1;\r
+               return cache[i];\r
+       end;\r
+\r
+end\r
+\r
+function merged(reader, merger)\r
+\r
+       local item1 = reader();\r
+       local merged = { item1 };\r
+       return function()\r
+               while true do\r
+                       if not item1 then return nil; end\r
+                       local item2 = reader();\r
+                       if not item2 then item1 = nil; return merged; end\r
+                       if merger(item1, item2) then\r
+                       --print("merged")\r
+                               item1 = item2;\r
+                               t_insert(merged, item1);\r
+                       else\r
+                       --print("unmerged", merged)\r
+                               item1 = item2;\r
+                               local tmp = merged;\r
+                               merged = { item1 };\r
+                               return tmp;\r
+                       end\r
+               end\r
+       end;\r
+\r
+end\r
+\r
+return _M;\r

diff --git a/tools/migration/prosody_files.lua b/tools/migration/prosody_files.lua
new file mode 100644 (file)
index 0000000..d9c2dd4
--- /dev/null
+++ b/tools/migration/prosody_files.lua
@@ -0,0 +1,133 @@
+\r
+local print = print;\r
+local assert = assert;\r
+local setmetatable = setmetatable;\r
+local tonumber = tonumber;\r
+local char = string.char;\r
+local coroutine = coroutine;\r
+local lfs = require "lfs";\r
+local loadfile = loadfile;\r
+local setfenv = setfenv;\r
+local pcall = pcall;\r
+local mtools = require "mtools";\r
+local next = next;\r
+local pairs = pairs;\r
+local json = require "util.json";\r
+\r
+prosody = {};\r
+local dm = require "util.datamanager"\r
+\r
+module "prosody_files"\r
+\r
+local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end\r
+local function is_file(path) return lfs.attributes(path, "mode") == "file"; end\r
+local function clean_path(path)\r
+       return path:gsub("\\", "/"):gsub("//+", "/");\r
+end\r
+local encode, decode; do\r
+       local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end });\r
+       decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end\r
+       encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end\r
+end\r
+local function decode_dir(x)\r
+       if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then\r
+               return decode(x);\r
+       end\r
+end\r
+local function decode_file(x)\r
+       if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then\r
+               return decode(x:gsub("%.dat$", ""));\r
+       end\r
+end\r
+local function prosody_dir(path, ondir, onfile, ...)\r
+       for x in lfs.dir(path) do\r
+               local xpath = path.."/"..x;\r
+               if decode_dir(x) and is_dir(xpath) then\r
+                       ondir(xpath, x, ...);\r
+               elseif decode_file(x) and is_file(xpath) then\r
+                       onfile(xpath, x, ...);\r
+               end\r
+       end\r
+end\r
+\r
+local function handle_root_file(path, name)\r
+       --print("root file: ", decode_file(name))\r
+       coroutine.yield { user = nil, host = nil, store = decode_file(name) };\r
+end\r
+local function handle_host_file(path, name, host)\r
+       --print("host file: ", decode_dir(host).."/"..decode_file(name))\r
+       coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) };\r
+end\r
+local function handle_store_file(path, name, store, host)\r
+       --print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store))\r
+       coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) };\r
+end\r
+local function handle_host_store(path, name, host)\r
+       prosody_dir(path, function() end, handle_store_file, name, host);\r
+end\r
+local function handle_host_dir(path, name)\r
+       prosody_dir(path, handle_host_store, handle_host_file, name);\r
+end\r
+local function handle_root_dir(path)\r
+       prosody_dir(path, handle_host_dir, handle_root_file);\r
+end\r
+\r
+local function decode_user(item)\r
+       local userdata = {\r
+               user = item[1].user;\r
+               host = item[1].host;\r
+               stores = {};\r
+       };\r
+       for i=1,#item do -- loop over stores\r
+               local result = {};\r
+               local store = item[i];\r
+               userdata.stores[store.store] = store.data;\r
+               store.user = nil; store.host = nil; store.store = nil;\r
+       end\r
+       return userdata;\r
+end\r
+\r
+function reader(input)\r
+       local path = clean_path(assert(input.path, "no input.path specified"));\r
+       assert(is_dir(path), "input.path is not a directory");\r
+       local iter = coroutine.wrap(function()handle_root_dir(path);end);\r
+       -- get per-user stores, sorted\r
+       local iter = mtools.sorted {\r
+               reader = function()\r
+                       local x = iter();\r
+                       if x then\r
+                               dm.set_data_path(path);\r
+                               x.data = assert(dm.load(x.user, x.host, x.store));\r
+                               return x;\r
+                       end\r
+               end;\r
+               sorter = function(a, b)\r
+                       local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";\r
+                       local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";\r
+                       return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);\r
+               end;\r
+       };\r
+       -- merge stores to get users\r
+       iter = mtools.merged(iter, function(a, b)\r
+               return (a.host == b.host and a.user == b.user);\r
+       end);\r
+\r
+       return function()\r
+               local x = iter();\r
+               return x and decode_user(x);\r
+       end\r
+end\r
+\r
+function writer(output)\r
+       local path = clean_path(assert(output.path, "no output.path specified"));\r
+       assert(is_dir(path), "output.path is not a directory");\r
+       return function(item)\r
+               if not item then return; end -- end of input\r
+               dm.set_data_path(path);\r
+               for store, data in pairs(item.stores) do\r
+                       assert(dm.store(item.user, item.host, store, data));\r
+               end\r
+       end\r
+end\r
+\r
+return _M;\r

diff --git a/tools/migration/prosody_sql.lua b/tools/migration/prosody_sql.lua
new file mode 100644 (file)
index 0000000..56990f4
--- /dev/null
+++ b/tools/migration/prosody_sql.lua
@@ -0,0 +1,174 @@
+\r
+local assert = assert;\r
+local DBI = require "DBI";\r
+local print = print;\r
+local type = type;\r
+local next = next;\r
+local pairs = pairs;\r
+local t_sort = table.sort;\r
+local json = require "util.json";\r
+local mtools = require "mtools";\r
+local tostring = tostring;\r
+local tonumber = tonumber;\r
+\r
+module "prosody_sql"\r
+\r
+local function create_table(connection, params)\r
+       local create_sql = "CREATE TABLE `prosody` (`host` TEXT, `user` TEXT, `store` TEXT, `key` TEXT, `type` TEXT, `value` TEXT);";\r
+       if params.driver == "PostgreSQL" then\r
+               create_sql = create_sql:gsub("`", "\"");\r
+       end\r
+       \r
+       local stmt = connection:prepare(create_sql);\r
+       if stmt then\r
+               local ok = stmt:execute();\r
+               local commit_ok = connection:commit();\r
+               if ok and commit_ok then\r
+                       local index_sql = "CREATE INDEX `prosody_index` ON `prosody` (`host`, `user`, `store`, `key`)";\r
+                       if params.driver == "PostgreSQL" then\r
+                               index_sql = index_sql:gsub("`", "\"");\r
+                       elseif params.driver == "MySQL" then\r
+                               index_sql = index_sql:gsub("`([,)])", "`(20)%1");\r
+                       end\r
+                       local stmt, err = connection:prepare(index_sql);\r
+                       local ok, commit_ok, commit_err;\r
+                       if stmt then\r
+                               ok, err = assert(stmt:execute());\r
+                               commit_ok, commit_err = assert(connection:commit());\r
+                       end\r
+               end\r
+       end\r
+end\r
+\r
+local function serialize(value)\r
+       local t = type(value);\r
+       if t == "string" or t == "boolean" or t == "number" then\r
+               return t, tostring(value);\r
+       elseif t == "table" then\r
+               local value,err = json.encode(value);\r
+               if value then return "json", value; end\r
+               return nil, err;\r
+       end\r
+       return nil, "Unhandled value type: "..t;\r
+end\r
+local function deserialize(t, value)\r
+       if t == "string" then return value;\r
+       elseif t == "boolean" then\r
+               if value == "true" then return true;\r
+               elseif value == "false" then return false; end\r
+       elseif t == "number" then return tonumber(value);\r
+       elseif t == "json" then\r
+               return json.decode(value);\r
+       end\r
+end\r
+\r
+local function decode_user(item)\r
+       local userdata = {\r
+               user = item[1][1].user;\r
+               host = item[1][1].host;\r
+               stores = {};\r
+       };\r
+       for i=1,#item do -- loop over stores\r
+               local result = {};\r
+               local store = item[i];\r
+               for i=1,#store do -- loop over store data\r
+                       local row = store[i];\r
+                       local k = row.key;\r
+                       local v = deserialize(row.type, row.value);\r
+                       if k and v then\r
+                               if k ~= "" then result[k] = v; elseif type(v) == "table" then\r
+                                       for a,b in pairs(v) do\r
+                                               result[a] = b;\r
+                                       end\r
+                               end\r
+                       end\r
+                       userdata.stores[store[1].store] = result;\r
+               end\r
+       end\r
+       return userdata;\r
+end\r
+\r
+function reader(input)\r
+       local dbh = assert(DBI.Connect(\r
+               assert(input.driver, "no input.driver specified"),\r
+               assert(input.database, "no input.database specified"),\r
+               input.username, input.password,\r
+               input.host, input.port\r
+       ));\r
+       assert(dbh:ping());\r
+       local stmt = assert(dbh:prepare("SELECT * FROM prosody"));\r
+       assert(stmt:execute());\r
+       local keys = {"host", "user", "store", "key", "type", "value"};\r
+       local f,s,val = stmt:rows(true);\r
+       -- get SQL rows, sorted\r
+       local iter = mtools.sorted {\r
+               reader = function() val = f(s, val); return val; end;\r
+               filter = function(x)\r
+                       for i=1,#keys do\r
+                               if not x[keys[i]] then return false; end -- TODO log error, missing field\r
+                       end\r
+                       if x.host  == "" then x.host  = nil; end\r
+                       if x.user  == "" then x.user  = nil; end\r
+                       if x.store == "" then x.store = nil; end\r
+                       return x;\r
+               end;\r
+               sorter = function(a, b)\r
+                       local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";\r
+                       local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";\r
+                       return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);\r
+               end;\r
+       };\r
+       -- merge rows to get stores\r
+       iter = mtools.merged(iter, function(a, b)\r
+               return (a.host == b.host and a.user == b.user and a.store == b.store);\r
+       end);\r
+       -- merge stores to get users\r
+       iter = mtools.merged(iter, function(a, b)\r
+               return (a[1].host == b[1].host and a[1].user == b[1].user);\r
+       end);\r
+       return function()\r
+               local x = iter();\r
+               return x and decode_user(x);\r
+       end;\r
+end\r
+\r
+function writer(output, iter)\r
+       local dbh = assert(DBI.Connect(\r
+               assert(output.driver, "no output.driver specified"),\r
+               assert(output.database, "no output.database specified"),\r
+               output.username, output.password,\r
+               output.host, output.port\r
+       ));\r
+       assert(dbh:ping());\r
+       create_table(dbh, output);\r
+       local stmt = assert(dbh:prepare("SELECT * FROM prosody"));\r
+       assert(stmt:execute());\r
+       local stmt = assert(dbh:prepare("DELETE FROM prosody"));\r
+       assert(stmt:execute());\r
+       local insert = assert(dbh:prepare("INSERT INTO `prosody` (`host`,`user`,`store`,`key`,`type`,`value`) VALUES (?,?,?,?,?,?)"));\r
+\r
+       return function(item)\r
+               if not item then assert(dbh:commit()) return dbh:close(); end -- end of input\r
+               local host = item.host or "";\r
+               local user = item.user or "";\r
+               for store, data in pairs(item.stores) do\r
+                       -- TODO transactions\r
+                       local extradata = {};\r
+                       for key, value in pairs(data) do\r
+                               if type(key) == "string" and key ~= "" then\r
+                                       local t, value = assert(serialize(value));\r
+                                       local ok, err = assert(insert:execute(host, user, store, key, t, value));\r
+                               else\r
+                                       extradata[key] = value;\r
+                               end\r
+                       end\r
+                       if next(extradata) ~= nil then\r
+                               local t, extradata = assert(serialize(extradata));\r
+                               local ok, err = assert(insert:execute(host, user, store, "", t, extradata));\r
+                       end\r
+               end\r
+       end;\r
+end\r
+\r
+\r
+return _M;\r