From fbe61c097fd9de3423cd1646685bca40c713711a Mon Sep 17 00:00:00 2001 From: syednabilashraf Date: Mon, 29 Jan 2024 22:54:21 +0600 Subject: [PATCH] wip --- .rockspec | 4 ++ Makefile | 13 +++++ lua/sync-remote/client/client.js | 64 +++++++++++++++++++++ lua/sync-remote/init.lua | 46 ++++++++++++++- lua/sync-remote/scripts/startSyncRemote.js | 56 +++++++++++++++++++ lua/sync-remote/scripts/watcher.sh | 2 +- lua/sync-remote/server/app/server.js | 28 ++++++++++ lua/sync-remote/server/app/watch.js | 65 ++++++++++++++++++++++ lua/sync-remote/server/run.sh | 10 ++++ lua/sync-remote/server/setup.sh | 22 ++++++++ sync-remote.nvim-dev-1.rockspec | 6 ++ 11 files changed, 312 insertions(+), 4 deletions(-) create mode 100644 .rockspec create mode 100644 Makefile create mode 100644 lua/sync-remote/client/client.js create mode 100644 lua/sync-remote/scripts/startSyncRemote.js create mode 100644 lua/sync-remote/server/app/server.js create mode 100644 lua/sync-remote/server/app/watch.js create mode 100644 lua/sync-remote/server/run.sh create mode 100644 lua/sync-remote/server/setup.sh create mode 100644 sync-remote.nvim-dev-1.rockspec diff --git a/.rockspec b/.rockspec new file mode 100644 index 0000000..5b18afd --- /dev/null +++ b/.rockspec @@ -0,0 +1,4 @@ +dependencies = { + "lua >= 5.4", + "luacheck >= 0.24.0-2" +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c2525da --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +# allows running lua modules directly +export PATH := lua_modules/bin:$(PATH) + +init: hooks install + +hooks: + git config core.hooksPath .githooks + +install: + luarocks --tree=lua_modules install --only-deps sync-remote.nvim-dev-1.rockspec + +lint: + luacheck --config .luacheckrc ./lua/**/*.lua diff --git a/lua/sync-remote/client/client.js b/lua/sync-remote/client/client.js new file mode 100644 index 0000000..64f4070 --- /dev/null +++ b/lua/sync-remote/client/client.js @@ -0,0 +1,64 @@ +const net = require('net'); +const fs = require('fs') +const { execSync, execFileSync } = require('child_process') +const path = require('path') + + +const handleFileModifiedEvent = (event, localRootDir, baseRsyncCommand) => { + event.data.files.forEach((file) => { + const localFilePath = localRootDir + file.name + + if (!file.exists) { + return fs.rmSync(localFilePath, { recursive: true, force: true }) + } + + const dirToMake = file.type === 'd' ? localFilePath : path.dirname(localFilePath) + fs.mkdirSync(dirToMake, { recursive: true }); + + if (file.type === "f") { + fs.writeFileSync(localFilePath, '') + } + + const rsyncCommand = `${baseRsyncCommand}${file.absolutePath} ${localFilePath}` + + execSync(rsyncCommand) + return console.log(rsyncCommand) + }) +} + + +const runClient = (serverPort, remoteHost, localRootDir, baseRsyncCommand) => { + const client = new net.Socket(); + + client.connect(serverPort, remoteHost, () => { + console.log(`Connected to server: ${remoteHost}:${serverPort}`); + const dataToSend = 'Hello from the client!'; + client.write(dataToSend); + + client.on('data', (eventPacket) => { + const start = Date.now() + const event = JSON.parse(eventPacket) + console.log('DATAJSON', JSON.stringify(event)) + if (event.type === "file_modified") { + handleFileModifiedEvent(event, localRootDir, baseRsyncCommand) + } + const end = Date.now() + console.log('total time', (end - start) / 1000) + }); + + client.on('close', () => { + console.log('Connection closed'); + }); + }); + + client.on('error', (err) => { + console.error(`Error: ${err.message}`); + }); + + // setTimeout(() => { + // client.end(); + // }, 5000); +} + +module.exports = runClient + diff --git a/lua/sync-remote/init.lua b/lua/sync-remote/init.lua index 33c7c8d..bcd4bd1 100644 --- a/lua/sync-remote/init.lua +++ b/lua/sync-remote/init.lua @@ -85,6 +85,7 @@ local function loadConfig() else config.local_root = config.local_root:gsub("~", os.getenv("HOME")) .. "/" config.remote_root = config.remote_root .. "/" + config.username, config.host = config.remote_root:match("([^@]+)@([^:/]+)") end end end @@ -101,18 +102,35 @@ local function isConfigFileLoaded() return true end +local function connectToRemote(callback) + local ssh_command = "ssh -f -N -M -o ControlPath=~/.ssh/control-syncremote " + .. config.username + .. "@" + .. config.host + + vim.fn.jobstart(ssh_command, { + on_exit = function() + vim.notify("Connected to " .. config.username .. "@" .. config.host, vim.log.levels.INFO) + callback() + end, + }) +end + function M.loadPlugin() vim.notify("Initializing sync-remote", vim.log.levels.INFO) loadConfig() + if config.remote_root and config.local_root then - loadWatcher(function() - vim.notify("Completed initializing!", vim.log.levels.INFO) + connectToRemote(function() + -- loadWatcher(function() + -- vim.notify("Completed initializing!", vim.log.levels.INFO) + -- end) end) end end local function sync(source, destination) - local rsync_command = "rsync -rzu --delete --no-whole-file --info=progress2 " + local rsync_command = "rsync -rzu -e 'ssh -o ControlPath=~/.ssh/config/control-syncremote' --delete --no-whole-file --info=progress2 " .. source .. "/ " .. destination @@ -194,11 +212,33 @@ function M.syncRemoteFileUp() end end +function M.saveAfterSync() + return false +end + +function M.autowritefile() + vim.notify("autowritefile") +end + +function M.autocreatefile() + vim.notify("autocreatefile") +end + function M.setup() vim.cmd([[command! SyncRemoteStart lua require('sync-remote').loadPlugin()]]) vim.cmd([[command! SyncRemoteFileUp lua require('sync-remote').syncRemoteFileUp()]]) vim.cmd([[command! SyncRemoteUp lua require('sync-remote').syncRemoteUp()]]) vim.cmd([[command! SyncRemoteDown lua require('sync-remote').syncRemoteDown()]]) + + -- vim.cmd([[ + -- augroup MyAutocommands + -- autocmd! + -- autocmd BufWritePre * lua require('sync-remote').autowritefile() + -- autocmd Filetype * lua require('sync-remote').autocreatefile() + -- augroup END + -- ]]) end return M + +-- ➜ ~ rsync -rvzu --filter=':- .gitignore' --exclude='.git' --include='**.gitignore' --info=progress2 -e 'ssh -o ControlPath=~/.ssh/control-syncremote' nabil.ashraf@172.27.220.10:/mnt/ephemeral/workspace/ /home/syednabilashraf/remotecms diff --git a/lua/sync-remote/scripts/startSyncRemote.js b/lua/sync-remote/scripts/startSyncRemote.js new file mode 100644 index 0000000..3800e2d --- /dev/null +++ b/lua/sync-remote/scripts/startSyncRemote.js @@ -0,0 +1,56 @@ +const { execSync } = require('child_process'); +const runClient = require('../client/client.js') + +const arguments = process.argv.slice(2); +// node startSyncRemote nabil.ashraf 172.27.220.10 /home/nabil.ashraf/remotecms/ /home/syednabilashraf/remotecms/ + +const remoteUser = arguments[0] +const remoteHost = arguments[1] +// const remoteRootDir = arguments[2] +const localRootDir = arguments[3] +// const remoteUser = 'nabil.ashraf'; +// const remoteHost = '172.27.220.10'; +// const remoteRootDir = "/home/nabil.ashraf/remotecms/" +// const localRootDir = "/home/syednabilashraf/remotecms/" + +let port = null; +const localServerDir = `${__dirname}/../server/app/`; +const remoteServerDir = '~/.sync-remote/'; +// const clientExecutablePath = `${__dirname}/../client/client.js` +const baseRsyncCommand = `rsync -rzu --filter=':- .gitignore' --exclude='.git' --include='**.gitignore' -e 'ssh -o ControlPath=~/.ssh/control-syncremote' ${remoteUser}@${remoteHost}:` + +const commands = { + createControlMaster: `ssh -f -N -M -o ControlPath=~/.ssh/control-syncremote ${remoteUser}@${remoteHost}`, + installServer: `rsync -rzu -e 'ssh -o ControlPath=~/.ssh/control-syncremote' ${localServerDir} ${remoteUser}@${remoteHost}:${remoteServerDir}`, + runServer: `bash ${__dirname}/../server/run.sh ${remoteUser} ${remoteHost}`, + getServerPortBufer: `ssh -f -o ControlPath=~/.ssh/control-syncremote ${remoteUser}@${remoteHost} "cat ${remoteServerDir}/port.log && rm ${remoteServerDir}/port.log"` +} + +const getPort = (buffer) => { + return buffer.toString().split(' ').pop() +} + +try { + execSync(commands.createControlMaster, { stdio: 'inherit' }) + console.log('Transferring server.js to remote machine...'); + + const start = Date.now() + execSync(commands.installServer); + + execSync(commands.runServer, { stdio: 'inherit' }); + + const portBuffer = execSync(commands.getServerPortBufer) + port = getPort(portBuffer) + if (!port) { + throw new Error("Failed to read port") + } + console.log("PORT", port) + const end2 = Date.now() + console.log('Finished executing server', (end2 - start) / 1000) + runClient(port, remoteHost, localRootDir, baseRsyncCommand) + + // process.exit() +} catch (error) { + console.error('Error:', error.message); +} + diff --git a/lua/sync-remote/scripts/watcher.sh b/lua/sync-remote/scripts/watcher.sh index 2703f22..92b5553 100644 --- a/lua/sync-remote/scripts/watcher.sh +++ b/lua/sync-remote/scripts/watcher.sh @@ -5,7 +5,7 @@ sync_remote() { local_full_path=${WATCHMAN_ROOT}/$1 remote_full_path=$remote_root/$1 if [ -e "$local_full_path" ]; then - rsync -rvzu --delete --no-whole-file $local_full_path $remote_full_path + rsync -rvzu -e 'ssh -o ControlPath=~/.ssh/config/control-syncremote' --delete --no-whole-file $local_full_path $remote_full_path fi } diff --git a/lua/sync-remote/server/app/server.js b/lua/sync-remote/server/app/server.js new file mode 100644 index 0000000..1e76980 --- /dev/null +++ b/lua/sync-remote/server/app/server.js @@ -0,0 +1,28 @@ +const net = require('net'); +const startWatcher = require('./watch.js') + +const remoteDir = "/home/nabil.ashraf/remotecms" +const port = 7001; + +const server = net.createServer((socket) => { + console.log(`Client connected: ${socket.remoteAddress}:${socket.remotePort}`); + startWatcher(remoteDir, socket) + + socket.on('data', (data) => { + console.log(`Received data from client: ${data.toString()}`); + const response = JSON.stringify({ + type: "connection_established", + data: "hello from server" + }); + socket.write(response); + }); + + socket.on('close', () => { + console.log('Client disconnected'); + }); +}); + +server.listen(port, () => { + console.log(`Server is listening on port ${port}`); +}); + diff --git a/lua/sync-remote/server/app/watch.js b/lua/sync-remote/server/app/watch.js new file mode 100644 index 0000000..5f56bce --- /dev/null +++ b/lua/sync-remote/server/app/watch.js @@ -0,0 +1,65 @@ +const watchman = require('fb-watchman'); +const path = require('path') + +const ignoredFiles = [".swo", ".swp", ".swx"] + +const isFileWatched = (file) => file.name.includes('.gitignore') || (!file.name.includes('.git') && !ignoredFiles.includes(path.extname(file.name))) + +const processFileForClient = (event) => (file) => { + let fileAbsolutePath = `${event.root}/${file.name}` + let fileRelativePath = file.name + + if (file.type === 'd') { + fileAbsolutePath += '/' + fileRelativePath += '/' + } + return { ...file, absolutePath: fileAbsolutePath, name: fileRelativePath } +} + +const handleWatchEvent = (event, socket) => { + if (event.is_fresh_instance) { + return; + } + const modifiedFiles = event.files + .filter(isFileWatched) + .map(processFileForClient(event)) + .reverse(); + + if (modifiedFiles.length) { + const responseEvent = { + type: "file_modified", + data: { + root: event.root, + files: modifiedFiles + } + } + + socket.write(JSON.stringify(responseEvent)); + } +} + +const startWatcher = (remoteDir, socket) => { + const client = new watchman.Client(); + client.command(['watch', remoteDir], (watchError, _watchEvent) => { + if (watchError) { + console.error('Error during watch command:', watchError); + return socket.end(); + } + console.log(`Started watching ${remoteDir}`) + + client.command(['subscribe', remoteDir, 'syncRemoteSub', { + fields: ['name', 'size', 'exists', 'type'], + }], (subscribeError, _subscribeEvent) => { + if (subscribeError) { + return console.error('Error during subscribe command:', subscribeError); + } + client.on('subscription', (event) => { + handleWatchEvent(kaku, socket) + }); + }); + }) +} + +module.exports = startWatcher + + diff --git a/lua/sync-remote/server/run.sh b/lua/sync-remote/server/run.sh new file mode 100644 index 0000000..fc8eb55 --- /dev/null +++ b/lua/sync-remote/server/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +REMOTE_USER=$1 +REMOTE_HOST=$2 +REMOTE_SERVER_PATH="~/.sync-remote/" + +ssh "$REMOTE_USER@$REMOTE_HOST" < ~/.sync-remote/port.log 2>&1 & +EOF diff --git a/lua/sync-remote/server/setup.sh b/lua/sync-remote/server/setup.sh new file mode 100644 index 0000000..36a3eb7 --- /dev/null +++ b/lua/sync-remote/server/setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +REMOTE_USER="nabil.ashraf" +REMOTE_HOST="172.39.39.114" +REMOTE_SERVER_PATH="~/.sync-remote/" +LOCAL_SERVER_PATH="$(cd "$(dirname $0)" && pwd)/app/" + +# rsync -rzu -e 'ssh -o ControlPath=~/.ssh/control-syncremote' $LOCAL_SERVER_PATH $REMOTE_USER@$REMOTE_HOST:$REMOTE_SERVER_PATH + +ssh "$REMOTE_USER@$REMOTE_HOST" <= 5.4", + "luacheck >= 0.24.0-2", +}