Skip to content

Commit

Permalink
ovl: fix push directory when dest path is absolute
Browse files Browse the repository at this point in the history
Properly fix push when dest dest path is absolute. Also added unitest.
Close #1.
  • Loading branch information
aitjcize committed May 11, 2024
1 parent 577351b commit dfef828
Show file tree
Hide file tree
Showing 4 changed files with 165 additions and 44 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ jobs:
run: git config --global --add safe.directory $PWD
- name: test
run: go test -v ./...
- name: unittest
run: |
source .venv/bin/activate
./scripts/ovl_unittest.py
- name: e2e_test
run: |
source .venv/bin/activate
Expand Down
Binary file added scripts/__pycache__/ovl.cpython-311.pyc
Binary file not shown.
101 changes: 57 additions & 44 deletions scripts/ovl.py
Original file line number Diff line number Diff line change
Expand Up @@ -686,7 +686,7 @@ def ParseMethodSubCommands(cls):


@ParseMethodSubCommands
class OverlordCLIClient:
class OverlordCliClient:
"""Overlord command line interface client."""

SUBCOMMANDS = []
Expand Down Expand Up @@ -1182,50 +1182,63 @@ def Shell(self, command=None):
else:
raise

@Command('push', 'push a file or directory to remote', [
Arg('srcs', nargs='+', metavar='SOURCE'),
Arg('dst', metavar='DESTINATION')])
def Push(self, args):
self.CheckClient()
def _RemoteDirExists(self, path):
return ast.literal_eval(self.CheckOutput(
'stat %s >/dev/null 2>&1 && echo True || echo False' % path))

@AutoRetry('push', _RETRY_TIMES)
def _push(src, dst):
src_base = os.path.basename(src)
def _RemoteGetPathType(self, path):
return self.CheckOutput('stat \'%s\' --printf \'%%F\' '
'2>/dev/null' % path).strip()

# Local file is a link
if os.path.islink(src):
pbar = ProgressBar(src_base)
link_path = os.readlink(src)
self.CheckOutput('mkdir -p %(dirname)s; '
'if [ -d "%(dst)s" ]; then '
'ln -sf "%(link_path)s" "%(dst)s/%(link_name)s"; '
'else ln -sf "%(link_path)s" "%(dst)s"; fi' %
dict(dirname=os.path.dirname(dst),
link_path=link_path, dst=dst,
link_name=src_base))
pbar.End()
return

mode = '0%o' % (0x1FF & os.stat(src).st_mode)
url = ('%s:%d/api/agent/upload/%s?dest=%s&perm=%s' %
(self._state.host, self._state.port,
urllib.parse.quote(self._selected_mid), dst, mode))
try:
UrlOpen(self._state, url + '&filename=%s' % src_base)
except urllib.error.HTTPError as e:
msg = json.loads(e.read()).get('error', None)
raise RuntimeError('push: %s' % msg) from None
@AutoRetry('push', _RETRY_TIMES)
def _PushSingleFile(self, src, dst):
src_base = os.path.basename(src)

# Local file is a link
if os.path.islink(src):
pbar = ProgressBar(src_base)
self._HTTPPostFile(url, src, pbar.SetProgress,
self._state.username, self._state.password)
link_path = os.readlink(src)
self.CheckOutput('mkdir -p %(dirname)s; '
'if [ -d "%(dst)s" ]; then '
'ln -sf "%(link_path)s" "%(dst)s/%(link_name)s"; '
'else ln -sf "%(link_path)s" "%(dst)s"; fi' %
dict(dirname=os.path.dirname(dst),
link_path=link_path, dst=dst,
link_name=src_base))
pbar.End()
return

mode = '0%o' % (0x1FF & os.stat(src).st_mode)
url = ('%s:%d/api/agent/upload/%s?dest=%s&perm=%s' %
(self._state.host, self._state.port,
urllib.parse.quote(self._selected_mid), dst, mode))
try:
UrlOpen(self._state, url + '&filename=%s' % src_base)
except urllib.error.HTTPError as e:
msg = json.loads(e.read()).get('error', None)
raise RuntimeError('push: %s' % msg) from None

pbar = ProgressBar(src_base)
self._HTTPPostFile(url, src, pbar.SetProgress,
self._state.username, self._state.password)
pbar.End()

@Command('push', 'push a file or directory to remote', [
Arg('srcs', nargs='+', metavar='SOURCE'),
Arg('dst', metavar='DESTINATION')])
def Push(self, args):
self.CheckClient()

def _push_single_target(src, dst):
if os.path.isdir(src):
dst_exists = ast.literal_eval(self.CheckOutput(
'stat %s >/dev/null 2>&1 && echo True || echo False' % dst))
for root, unused_x, files in os.walk(src):
src = os.path.abspath(src)
base_dir_name = os.path.basename(src)
dst_exists = self._RemoteDirExists(dst)

if dst_exists and self._RemoteGetPathType(dst) != 'directory':
raise RuntimeError('push: %s: Not a directory' % dst)

for subpath, unused_x, files in os.walk(src):
# If destination directory does not exist, we should strip the first
# layer of directory. For example: src_dir contains a single file 'A'
#
Expand All @@ -1236,16 +1249,16 @@ def _push_single_target(src, dst):
# If dest_dir does not exist, the resulting directory structure should
# be:
# dest_dir/A
dst_root = root if dst_exists else root[len(src):].lstrip('/')
dst_root = base_dir_name if dst_exists else ''
relpath = os.path.relpath(subpath, src)
for name in files:
_push(os.path.join(root, name),
os.path.join(dst, dst_root, name))
self._PushSingleFile(os.path.join(subpath, name),
os.path.join(dst, dst_root, relpath, name))
else:
_push(src, dst)
self._PushSingleFile(src, dst)

if len(args.srcs) > 1:
dst_type = self.CheckOutput('stat \'%s\' --printf \'%%F\' '
'2>/dev/null' % args.dst).strip()
dst_type = self._RemoteGetPathType(args.dst)
if not dst_type:
raise RuntimeError('push: %s: No such file or directory' % args.dst)
if dst_type != 'directory':
Expand Down Expand Up @@ -1446,7 +1459,7 @@ def main():
handler.setFormatter(formatter)
logger.addHandler(handler)

ovl = OverlordCLIClient()
ovl = OverlordCliClient()
try:
ovl.Main()
except KeyboardInterrupt:
Expand Down
104 changes: 104 additions & 0 deletions scripts/ovl_unittest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
#!/usr/bin/env python3

import argparse
import os
import unittest

from ovl import OverlordCliClient


class PushUnittest(unittest.TestCase):
"""Test cases for push command."""
def setUp(self):
self.cli = OverlordCliClient()
self.results = []
self.git_root = os.path.normpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)), '..'))

self.cli.CheckClient = lambda: None
self.cli._PushSingleFile = self.PushSingleFile

def PushSingleFile(self, src, dst):
"""Mock function for _PushSingleFile."""
self.results.append((src, dst))

def test_push_src_abs_dir_remote_exists_dir(self):
"""Test push src dir to remote dir when remote dir exists."""

dst = '/home/aitjcize/some_dir'
self.cli._RemoteDirExists = lambda path: True
self.cli._RemoteGetPathType = lambda path: 'directory'
args = argparse.Namespace(srcs=[os.path.join(self.git_root, 'cmd')], dst=dst)
self.cli.Push(args)

self.assertEqual(sorted(self.results), sorted([
(os.path.join(self.git_root, 'cmd/ghost/main.go'),
os.path.join(dst, 'cmd/ghost/main.go')),
(os.path.join(self.git_root, 'cmd/overlordd/main.go'),
os.path.join(dst, 'cmd/overlordd/main.go')),
]))

def test_push_src_abs_dir_remote_exists_but_not_dir(self):
"""Test push src dir to remote dir when remote path exists but not a
directory."""

dst = '/home/aitjcize/some_dir'
self.cli._RemoteDirExists = lambda path: True
self.cli._RemoteGetPathType = lambda path: 'file'
args = argparse.Namespace(srcs=[os.path.join(self.git_root, 'cmd')], dst=dst)

self.assertRaises(RuntimeError, self.cli.Push, args)

def test_push_src_abs_dir_remote_does_not_exist(self):
"""Test push src dir to remote dir when remote dir does not exist."""

dst = '/home/aitjcize/some_dir'
self.cli._RemoteDirExists = lambda path: False
self.cli._RemoteGetPathType = lambda path: 'directory'
args = argparse.Namespace(srcs=[os.path.join(self.git_root, 'cmd')], dst=dst)
self.cli.Push(args)

self.assertEqual(sorted(self.results), sorted([
(os.path.join(self.git_root, 'cmd/ghost/main.go'),
os.path.join(dst, 'ghost/main.go')),
(os.path.join(self.git_root, 'cmd/overlordd/main.go'),
os.path.join(dst, 'overlordd/main.go')),
]))

def test_push_rel_dir_remote_rel_dir(self):
"""Test push relative src dir to relative remote dir."""

os.chdir(self.git_root)
dst = 'some_dir'
self.cli._RemoteDirExists = lambda path: True
self.cli._RemoteGetPathType = lambda path: 'directory'
args = argparse.Namespace(srcs=['cmd'], dst=dst)
self.cli.Push(args)

self.assertEqual(sorted(self.results), sorted([
(os.path.join(self.git_root, 'cmd/ghost/main.go'),
os.path.join(dst, 'cmd/ghost/main.go')),
(os.path.join(self.git_root, 'cmd/overlordd/main.go'),
os.path.join(dst, 'cmd/overlordd/main.go')),
]))

def test_push_rel_dir_remote_rel_does_not_exist(self):
"""Test push relative src dir to relative remote dir when remote dir does
not exist."""

os.chdir(self.git_root)
dst = 'some_dir'
self.cli._RemoteDirExists = lambda path: False
self.cli._RemoteGetPathType = lambda path: 'directory'
args = argparse.Namespace(srcs=['cmd'], dst=dst)
self.cli.Push(args)

self.assertEqual(sorted(self.results), sorted([
(os.path.join(self.git_root, 'cmd/ghost/main.go'),
os.path.join(dst, 'ghost/main.go')),
(os.path.join(self.git_root, 'cmd/overlordd/main.go'),
os.path.join(dst, 'overlordd/main.go')),
]))

if __name__ == '__main__':
unittest.main()

0 comments on commit dfef828

Please sign in to comment.